diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index f201dc1b8..cc6775e82 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -1,34 +1,40 @@ name: 🌥 Docker Push on: - release: - types: [published] + workflow_run: + workflows: ["🎉 Release Binary"] + types: + - completed workflow_dispatch: jobs: docker: runs-on: ubuntu-latest steps: - - - name: Checkout + - name: Git Checkout uses: actions/checkout@v2 - - - name: Set up QEMU + + - name: Get Github tag + id: meta + run: | + echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/nuclei/releases/latest" | jq -r .tag_name)" + + - name: Set up QEMU uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx + + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub + + - name: Login to DockerHub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - - - name: Build and push + + - name: Build and push uses: docker/build-push-action@v2 with: context: . platforms: linux/amd64,linux/arm64,linux/arm push: true - tags: projectdiscovery/nuclei:latest \ No newline at end of file + tags: projectdiscovery/nuclei:latest,projectdiscovery/nuclei:${{ steps.meta.outputs.tag }} \ No newline at end of file diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index c0748e870..3a9983470 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -1,30 +1,26 @@ name: 🎉 Release Binary on: create: - tags: - - v* workflow_dispatch: jobs: release: runs-on: ubuntu-latest + if: ${{ startsWith(github.ref, 'refs/tags/v') }} + steps: - - - name: "Check out code" - uses: actions/checkout@v2 + - uses: actions/checkout@v2 with: fetch-depth: 0 - - - name: "Set up Go" - uses: actions/setup-go@v2 + + - uses: actions/setup-go@v2 with: go-version: 1.17 - - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: "Create release on GitHub" - uses: goreleaser/goreleaser-action@v2 + + - uses: goreleaser/goreleaser-action@v2 with: args: "release --rm-dist" version: latest - workdir: v2/ \ No newline at end of file + workdir: v2/ + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7a7431ac7..7d3467d34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -FROM golang:1.17.1-alpine as build-env -RUN GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei +FROM golang:1.17.2-alpine as build-env +RUN go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest FROM alpine:3.14 -RUN apk add --no-cache bind-tools ca-certificates +RUN apk add --no-cache bind-tools ca-certificates chromium COPY --from=build-env /go/bin/nuclei /usr/local/bin/nuclei ENTRYPOINT ["nuclei"] diff --git a/README.md b/README.md index 17d7d211c..bfa68972e 100644 --- a/README.md +++ b/README.md @@ -90,30 +90,29 @@ TARGET: -l, -list string path to file containing a list of target URLs/hosts to scan (one per line) TEMPLATES: - -tl list all available templates -t, -templates string[] template or template directory paths to include in the scan - -w, -workflows string[] list of workflows to run - -nt, -new-templates run newly added templates only + -nt, -new-templates run only new templates added in latest nuclei-templates release + -w, -workflows string[] workflow or workflow directory paths to include in the scan -validate validate the passed templates to nuclei + -tl list all available templates FILTERING: - -tags string[] execute a subset of templates that contain the provided tags - -include-tags string[] tags from the default deny list that permit executing more intrusive templates - -etags, -exclude-tags string[] exclude templates with the provided tags - -include-templates string[] templates to be executed even if they are excluded either by default or configuration - -exclude-templates, -exclude string[] template or template directory paths to exclude - -severity, -impact value[] Templates to run based on severity. Possible values: info, low, medium, high, critical - -author string[] execute templates that are (co-)created by the specified authors + -tags string[] execute a subset of templates that contain the provided tags + -etags, -exclude-tags string[] exclude templates with the provided tags + -itags, -include-tags string[] tags from the default deny list that permit executing more intrusive templates + -et, -exclude-templates string[] template or template directory paths to exclude + -it, -include-templates string[] templates to be executed even if they are excluded either by default or configuration + -s, -severity value[] Templates to run based on severity. Possible values - info,low,medium,high,critical + -es, -exclude-severity value[] Templates to exclude based on severity. Possible values - info,low,medium,high,critical + -a, -author string[] execute templates that are (co-)created by the specified authors OUTPUT: -o, -output string output file to write found issues/vulnerabilities -silent display findings only - -v, -verbose show verbose output - -vv display extra verbose information -nc, -no-color disable output content coloring (ANSI escape codes) -json write output in JSONL(ines) format -irr, -include-rr include request/response pairs in the JSONL output (for findings only) - -nm, -no-meta don't display match metadata in CLI output + -nm, -no-meta don't display match metadata -nts, -no-timestamp don't display timestamp metadata in CLI output -rdb, -report-db string local nuclei reporting database (always use this to persist report data) -me, -markdown-export string directory to export results in markdown format @@ -125,37 +124,39 @@ CONFIGURATIONS: -H, -header string[] custom headers in header:value format -V, -var value custom vars in var=value format -r, -resolvers string file containing resolver list for nuclei - -system-resolvers use system DNS resolving as error fallback + -sr, -system-resolvers use system DNS resolving as error fallback -passive enable passive HTTP response processing mode - -env-vars enable environment variables support + -ev, -env-vars enable environment variables to be used in template INTERACTSH: - -no-interactsh disable interactsh server for OOB testing - -interactsh-url string interactsh server url for self-hosted instance (default "https://interact.sh") - -interactsh-token string authentication token for self-hosted interactsh server - -interactions-cache-size int number of requests to keep in the interactions cache (default 5000) - -interactions-eviction int number of seconds to wait before evicting requests from cache (default 60) - -interactions-poll-duration int number of seconds to wait before each interaction poll request (default 5) - -interactions-cooldown-period int extra time for interaction polling before exiting (default 5) + -iserver, -interactsh-server string interactsh server url for self-hosted instance (default "https://interactsh.com") + -itoken, -interactsh-token string authentication token for self-hosted interactsh server + -interactions-cache-size int number of requests to keep in the interactions cache (default 5000) + -interactions-eviction int number of seconds to wait before evicting requests from cache (default 60) + -interactions-poll-duration int number of seconds to wait before each interaction poll request (default 5) + -interactions-cooldown-period int extra time for interaction polling before exiting (default 5) + -ni, -no-interactsh disable interactsh server for OAST testing, exclude OAST based templates RATE-LIMIT: -rl, -rate-limit int maximum number of requests to send per second (default 150) -rlm, -rate-limit-minute int maximum number of requests to send per minute -bs, -bulk-size int maximum number of hosts to be analyzed in parallel per template (default 25) - -c, -concurrency int maximum number of templates to be executed in parallel (default 10) + -c, -concurrency int maximum number of templates to be executed in parallel (default 25) OPTIMIZATIONS: -timeout int time to wait in seconds before timeout (default 5) -retries int number of times to retry a failed request (default 1) - -max-host-error int max errors for a host before skipping from scan (default 30) + -mhe, -max-host-error int max errors for a host before skipping from scan (default 30) -project use a project folder to avoid sending same request multiple times - -project-path string set a specific project path (default "$TMPDIR/") + -project-path string set a specific project path -spm, -stop-at-first-path stop processing HTTP requests after the first match (may break template/workflow logic) + -stream Stream mode - start elaborating without sorting the input HEADLESS: - -headless enable templates that require headless browser support - -page-timeout int seconds to wait for each page in headless mode (default 20) - -show-browser show the browser on the screen when running templates with headless mode + -headless enable templates that require headless browser support + -page-timeout int seconds to wait for each page in headless mode (default 20) + -sb, -show-browser show the browser on the screen when running templates with headless mode + -sc, -system-chrome Use local installed chrome browser instead of nuclei installed DEBUG: -debug show all requests and responses @@ -163,22 +164,24 @@ DEBUG: -debug-resp show all received responses -proxy, -proxy-url string URL of the HTTP proxy server -proxy-socks-url string URL of the SOCKS proxy server - -trace-log string file to write sent requests trace log + -tlog, -trace-log string file to write sent requests trace log -version show nuclei version + -v, -verbose show verbose output + -vv display extra verbose information -tv, -templates-version shows the version of the installed nuclei-templates UPDATE: - -update update nuclei to the latest released version - -ut, -update-templates update the community templates to latest released version - -nut, -no-update-templates do not check for nuclei-templates updates - -ud, -update-directory string overwrite the default nuclei-templates directory (default "$HOME/nuclei-templates") + -update update nuclei engine to the latest released version + -ut, -update-templates update nuclei-templates to latest released version + -ud, -update-directory string overwrite the default directory to install nuclei-templates + -duc, -disable-update-check disable automatic nuclei/templates update check STATISTICS: -stats display statistics about the running scan - -stats-json write statistics data to an output file in JSONL(ines) format + -sj, -stats-json write statistics data to an output file in JSONL(ines) format -si, -stats-interval int number of seconds to wait between showing a statistics update (default 5) - -metrics expose nuclei metrics on a port - -metrics-port int port to expose nuclei metrics on (default 9092) + -m, -metrics expose nuclei metrics on a port + -mp, -metrics-port int port to expose nuclei metrics on (default 9092) ``` ### Running Nuclei diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 86191bdba..30ed998fb 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -230,6 +230,19 @@ Workflows is a list of workflows to execute for a template.
+
+ +self-contained bool + +
+
+ +Self Contained marks Requests for the template as self-contained + +
+ +
+ @@ -823,14 +836,14 @@ in a combined manner allowing multirequest based matchers. Attack is the type of payload combinations to perform. -Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates +batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. Valid values: - - sniper + - batteringram - pitchfork @@ -1192,6 +1205,19 @@ StopAtFirstMatch stops the execution of the requests and template as soon as a m
+
+ +skip-variables-check bool + +
+
+ +SkipVariablesCheck skips the check for unresolved variables in request + +
+ +
+ @@ -2026,6 +2052,19 @@ Recursion determines if resolver should recurse all records to get fresh results
+
+ +resolvers []string + +
+
+ +Resolvers to use for the dns requests + +
+ +
+ @@ -2286,14 +2325,14 @@ host: Attack is the type of payload combinations to perform. -Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates +Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates permutations and combinations for all payloads. Valid values: - - sniper + - batteringram - pitchfork diff --git a/integration_tests/http/self-contained.yaml b/integration_tests/http/self-contained.yaml new file mode 100644 index 000000000..4ecacbbd9 --- /dev/null +++ b/integration_tests/http/self-contained.yaml @@ -0,0 +1,18 @@ +id: example-self-contained-input + +info: + name: example-self-contained + author: pd-team + severity: info + +self-contained: true +requests: + - raw: + - | + GET http://localhost:5431/ HTTP/1.1 + Host: {{Hostname}} + + matchers: + - type: word + words: + - This is self-contained response \ No newline at end of file diff --git a/integration_tests/network/self-contained.yaml b/integration_tests/network/self-contained.yaml new file mode 100644 index 000000000..fad3e2ac8 --- /dev/null +++ b/integration_tests/network/self-contained.yaml @@ -0,0 +1,16 @@ +id: example-self-contained-input + +info: + name: example-self-contained + author: pd-team + severity: info + +self-contained: true +network: + - host: + - "localhost:5431" + + matchers: + - type: word + words: + - "Authentication successful" \ No newline at end of file diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index 28c3e2de0..56253b7ed 100755 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -373,6 +373,14 @@ "type": "boolean", "title": "recurse all servers", "description": "Recursion determines if resolver should recurse all records to get fresh results" + }, + "resolvers": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Resolvers", + "description": "Define resolvers to use within the template" } }, "additionalProperties": false, @@ -598,7 +606,7 @@ }, "attack": { "enum": [ - "sniper", + "batteringram", "pitchfork", "clusterbomb" ], @@ -712,6 +720,11 @@ "type": "boolean", "title": "stop at first match", "description": "Stop the execution after a match is found" + }, + "skip-variables-check": { + "type": "boolean", + "title": "skip variable checks", + "description": "Skips the check for unresolved variables in request" } }, "additionalProperties": false, @@ -764,7 +777,7 @@ }, "attack": { "enum": [ - "sniper", + "batteringram", "pitchfork", "clusterbomb" ], @@ -898,6 +911,11 @@ "type": "array", "title": "list of workflows to execute", "description": "List of workflows to execute for template" + }, + "self-contained": { + "type": "boolean", + "title": "mark requests as self-contained", + "description": "Mark Requests for the template as self-contained" } }, "additionalProperties": false, diff --git a/v2/cmd/functional-test/main.go b/v2/cmd/functional-test/main.go index bc95f20b5..fc96fde80 100644 --- a/v2/cmd/functional-test/main.go +++ b/v2/cmd/functional-test/main.go @@ -15,6 +15,7 @@ import ( ) var ( + debug = os.Getenv("DEBUG") == "true" success = aurora.Green("[✓]").String() failed = aurora.Red("[✘]").String() errored = false @@ -65,11 +66,11 @@ func runIndividualTestCase(testcase string) error { if len(parts) > 1 { finalArgs = parts[1:] } - mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, finalArgs) + mainOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*mainNucleiBinary, debug, finalArgs) if err != nil { return errors.Wrap(err, "could not run nuclei main test") } - devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, finalArgs) + devOutput, err := testutils.RunNucleiBinaryAndGetLoadedTemplates(*devNucleiBinary, debug, finalArgs) if err != nil { return errors.Wrap(err, "could not run nuclei dev test") } diff --git a/v2/cmd/functional-test/run.sh b/v2/cmd/functional-test/run.sh index 030b25443..8f6c635c4 100644 --- a/v2/cmd/functional-test/run.sh +++ b/v2/cmd/functional-test/run.sh @@ -7,7 +7,7 @@ echo 'Building Nuclei binary from current branch' go build -o nuclei_dev ../nuclei echo 'Installing latest release of nuclei' -GO111MODULE=on go get -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei +GO111MODULE=on go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei echo 'Starting Nuclei functional test' ./functional-test -main nuclei -dev ./nuclei_dev -testcases testcases.txt \ No newline at end of file diff --git a/v2/cmd/integration-test/dns.go b/v2/cmd/integration-test/dns.go index 7661f48f9..3e1ae8146 100644 --- a/v2/cmd/integration-test/dns.go +++ b/v2/cmd/integration-test/dns.go @@ -14,7 +14,7 @@ type dnsBasic struct{} func (h *dnsBasic) Execute(filePath string) error { var routerErr error - results, err := testutils.RunNucleiAndGetResults(filePath, "one.one.one.one", debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "one.one.one.one", debug) if err != nil { return err } diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 8188c27af..c3e23a7a0 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -7,7 +7,6 @@ import ( "net" "net/http" "net/http/httptest" - "net/http/httputil" "strings" "github.com/julienschmidt/httprouter" @@ -32,14 +31,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/raw-unsafe-request.yaml": &httpRawUnsafeRequest{}, "http/request-condition.yaml": &httpRequestCondition{}, "http/request-condition-new.yaml": &httpRequestCondition{}, -} - -func httpDebugRequestDump(r *http.Request) { - if debug { - if dump, err := httputil.DumpRequest(r, true); err == nil { - fmt.Printf("\nRequest dump: \n%s\n\n", string(dump)) - } - } + "http/self-contained.yaml": &httpRequestSelContained{}, } type httpGetHeaders struct{} @@ -47,16 +39,15 @@ type httpGetHeaders struct{} // Execute executes a test case and returns an error if occurred func (h *httpGetHeaders) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if strings.EqualFold(r.Header.Get("test"), "nuclei") { fmt.Fprintf(w, "This is test headers matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -71,16 +62,15 @@ type httpGetQueryString struct{} // Execute executes a test case and returns an error if occurred func (h *httpGetQueryString) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") { fmt.Fprintf(w, "This is test querystring matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -95,18 +85,16 @@ type httpGetRedirects struct{} // Execute executes a test case and returns an error if occurred func (h *httpGetRedirects) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { http.Redirect(w, r, "/redirected", http.StatusFound) - })) - router.GET("/redirected", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + }) + router.GET("/redirected", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { fmt.Fprintf(w, "This is test redirects matcher text") - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -121,14 +109,13 @@ type httpGet struct{} // Execute executes a test case and returns an error if occurred func (h *httpGet) Execute(filePath string) error { router := httprouter.New() - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { fmt.Fprintf(w, "This is test matcher text") - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -145,8 +132,7 @@ func (h *httpPostBody) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if err := r.ParseForm(); err != nil { routerErr = err return @@ -154,11 +140,11 @@ func (h *httpPostBody) Execute(filePath string) error { if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") { fmt.Fprintf(w, "This is test post-body matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -178,9 +164,7 @@ func (h *httpPostJSONBody) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) - + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { type doc struct { Username string `json:"username"` Password string `json:"password"` @@ -193,11 +177,11 @@ func (h *httpPostJSONBody) Execute(filePath string) error { if strings.EqualFold(obj.Username, "test") && strings.EqualFold(obj.Password, "nuclei") { fmt.Fprintf(w, "This is test post-json-body matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -217,8 +201,7 @@ func (h *httpPostMultipartBody) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if err := r.ParseMultipartForm(1 * 1024); err != nil { routerErr = err return @@ -236,11 +219,11 @@ func (h *httpPostMultipartBody) Execute(filePath string) error { if strings.EqualFold(password[0], "nuclei") && strings.EqualFold(file[0].Filename, "username") { fmt.Fprintf(w, "This is test post-multipart matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -260,8 +243,7 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if err := r.ParseForm(); err != nil { routerErr = err return @@ -269,17 +251,16 @@ func (h *httpRawDynamicExtractor) Execute(filePath string) error { if strings.EqualFold(r.Form.Get("testing"), "parameter") { fmt.Fprintf(w, "Token: 'nuclei'") } - })) - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + }) + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if strings.EqualFold(r.URL.Query().Get("username"), "nuclei") { fmt.Fprintf(w, "Test is test-dynamic-extractor-raw matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -299,16 +280,15 @@ func (h *httpRawGetQuery) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if strings.EqualFold(r.URL.Query().Get("test"), "nuclei") { fmt.Fprintf(w, "Test is test raw-get-query-matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -328,15 +308,13 @@ func (h *httpRawGet) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) - + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { fmt.Fprintf(w, "Test is test raw-get-matcher text") - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -356,8 +334,7 @@ func (h *httpRawPayload) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if err := r.ParseForm(); err != nil { routerErr = err return @@ -368,11 +345,11 @@ func (h *httpRawPayload) Execute(filePath string) error { if strings.EqualFold(r.Form.Get("username"), "test") && (strings.EqualFold(r.Form.Get("password"), "nuclei") || strings.EqualFold(r.Form.Get("password"), "guest")) { fmt.Fprintf(w, "Test is raw-payload matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -392,8 +369,7 @@ func (h *httpRawPostBody) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if err := r.ParseForm(); err != nil { routerErr = err return @@ -401,11 +377,11 @@ func (h *httpRawPostBody) Execute(filePath string) error { if strings.EqualFold(r.Form.Get("username"), "test") && strings.EqualFold(r.Form.Get("password"), "nuclei") { fmt.Fprintf(w, "Test is test raw-post-body-matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -425,8 +401,7 @@ func (h *httpRawCookieReuse) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.POST("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.POST("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if err := r.ParseForm(); err != nil { routerErr = err return @@ -434,9 +409,8 @@ func (h *httpRawCookieReuse) Execute(filePath string) error { if strings.EqualFold(r.Form.Get("testing"), "parameter") { http.SetCookie(w, &http.Cookie{Name: "nuclei", Value: "test"}) } - })) - router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + }) + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { if err := r.ParseForm(); err != nil { routerErr = err return @@ -450,11 +424,11 @@ func (h *httpRawCookieReuse) Execute(filePath string) error { if strings.EqualFold(cookie.Value, "test") { fmt.Fprintf(w, "Test is test-cookie-reuse matcher text") } - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -479,7 +453,7 @@ func (h *httpRawUnsafeRequest) Execute(filePath string) error { }) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, "http://"+ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "http://"+ts.URL, debug) if err != nil { return err } @@ -499,18 +473,48 @@ func (h *httpRequestCondition) Execute(filePath string) error { router := httprouter.New() var routerErr error - router.GET("/200", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + router.GET("/200", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { w.WriteHeader(200) - })) - router.GET("/400", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) + }) + router.GET("/400", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { w.WriteHeader(400) - })) + }) ts := httptest.NewServer(router) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if routerErr != nil { + return routerErr + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type httpRequestSelContained struct{} + +// Execute executes a test case and returns an error if occurred +func (h *httpRequestSelContained) Execute(filePath string) error { + router := httprouter.New() + var routerErr error + + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + _, _ = w.Write([]byte("This is self-contained response")) + }) + server := &http.Server{ + Addr: fmt.Sprintf("localhost:%d", defaultStaticPort), + Handler: router, + } + go func() { + _ = server.ListenAndServe() + }() + defer server.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug) if err != nil { return err } diff --git a/v2/cmd/integration-test/network.go b/v2/cmd/integration-test/network.go index 5bb0fe519..e170b07b9 100644 --- a/v2/cmd/integration-test/network.go +++ b/v2/cmd/integration-test/network.go @@ -7,11 +7,14 @@ import ( ) var networkTestcases = map[string]testutils.TestCase{ - "network/basic.yaml": &networkBasic{}, - "network/hex.yaml": &networkBasic{}, - "network/multi-step.yaml": &networkMultiStep{}, + "network/basic.yaml": &networkBasic{}, + "network/hex.yaml": &networkBasic{}, + "network/multi-step.yaml": &networkMultiStep{}, + "network/self-contained.yaml": &networkRequestSelContained{}, } +const defaultStaticPort = 5431 + type networkBasic struct{} // Execute executes a test case and returns an error if occurred @@ -32,7 +35,7 @@ func (h *networkBasic) Execute(filePath string) error { }) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) if err != nil { return err } @@ -75,7 +78,39 @@ func (h *networkMultiStep) Execute(filePath string) error { }) defer ts.Close() - results, err := testutils.RunNucleiAndGetResults(filePath, ts.URL, debug) + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + if routerErr != nil { + return routerErr + } + + var expectedResultsSize int + if debug { + expectedResultsSize = 3 + } else { + expectedResultsSize = 1 + } + if len(results) != expectedResultsSize { + return errIncorrectResultsCount(results) + } + return nil +} + +type networkRequestSelContained struct{} + +// Execute executes a test case and returns an error if occurred +func (h *networkRequestSelContained) Execute(filePath string) error { + var routerErr error + + ts := testutils.NewTCPServer(func(conn net.Conn) { + defer conn.Close() + + _, _ = conn.Write([]byte("Authentication successful")) + }, defaultStaticPort) + defer ts.Close() + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug) if err != nil { return err } diff --git a/v2/cmd/integration-test/workflow.go b/v2/cmd/integration-test/workflow.go index 08d6e1385..5f39b4ebf 100644 --- a/v2/cmd/integration-test/workflow.go +++ b/v2/cmd/integration-test/workflow.go @@ -23,7 +23,6 @@ type workflowBasic struct{} func (h *workflowBasic) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) @@ -45,7 +44,6 @@ type workflowConditionMatched struct{} func (h *workflowConditionMatched) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) @@ -67,7 +65,6 @@ type workflowConditionUnmatch struct{} func (h *workflowConditionUnmatch) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) @@ -89,7 +86,6 @@ type workflowMatcherName struct{} func (h *workflowMatcherName) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - httpDebugRequestDump(r) fmt.Fprintf(w, "This is test matcher text") }) ts := httptest.NewServer(router) diff --git a/v2/cmd/nuclei/issue-tracker-config.yaml b/v2/cmd/nuclei/issue-tracker-config.yaml index fd3cab274..5364db5f5 100644 --- a/v2/cmd/nuclei/issue-tracker-config.yaml +++ b/v2/cmd/nuclei/issue-tracker-config.yaml @@ -7,7 +7,7 @@ # github contains configuration options for github issue tracker #github: -# # base-url is the optional self-hosted github application url +# # base-url (optional) is the self-hosted github application url # base-url: "" # # username is the username of the github user # username: "" @@ -17,12 +17,14 @@ # token: "" # # project-name is the name of the repository. # project-name: "" -# # issue-label is the label of the created issue type +# # issue-label (optional) is the label of the created issue type # issue-label: "" +# # severity-as-label (optional) sets the sevetiry as the label of the created issue type +# severity-as-label: false # gitlab contains configuration options for gitlab issue tracker #gitlab: -# # base-url is the optional self-hosted gitlab application url +# # base-url (optional) is the self-hosted gitlab application url # base-url: "" # # username is the username of the gitlab user # username: "" @@ -30,14 +32,16 @@ # token: "" # # project-id is the ID of the repository. # project-id: "" -# # issue-label is the label of the created issue type +# # issue-label (optional) is the label of the created issue type # issue-label: "" +# # severity-as-label (optional) sets the sevetiry as the label of the created issue type +# severity-as-label: false # jira contains configuration options for jira issue tracker #jira: -# # cloud is the boolean which tells if Jira instance is running in the cloud or on-prem version is used +# # cloud (optional) is the boolean which tells if Jira instance is running in the cloud or on-prem version is used # cloud: true -# # update-existing is the boolean which tells if the existing, opened issue should be updated or new one should be created +# # update-existing (optional) is the boolean which tells if the existing, opened issue should be updated or new one should be created # update-existing: false # # URL is the jira application url # url: "" @@ -60,11 +64,11 @@ # port: 9200 # # IndexName is the name of the elasticsearch index # index-name: nuclei -# # SSL enables ssl for elasticsearch connection +# # SSL (optional) enables ssl for elasticsearch connection # ssl: false -# # SSLVerification disables SSL verification for elasticsearch +# # SSLVerification (optional) disables SSL verification for elasticsearch # ssl-verification: false # # Username for the elasticsearch instance # username: test # # Password is the password for elasticsearch instance -# password: test \ No newline at end of file +# password: test diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 23fd7b1db..d2f8ebb9f 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -53,92 +53,80 @@ on extensive configurability, massive extensibility and ease of use.`) ) createGroup(flagSet, "templates", "Templates", - flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), - flagSet.StringSliceVarP(&options.Templates, "templates", "t", []string{}, "template or template directory paths to include in the scan"), - flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "list of workflows to run"), - - flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run newly added templates only"), + flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"), + flagSet.StringSliceVarP(&options.Workflows, "workflows", "w", []string{}, "workflow or workflow directory paths to include in the scan"), flagSet.BoolVar(&options.Validate, "validate", false, "validate the passed templates to nuclei"), + flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"), ) createGroup(flagSet, "filters", "Filtering", flagSet.NormalizedStringSliceVar(&options.Tags, "tags", []string{}, "execute a subset of templates that contain the provided tags"), - flagSet.NormalizedStringSliceVar(&options.IncludeTags, "include-tags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list + flagSet.NormalizedStringSliceVarP(&options.IncludeTags, "include-tags", "itags", []string{}, "tags from the default deny list that permit executing more intrusive templates"), // TODO show default deny list flagSet.NormalizedStringSliceVarP(&options.ExcludeTags, "exclude-tags", "etags", []string{}, "exclude templates with the provided tags"), - - flagSet.StringSliceVar(&options.IncludeTemplates, "include-templates", []string{}, "templates to be executed even if they are excluded either by default or configuration"), - flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude", "exclude-templates", []string{}, "template or template directory paths to exclude"), - - flagSet.VarP(&options.Severities, "impact", "severity", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), - flagSet.NormalizedStringSliceVar(&options.Author, "author", []string{}, "execute templates that are (co-)created by the specified authors"), + flagSet.StringSliceVarP(&options.IncludeTemplates, "include-templates", "it", []string{}, "templates to be executed even if they are excluded either by default or configuration"), + flagSet.StringSliceVarP(&options.ExcludedTemplates, "exclude-templates", "et", []string{}, "template or template directory paths to exclude"), + flagSet.VarP(&options.Severities, "severity", "s", fmt.Sprintf("Templates to run based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), + flagSet.VarP(&options.ExcludeSeverities, "exclude-severity", "es", fmt.Sprintf("Templates to exclude based on severity. Possible values: %s", severity.GetSupportedSeverities().String())), + flagSet.NormalizedStringSliceVarP(&options.Author, "author", "a", []string{}, "execute templates that are (co-)created by the specified authors"), ) createGroup(flagSet, "output", "Output", flagSet.StringVarP(&options.Output, "output", "o", "", "output file to write found issues/vulnerabilities"), - flagSet.BoolVar(&options.Silent, "silent", false, "display findings only"), - flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), - flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display extra verbose information"), flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"), - flagSet.BoolVar(&options.JSON, "json", false, "write output in JSONL(ines) format"), flagSet.BoolVarP(&options.JSONRequests, "include-rr", "irr", false, "include request/response pairs in the JSONL output (for findings only)"), - flagSet.BoolVarP(&options.NoMeta, "no-meta", "nm", false, "don't display match metadata"), flagSet.BoolVarP(&options.NoTimestamp, "no-timestamp", "nts", false, "don't display timestamp metadata in CLI output"), flagSet.StringVarP(&options.ReportingDB, "report-db", "rdb", "", "local nuclei reporting database (always use this to persist report data)"), - - flagSet.StringVarP(&options.DiskExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"), + flagSet.StringVarP(&options.MarkdownExportDirectory, "markdown-export", "me", "", "directory to export results in markdown format"), flagSet.StringVarP(&options.SarifExport, "sarif-export", "se", "", "file to export results in SARIF format"), ) createGroup(flagSet, "configs", "Configurations", flagSet.StringVar(&cfgFile, "config", "", "path to the nuclei configuration file"), flagSet.StringVarP(&options.ReportingConfig, "report-config", "rc", "", "nuclei reporting module configuration file"), // TODO merge into the config file or rename to issue-tracking - flagSet.StringSliceVarP(&options.CustomHeaders, "header", "H", []string{}, "custom headers in header:value format"), - flagSet.RuntimeMapVarP(&options.Vars, "var", "V", []string{}, "custom vars in var=value format"), - flagSet.StringVarP(&options.ResolversFile, "resolvers", "r", "", "file containing resolver list for nuclei"), - flagSet.BoolVar(&options.SystemResolvers, "system-resolvers", false, "use system DNS resolving as error fallback"), + flagSet.BoolVarP(&options.SystemResolvers, "system-resolvers", "sr", false, "use system DNS resolving as error fallback"), flagSet.BoolVar(&options.OfflineHTTP, "passive", false, "enable passive HTTP response processing mode"), - flagSet.BoolVar(&options.EnvironmentVariables, "env-vars", false, "enable environment variables support"), + flagSet.BoolVarP(&options.EnvironmentVariables, "env-vars", "ev", false, "enable environment variables to be used in template"), ) createGroup(flagSet, "interactsh", "interactsh", - flagSet.BoolVar(&options.NoInteractsh, "no-interactsh", false, "disable interactsh server for OOB testing"), - flagSet.StringVar(&options.InteractshURL, "interactsh-url", "https://interact.sh", "interactsh server url for self-hosted instance"), - flagSet.StringVar(&options.InteractshToken, "interactsh-token", "", "authentication token for self-hosted interactsh server"), + flagSet.StringVarP(&options.InteractshURL, "interactsh-server", "iserver", "https://interactsh.com", "interactsh server url for self-hosted instance"), + flagSet.StringVarP(&options.InteractshToken, "interactsh-token", "itoken", "", "authentication token for self-hosted interactsh server"), flagSet.IntVar(&options.InteractionsCacheSize, "interactions-cache-size", 5000, "number of requests to keep in the interactions cache"), flagSet.IntVar(&options.InteractionsEviction, "interactions-eviction", 60, "number of seconds to wait before evicting requests from cache"), flagSet.IntVar(&options.InteractionsPollDuration, "interactions-poll-duration", 5, "number of seconds to wait before each interaction poll request"), flagSet.IntVar(&options.InteractionsColldownPeriod, "interactions-cooldown-period", 5, "extra time for interaction polling before exiting"), + flagSet.BoolVarP(&options.NoInteractsh, "no-interactsh", "ni", false, "disable interactsh server for OAST testing, exclude OAST based templates"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum number of requests to send per second"), flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), flagSet.IntVarP(&options.BulkSize, "bulk-size", "bs", 25, "maximum number of hosts to be analyzed in parallel per template"), - flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 10, "maximum number of templates to be executed in parallel"), + flagSet.IntVarP(&options.TemplateThreads, "concurrency", "c", 25, "maximum number of templates to be executed in parallel"), ) createGroup(flagSet, "optimization", "Optimizations", flagSet.IntVar(&options.Timeout, "timeout", 5, "time to wait in seconds before timeout"), flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"), - flagSet.IntVar(&options.MaxHostError, "max-host-error", 30, "max errors for a host before skipping from scan"), - + flagSet.IntVarP(&options.MaxHostError, "max-host-error", "mhe", 30, "max errors for a host before skipping from scan"), flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"), flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"), - flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-path", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"), + flagSet.BoolVar(&options.Stream, "stream", false, "Stream mode - start elaborating without sorting the input"), ) createGroup(flagSet, "headless", "Headless", flagSet.BoolVar(&options.Headless, "headless", false, "enable templates that require headless browser support"), flagSet.IntVar(&options.PageTimeout, "page-timeout", 20, "seconds to wait for each page in headless mode"), - flagSet.BoolVar(&options.ShowBrowser, "show-browser", false, "show the browser on the screen when running templates with headless mode"), + flagSet.BoolVarP(&options.ShowBrowser, "show-browser", "sb", false, "show the browser on the screen when running templates with headless mode"), + flagSet.BoolVarP(&options.UseInstalledChrome, "system-chrome", "sc", false, "Use local installed chrome browser instead of nuclei installed"), ) createGroup(flagSet, "debug", "Debug", @@ -150,27 +138,26 @@ on extensive configurability, massive extensibility and ease of use.`) TODO should auto-set the HTTP_PROXY variable for the process? */ flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), - - flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"), - + flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), + flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), + flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), flagSet.BoolVarP(&options.TemplatesVersion, "templates-version", "tv", false, "shows the version of the installed nuclei-templates"), ) createGroup(flagSet, "update", "Update", - flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei to the latest released version"), - flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update the community templates to latest released version"), - flagSet.BoolVarP(&options.NoUpdateTemplates, "no-update-templates", "nut", false, "do not check for nuclei-templates updates"), - flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default nuclei-templates directory"), + flagSet.BoolVar(&options.UpdateNuclei, "update", false, "update nuclei engine to the latest released version"), + flagSet.BoolVarP(&options.UpdateTemplates, "update-templates", "ut", false, "update nuclei-templates to latest released version"), + flagSet.StringVarP(&options.TemplatesDirectory, "update-directory", "ud", templatesDirectory, "overwrite the default directory to install nuclei-templates"), + flagSet.BoolVarP(&options.NoUpdateTemplates, "disable-update-check", "duc", false, "disable automatic nuclei/templates update check"), ) createGroup(flagSet, "stats", "Statistics", flagSet.BoolVar(&options.EnableProgressBar, "stats", false, "display statistics about the running scan"), - flagSet.BoolVar(&options.StatsJSON, "stats-json", false, "write statistics data to an output file in JSONL(ines) format"), + flagSet.BoolVarP(&options.StatsJSON, "stats-json", "sj", false, "write statistics data to an output file in JSONL(ines) format"), flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"), - - flagSet.BoolVar(&options.Metrics, "metrics", false, "expose nuclei metrics on a port"), - flagSet.IntVar(&options.MetricsPort, "metrics-port", 9092, "port to expose nuclei metrics on"), + flagSet.BoolVarP(&options.Metrics, "metrics", "m", false, "expose nuclei metrics on a port"), + flagSet.IntVarP(&options.MetricsPort, "metrics-port", "mp", 9092, "port to expose nuclei metrics on"), ) _ = flagSet.Parse() @@ -188,3 +175,10 @@ func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags currentFlag.Group(groupName) } } + +/* +HacktoberFest update: Below, you can find our ticket recommendations. Tasks with the "good first issue" label are suitable for first time contributors. If you have other ideas, or need help with getting started, join our Discord channel or reach out to @forgedhallpass. + +https://github.com/issues?q=is%3Aopen+is%3Aissue+user%3Aprojectdiscovery+label%3AHacktoberfest + +*/ diff --git a/v2/go.mod b/v2/go.mod index 3f5502466..e5e74498f 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -31,15 +31,17 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e - github.com/projectdiscovery/goflags v0.0.7 + github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 + github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 + github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa - github.com/projectdiscovery/interactsh v0.0.4 + github.com/projectdiscovery/interactsh v0.0.6 github.com/projectdiscovery/nuclei-updatecheck-api v0.0.0-20210914222811-0a072d262f77 github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a github.com/projectdiscovery/retryablehttp-go v1.0.2 - github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 + github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d github.com/projectdiscovery/yamldoc-go v1.0.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 @@ -63,6 +65,7 @@ require ( golang.org/x/text v0.3.7 google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 + moul.io/http2curl v1.0.0 ) require ( @@ -72,6 +75,9 @@ require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antchfx/xpath v1.1.6 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/bits-and-blooms/bloom/v3 v3.0.1 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -84,16 +90,18 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.0.0 // indirect - github.com/google/uuid v1.2.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/css v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/itchyny/timefmt-go v0.1.3 // indirect github.com/jasonlvhit/gocron v0.0.1 // indirect github.com/karlseguin/ccache/v2 v2.0.8 // indirect - github.com/klauspost/compress v1.13.4 // indirect + github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/mattn/go-isatty v0.0.13 // indirect + github.com/microcosm-cc/bluemonday v1.0.15 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -103,6 +111,7 @@ require ( github.com/projectdiscovery/mapcidr v0.0.8 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/tklauser/go-sysconf v0.3.7 // indirect github.com/tklauser/numcpus v0.2.3 // indirect github.com/trivago/tgo v1.0.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index d908b8fe3..b95089c7e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -31,13 +31,21 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= +github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Ice3man543/nvd v1.0.8 h1:2CBEgOxyWAkQocnnmEMmRtVPWooPRvcuHFLWj48EM4c= github.com/Ice3man543/nvd v1.0.8/go.mod h1:0DxLJk6revOcJKiZxa2K+rNF/HO1zJO97lqQtXhXfSc= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -48,13 +56,23 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c h1:oJsq4z4xKgZWWOhrSZuLZ5KyYfRFytddLL1E5+psfIY= github.com/alecthomas/jsonschema v0.0.0-20210818095345-1014919a589c/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= @@ -65,15 +83,36 @@ github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bloom/v3 v3.0.1 h1:Inlf0YXbgehxVjMPmCGv86iMCKMGPPrPSHtBF5yRHwA= +github.com/bits-and-blooms/bloom/v3 v3.0.1/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= @@ -82,6 +121,8 @@ github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaq github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= github.com/c4milo/unpackit v0.1.0/go.mod h1:pvXCMYlSV8zwGFWMaT+PWYkAB/cvDjN2mv9r7ZRSxEo= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -89,17 +130,38 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/errors v1.8.6/go.mod h1:hOm5fabihW+xEyY1kuypGwqT+Vt7rafg04ytBtIpeIQ= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= +github.com/cockroachdb/pebble v0.0.0-20210827150156-ff43a5880feb/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.1.1/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= @@ -109,10 +171,12 @@ github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= @@ -120,33 +184,76 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI= github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-rod/rod v0.91.1/go.mod h1:/W4lcZiCALPD603MnJGIvhtywP3R6yRB9EDfFfsHiiI= github.com/go-rod/rod v0.101.7 h1:kbI5CNvcRhf7feybBln4xDutsM0mbsF0ENNZfKcF6WA= github.com/go-rod/rod v0.101.7/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= @@ -179,6 +286,7 @@ github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -209,32 +317,73 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= github.com/itchyny/gojq v0.12.4/go.mod h1:EQUSKgW/YaOxmXpAwGiowFDO4i2Rmtk5+9dFyeiymAg= @@ -243,15 +392,26 @@ github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrn github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA= @@ -260,13 +420,25 @@ github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLM github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= +github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= +github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= +github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= -github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -277,25 +449,50 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= +github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= +github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY= +github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -304,34 +501,70 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/ginkgo v1.16.2 h1:HFB2fbVIlhIfCfOW81bZFbiC/RvnpXSdhbF2/DJr134= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= +github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8= +github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/owenrumney/go-sarif v1.0.11 h1:7k4TLSi6h3vAozSECjO0arcQoeUNDMgvA7LDac95sJo= github.com/owenrumney/go-sarif v1.0.11/go.mod h1:hTBFbxU7GuVRUvwMx+eStp9M/Oun4xHCS3vqpPvket8= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= @@ -339,20 +572,31 @@ github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydK github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/fastdialer v0.0.12/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= +github.com/projectdiscovery/fastdialer v0.0.13-0.20210824195254-0113c1406542/go.mod h1:TuapmLiqtunJOxpM7g0tpTy/TUF/0S+XFyx0B0Wx0DQ= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e h1:xMAFYJgRxopAwKrj7HDwMBKJGCGDbHqopS8f959xges= github.com/projectdiscovery/fastdialer v0.0.13-0.20210917073912-cad93d88e69e/go.mod h1:O1l6+vAQy1QRo9FqyuyJ57W3CwpIXXg7oGo14Le6ZYQ= -github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= +github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08 h1:NwD1R/du1dqrRKN3SJl9kT6tN3K9puuWFXEvYF2ihew= +github.com/projectdiscovery/filekv v0.0.0-20210915124239-3467ef45dd08/go.mod h1:paLCnwV8sL7ppqIwVQodQrk3F6mnWafwTDwRd7ywZwQ= +github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20210914153648-31f843feaad4/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7UhrAKnccZttr78CAmG768sSCd+MBn4ayLVDeqA= +github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= +github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240 h1:b7zDUSsgN5f4/IlhKF6RVGsp/NkHIuty0o1YjzAMKUs= +github.com/projectdiscovery/goflags v0.0.8-0.20211007103353-9b9229e8a240/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= +github.com/projectdiscovery/hmap v0.0.2-0.20210727180307-d63d35146e97/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= +github.com/projectdiscovery/hmap v0.0.2-0.20210825180603-fca7166c158f/go.mod h1:RLM8b1z2HEq74u5AXN1Lbvfq+1BZWpnTQJcwLnMLA54= github.com/projectdiscovery/hmap v0.0.2-0.20210917073634-bfb0e9c03800/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= -github.com/projectdiscovery/interactsh v0.0.4 h1:3BtCZrrTovGYiqdFktXJ4NxKAQFvUvzcEI5pJIuShM8= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= +github.com/projectdiscovery/interactsh v0.0.6 h1:z2nVuc2FDRtOVjoYySiG0by/JA1Dwmgzb2rBFJEFR/c= +github.com/projectdiscovery/interactsh v0.0.6/go.mod h1:dB/c1A9I2trIHfMbU/wNzxQkGara0UE33zDtGcdBh+U= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= @@ -379,33 +623,72 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjB github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d h1:YBYwsm8MrSp9t7mLehyqGwUKZWB08fG+YRePQRo5iFw= +github.com/projectdiscovery/stringsutil v0.0.0-20211013053023-e7b2e104d80d/go.mod h1:JK4F9ACNPgO+Lbm80khX2q1ABInBMbwIOmbsEE61Sn4= github.com/projectdiscovery/yamldoc-go v1.0.2 h1:SKb7PHgSOXm27Zci05ba0FxpyQiu6bGEiVMEcjCK1rQ= github.com/projectdiscovery/yamldoc-go v1.0.2/go.mod h1:7uSxfMXaBmzvw8m5EhOEjB6nhz0rK/H9sUjq1ciZu24= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/ksuid v1.0.3/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil/v3 v3.21.7 h1:PnTqQamUjwEDSgn+nBGu0qSDV/CfvyiR/gwTH3i7HTU= github.com/shirou/gopsutil/v3 v3.21.7/go.mod h1:RGl11Y7XMTQPmHh8F0ayC6haKNBgH4PXMJuTAcMOlz4= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -413,11 +696,17 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -441,23 +730,36 @@ github.com/tklauser/go-sysconf v0.3.7 h1:HT7h4+536gjqeq1ZIJPgOl1rg1XFatQGVZWp7Py github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4= github.com/tklauser/numcpus v0.2.3 h1:nQ0QYpiritP6ViFhrKYsiv6VVxOpum2Gks5GhnJbS/8= github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY= github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0= @@ -475,39 +777,67 @@ github.com/ysmood/gson v0.6.4/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3R github.com/ysmood/leakless v0.6.12/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw= github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zclconf/go-cty v1.8.4 h1:pwhhz5P+Fjxse7S7UriBrMu6AUJSZM5pKqGem1PjGAs= github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -515,6 +845,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20210826195003-46c773283d9d/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -527,29 +859,42 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -570,10 +915,13 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -595,27 +943,38 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -629,6 +988,7 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -642,7 +1002,9 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= @@ -658,16 +1020,23 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -677,6 +1046,8 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -685,6 +1056,8 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -701,17 +1074,21 @@ golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -729,6 +1106,7 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -737,11 +1115,13 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -766,10 +1146,17 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -778,6 +1165,7 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -792,18 +1180,27 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nRnFU= gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -814,6 +1211,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -821,7 +1219,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= +moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index cba1af5bf..59791fe63 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" @@ -21,11 +22,6 @@ func ParseOptions(options *types.Options) { // Check if stdin pipe was given options.Stdin = hasStdin() - // if VerboseVerbose is set, it implicitly enables the Verbose option as well - if options.VerboseVerbose { - options.Verbose = true - } - // Read the inputs and configure the logging configureOutput(options) @@ -61,6 +57,14 @@ func ParseOptions(options *types.Options) { // Load the resolvers if user asked for them loadResolvers(options) + // removes all cli variables containing payloads and add them to the internal struct + for key, value := range options.Vars.AsMap() { + if fileutil.FileExists(value.(string)) { + _ = options.Vars.Del(key) + options.AddVarPayload(key, value) + } + } + err := protocolinit.Init(options) if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) @@ -95,6 +99,7 @@ func validateOptions(options *types.Options) error { } if options.Validate { + options.Headless = true // required for correct validation of headless templates validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) } @@ -117,7 +122,7 @@ func isValidURL(urlString string) bool { // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output - if options.Verbose || options.VerboseVerbose { + if options.Verbose { gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) } if options.Debug { diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index f24055ccf..6b14b5f91 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -7,11 +7,20 @@ import ( "go.uber.org/atomic" ) +// processSelfContainedTemplates execute a self-contained template. +func (r *Runner) processSelfContainedTemplates(template *templates.Template) bool { + match, err := template.Executer.Execute("") + if err != nil { + gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) + } + return match +} + // processTemplateWithList execute a template against the list of user provided targets func (r *Runner) processTemplateWithList(template *templates.Template) bool { results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) - r.hostMap.Scan(func(k, _ []byte) error { + processItem := func(k, _ []byte) error { URL := string(k) // Skip if the host has had errors @@ -29,7 +38,13 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { results.CAS(false, match) }(URL) return nil - }) + } + if r.options.Stream { + _ = r.hostMapStream.Scan(processItem) + } else { + r.hostMap.Scan(processItem) + } + wg.Wait() return results.Load() } @@ -39,7 +54,7 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { results := &atomic.Bool{} wg := sizedwaitgroup.New(r.options.BulkSize) - r.hostMap.Scan(func(k, _ []byte) error { + processItem := func(k, _ []byte) error { URL := string(k) // Skip if the host has had errors @@ -53,7 +68,14 @@ func (r *Runner) processWorkflowWithList(template *templates.Template) bool { results.CAS(false, match) }(URL) return nil - }) + } + + if r.options.Stream { + _ = r.hostMapStream.Scan(processItem) + } else { + r.hostMap.Scan(processItem) + } + wg.Wait() return results.Load() } diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index e2f821134..b3f806952 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -16,6 +16,8 @@ import ( "go.uber.org/ratelimit" "gopkg.in/yaml.v2" + "github.com/projectdiscovery/filekv" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" "github.com/projectdiscovery/nuclei/v2/internal/colorizer" @@ -34,7 +36,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/templates" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -45,6 +47,7 @@ import ( // Runner is a client for running the enumeration process. type Runner struct { hostMap *hybrid.HybridMap + hostMapStream *filekv.FileDB output output.Writer interactsh *interactsh.Client inputCount int64 @@ -119,6 +122,20 @@ func New(options *types.Options) (*Runner, error) { } runner.hostMap = hm + if options.Stream { + fkvOptions := filekv.DefaultOptions + if tmpFileName, err := fileutil.GetTempFileName(); err != nil { + return nil, errors.Wrap(err, "could not create temporary input file") + } else { + fkvOptions.Path = tmpFileName + } + fkv, err := filekv.Open(fkvOptions) + if err != nil { + return nil, errors.Wrap(err, "could not create temporary unsorted input file") + } + runner.hostMapStream = fkv + } + runner.inputCount = 0 dupeCount := 0 @@ -138,6 +155,9 @@ func New(options *types.Options) (*Runner, error) { runner.inputCount++ // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) + if options.Stream { + _ = runner.hostMapStream.Set([]byte(url), nil) + } } } @@ -158,6 +178,9 @@ func New(options *types.Options) (*Runner, error) { runner.inputCount++ // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) + if options.Stream { + _ = runner.hostMapStream.Set([]byte(url), nil) + } } } @@ -180,6 +203,9 @@ func New(options *types.Options) (*Runner, error) { runner.inputCount++ // nolint:errcheck // ignoring error runner.hostMap.Set(url, nil) + if options.Stream { + _ = runner.hostMapStream.Set([]byte(url), nil) + } } input.Close() } @@ -262,12 +288,12 @@ func createReportingOptions(options *types.Options) (*reporting.Options, error) } file.Close() } - if options.DiskExportDirectory != "" { + if options.MarkdownExportDirectory != "" { if reportingOptions != nil { - reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory} + reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory} } else { reportingOptions = &reporting.Options{} - reportingOptions.DiskExporter = &disk.Options{Directory: options.DiskExportDirectory} + reportingOptions.MarkdownExporter = &markdown.Options{Directory: options.MarkdownExportDirectory} } } if options.SarifExport != "" { @@ -290,6 +316,9 @@ func (r *Runner) Close() { if r.projectFile != nil { r.projectFile.Close() } + if r.options.Stream { + r.hostMapStream.Close() + } protocolinit.Close() } @@ -344,6 +373,7 @@ func (r *Runner) RunEnumeration() error { IncludeTemplates: r.options.IncludeTemplates, Authors: r.options.Author, Severities: r.options.Severities, + ExcludeSeverities: r.options.ExcludeSeverities, IncludeTags: r.options.IncludeTags, TemplatesDirectory: r.options.TemplatesDirectory, Catalog: r.catalog, @@ -508,7 +538,9 @@ func (r *Runner) RunEnumeration() error { go func(template *templates.Template) { defer wgtemplates.Done() - if len(template.Workflows) > 0 { + if template.SelfContained { + results.CAS(false, r.processSelfContainedTemplates(template)) + } else if len(template.Workflows) > 0 { results.CAS(false, r.processWorkflowWithList(template)) } else { results.CAS(false, r.processTemplateWithList(template)) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 2b4cc06d6..217f34008 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -48,7 +48,7 @@ var reVersion = regexp.MustCompile(`\d+\.\d+\.\d+`) // // If the path exists but does not contain the latest version of public templates, // the new version is downloaded from GitHub to the templates' directory, overwriting the old content. -func (r *Runner) updateTemplates() error { +func (r *Runner) updateTemplates() error { // TODO this method does more than just update templates. Should be refactored. home, err := os.UserHomeDir() if err != nil { return err @@ -75,6 +75,7 @@ func (r *Runner) updateTemplates() error { if r.options.NoUpdateTemplates && !r.options.UpdateTemplates { return nil } + client.InitNucleiVersion(config.Version) r.fetchLatestVersionsFromGithub(configDir) // also fetch the latest versions @@ -121,59 +122,74 @@ func (r *Runner) updateTemplates() error { return nil } - // Get the configuration currently on disk. - verText := r.templatesConfig.TemplateVersion - indices := reVersion.FindStringIndex(verText) - if indices == nil { - return fmt.Errorf("invalid release found with tag %s", err) - } - if indices[0] > 0 { - verText = verText[indices[0]:] - } - - oldVersion, err := semver.Make(verText) + latestVersion, currentVersion, err := getVersions(r) if err != nil { return err } - version, err := semver.Parse(r.templatesConfig.NucleiTemplatesLatestVersion) - if err != nil { - return err - } - - if version.EQ(oldVersion) { + if latestVersion.EQ(currentVersion) { if r.options.UpdateTemplates { gologger.Info().Msgf("No new updates found for nuclei templates") } return config.WriteConfiguration(r.templatesConfig) } - if version.GT(oldVersion) { - gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", oldVersion, version.String()) + if err := updateTemplates(latestVersion, currentVersion, r, ctx); err != nil { + return err + } + return nil +} + +func updateTemplates(latestVersion semver.Version, currentVersion semver.Version, runner *Runner, ctx context.Context) error { + if latestVersion.GT(currentVersion) { + gologger.Info().Msgf("Your current nuclei-templates v%s are outdated. Latest is v%s\n", currentVersion, latestVersion.String()) gologger.Info().Msgf("Downloading latest release...") - if r.options.TemplatesDirectory != "" { - r.templatesConfig.TemplatesDirectory = r.options.TemplatesDirectory + if runner.options.TemplatesDirectory != "" { + runner.templatesConfig.TemplatesDirectory = runner.options.TemplatesDirectory } - r.templatesConfig.TemplateVersion = version.String() + runner.templatesConfig.TemplateVersion = latestVersion.String() - gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", version.String(), r.templatesConfig.TemplatesDirectory) + gologger.Verbose().Msgf("Downloading nuclei-templates (v%s) to %s\n", latestVersion.String(), runner.templatesConfig.TemplatesDirectory) - asset, err := r.getLatestReleaseFromGithub(r.templatesConfig.NucleiTemplatesLatestVersion) + asset, err := runner.getLatestReleaseFromGithub(runner.templatesConfig.NucleiTemplatesLatestVersion) if err != nil { return err } - if _, err := r.downloadReleaseAndUnzip(ctx, version.String(), asset.GetZipballURL()); err != nil { + if _, err := runner.downloadReleaseAndUnzip(ctx, latestVersion.String(), asset.GetZipballURL()); err != nil { return err } - if err := config.WriteConfiguration(r.templatesConfig); err != nil { + if err := config.WriteConfiguration(runner.templatesConfig); err != nil { return err } - gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", version.String()) + gologger.Info().Msgf("Successfully updated nuclei-templates (v%s). GoodLuck!\n", latestVersion.String()) } return nil } +func getVersions(runner *Runner) (semver.Version, semver.Version, error) { + // Get the configuration currently on disk. + verText := runner.templatesConfig.TemplateVersion + indices := reVersion.FindStringIndex(verText) + if indices == nil { + return semver.Version{}, semver.Version{}, fmt.Errorf("invalid release found with tag %s", verText) + } + if indices[0] > 0 { + verText = verText[indices[0]:] + } + + currentVersion, err := semver.Make(verText) + if err != nil { + return semver.Version{}, semver.Version{}, err + } + + latestVersion, err := semver.Parse(runner.templatesConfig.NucleiTemplatesLatestVersion) + if err != nil { + return semver.Version{}, semver.Version{}, err + } + return latestVersion, currentVersion, nil +} + // readInternalConfigurationFile reads the internal configuration file for nuclei func (r *Runner) readInternalConfigurationFile(home, configDir string) error { templatesConfigFile := filepath.Join(configDir, nucleiConfigFilename) @@ -210,9 +226,9 @@ func (r *Runner) checkNucleiIgnoreFileUpdates(configDir string) bool { // getLatestReleaseFromGithub returns the latest release from GitHub func (r *Runner) getLatestReleaseFromGithub(latestTag string) (*github.RepositoryRelease, error) { - client := github.NewClient(nil) + gitHubClient := github.NewClient(nil) - release, _, err := client.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag) + release, _, err := gitHubClient.Repositories.GetReleaseByTag(context.Background(), userName, repoName, "v"+latestTag) if err != nil { return nil, err } @@ -361,7 +377,7 @@ func (r *Runner) compareAndWriteTemplates(zipReader *zip.Reader) (*templateUpdat for templatePath, templateChecksums := range templateChecksumsMap { _, ok := results.checksums[templatePath] if !ok && templateChecksums[0] == templateChecksums[1] { - os.Remove(templatePath) + _ = os.Remove(templatePath) results.deletions = append(results.deletions, strings.TrimPrefix(strings.TrimPrefix(templatePath, r.templatesConfig.TemplatesDirectory), string(os.PathSeparator))) } } @@ -456,7 +472,7 @@ func (r *Runner) printUpdateChangelog(results *templateUpdateResults, version st // fetchLatestVersionsFromGithub fetches the latest versions of nuclei repos from GitHub // -// This fetches latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions +// This fetches the latest nuclei/templates/ignore from https://version-check.nuclei.sh/versions // If you want to disable this automatic update check, use -nut flag. func (r *Runner) fetchLatestVersionsFromGithub(configDir string) { versions, err := client.GetLatestNucleiTemplatesVersion() diff --git a/v2/internal/testutils/integration.go b/v2/internal/testutils/integration.go index bb5d557f1..8c5491d28 100644 --- a/v2/internal/testutils/integration.go +++ b/v2/internal/testutils/integration.go @@ -2,6 +2,7 @@ package testutils import ( "errors" + "fmt" "net" "os" "os/exec" @@ -9,20 +10,36 @@ import ( "strings" ) -// RunNucleiAndGetResults returns a list of results for a template -func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - cmd := exec.Command("./nuclei", "-t", template, "-target", url, "-silent") +// RunNucleiTemplateAndGetResults returns a list of results for a template +func RunNucleiTemplateAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { + return runNucleiAndGetResults(true, template, url, debug, extra...) +} + +// RunNucleiWorkflowAndGetResults returns a list of results for a workflow +func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { + return runNucleiAndGetResults(false, template, url, debug, extra...) +} + +func runNucleiAndGetResults(isTemplate bool, template, url string, debug bool, extra ...string) ([]string, error) { + var templateOrWorkflowFlag string + if isTemplate { + templateOrWorkflowFlag = "-t" + } else { + templateOrWorkflowFlag = "-w" + } + + cmd := exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-silent") if debug { - cmd = exec.Command("./nuclei", "-t", template, "-target", url, "-debug") + cmd = exec.Command("./nuclei", templateOrWorkflowFlag, template, "-target", url, "-debug") cmd.Stderr = os.Stderr + fmt.Println(cmd.String()) } cmd.Args = append(cmd.Args, extra...) - data, err := cmd.Output() if err != nil { return nil, err } - parts := []string{} + var parts []string items := strings.Split(string(data), "\n") for _, i := range items { if i != "" { @@ -35,9 +52,11 @@ func RunNucleiAndGetResults(template, url string, debug bool, extra ...string) ( var templateLoaded = regexp.MustCompile(`(?:Templates|Workflows) loaded[^:]*: (\d+)`) // RunNucleiBinaryAndGetLoadedTemplates returns a list of results for a template -func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (string, error) { +func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, debug bool, args []string) (string, error) { cmd := exec.Command(nucleiBinary, args...) - + if debug { + fmt.Println(cmd.String()) + } data, err := cmd.CombinedOutput() if err != nil { return "", err @@ -49,29 +68,6 @@ func RunNucleiBinaryAndGetLoadedTemplates(nucleiBinary string, args []string) (s return matches[0][1], nil } -// RunNucleiWorkflowAndGetResults returns a list of results for a workflow -func RunNucleiWorkflowAndGetResults(template, url string, debug bool, extra ...string) ([]string, error) { - cmd := exec.Command("./nuclei", "-w", template, "-target", url, "-silent") - if debug { - cmd = exec.Command("./nuclei", "-w", template, "-target", url, "-debug") - cmd.Stderr = os.Stderr - } - cmd.Args = append(cmd.Args, extra...) - - data, err := cmd.Output() - if err != nil { - return nil, err - } - parts := []string{} - items := strings.Split(string(data), "\n") - for _, i := range items { - if i != "" { - parts = append(parts, i) - } - } - return parts, nil -} - // TestCase is a single integration test case type TestCase interface { // Execute executes a test case and returns any errors if occurred @@ -85,10 +81,14 @@ type TCPServer struct { } // NewTCPServer creates a new TCP server from a handler -func NewTCPServer(handler func(conn net.Conn)) *TCPServer { +func NewTCPServer(handler func(conn net.Conn), port ...int) *TCPServer { server := &TCPServer{} - l, err := net.Listen("tcp", "127.0.0.1:0") + var gotPort int + if len(port) > 0 { + gotPort = port[0] + } + l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", gotPort)) if err != nil { panic(err) } diff --git a/v2/pkg/catalog/config/config.go b/v2/pkg/catalog/config/config.go index 76160a2d7..8a996b62d 100644 --- a/v2/pkg/catalog/config/config.go +++ b/v2/pkg/catalog/config/config.go @@ -6,8 +6,9 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" "gopkg.in/yaml.v2" + + "github.com/projectdiscovery/gologger" ) // Config contains the internal nuclei engine configuration @@ -25,7 +26,7 @@ type Config struct { const nucleiConfigFilename = ".templates-config.json" // Version is the current version of nuclei -const Version = `2.5.2` +const Version = `2.5.3` func getConfigDetails() (string, error) { homeDir, err := os.UserHomeDir() diff --git a/v2/pkg/catalog/loader/filter/tag_filter.go b/v2/pkg/catalog/loader/filter/tag_filter.go index bfc75bd17..ed8722c90 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter.go +++ b/v2/pkg/catalog/loader/filter/tag_filter.go @@ -9,11 +9,12 @@ import ( // TagFilter is used to filter nuclei templates for tag based execution type TagFilter struct { - allowedTags map[string]struct{} - severities map[severity.Severity]struct{} - authors map[string]struct{} - block map[string]struct{} - matchAllows map[string]struct{} + allowedTags map[string]struct{} + severities map[severity.Severity]struct{} + excludeSeverities map[severity.Severity]struct{} + authors map[string]struct{} + block map[string]struct{} + matchAllows map[string]struct{} } // ErrExcluded is returned for excluded templates @@ -54,15 +55,21 @@ func (tagFilter *TagFilter) Match(templateTags, templateAuthors []string, templa } func isSeverityMatch(tagFilter *TagFilter, templateSeverity severity.Severity) bool { - if len(tagFilter.severities) == 0 || templateSeverity == severity.Undefined { + if (len(tagFilter.excludeSeverities) == 0 && len(tagFilter.severities) == 0) || templateSeverity == severity.Undefined { return true } - if _, ok := tagFilter.severities[templateSeverity]; ok { - return true + included := true + if len(tagFilter.severities) > 0 { + _, included = tagFilter.severities[templateSeverity] } - return false + excluded := false + if len(tagFilter.excludeSeverities) > 0 { + _, excluded = tagFilter.excludeSeverities[templateSeverity] + } + + return included && !excluded } func isAuthorMatch(tagFilter *TagFilter, templateAuthors []string) bool { @@ -110,23 +117,25 @@ func isTagMatch(tagFilter *TagFilter, templateTags []string) bool { } type Config struct { - Tags []string - ExcludeTags []string - Authors []string - Severities severity.Severities - IncludeTags []string + Tags []string + ExcludeTags []string + Authors []string + Severities severity.Severities + ExcludeSeverities severity.Severities + IncludeTags []string } // New returns a tag filter for nuclei tag based execution // -// It takes into account Tags, Severities, Authors, IncludeTags, ExcludeTags. +// It takes into account Tags, Severities, ExcludeSeverities, Authors, IncludeTags, ExcludeTags. func New(config *Config) *TagFilter { filter := &TagFilter{ - allowedTags: make(map[string]struct{}), - authors: make(map[string]struct{}), - severities: make(map[severity.Severity]struct{}), - block: make(map[string]struct{}), - matchAllows: make(map[string]struct{}), + allowedTags: make(map[string]struct{}), + authors: make(map[string]struct{}), + severities: make(map[severity.Severity]struct{}), + excludeSeverities: make(map[severity.Severity]struct{}), + block: make(map[string]struct{}), + matchAllows: make(map[string]struct{}), } for _, tag := range config.ExcludeTags { for _, val := range splitCommaTrim(tag) { @@ -140,6 +149,11 @@ func New(config *Config) *TagFilter { filter.severities[tag] = struct{}{} } } + for _, tag := range config.ExcludeSeverities { + if _, ok := filter.excludeSeverities[tag]; !ok { + filter.excludeSeverities[tag] = struct{}{} + } + } for _, tag := range config.Authors { for _, val := range splitCommaTrim(tag) { if _, ok := filter.authors[val]; !ok { diff --git a/v2/pkg/catalog/loader/filter/tag_filter_test.go b/v2/pkg/catalog/loader/filter/tag_filter_test.go index 8acc9a356..22d18b189 100644 --- a/v2/pkg/catalog/loader/filter/tag_filter_test.go +++ b/v2/pkg/catalog/loader/filter/tag_filter_test.go @@ -73,6 +73,16 @@ func TestTagBasedFilter(t *testing.T) { matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil) require.True(t, matched, "could not get correct match") }) + t.Run("match-exclude-severity", func(t *testing.T) { + filter := New(&Config{ + ExcludeSeverities: severity.Severities{severity.Low}, + }) + matched, _ := filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.High, nil) + require.True(t, matched, "could not get correct match") + + matched, _ = filter.Match([]string{"fuzz"}, []string{"pdteam"}, severity.Low, nil) + require.False(t, matched, "could not get correct match") + }) t.Run("match-exclude-with-tags", func(t *testing.T) { filter := New(&Config{ Tags: []string{"tag"}, diff --git a/v2/pkg/catalog/loader/loader.go b/v2/pkg/catalog/loader/loader.go index fee382447..eac22b4ae 100644 --- a/v2/pkg/catalog/loader/loader.go +++ b/v2/pkg/catalog/loader/loader.go @@ -19,11 +19,12 @@ type Config struct { ExcludeTemplates []string IncludeTemplates []string - Tags []string - ExcludeTags []string - Authors []string - Severities severity.Severities - IncludeTags []string + Tags []string + ExcludeTags []string + Authors []string + Severities severity.Severities + ExcludeSeverities severity.Severities + IncludeTags []string Catalog *catalog.Catalog ExecutorOptions protocols.ExecuterOptions @@ -49,11 +50,12 @@ func New(config *Config) (*Store, error) { store := &Store{ config: config, tagFilter: filter.New(&filter.Config{ - Tags: config.Tags, - ExcludeTags: config.ExcludeTags, - Authors: config.Authors, - Severities: config.Severities, - IncludeTags: config.IncludeTags, + Tags: config.Tags, + ExcludeTags: config.ExcludeTags, + Authors: config.Authors, + Severities: config.Severities, + ExcludeSeverities: config.ExcludeSeverities, + IncludeTags: config.IncludeTags, }), pathFilter: filter.NewPathFilter(&filter.PathFilterConfig{ IncludedTemplates: config.IncludeTemplates, diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index 056f4ace7..787cd8ff6 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -224,6 +224,15 @@ var functions = map[string]govaluate.ExpressionFunction{ } return rand.Intn(max-min) + min, nil }, + "unixtime": func(args ...interface{}) (interface{}, error) { + seconds := 0 + if len(args) >= 1 { + seconds = int(args[0].(float64)) + } + now := time.Now() + offset := now.Add(time.Duration(seconds) * time.Second) + return offset.Unix(), nil + }, // Time Functions "waitfor": func(args ...interface{}) (interface{}, error) { seconds := args[0].(float64) diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 9a1144438..fff0055de 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -3,6 +3,9 @@ package matchers import ( "encoding/hex" "strings" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" ) // MatchStatusCode matches a status code check against a corpus @@ -38,15 +41,25 @@ func (m *Matcher) MatchSize(length int) bool { } // MatchWords matches a word check against a corpus. -func (m *Matcher) MatchWords(corpus string) bool { +func (m *Matcher) MatchWords(corpus string, dynamicValues map[string]interface{}) (bool, []string) { + var matchedWords []string // Iterate over all the words accepted as valid for i, word := range m.Words { + if dynamicValues == nil { + dynamicValues = make(map[string]interface{}) + } + + var err error + word, err = expressions.Evaluate(word, dynamicValues) + if err != nil { + continue + } // Continue if the word doesn't match if !strings.Contains(corpus, word) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue @@ -54,19 +67,22 @@ func (m *Matcher) MatchWords(corpus string) bool { // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, []string{word} } + matchedWords = append(matchedWords, word) + // If we are at the end of the words, return with true if len(m.Words)-1 == i { - return true + return true, matchedWords } } - return false + return false, []string{} } // MatchRegex matches a regex check against a corpus -func (m *Matcher) MatchRegex(corpus string) bool { +func (m *Matcher) MatchRegex(corpus string) (bool, []string) { + var matchedRegexes []string // Iterate over all the regexes accepted as valid for i, regex := range m.regexCompiled { // Continue if the regex doesn't match @@ -74,36 +90,47 @@ func (m *Matcher) MatchRegex(corpus string) bool { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue } + currentMatches := regex.FindAllString(corpus, -1) // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, currentMatches } + matchedRegexes = append(matchedRegexes, currentMatches...) + // If we are at the end of the regex, return with true if len(m.regexCompiled)-1 == i { - return true + return true, matchedRegexes } } - return false + return false, []string{} } // MatchBinary matches a binary check against a corpus -func (m *Matcher) MatchBinary(corpus string) bool { +func (m *Matcher) MatchBinary(corpus string) (bool, []string) { + var matchedBinary []string // Iterate over all the words accepted as valid for i, binary := range m.Binary { // Continue if the word doesn't match - hexa, _ := hex.DecodeString(binary) + hexa, err := hex.DecodeString(binary) + if err != nil { + gologger.Warning().Msgf("Could not hex encode the given binary matcher value: '%s'", binary) + if m.condition == ANDCondition { + return false, []string{} + } + continue + } if !strings.Contains(corpus, string(hexa)) { // If we are in an AND request and a match failed, // return false as the AND condition fails on any single mismatch. if m.condition == ANDCondition { - return false + return false, []string{} } // Continue with the flow since it's an OR Condition. continue @@ -111,15 +138,17 @@ func (m *Matcher) MatchBinary(corpus string) bool { // If the condition was an OR, return on the first match. if m.condition == ORCondition { - return true + return true, []string{string(hexa)} } + matchedBinary = append(matchedBinary, string(hexa)) + // If we are at the end of the words, return with true if len(m.Binary)-1 == i { - return true + return true, matchedBinary } } - return false + return false, []string{} } // MatchDSL matches on a generic map result diff --git a/v2/pkg/operators/matchers/match_test.go b/v2/pkg/operators/matchers/match_test.go index d6c7dc8a0..4c2f9bc0c 100644 --- a/v2/pkg/operators/matchers/match_test.go +++ b/v2/pkg/operators/matchers/match_test.go @@ -6,27 +6,60 @@ import ( "github.com/stretchr/testify/require" ) -func TestANDCondition(t *testing.T) { +func TestWordANDCondition(t *testing.T) { m := &Matcher{condition: ANDCondition, Words: []string{"a", "b"}} - matched := m.MatchWords("a b") - require.True(t, matched, "Could not match valid AND condition") + isMatched, matched := m.MatchWords("a b", nil) + require.True(t, isMatched, "Could not match words with valid AND condition") + require.Equal(t, m.Words, matched) - matched = m.MatchWords("b") - require.False(t, matched, "Could match invalid AND condition") + isMatched, matched = m.MatchWords("b", nil) + require.False(t, isMatched, "Could match words with invalid AND condition") + require.Equal(t, []string{}, matched) +} + +func TestRegexANDCondition(t *testing.T) { + m := &Matcher{Type: "regex", Condition: "and", Regex: []string{"[a-z]{3}", "\\d{2}"}} + err := m.CompileMatchers() + require.Nil(t, err) + + isMatched, matched := m.MatchRegex("abc abcd 123") + require.True(t, isMatched, "Could not match regex with valid AND condition") + require.Equal(t, []string{"abc", "abc", "12"}, matched) + + isMatched, matched = m.MatchRegex("bc 1") + require.False(t, isMatched, "Could match regex with invalid AND condition") + require.Equal(t, []string{}, matched) } func TestORCondition(t *testing.T) { m := &Matcher{condition: ORCondition, Words: []string{"a", "b"}} - matched := m.MatchWords("a b") - require.True(t, matched, "Could not match valid OR condition") + isMatched, matched := m.MatchWords("a b", nil) + require.True(t, isMatched, "Could not match valid word OR condition") + require.Equal(t, []string{"a"}, matched) - matched = m.MatchWords("b") - require.True(t, matched, "Could not match valid OR condition") + isMatched, matched = m.MatchWords("b", nil) + require.True(t, isMatched, "Could not match valid word OR condition") + require.Equal(t, []string{"b"}, matched) - matched = m.MatchWords("c") - require.False(t, matched, "Could match invalid OR condition") + isMatched, matched = m.MatchWords("c", nil) + require.False(t, isMatched, "Could match invalid word OR condition") + require.Equal(t, []string{}, matched) +} + +func TestRegexOrCondition(t *testing.T) { + m := &Matcher{Type: "regex", Condition: "or", Regex: []string{"[a-z]{3}", "\\d{2}"}} + err := m.CompileMatchers() + require.Nil(t, err) + + isMatched, matched := m.MatchRegex("ab 123") + require.True(t, isMatched, "Could not match valid regex OR condition") + require.Equal(t, []string{"12"}, matched) + + isMatched, matched = m.MatchRegex("bc 1") + require.False(t, isMatched, "Could match invalid regex OR condition") + require.Equal(t, []string{}, matched) } func TestHexEncoding(t *testing.T) { @@ -34,6 +67,7 @@ func TestHexEncoding(t *testing.T) { err := m.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := m.MatchWords("PING") - require.True(t, matched, "Could not match valid Hex condition") + isMatched, matched := m.MatchWords("PING", nil) + require.True(t, isMatched, "Could not match valid Hex condition") + require.Equal(t, m.Words, matched) } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index a0d176477..88ee413e1 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -165,6 +165,14 @@ func (m *Matcher) Result(data bool) bool { return data } +// ResultWithMatchedSnippet returns true and the matched snippet, or false and an empty string +func (m *Matcher) ResultWithMatchedSnippet(data bool, matchedSnippet []string) (bool, []string) { + if m.Negative { + return !data, []string{} + } + return data, matchedSnippet +} + // GetType returns the type of the matcher func (m *Matcher) GetType() MatcherType { return m.matcherType diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 493eddf06..af13b4d93 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -1,6 +1,8 @@ package operators import ( + "strconv" + "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" @@ -32,19 +34,19 @@ type Operators struct { } // Compile compiles the operators as well as their corresponding matchers and extractors -func (r *Operators) Compile() error { - if r.MatchersCondition != "" { - r.matchersCondition = matchers.ConditionTypes[r.MatchersCondition] +func (operators *Operators) Compile() error { + if operators.MatchersCondition != "" { + operators.matchersCondition = matchers.ConditionTypes[operators.MatchersCondition] } else { - r.matchersCondition = matchers.ORCondition + operators.matchersCondition = matchers.ORCondition } - for _, matcher := range r.Matchers { + for _, matcher := range operators.Matchers { if err := matcher.CompileMatchers(); err != nil { return errors.Wrap(err, "could not compile matcher") } } - for _, extractor := range r.Extractors { + for _, extractor := range operators.Extractors { if err := extractor.CompileExtractors(); err != nil { return errors.Wrap(err, "could not compile extractor") } @@ -53,8 +55,8 @@ func (r *Operators) Compile() error { } // GetMatchersCondition returns the condition for the matchers -func (r *Operators) GetMatchersCondition() matchers.ConditionType { - return r.matchersCondition +func (operators *Operators) GetMatchersCondition() matchers.ConditionType { + return operators.matchersCondition } // Result is a result structure created from operators running on data. @@ -64,7 +66,7 @@ type Result struct { // Extracted is true if any result type values were extracted Extracted bool // Matches is a map of matcher names that we matched - Matches map[string]struct{} + Matches map[string][]string // Extracts contains all the data extracted from inputs Extracts map[string][]string // OutputExtracts is the list of extracts to be displayed on screen. @@ -100,24 +102,24 @@ func (r *Result) Merge(result *Result) { } // MatchFunc performs matching operation for a matcher on model and returns true or false. -type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) bool +type MatchFunc func(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) // ExtractFunc performs extracting operation for an extractor on model and returns true or false. type ExtractFunc func(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // Execute executes the operators on data and returns a result structure -func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc) (*Result, bool) { - matcherCondition := r.GetMatchersCondition() +func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc, extract ExtractFunc, isDebug bool) (*Result, bool) { + matcherCondition := operators.GetMatchersCondition() var matches bool result := &Result{ - Matches: make(map[string]struct{}), + Matches: make(map[string][]string), Extracts: make(map[string][]string), DynamicValues: make(map[string]interface{}), } // Start with the extractors first and evaluate them. - for _, extractor := range r.Extractors { + for _, extractor := range operators.Extractors { var extractorResults []string for match := range extract(data, extractor) { @@ -136,23 +138,22 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac } } - for _, matcher := range r.Matchers { - // Check if the matcher matched - if !match(data, matcher) { - // If the condition is AND we haven't matched, try next request. - if matcherCondition == matchers.ANDCondition { - if len(result.DynamicValues) > 0 { - return result, true + for matcherIndex, matcher := range operators.Matchers { + if isMatch, matched := match(data, matcher); isMatch { + if isDebug { // matchers without an explicit name or with AND condition should only be made visible if debug is enabled + matcherName := getMatcherName(matcher, matcherIndex) + result.Matches[matcherName] = matched + } else { // if it's a "named" matcher with OR condition, then display it + if matcherCondition == matchers.ORCondition && matcher.Name != "" { + result.Matches[matcher.Name] = matched } - return nil, false - } - } else { - // If the matcher has matched, and it's an OR - // write the first output then move to next matcher. - if matcherCondition == matchers.ORCondition && matcher.Name != "" { - result.Matches[matcher.Name] = struct{}{} } matches = true + } else if matcherCondition == matchers.ANDCondition { + if len(result.DynamicValues) > 0 { + return result, true + } + return nil, false } } @@ -162,8 +163,8 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac return result, true } - // Don't print if we have matchers and they have not matched, regardless of extractor - if len(r.Matchers) > 0 && !matches { + // Don't print if we have matchers, and they have not matched, regardless of extractor + if len(operators.Matchers) > 0 && !matches { return nil, false } // Write a final string of output if matcher type is @@ -174,12 +175,20 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac return nil, false } +func getMatcherName(matcher *matchers.Matcher, matcherIndex int) string { + if matcher.Name != "" { + return matcher.Name + } else { + return matcher.Type + "-" + strconv.Itoa(matcherIndex+1) // making the index start from 1 to be more readable + } +} + // ExecuteInternalExtractors executes internal dynamic extractors -func (r *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} { +func (operators *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} { dynamicValues := make(map[string]interface{}) // Start with the extractors first and evaluate them. - for _, extractor := range r.Extractors { + for _, extractor := range operators.Extractors { if !extractor.Internal { continue } diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 8f42646cf..b90be29c3 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -59,15 +59,15 @@ type InternalWrappedEvent struct { // ResultEvent is a wrapped result event for a single nuclei output. type ResultEvent struct { // TemplateID is the ID of the template for the result. - TemplateID string `json:"templateID"` + TemplateID string `json:"template-id"` // TemplatePath is the path of template TemplatePath string `json:"-"` // Info contains information block of the template for the result. Info model.Info `json:"info,inline"` // MatcherName is the name of the matcher matched if any. - MatcherName string `json:"matcher_name,omitempty"` + MatcherName string `json:"matcher-name,omitempty"` // ExtractorName is the name of the extractor matched if any. - ExtractorName string `json:"extractor_name,omitempty"` + ExtractorName string `json:"extractor-name,omitempty"` // Type is the type of the result event. Type string `json:"type"` // Host is the host input on which match was found. @@ -75,9 +75,9 @@ type ResultEvent struct { // Path is the path input on which match was found. Path string `json:"path,omitempty"` // Matched contains the matched input in its transformed form. - Matched string `json:"matched,omitempty"` + Matched string `json:"matched-at,omitempty"` // ExtractedResults contains the extraction result from the inputs. - ExtractedResults []string `json:"extracted_results,omitempty"` + ExtractedResults []string `json:"extracted-results,omitempty"` // Request is the optional, dumped request for the match. Request string `json:"request,omitempty"` // Response is the optional, dumped response for the match. @@ -90,7 +90,9 @@ type ResultEvent struct { Timestamp time.Time `json:"timestamp"` // Interaction is the full details of interactsh interaction. Interaction *server.Interaction `json:"interaction,omitempty"` - + // CURLCommand is an optional curl command to reproduce the request + // Only applicable if the report is for HTTP. + CURLCommand string `json:"curl-command,omitempty"` FileToIndexPosition map[string]int `json:"-"` } diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index ac523c8d6..c060f341d 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -67,7 +67,7 @@ func (e *Executer) Execute(input string) (bool, error) { dynamicValues := make(map[string]interface{}) err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { - result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) + result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) if matched && result != nil { event.OperatorsResult = result event.InternalEvent["template-id"] = operator.templateID @@ -98,7 +98,7 @@ func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEve dynamicValues := make(map[string]interface{}) err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { - result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) + result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract, e.options.Options.Debug || e.options.Options.DebugResponse) if matched && result != nil { event.OperatorsResult = result event.InternalEvent["template-id"] = operator.templateID diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go index 7f27b88a3..5a4ba2f3a 100644 --- a/v2/pkg/protocols/common/expressions/expressions.go +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -18,6 +18,21 @@ var templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}["'\)\}]*`) // The provided keys from finalValues will be used as variable names // for substitution inside the expression. func Evaluate(data string, base map[string]interface{}) (string, error) { + return evaluate(data, base) +} + +// EvaluateByte checks if the match contains a dynamic variable, for each +// found one we will check if it's an expression and can +// be compiled, it will be evaluated and the results will be returned. +// +// The provided keys from finalValues will be used as variable names +// for substitution inside the expression. +func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) { + finalData, err := evaluate(string(data), base) + return []byte(finalData), err +} + +func evaluate(data string, base map[string]interface{}) (string, error) { data = replacer.Replace(data, base) dynamicValues := make(map[string]interface{}) @@ -37,30 +52,3 @@ func Evaluate(data string, base map[string]interface{}) (string, error) { // Replacer dynamic values if any in raw request and parse it return replacer.Replace(data, dynamicValues), nil } - -// EvaluateByte checks if the match contains a dynamic variable, for each -// found one we will check if it's an expression and can -// be compiled, it will be evaluated and the results will be returned. -// -// The provided keys from finalValues will be used as variable names -// for substitution inside the expression. -func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) { - final := replacer.Replace(string(data), base) - - dynamicValues := make(map[string]interface{}) - for _, match := range templateExpressionRegex.FindAllString(final, -1) { - expr := generators.TrimDelimiters(match) - - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) - if err != nil { - continue - } - result, err := compiled.Evaluate(base) - if err != nil { - continue - } - dynamicValues[expr] = result - } - // Replacer dynamic values if any in raw request and parse it - return []byte(replacer.Replace(final, dynamicValues)), nil -} diff --git a/v2/pkg/protocols/common/expressions/variables.go b/v2/pkg/protocols/common/expressions/variables.go new file mode 100644 index 000000000..48ac546ed --- /dev/null +++ b/v2/pkg/protocols/common/expressions/variables.go @@ -0,0 +1,56 @@ +package expressions + +import ( + "errors" + "regexp" + "strings" +) + +var unresolvedVariablesRegex = regexp.MustCompile(`(?:%7[B|b]|\{){2}([^}]+)(?:%7[D|d]|\}){2}["'\)\}]*`) + +// ContainsUnresolvedVariables returns an error with variable names if the passed +// input contains unresolved {{}} variables. +func ContainsUnresolvedVariables(data string) error { + matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { + return nil + } + errorString := &strings.Builder{} + errorString.WriteString("unresolved variables found: ") + + for i, match := range matches { + if len(match) < 2 { + continue + } + errorString.WriteString(match[1]) + if i != len(matches)-1 { + errorString.WriteString(",") + } + } + errorMessage := errorString.String() + return errors.New(errorMessage) +} + +func ContainsVariablesWithNames(data string, names map[string]interface{}) error { + matches := unresolvedVariablesRegex.FindAllStringSubmatch(data, -1) + if len(matches) == 0 { + return nil + } + errorString := &strings.Builder{} + errorString.WriteString("unresolved variables with values found: ") + + for i, match := range matches { + if len(match) < 2 { + continue + } + matchName := match[1] + if _, ok := names[matchName]; !ok { + errorString.WriteString(matchName) + if i != len(matches)-1 { + errorString.WriteString(",") + } + } + } + errorMessage := errorString.String() + return errors.New(errorMessage) +} diff --git a/v2/pkg/protocols/common/expressions/variables_test.go b/v2/pkg/protocols/common/expressions/variables_test.go new file mode 100644 index 000000000..61d071351 --- /dev/null +++ b/v2/pkg/protocols/common/expressions/variables_test.go @@ -0,0 +1,24 @@ +package expressions + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUnresolvedVariablesCheck(t *testing.T) { + tests := []struct { + data string + err error + }{ + {"{{test}}", errors.New("unresolved variables found: test")}, + {"{{test}}/{{another}}", errors.New("unresolved variables found: test,another")}, + {"test", nil}, + {"%7b%7btest%7d%7d", errors.New("unresolved variables found: test")}, + } + for _, test := range tests { + err := ContainsUnresolvedVariables(test.data) + require.Equal(t, test.err, err, "could not get unresolved variables") + } +} diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index dcca16f55..a63fbcb2b 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -2,6 +2,8 @@ package generators +import "github.com/pkg/errors" + // Generator is the generator struct for generating payloads type Generator struct { Type Type @@ -12,8 +14,8 @@ type Generator struct { type Type int const ( - // Sniper replaces each variable with values at a time. - Sniper Type = iota + 1 + // Batteringram replaces same payload into all of the defined payload positions at once. + BatteringRam Type = iota + 1 // PitchFork replaces variables with positional value from multiple wordlists PitchFork // ClusterBomb replaces variables with all possible combinations of values @@ -22,9 +24,9 @@ const ( // StringToType is a table for conversion of attack type from string. var StringToType = map[string]Type{ - "sniper": Sniper, - "pitchfork": PitchFork, - "clusterbomb": ClusterBomb, + "batteringram": BatteringRam, + "pitchfork": PitchFork, + "clusterbomb": ClusterBomb, } // New creates a new generator structure for payload generation @@ -41,6 +43,12 @@ func New(payloads map[string]interface{}, payloadType Type, templatePath string) generator.Type = payloadType generator.payloads = compiled + // Validate the batteringram payload set + if payloadType == BatteringRam { + if len(payloads) != 1 { + return nil, errors.New("batteringram must have single payload set") + } + } return generator, nil } @@ -87,7 +95,7 @@ func (i *Iterator) Remaining() int { func (i *Iterator) Total() int { count := 0 switch i.Type { - case Sniper: + case BatteringRam: for _, p := range i.payloads { count += len(p.values) } @@ -110,19 +118,19 @@ func (i *Iterator) Total() int { // Value returns the next value for an iterator func (i *Iterator) Value() (map[string]interface{}, bool) { switch i.Type { - case Sniper: - return i.sniperValue() + case BatteringRam: + return i.batteringRamValue() case PitchFork: return i.pitchforkValue() case ClusterBomb: return i.clusterbombValue() default: - return i.sniperValue() + return i.batteringRamValue() } } -// sniperValue returns a list of all payloads for the iterator -func (i *Iterator) sniperValue() (map[string]interface{}, bool) { +// batteringRamValue returns a list of all payloads for the iterator +func (i *Iterator) batteringRamValue() (map[string]interface{}, bool) { values := make(map[string]interface{}, 1) currentIndex := i.msbIterator @@ -132,7 +140,7 @@ func (i *Iterator) sniperValue() (map[string]interface{}, bool) { if i.msbIterator == len(i.payloads) { return nil, false } - return i.sniperValue() + return i.batteringRamValue() } values[payload.name] = payload.value() payload.incrementPosition() diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index a19f53114..de37ca62d 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -6,11 +6,10 @@ import ( "github.com/stretchr/testify/require" ) -func TestSniperGenerator(t *testing.T) { +func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} - moreUsernames := []string{"login", "test"} - generator, err := New(map[string]interface{}{"username": usernames, "aliases": moreUsernames}, Sniper, "") + generator, err := New(map[string]interface{}{"username": usernames}, BatteringRam, "") require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -22,7 +21,7 @@ func TestSniperGenerator(t *testing.T) { } count++ } - require.Equal(t, len(usernames)+len(moreUsernames), count, "could not get correct sniper counts") + require.Equal(t, len(usernames), count, "could not get correct batteringram counts") } func TestPitchforkGenerator(t *testing.T) { diff --git a/v2/pkg/protocols/common/generators/options.go b/v2/pkg/protocols/common/generators/options.go new file mode 100644 index 000000000..868ccef7c --- /dev/null +++ b/v2/pkg/protocols/common/generators/options.go @@ -0,0 +1,20 @@ +package generators + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// BuildPayloadFromOptions returns a map with the payloads provided via CLI +func BuildPayloadFromOptions(options *types.Options) map[string]interface{} { + m := make(map[string]interface{}) + // merge with vars + if !options.Vars.IsEmpty() { + m = MergeMaps(m, options.Vars.AsMap()) + } + + // merge with env vars + if options.EnvironmentVariables { + m = MergeMaps(EnvVars(), m) + } + return m +} diff --git a/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go new file mode 100644 index 000000000..c94eb15a3 --- /dev/null +++ b/v2/pkg/protocols/common/helpers/eventcreator/eventcreator.go @@ -0,0 +1,29 @@ +package eventcreator + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +// CreateEvent wraps the outputEvent with the result of the operators defined on the request +func CreateEvent(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool) *output.InternalWrappedEvent { + return CreateEventWithAdditionalOptions(request, outputEvent, isResponseDebug, func(internalWrappedEvent *output.InternalWrappedEvent) {}) +} + +// CreateEventWithAdditionalOptions wraps the outputEvent with the result of the operators defined on the request and enables extending the resulting event with additional attributes or values. +func CreateEventWithAdditionalOptions(request protocols.Request, outputEvent output.InternalEvent, isResponseDebug bool, + addAdditionalOptions func(internalWrappedEvent *output.InternalWrappedEvent)) *output.InternalWrappedEvent { + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + for _, compiledOperator := range request.GetCompiledOperators() { + if compiledOperator != nil { + result, ok := compiledOperator.Execute(outputEvent, request.Match, request.Extract, isResponseDebug) + if ok && result != nil { + event.OperatorsResult = result + addAdditionalOptions(event) + event.Results = append(event.Results, request.MakeResultEvent(event)...) + } + } + } + + return event +} diff --git a/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go new file mode 100644 index 000000000..1cb914e97 --- /dev/null +++ b/v2/pkg/protocols/common/helpers/responsehighlighter/response_highlighter.go @@ -0,0 +1,35 @@ +package responsehighlighter + +import ( + "strconv" + "strings" + + "github.com/logrusorgru/aurora" + + "github.com/projectdiscovery/nuclei/v2/pkg/operators" +) + +var colorizer = aurora.NewAurora(true) + +func Highlight(operatorResult *operators.Result, response string, noColor bool) string { + result := response + if operatorResult != nil && !noColor { + for _, matches := range operatorResult.Matches { + if len(matches) > 0 { + for _, currentMatch := range matches { + result = strings.ReplaceAll(result, currentMatch, colorizer.Green(currentMatch).String()) + } + } + } + } + + return result +} + +func CreateStatusCodeSnippet(response string, statusCode int) string { + if strings.HasPrefix(response, "HTTP/") { + strStatusCode := strconv.Itoa(statusCode) + return response[:strings.Index(response, strStatusCode)+len(strStatusCode)] + } + return "" +} diff --git a/v2/pkg/protocols/common/interactsh/interactsh.go b/v2/pkg/protocols/common/interactsh/interactsh.go index 1c3310365..5c2914ff5 100644 --- a/v2/pkg/protocols/common/interactsh/interactsh.go +++ b/v2/pkg/protocols/common/interactsh/interactsh.go @@ -145,7 +145,7 @@ func (c *Client) processInteractionForRequest(interaction *server.Interaction, d data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol data.Event.InternalEvent["interactsh_request"] = interaction.RawRequest data.Event.InternalEvent["interactsh_response"] = interaction.RawResponse - result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc) + result, matched := data.Operators.Execute(data.Event.InternalEvent, data.MatchFunc, data.ExtractFunc, false) if !matched || result == nil { return false // if we don't match, return } @@ -256,7 +256,7 @@ func (c *Client) RequestEvent(interactshURL string, data *RequestData) { // HasMatchers returns true if an operator has interactsh part // matchers or extractors. // -// Used by requests to show result or not depending on presence of interact.sh +// Used by requests to show result or not depending on presence of interactsh.com // data part matchers. func HasMatchers(op *operators.Operators) bool { if op == nil { diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 3881d1a5a..15f42dc02 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -72,45 +72,55 @@ type Request struct { // description: | // Recursion determines if resolver should recurse all records to get fresh results. Recursion bool `yaml:"recursion,omitempty" jsonschema:"title=recurse all servers,description=Recursion determines if resolver should recurse all records to get fresh results"` + // Resolvers to use for the dns requests + Resolvers []string `yaml:"resolvers,omitempty" jsonschema:"title=Resolvers,description=Define resolvers to use within the template"` +} + +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + dnsClientOptions := &dnsclientpool.Configuration{ + Retries: request.Retries, + } + if len(request.Resolvers) > 0 { + dnsClientOptions.Resolvers = request.Resolvers + } // Create a dns client for the class - client, err := dnsclientpool.Get(options.Options, &dnsclientpool.Configuration{ - Retries: r.Retries, - }) + client, err := dnsclientpool.Get(options.Options, dnsClientOptions) if err != nil { return errors.Wrap(err, "could not get dns client") } - r.dnsClient = client + request.dnsClient = client - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } - r.class = classToInt(r.Class) - r.options = options - r.question = questionTypeToInt(r.Type) + request.class = classToInt(request.Class) + request.options = options + request.question = questionTypeToInt(request.Type) return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } // Make returns the request to be sent for the protocol -func (r *Request) Make(domain string) (*dns.Msg, error) { - if r.question != dns.TypePTR && net.ParseIP(domain) != nil { +func (request *Request) Make(domain string) (*dns.Msg, error) { + if request.question != dns.TypePTR && net.ParseIP(domain) != nil { return nil, errors.New("cannot use IP address as DNS input") } domain = dns.Fqdn(domain) @@ -118,16 +128,24 @@ func (r *Request) Make(domain string) (*dns.Msg, error) { // Build a request on the specified URL req := new(dns.Msg) req.Id = dns.Id() - req.RecursionDesired = r.Recursion + req.RecursionDesired = request.Recursion var q dns.Question - final := replacer.Replace(r.Name, map[string]interface{}{"FQDN": domain}) + final := replacer.Replace(request.Name, map[string]interface{}{"FQDN": domain}) q.Name = dns.Fqdn(final) - q.Qclass = r.class - q.Qtype = r.question + q.Qclass = request.class + q.Qtype = request.question req.Question = append(req.Question, q) + + req.SetEdns0(4096, false) + + switch request.question { + case dns.TypeTXT: + req.AuthenticatedData = true + } + return req, nil } diff --git a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go index 999f2a2c9..46308c9fe 100644 --- a/v2/pkg/protocols/dns/dnsclientpool/clientpool.go +++ b/v2/pkg/protocols/dns/dnsclientpool/clientpool.go @@ -44,21 +44,24 @@ func Init(options *types.Options) error { type Configuration struct { // Retries contains the retries for the dns client Retries int + // Resolvers contains the specific per request resolvers + Resolvers []string } // Hash returns the hash of the configuration to allow client pooling func (c *Configuration) Hash() string { builder := &strings.Builder{} - builder.Grow(8) builder.WriteString("r") builder.WriteString(strconv.Itoa(c.Retries)) + builder.WriteString("l") + builder.WriteString(strings.Join(c.Resolvers, "")) hash := builder.String() return hash } // Get creates or gets a client for the protocol based on custom configuration func Get(options *types.Options, configuration *Configuration) (*retryabledns.Client, error) { - if !(configuration.Retries > 1) { + if !(configuration.Retries > 1) && len(configuration.Resolvers) == 0 { return normalClient, nil } hash := configuration.Hash() @@ -72,6 +75,8 @@ func Get(options *types.Options, configuration *Configuration) (*retryabledns.Cl resolvers := defaultResolvers if options.ResolversFile != "" { resolvers = options.InternalResolversList + } else if len(configuration.Resolvers) > 0 { + resolvers = configuration.Resolvers } client := retryabledns.New(resolvers, configuration.Retries) diff --git a/v2/pkg/protocols/dns/operators.go b/v2/pkg/protocols/dns/operators.go index 5e5137420..b9f0454b3 100644 --- a/v2/pkg/protocols/dns/operators.go +++ b/v2/pkg/protocols/dns/operators.go @@ -10,11 +10,12 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "": @@ -23,28 +24,32 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } switch matcher.GetType() { case matchers.StatusMatcher: - return matcher.Result(matcher.MatchStatusCode(item.(int))) + statusCode, ok := item.(int) + if !ok { + return false, []string{} + } + return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(types.ToString(item)))) + return matcher.Result(matcher.MatchSize(len(types.ToString(item)))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(types.ToString(item))) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(types.ToString(item), nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(types.ToString(item))) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(types.ToString(item))) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(types.ToString(item))) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(types.ToString(item))) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { part := extractor.Part switch part { case "body", "all": @@ -67,77 +72,29 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext } // responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 11) - - // Some data regarding the request metadata - data["host"] = host - data["matched"] = matched - data["request"] = req.String() - - data["rcode"] = resp.Rcode - buffer := &bytes.Buffer{} - for _, question := range resp.Question { - buffer.WriteString(question.String()) +func (request *Request) responseToDSLMap(req, resp *dns.Msg, host, matched string) output.InternalEvent { + return output.InternalEvent{ + "host": host, + "matched": matched, + "request": req.String(), + "rcode": resp.Rcode, + "question": questionToString(resp.Question), + "extra": rrToString(resp.Extra), + "answer": rrToString(resp.Answer), + "ns": rrToString(resp.Ns), + "raw": resp.String(), + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, } - data["question"] = buffer.String() - buffer.Reset() - - for _, extra := range resp.Extra { - buffer.WriteString(extra.String()) - } - data["extra"] = buffer.String() - buffer.Reset() - - for _, answer := range resp.Answer { - buffer.WriteString(answer.String()) - } - data["answer"] = buffer.String() - buffer.Reset() - - for _, ns := range resp.Ns { - buffer.WriteString(ns.String()) - } - data["ns"] = buffer.String() - buffer.Reset() - - rawData := resp.String() - data["raw"] = rawData - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath - return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), @@ -152,3 +109,19 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out } return data } + +func rrToString(resourceRecords []dns.RR) string { // TODO rewrite with generics when available + buffer := &bytes.Buffer{} + for _, resourceRecord := range resourceRecords { + buffer.WriteString(resourceRecord.String()) + } + return buffer.String() +} + +func questionToString(resourceRecords []dns.Question) string { + buffer := &bytes.Buffer{} + for _, resourceRecord := range resourceRecords { + buffer.WriteString(resourceRecord.String()) + } + return buffer.String() +} diff --git a/v2/pkg/protocols/dns/operators_test.go b/v2/pkg/protocols/dns/operators_test.go index 286dd582f..c8d06124a 100644 --- a/v2/pkg/protocols/dns/operators_test.go +++ b/v2/pkg/protocols/dns/operators_test.go @@ -87,8 +87,9 @@ func TestDNSOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatch, matched := request.Match(event, matcher) + require.True(t, isMatch, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("rcode", func(t *testing.T) { @@ -100,8 +101,9 @@ func TestDNSOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile rcode matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid rcode response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid rcode response") + require.Equal(t, []string{}, matched) }) t.Run("negative", func(t *testing.T) { @@ -114,8 +116,9 @@ func TestDNSOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -127,8 +130,9 @@ func TestDNSOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } @@ -232,13 +236,15 @@ func TestDNSMakeResult(t *testing.T) { event := request.responseToDSLMap(req, resp, "one.one.one.one", "one.one.one.one") finalEvent := &output.InternalWrappedEvent{InternalEvent: event} if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) } } require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") - require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") - require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") + resultEvent := finalEvent.Results[0] + require.Equal(t, "test", resultEvent.MatcherName, "could not get correct matcher name of results") + require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results") + require.Equal(t, "one.one.one.one", resultEvent.Matched, "could not get matched value") } diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index ae3041f33..28a3309b0 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -4,15 +4,19 @@ import ( "net/url" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" ) var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // Parse the URL and return domain if URL. var domain string if isURL(input) { @@ -22,49 +26,49 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused } // Compile each request for the template based on the URL - compiledRequest, err := r.Make(domain) + compiledRequest, err := request.Make(domain) if err != nil { - r.options.Output.Request(r.options.TemplateID, domain, "dns", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not build request") } - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", r.options.TemplateID, domain) - gologger.Print().Msgf("%s", compiledRequest.String()) + requestString := compiledRequest.String() + if varErr := expressions.ContainsUnresolvedVariables(requestString); varErr != nil { + gologger.Warning().Msgf("[%s] Could not make dns request for %s: %v\n", request.options.TemplateID, domain, varErr) + return nil + } + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Str("domain", domain).Msgf("[%s] Dumped DNS request for %s", request.options.TemplateID, domain) + gologger.Print().Msgf("%s", requestString) } // Send the request to the target servers - resp, err := r.dnsClient.Do(compiledRequest) + resp, err := request.dnsClient.Do(compiledRequest) if err != nil { - r.options.Output.Request(r.options.TemplateID, domain, "dns", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + request.options.Progress.IncrementFailedRequestsBy(1) } if resp == nil { return errors.Wrap(err, "could not send dns request") } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() - r.options.Output.Request(r.options.TemplateID, domain, "dns", err) - gologger.Verbose().Msgf("[%s] Sent DNS request to %s", r.options.TemplateID, domain) + request.options.Output.Request(request.options.TemplateID, domain, "dns", err) + gologger.Verbose().Msgf("[%s] Sent DNS request to %s", request.options.TemplateID, domain) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped DNS response for %s", r.options.TemplateID, domain) - gologger.Print().Msgf("%s", resp.String()) - } - outputEvent := r.responseToDSLMap(compiledRequest, resp, input, input) + outputEvent := request.responseToDSLMap(compiledRequest, resp, input, input) for k, v := range previous { outputEvent[k] = v } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped DNS response for %s", request.options.TemplateID, domain) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, resp.String(), request.options.Options.NoColor)) } + callback(event) return nil } diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index 2d1e64393..e61965025 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -56,54 +56,54 @@ type Request struct { var defaultDenylist = []string{".3g2", ".3gp", ".7z", ".apk", ".arj", ".avi", ".axd", ".bmp", ".css", ".csv", ".deb", ".dll", ".doc", ".drv", ".eot", ".exe", ".flv", ".gif", ".gifv", ".gz", ".h264", ".ico", ".iso", ".jar", ".jpeg", ".jpg", ".lock", ".m4a", ".m4v", ".map", ".mkv", ".mov", ".mp3", ".mp4", ".mpeg", ".mpg", ".msi", ".ogg", ".ogm", ".ogv", ".otf", ".pdf", ".pkg", ".png", ".ppt", ".psd", ".rar", ".rm", ".rpm", ".svg", ".swf", ".sys", ".tar.gz", ".tar", ".tif", ".tiff", ".ttf", ".vob", ".wav", ".webm", ".wmv", ".woff", ".woff2", ".xcf", ".xls", ".xlsx", ".zip"} // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } // By default, use 5 MB as max size to read. - if r.MaxSize == 0 { - r.MaxSize = 5 * 1024 * 1024 + if request.MaxSize == 0 { + request.MaxSize = 5 * 1024 * 1024 } - r.options = options + request.options = options - r.extensions = make(map[string]struct{}) - r.extensionDenylist = make(map[string]struct{}) + request.extensions = make(map[string]struct{}) + request.extensionDenylist = make(map[string]struct{}) - for _, extension := range r.Extensions { + for _, extension := range request.Extensions { if extension == "all" { - r.allExtensions = true + request.allExtensions = true } else { if !strings.HasPrefix(extension, ".") { extension = "." + extension } - r.extensions[extension] = struct{}{} + request.extensions[extension] = struct{}{} } } for _, extension := range defaultDenylist { if !strings.HasPrefix(extension, ".") { extension = "." + extension } - r.extensionDenylist[extension] = struct{}{} + request.extensionDenylist[extension] = struct{}{} } - for _, extension := range r.ExtensionDenylist { + for _, extension := range request.ExtensionDenylist { if !strings.HasPrefix(extension, ".") { extension = "." + extension } - r.extensionDenylist[extension] = struct{}{} + request.extensionDenylist[extension] = struct{}{} } return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go index ec1cc2585..f4deaa100 100644 --- a/v2/pkg/protocols/file/find.go +++ b/v2/pkg/protocols/file/find.go @@ -7,50 +7,51 @@ import ( "github.com/karrick/godirwalk" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" ) // getInputPaths parses the specified input paths and returns a compiled // list of finished absolute paths to the files evaluating any allowlist, denylist, // glob, file or folders, etc. -func (r *Request) getInputPaths(target string, callback func(string)) error { +func (request *Request) getInputPaths(target string, callback func(string)) error { processed := make(map[string]struct{}) // Template input includes a wildcard - if strings.Contains(target, "*") && !r.NoRecursive { - if err := r.findGlobPathMatches(target, processed, callback); err != nil { + if strings.Contains(target, "*") && !request.NoRecursive { + if err := request.findGlobPathMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find glob matches") } return nil } // Template input is either a file or a directory - file, err := r.findFileMatches(target, processed, callback) + file, err := request.findFileMatches(target, processed, callback) if err != nil { return errors.Wrap(err, "could not find file") } if file { return nil } - if r.NoRecursive { + if request.NoRecursive { return nil // we don't process dirs in no-recursive mode } // Recursively walk down the Templates directory and run all // the template file checks - if err := r.findDirectoryMatches(target, processed, callback); err != nil { + if err := request.findDirectoryMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find directory matches") } return nil } // findGlobPathMatches returns the matched files from a glob path -func (r *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { matches, err := filepath.Glob(absPath) if err != nil { return errors.Errorf("wildcard found, but unable to glob: %s\n", err) } for _, match := range matches { - if !r.validatePath(match) { + if !request.validatePath(match) { continue } if _, ok := processed[match]; !ok { @@ -63,7 +64,7 @@ func (r *Request) findGlobPathMatches(absPath string, processed map[string]struc // findFileMatches finds if a path is an absolute file. If the path // is a file, it returns true otherwise false with no errors. -func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { +func (request *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { info, err := os.Stat(absPath) if err != nil { return false, err @@ -72,7 +73,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, return false, nil } if _, ok := processed[absPath]; !ok { - if !r.validatePath(absPath) { + if !request.validatePath(absPath) { return false, nil } processed[absPath] = struct{}{} @@ -82,7 +83,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, } // findDirectoryMatches finds matches for templates from a directory -func (r *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { err := godirwalk.Walk(absPath, &godirwalk.Options{ Unsorted: true, ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { @@ -92,7 +93,7 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru if d.IsDir() { return nil } - if !r.validatePath(path) { + if !request.validatePath(path) { return nil } if _, ok := processed[path]; !ok { @@ -106,17 +107,17 @@ func (r *Request) findDirectoryMatches(absPath string, processed map[string]stru } // validatePath validates a file path for blacklist and whitelist options -func (r *Request) validatePath(item string) bool { +func (request *Request) validatePath(item string) bool { extension := filepath.Ext(item) - if len(r.extensions) > 0 { - if _, ok := r.extensions[extension]; ok { + if len(request.extensions) > 0 { + if _, ok := request.extensions[extension]; ok { return true - } else if !r.allExtensions { + } else if !request.allExtensions { return false } } - if _, ok := r.extensionDenylist[extension]; ok { + if _, ok := request.extensionDenylist[extension]; ok { gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, extension) return false } diff --git a/v2/pkg/protocols/file/operators.go b/v2/pkg/protocols/file/operators.go index 6631aeba1..c0d95c5b0 100644 --- a/v2/pkg/protocols/file/operators.go +++ b/v2/pkg/protocols/file/operators.go @@ -6,14 +6,16 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "data", "": @@ -22,27 +24,27 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { case "body", "all", "data", "": @@ -64,49 +66,27 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext return nil } -// responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(raw, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 5) - - // Some data regarding the request metadata - data["path"] = host - data["matched"] = matched - data["raw"] = raw - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath - return data +// responseToDSLMap converts a file response to a map for use in DSL matching +func (request *Request) responseToDSLMap(raw, inputFilePath, matchedFileName string) output.InternalEvent { + return output.InternalEvent{ + "path": inputFilePath, + "matched": matchedFileName, + "raw": raw, + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, + } } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + results := protocols.MakeDefaultResultEvent(request, wrapped) - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } raw, ok := wrapped.InternalEvent["raw"] if !ok { return results } + rawStr, ok := raw.(string) if !ok { return results @@ -133,7 +113,11 @@ func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*outpu return results } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), @@ -141,7 +125,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out Type: "file", Path: types.ToString(wrapped.InternalEvent["path"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), - Host: types.ToString(wrapped.InternalEvent["matched"]), + Host: types.ToString(wrapped.InternalEvent["host"]), ExtractedResults: wrapped.OperatorsResult.OutputExtracts, Response: types.ToString(wrapped.InternalEvent["raw"]), Timestamp: time.Now(), diff --git a/v2/pkg/protocols/file/operators_test.go b/v2/pkg/protocols/file/operators_test.go index 7c5be06d9..b03d641b5 100644 --- a/v2/pkg/protocols/file/operators_test.go +++ b/v2/pkg/protocols/file/operators_test.go @@ -72,8 +72,9 @@ func TestFileOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -86,8 +87,9 @@ func TestFileOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -99,8 +101,9 @@ func TestFileOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } @@ -156,7 +159,62 @@ func TestFileOperatorExtract(t *testing.T) { }) } -func TestFileMakeResult(t *testing.T) { +func TestFileMakeResultWithOrMatcher(t *testing.T) { + expectedValue := []string{"1.1.1.1"} + namedMatcherName := "test" + + finalEvent := testFileMakeResultOperators(t, "or") + require.Equal(t, namedMatcherName, finalEvent.Results[0].MatcherName) + require.Equal(t, expectedValue, finalEvent.OperatorsResult.Matches[namedMatcherName], "could not get matched value") +} + +func TestFileMakeResultWithAndMatcher(t *testing.T) { + finalEvent := testFileMakeResultOperators(t, "and") + require.Equal(t, "", finalEvent.Results[0].MatcherName) + require.Empty(t, finalEvent.OperatorsResult.Matches) +} + +func testFileMakeResultOperators(t *testing.T, matcherCondition string) *output.InternalWrappedEvent { + expectedValue := []string{"1.1.1.1"} + namedMatcherName := "test" + matcher := []*matchers.Matcher{ + { + Part: "raw", + Type: "word", + Words: expectedValue, + }, + { + Name: namedMatcherName, + Part: "raw", + Type: "word", + Words: expectedValue, + }, + } + + expectedValues := map[string][]string{ + "word-1": expectedValue, + namedMatcherName: expectedValue, + } + + finalEvent := testFileMakeResult(t, matcher, matcherCondition, true) + for matcherName, matchedValues := range expectedValues { + var matchesOne = false + for i := 0; i <= len(expectedValue); i++ { + resultEvent := finalEvent.Results[i] + if matcherName == resultEvent.MatcherName { + matchesOne = true + } + } + require.True(t, matchesOne) + require.Equal(t, matchedValues, finalEvent.OperatorsResult.Matches[matcherName], "could not get matched value") + } + + finalEvent = testFileMakeResult(t, matcher, matcherCondition, false) + require.Equal(t, 1, len(finalEvent.Results)) + return finalEvent +} + +func testFileMakeResult(t *testing.T, matchers []*matchers.Matcher, matcherCondition string, isDebug bool) *output.InternalWrappedEvent { options := testutils.DefaultOptions testutils.Init(options) @@ -168,12 +226,8 @@ func TestFileMakeResult(t *testing.T) { Extensions: []string{"*", ".lock"}, ExtensionDenylist: []string{".go"}, Operators: operators.Operators{ - Matchers: []*matchers.Matcher{{ - Name: "test", - Part: "raw", - Type: "word", - Words: []string{"1.1.1.1"}, - }}, + MatchersCondition: matcherCondition, + Matchers: matchers, Extractors: []*extractors.Extractor{{ Part: "raw", Type: "regex", @@ -188,20 +242,24 @@ func TestFileMakeResult(t *testing.T) { err := request.Compile(executerOpts) require.Nil(t, err, "could not compile file request") - resp := "test-data\r\n1.1.1.1\r\n" - event := request.responseToDSLMap(resp, "one.one.one.one", "one.one.one.one") + matchedFileName := "test.txt" + fileContent := "test-data\r\n1.1.1.1\r\n" + + event := request.responseToDSLMap(fileContent, "/tmp", matchedFileName) require.Len(t, event, 6, "could not get correct number of items in dsl map") - require.Equal(t, resp, event["raw"], "could not get correct resp") + require.Equal(t, fileContent, event["raw"], "could not get correct resp") finalEvent := &output.InternalWrappedEvent{InternalEvent: event} if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, isDebug) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) } } - require.Equal(t, 1, len(finalEvent.Results), "could not get correct number of results") - require.Equal(t, "test", finalEvent.Results[0].MatcherName, "could not get correct matcher name of results") - require.Equal(t, "1.1.1.1", finalEvent.Results[0].ExtractedResults[0], "could not get correct extracted results") + resultEvent := finalEvent.Results[0] + require.Equal(t, "1.1.1.1", resultEvent.ExtractedResults[0], "could not get correct extracted results") + require.Equal(t, matchedFileName, resultEvent.Matched, "could not get matched value") + + return finalEvent } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index d272e256b..61bb87185 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -5,75 +5,74 @@ import ( "os" "github.com/pkg/errors" + "github.com/remeh/sizedwaitgroup" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" - "github.com/remeh/sizedwaitgroup" ) var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - wg := sizedwaitgroup.New(r.options.Options.BulkSize) +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + wg := sizedwaitgroup.New(request.options.Options.BulkSize) - err := r.getInputPaths(input, func(data string) { + err := request.getInputPaths(input, func(data string) { wg.Add() - go func(data string) { + go func(filePath string) { defer wg.Done() - file, err := os.Open(data) + file, err := os.Open(filePath) if err != nil { - gologger.Error().Msgf("Could not open file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not open file path %s: %s\n", filePath, err) return } defer file.Close() stat, err := file.Stat() if err != nil { - gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not stat file path %s: %s\n", filePath, err) return } - if stat.Size() >= int64(r.MaxSize) { - gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data) + if stat.Size() >= int64(request.MaxSize) { + gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", filePath) return } buffer, err := ioutil.ReadAll(file) if err != nil { - gologger.Error().Msgf("Could not read file path %s: %s\n", data, err) + gologger.Error().Msgf("Could not read file path %s: %s\n", filePath, err) return } dataStr := tostring.UnsafeToString(buffer) - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped file request for %s", r.options.TemplateID, data) - gologger.Print().Msgf("%s", dataStr) - } - gologger.Verbose().Msgf("[%s] Sent FILE request to %s", r.options.TemplateID, data) - outputEvent := r.responseToDSLMap(dataStr, input, data) + + gologger.Verbose().Msgf("[%s] Sent FILE request to %s", request.options.TemplateID, filePath) + outputEvent := request.responseToDSLMap(dataStr, input, filePath) for k, v := range previous { outputEvent[k] = v } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped file request for %s", request.options.TemplateID, filePath) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, dataStr, request.options.Options.NoColor)) } + callback(event) }(data) }) wg.Wait() if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "file", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "file", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() return nil } diff --git a/v2/pkg/protocols/headless/engine/action.go b/v2/pkg/protocols/headless/engine/action.go index 0be71fff8..5f97217b2 100644 --- a/v2/pkg/protocols/headless/engine/action.go +++ b/v2/pkg/protocols/headless/engine/action.go @@ -50,6 +50,8 @@ const ( ActionDebug // ActionSleep executes a sleep for a specified duration ActionSleep + // ActionWaitVisible waits until an element appears. + ActionWaitVisible ) // ActionStringToAction converts an action from string to internal representation @@ -75,6 +77,7 @@ var ActionStringToAction = map[string]ActionType{ "keyboard": ActionKeyboard, "debug": ActionDebug, "sleep": ActionSleep, + "waitvisible": ActionWaitVisible, } // ActionToActionString converts an action from internal representation to string @@ -100,6 +103,7 @@ var ActionToActionString = map[ActionType]string{ ActionKeyboard: "keyboard", ActionDebug: "debug", ActionSleep: "sleep", + ActionWaitVisible: "waitvisible", } // Action is an action taken by the browser to reach a navigation diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index 073743840..a506cbda4 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -34,6 +34,7 @@ func New(options *types.Options) (*Browser, error) { return nil, errors.Wrap(err, "could not create temporary directory") } previouspids := findChromeProcesses() + chromeLauncher := launcher.New(). Leakless(false). Set("disable-gpu", "true"). @@ -49,6 +50,14 @@ func New(options *types.Options) (*Browser, error) { Delete("use-mock-keychain"). UserDataDir(dataStore) + if options.UseInstalledChrome { + if chromePath, hasChrome := launcher.LookPath(); hasChrome { + chromeLauncher.Bin(chromePath) + } else { + return nil, errors.New("the chrome browser is not installed") + } + } + if options.ShowBrowser { chromeLauncher = chromeLauncher.Headless(false) } else { diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index c9ec6e0ce..b4747d541 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -3,6 +3,7 @@ package engine import ( "crypto/tls" "net/http" + "net/url" "time" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" @@ -22,5 +23,12 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, }, } + + if options.ProxyURL != "" { + if proxyURL, err := url.Parse(options.ProxyURL); err == nil { + transport.Proxy = http.ProxyURL(proxyURL) + } + } + return &http.Client{Transport: transport, Timeout: time.Duration(options.Timeout*3) * time.Second} } diff --git a/v2/pkg/protocols/headless/engine/page.go b/v2/pkg/protocols/headless/engine/page.go index 0ae5f0ac7..afd59fd29 100644 --- a/v2/pkg/protocols/headless/engine/page.go +++ b/v2/pkg/protocols/headless/engine/page.go @@ -8,7 +8,7 @@ import ( "github.com/go-rod/rod/lib/proto" ) -// Page is a single page in an isolated browser instanace +// Page is a single page in an isolated browser instance type Page struct { page *rod.Page rules []requestRule diff --git a/v2/pkg/protocols/headless/engine/page_actions.go b/v2/pkg/protocols/headless/engine/page_actions.go index 2ec2365b9..fc6e821ea 100644 --- a/v2/pkg/protocols/headless/engine/page_actions.go +++ b/v2/pkg/protocols/headless/engine/page_actions.go @@ -1,6 +1,7 @@ package engine import ( + "context" "io/ioutil" "net" "net/url" @@ -11,6 +12,7 @@ import ( "github.com/go-rod/rod" "github.com/go-rod/rod/lib/proto" + "github.com/go-rod/rod/lib/utils" "github.com/pkg/errors" "github.com/segmentio/ksuid" "github.com/valyala/fasttemplate" @@ -46,7 +48,7 @@ func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]s case ActionGetResource: err = p.GetResource(act, outData) case ActionExtract: - err = p.SelectInputElement(act, outData) + err = p.ExtractElement(act, outData) case ActionWaitEvent: err = p.WaitEvent(act, outData) case ActionFilesInput: @@ -67,6 +69,8 @@ func (p *Page) ExecuteActions(baseURL *url.URL, actions []*Action) (map[string]s err = p.DebugAction(act, outData) case ActionSleep: err = p.SleepAction(act, outData) + case ActionWaitVisible: + err = p.WaitVisible(act, outData) default: continue } @@ -83,6 +87,83 @@ type requestRule struct { Args map[string]string } +const elementDidNotAppearMessage = "Element did not appear in the given amount of time" + +// WaitVisible waits until an element appears. +func (p *Page) WaitVisible(act *Action, out map[string]string) error { + timeout, err := getTimeout(act) + if err != nil { + return errors.Wrap(err, "Wrong timeout given") + } + + pollTime, err := getPollTime(act) + if err != nil { + return errors.Wrap(err, "Wrong polling time given") + } + + element, _ := p.Sleeper(pollTime, timeout). + Timeout(timeout). + pageElementBy(act.Data) + + if element != nil { + if err := element.WaitVisible(); err != nil { + return errors.Wrap(err, elementDidNotAppearMessage) + } + } else { + return errors.New(elementDidNotAppearMessage) + } + + return nil +} + +func (p *Page) Sleeper(pollTimeout, timeout time.Duration) *Page { + page := *p + page.page = page.Page().Sleeper(func() utils.Sleeper { + return createBackOffSleeper(pollTimeout, timeout) + }) + return &page +} + +func (p *Page) Timeout(timeout time.Duration) *Page { + page := *p + page.page = page.Page().Timeout(timeout) + return &page +} + +func createBackOffSleeper(pollTimeout, timeout time.Duration) utils.Sleeper { + backoffSleeper := utils.BackoffSleeper(pollTimeout, timeout, func(duration time.Duration) time.Duration { + return duration + }) + + return func(ctx context.Context) error { + if ctx.Err() != nil { + return ctx.Err() + } + + return backoffSleeper(ctx) + } +} + +func getTimeout(act *Action) (time.Duration, error) { + return geTimeParameter(act, "timeout", 3, time.Second) +} + +func getPollTime(act *Action) (time.Duration, error) { + return geTimeParameter(act, "pollTime", 100, time.Millisecond) +} + +func geTimeParameter(act *Action, parameterName string, defaultValue time.Duration, duration time.Duration) (time.Duration, error) { + pollTimeString := act.GetArg(parameterName) + if pollTimeString == "" { + return defaultValue * duration, nil + } + timeout, err := strconv.Atoi(pollTimeString) + if err != nil { + return time.Duration(0), err + } + return time.Duration(timeout) * duration, nil +} + // ActionAddHeader executes a AddHeader action. func (p *Page) ActionAddHeader(act *Action, out map[string]string /*TODO review unused parameter*/) error { in := act.GetArg("part") diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 658c32908..902b978ac 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -2,19 +2,517 @@ package engine import ( "fmt" + "io/ioutil" "net/http" "net/http/httptest" "net/url" + "os" "strings" "testing" "time" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/stretchr/testify/require" ) func TestActionNavigate(t *testing.T) { + response := ` + + + Nuclei Test Page + + +

Nuclei Test

+ + ` + + actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}} + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + }) +} + +func TestActionScript(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + + timeout := 2 * time.Second + + t.Run("run-and-results", func(t *testing.T) { + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + } + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") + }) + }) + + t.Run("hook", func(t *testing.T) { + actions := []*Action{ + {ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, + } + testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") + }) + }) +} + +func TestActionClick(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("button") + val := el.MustAttribute("a") + require.Equal(t, "ok", *val, "could not click button") + }) +} + +func TestActionRightClick(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("button") + val := el.MustAttribute("a") + require.Equal(t, "ok", *val, "could not click button") + }) +} + +func TestActionTextInput(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + val := el.MustAttribute("event") + require.Equal(t, "input-change", *val, "could not get input change") + require.Equal(t, "test", el.MustText(), "could not get input change value") + }) +} + +func TestActionHeadersChange(t *testing.T) { + actions := []*Action{ + {ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test") == "Hello" { + _, _ = fmt.Fprintln(w, `found`) + } + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + }) +} + +func TestActionScreenshot(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "screenshot", Data: map[string]string{"to": "test"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page() + require.FileExists(t, "test.png", el, "could not get screenshot file") + _ = os.Remove("test.png") + }) +} + +func TestActionTimeInput(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "time", Data: map[string]string{"selector": "input", "value": "2006-01-02T15:04:05Z"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "2006-01-02", el.MustText(), "could not get input time value") + }) +} + +func TestActionSelectInput(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "select", Data: map[string]string{"by": "x", "xpath": "//select[@id='test']", "value": "Test2", "selected": "true"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("select") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") + }) +} + +func TestActionFilesInput(t *testing.T) { + response := ` + + + Nuclei Test Page + + Nuclei Test Page + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "files", Data: map[string]string{"selector": "input", "value": "test1.pdf"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") + el := page.Page().MustElement("input") + require.Equal(t, "C:\\fakepath\\test1.pdf", el.MustText(), "could not get input file") + }) +} + +func TestActionWaitLoad(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("button") + style, attributeErr := el.Attribute("style") + require.Nil(t, attributeErr) + require.Equal(t, "color: red;", *style, "could not get color") + }) +} + +func TestActionGetResource(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "getresource", Data: map[string]string{"by": "x", "xpath": "//img[@id='test']"}, Name: "src"}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, len(out["src"]), 3159, "could not find resource") + }) +} + +func TestActionExtract(t *testing.T) { + response := ` + + + Nuclei Test Page + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "extract", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}, Name: "extract"}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "Wait for me!", out["extract"], "could not extract text") + }) +} + +func TestActionSetMethod(t *testing.T) { + response := ` + + + Nuclei Test Page + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "setmethod", Data: map[string]string{"part": "x", "method": "SET"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "SET", page.rules[0].Args["method"], "could not find resource") + }) +} + +func TestActionAddHeader(t *testing.T) { + actions := []*Action{ + {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test") == "Hello" { + _, _ = fmt.Fprintln(w, `found`) + } + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + }) +} + +func TestActionDeleteHeader(t *testing.T) { + actions := []*Action{ + {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test1", "value": "Hello"}}, + {ActionType: "addheader", Data: map[string]string{"part": "request", "key": "Test2", "value": "World"}}, + {ActionType: "deleteheader", Data: map[string]string{"part": "request", "key": "Test2"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + + handler := func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Test1") == "Hello" && r.Header.Get("Test2") == "" { + _, _ = fmt.Fprintln(w, `header deleted`) + } + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "header deleted", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not delete header correctly") + }) +} + +func TestActionSetBody(t *testing.T) { + actions := []*Action{ + {ActionType: "setbody", Data: map[string]string{"part": "request", "body": "hello"}}, + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + } + + handler := func(w http.ResponseWriter, r *http.Request) { + body, _ := ioutil.ReadAll(r.Body) + _, _ = fmt.Fprintln(w, string(body)) + } + + testHeadless(t, actions, 20*time.Second, handler, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.Equal(t, "hello", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") + }) +} + +func TestActionKeyboard(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitload"}, + {ActionType: "click", Data: map[string]string{"selector": "input"}}, + {ActionType: "keyboard", Data: map[string]string{"keys": "Test2"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + el := page.Page().MustElement("input") + require.Equal(t, "Test2", el.MustText(), "could not get input change value") + }) +} + +func TestActionSleep(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "sleep", Data: map[string]string{"duration": "2"}}, + } + + testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + require.True(t, page.Page().MustElement("button").MustVisible(), "could not get button") + }) +} + +func TestActionWaitVisible(t *testing.T) { + response := ` + + + Nuclei Test Page + + + + ` + + actions := []*Action{ + {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, + {ActionType: "waitvisible", Data: map[string]string{"by": "x", "xpath": "//button[@id='test']"}}, + } + + t.Run("wait for an element being visible", func(t *testing.T) { + testHeadlessSimpleResponse(t, response, actions, 2*time.Second, func(page *Page, err error, out map[string]string) { + require.Nil(t, err, "could not run page actions") + + page.Page().MustElement("button").MustVisible() + }) + }) + + t.Run("timeout because of element not visible", func(t *testing.T) { + testHeadlessSimpleResponse(t, response, actions, time.Second/2, func(page *Page, err error, out map[string]string) { + require.Error(t, err) + require.Contains(t, err.Error(), "Element did not appear in the given amount of time") + }) + }) +} + +func testHeadlessSimpleResponse(t *testing.T, response string, actions []*Action, timeout time.Duration, assert func(page *Page, pageErr error, out map[string]string)) { + t.Helper() + testHeadless(t, actions, timeout, func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintln(w, response) + }, assert) +} + +func testHeadless(t *testing.T, actions []*Action, timeout time.Duration, handler func(w http.ResponseWriter, r *http.Request), assert func(page *Page, pageErr error, extractedData map[string]string)) { + t.Helper() _ = protocolstate.Init(&types.Options{}) browser, err := New(&types.Options{ShowBrowser: false}) @@ -25,256 +523,15 @@ func TestActionNavigate(t *testing.T) { require.Nil(t, err, "could not create browser instance") defer instance.Close() - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - -

Nuclei Test

- - `) - })) + ts := httptest.NewServer(http.HandlerFunc(handler)) defer ts.Close() parsed, err := url.Parse(ts.URL) require.Nil(t, err, "could not parse URL") + extractedData, page, err := instance.Run(parsed, actions, timeout) + assert(page, err, extractedData) - actions := []*Action{{ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: "waitload"}} - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") -} - -func TestActionScript(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - t.Run("run-and-results", func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, - } - out, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - require.Equal(t, "some-data", out["test"], "could not run js and get results correctly") - }) - - t.Run("hook", func(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - Nuclei Test Page - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - actions := []*Action{ - {ActionType: "script", Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "script", Name: "test", Data: map[string]string{"code": "window.test"}}, - } - out, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - require.Equal(t, "some-data", out["test"], "could not run js and get results correctly with js hook") - }) -} - -func TestActionClick(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "click", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking + if page != nil { + page.Close() } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("button") - val := el.MustAttribute("a") - require.Equal(t, "ok", *val, "could not click button") -} - -func TestActionRightClick(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - Nuclei Test Page - - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "rightclick", Data: map[string]string{"selector": "button"}}, // Use css selector for clicking - } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("button") - val := el.MustAttribute("a") - require.Equal(t, "ok", *val, "could not click button") -} - -func TestActionTextInput(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, ` - - - Nuclei Test Page - - Nuclei Test Page - - `) - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - actions := []*Action{ - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - {ActionType: "text", Data: map[string]string{"selector": "input", "value": "test"}}, - } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - - require.Equal(t, "Nuclei Test Page", page.Page().MustInfo().Title, "could not navigate correctly") - el := page.Page().MustElement("input") - val := el.MustAttribute("event") - require.Equal(t, "input-change", *val, "could not get input change") - require.Equal(t, "test", el.MustText(), "could not get input change value") -} - -func TestActionHeadersChange(t *testing.T) { - _ = protocolstate.Init(&types.Options{}) - - browser, err := New(&types.Options{ShowBrowser: false}) - require.Nil(t, err, "could not create browser") - defer browser.Close() - - instance, err := browser.NewInstance() - require.Nil(t, err, "could not create browser instance") - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Test") == "Hello" { - fmt.Fprintln(w, `found`) - } - })) - defer ts.Close() - - parsed, err := url.Parse(ts.URL) - require.Nil(t, err, "could not parse URL") - - actions := []*Action{ - {ActionType: "setheader", Data: map[string]string{"part": "request", "key": "Test", "value": "Hello"}}, - {ActionType: "navigate", Data: map[string]string{"url": "{{BaseURL}}"}}, - {ActionType: "waitload"}, - } - _, page, err := instance.Run(parsed, actions, 20*time.Second) - require.Nil(t, err, "could not run page actions") - defer page.Close() - - require.Equal(t, "found", strings.ToLower(strings.TrimSpace(page.Page().MustElement("html").MustText())), "could not set header correctly") } diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 8af250b2b..43f62f437 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -32,24 +32,24 @@ type Step struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators +func (request *Request) Compile(options *protocols.ExecuterOptions) error { + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } - r.options = options + request.options = options return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } diff --git a/v2/pkg/protocols/headless/operators.go b/v2/pkg/protocols/headless/operators.go index 776adb89c..81a81a6ab 100644 --- a/v2/pkg/protocols/headless/operators.go +++ b/v2/pkg/protocols/headless/operators.go @@ -4,14 +4,16 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "resp", "": @@ -20,27 +22,27 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { case "body", "resp", "": @@ -62,50 +64,29 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext return nil } -// responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 5) - - // Some data regarding the request metadata - data["host"] = host - data["matched"] = matched - data["req"] = req - data["data"] = resp - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath - return data +// responseToDSLMap converts a headless response to a map for use in DSL matching +func (request *Request) responseToDSLMap(resp, req, host, matched string) output.InternalEvent { + return output.InternalEvent{ + "host": host, + "matched": matched, + "req": req, + "data": resp, + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, + } } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 54345d468..55394ed7f 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -6,75 +6,72 @@ import ( "time" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" ) var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { - instance, err := r.options.Browser.NewInstance() +func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent /*TODO review unused parameter*/, callback protocols.OutputEventCallback) error { + instance, err := request.options.Browser.NewInstance() if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "headless", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer instance.Close() parsed, err := url.Parse(input) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "headless", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } - out, page, err := instance.Run(parsed, r.Steps, time.Duration(r.options.Options.PageTimeout)*time.Second) + out, page, err := instance.Run(parsed, request.Steps, time.Duration(request.options.Options.PageTimeout)*time.Second) if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "headless", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "headless", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could get html element") } defer page.Close() - r.options.Output.Request(r.options.TemplateID, input, "headless", nil) - r.options.Progress.IncrementRequests() + request.options.Output.Request(request.options.TemplateID, input, "headless", nil) + request.options.Progress.IncrementRequests() gologger.Verbose().Msgf("Sent Headless request to %s", input) reqBuilder := &strings.Builder{} - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped Headless request for %s", r.options.TemplateID, input) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped Headless request for %s", request.options.TemplateID, input) - for _, act := range r.Steps { + for _, act := range request.Steps { reqBuilder.WriteString(act.String()) reqBuilder.WriteString("\n") } gologger.Print().Msgf("%s", reqBuilder.String()) } - var respBody string + var responseBody string html, err := page.Page().Element("html") if err == nil { - respBody, _ = html.HTML() + responseBody, _ = html.HTML() } - outputEvent := r.responseToDSLMap(respBody, reqBuilder.String(), input, input) + outputEvent := request.responseToDSLMap(responseBody, reqBuilder.String(), input, input) for k, v := range out { outputEvent[k] = v } - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Debug().Msgf("[%s] Dumped Headless response for %s", r.options.TemplateID, input) - gologger.Print().Msgf("%s", respBody) + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Headless response for %s", request.options.TemplateID, input) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, responseBody, request.options.Options.NoColor)) } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.Results = r.MakeResultEvent(event) - } - } callback(event) return nil } diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index e1f39166b..ea83aea72 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -1,6 +1,7 @@ package http import ( + "bufio" "context" "fmt" "io" @@ -20,6 +21,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/race" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/raw" + "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/rawhttp" "github.com/projectdiscovery/retryablehttp-go" ) @@ -35,11 +37,25 @@ type generatedRequest struct { meta map[string]interface{} pipelinedClient *rawhttp.PipelineClient request *retryablehttp.Request + dynamicValues map[string]interface{} +} + +func (g *generatedRequest) URL() string { + if g.request != nil { + return g.request.URL.String() + } + if g.rawRequest != nil { + return g.rawRequest.FullURL + } + return "" } // Make creates a http request for the provided input. // It returns io.EOF as error when all the requests have been exhausted. func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) { + if r.request.SelfContained { + return r.makeSelfContainedRequest(dynamicValues, interactURL) + } // We get the next payload for the request. data, payloads, ok := r.nextValue() if !ok { @@ -47,6 +63,14 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa } ctx := context.Background() + if interactURL != "" { + data = r.options.Interactsh.ReplaceMarkers(data, interactURL) + + for payloadName, payloadValue := range payloads { + payloads[payloadName] = r.options.Interactsh.ReplaceMarkers(types.ToString(payloadValue), interactURL) + } + } + parsed, err := url.Parse(baseURL) if err != nil { return nil, err @@ -54,22 +78,22 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa data, parsed = baseURLWithTemplatePrefs(data, parsed) - trailingSlash := false isRawRequest := len(r.request.Raw) > 0 + + // If the request is not a raw request, and the URL input path is suffixed with + // a trailing slash, and our Input URL is also suffixed with a trailing slash, + // mark trailingSlash bool as true which will be later used during variable generation + // to generate correct path removed slash which would otherwise generate // invalid sequence. + // TODO: Figure out a cleaner way to do this sanitization. + trailingSlash := false if !isRawRequest && strings.HasSuffix(parsed.Path, "/") && strings.Contains(data, "{{BaseURL}}/") { trailingSlash = true } - values := generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash)) - // merge with vars - if !r.options.Options.Vars.IsEmpty() { - values = generators.MergeMaps(values, r.options.Options.Vars.AsMap()) - } - - // merge with env vars - if r.options.Options.EnvironmentVariables { - values = generators.MergeMaps(generators.EnvVars(), values) - } + values := generators.MergeMaps( + generators.MergeMaps(dynamicValues, generateVariables(parsed, trailingSlash)), + generators.BuildPayloadFromOptions(r.request.options.Options), + ) // If data contains \n it's a raw request, process it like raw. Else // continue with the template based request flow. @@ -79,6 +103,48 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) } +func (r *requestGenerator) makeSelfContainedRequest(dynamicValues map[string]interface{}, interactURL string) (*generatedRequest, error) { + // We get the next payload for the request. + data, payloads, ok := r.nextValue() + if !ok { + return nil, io.EOF + } + ctx := context.Background() + + isRawRequest := r.request.isRaw() + + // If the request is a raw request, get the URL from the request + // header and use it to make the request. + if isRawRequest { + // Get the hostname from the URL section to build the request. + reader := bufio.NewReader(strings.NewReader(data)) + s, err := reader.ReadString('\n') + if err != nil { + return nil, fmt.Errorf("could not read request: %s", err) + } + + parts := strings.Split(s, " ") + if len(parts) < 3 { + return nil, fmt.Errorf("malformed request supplied") + } + parsed, err := url.Parse(parts[1]) + if err != nil { + return nil, fmt.Errorf("could not parse request URL: %s", err) + } + values := generators.MergeMaps( + generators.MergeMaps(dynamicValues, generateVariables(parsed, false)), + generators.BuildPayloadFromOptions(r.request.options.Options), + ) + + return r.makeHTTPRequestFromRaw(ctx, parsed.String(), data, values, payloads, interactURL) + } + values := generators.MergeMaps( + dynamicValues, + generators.BuildPayloadFromOptions(r.request.options.Options), + ) + return r.makeHTTPRequestFromModel(ctx, data, values, payloads, interactURL) +} + // Total returns the total number of requests for the generator func (r *requestGenerator) Total() int { if r.payloadIterator != nil { @@ -136,7 +202,7 @@ func (r *requestGenerator) makeHTTPRequestFromModel(ctx context.Context, data st if err != nil { return nil, err } - return &generatedRequest{request: request, meta: generatorValues, original: r.request}, nil + return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil } // makeHTTPRequestFromRaw creates a *http.Request from a raw request @@ -197,7 +263,7 @@ func (r *requestGenerator) handleRawWithPayloads(ctx context.Context, rawRequest return nil, err } - return &generatedRequest{request: request, meta: generatorValues, original: r.request}, nil + return &generatedRequest{request: request, meta: generatorValues, original: r.request, dynamicValues: finalValues}, nil } // fillRequest fills various headers in the request with values diff --git a/v2/pkg/protocols/http/cluster.go b/v2/pkg/protocols/http/cluster.go index bdfa07358..8c1674be3 100644 --- a/v2/pkg/protocols/http/cluster.go +++ b/v2/pkg/protocols/http/cluster.go @@ -9,20 +9,20 @@ import ( // This used by the clustering engine to decide whether two requests // are similar enough to be considered one and can be checked by // just adding the matcher/extractors for the request and the correct IDs. -func (r *Request) CanCluster(other *Request) bool { - if len(r.Payloads) > 0 || len(r.Raw) > 0 || len(r.Body) > 0 || r.Unsafe || r.ReqCondition || r.Name != "" { +func (request *Request) CanCluster(other *Request) bool { + if len(request.Payloads) > 0 || len(request.Raw) > 0 || len(request.Body) > 0 || request.Unsafe || request.ReqCondition || request.Name != "" { return false } - if r.Method != other.Method || - r.MaxRedirects != other.MaxRedirects || - r.CookieReuse != other.CookieReuse || - r.Redirects != other.Redirects { + if request.Method != other.Method || + request.MaxRedirects != other.MaxRedirects || + request.CookieReuse != other.CookieReuse || + request.Redirects != other.Redirects { return false } - if !compare.StringSlice(r.Path, other.Path) { + if !compare.StringSlice(request.Path, other.Path) { return false } - if !compare.StringMap(r.Headers, other.Headers) { + if !compare.StringMap(request.Headers, other.Headers) { return false } return true diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index d681c1b5a..304f1a497 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -1,12 +1,15 @@ package http import ( + "fmt" "strings" "github.com/pkg/errors" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" "github.com/projectdiscovery/rawhttp" @@ -43,13 +46,13 @@ type Request struct { // description: | // Attack is the type of payload combinations to perform. // - // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates + // batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. // values: - // - "sniper" + // - "batteringram" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Method is the HTTP Request Method. // values: @@ -132,6 +135,11 @@ type Request struct { generator *generators.Generator // optional, only enabled when using payloads httpClient *retryablehttp.Client rawhttpClient *rawhttp.Client + dynamicValues map[string]interface{} + + // description: | + // SelfContained specifies if the request is self contained. + SelfContained bool `yaml:"-" json:"-"` // description: | // CookieReuse is an optional setting that enables cookie reuse for @@ -166,24 +174,31 @@ type Request struct { // description: | // StopAtFirstMatch stops the execution of the requests and template as soon as a match is found. StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop the execution after a match is found"` + // description: | + // SkipVariablesCheck skips the check for unresolved variables in request + SkipVariablesCheck bool `yaml:"skip-variables-check,omitempty" jsonschema:"title=skip variable checks,description=Skips the check for unresolved variables in request"` } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID +} + +func (request *Request) isRaw() bool { + return len(request.Raw) > 0 } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { connectionConfiguration := &httpclientpool.Configuration{ - Threads: r.Threads, - MaxRedirects: r.MaxRedirects, - FollowRedirects: r.Redirects, - CookieReuse: r.CookieReuse, + Threads: request.Threads, + MaxRedirects: request.MaxRedirects, + FollowRedirects: request.Redirects, + CookieReuse: request.CookieReuse, } // if the headers contain "Connection" we need to disable the automatic keep alive of the standard library - if _, hasConnectionHeader := r.Headers["Connection"]; hasConnectionHeader { + if _, hasConnectionHeader := request.Headers["Connection"]; hasConnectionHeader { connectionConfiguration.Connection = &httpclientpool.ConnectionConfiguration{DisableKeepAlive: false} } @@ -191,76 +206,110 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if err != nil { return errors.Wrap(err, "could not get dns client") } - r.customHeaders = make(map[string]string) - r.httpClient = client - r.options = options - for _, option := range r.options.Options.CustomHeaders { + request.customHeaders = make(map[string]string) + request.httpClient = client + request.options = options + for _, option := range request.options.Options.CustomHeaders { parts := strings.SplitN(option, ":", 2) if len(parts) != 2 { continue } - r.customHeaders[parts[0]] = strings.TrimSpace(parts[1]) + request.customHeaders[parts[0]] = strings.TrimSpace(parts[1]) } - if r.Body != "" && !strings.Contains(r.Body, "\r\n") { - r.Body = strings.ReplaceAll(r.Body, "\n", "\r\n") + if request.Body != "" && !strings.Contains(request.Body, "\r\n") { + request.Body = strings.ReplaceAll(request.Body, "\n", "\r\n") } - if len(r.Raw) > 0 { - for i, raw := range r.Raw { + if len(request.Raw) > 0 { + for i, raw := range request.Raw { if !strings.Contains(raw, "\r\n") { - r.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n") + request.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n") } } - r.rawhttpClient = httpclientpool.GetRawHTTP(options.Options) + request.rawhttpClient = httpclientpool.GetRawHTTP(options.Options) } - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if compileErr := compiled.Compile(); compileErr != nil { return errors.Wrap(compileErr, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } - if len(r.Payloads) > 0 { - attackType := r.AttackType - if attackType == "" { - attackType = "sniper" + // Resolve payload paths from vars if they exists + for name, payload := range request.options.Options.VarsPayload() { + payloadStr, ok := payload.(string) + // check if inputs contains the payload + var hasPayloadName bool + // search for markers in all request parts + var inputs []string + inputs = append(inputs, request.Method, request.Body) + inputs = append(inputs, request.Raw...) + for k, v := range request.customHeaders { + inputs = append(inputs, fmt.Sprintf("%s: %s", k, v)) + } + for k, v := range request.Headers { + inputs = append(inputs, fmt.Sprintf("%s: %s", k, v)) + } + + for _, input := range inputs { + if expressions.ContainsVariablesWithNames(input, map[string]interface{}{name: payload}) == nil { + hasPayloadName = true + break + } + } + if ok && hasPayloadName && fileutil.FileExists(payloadStr) { + if request.Payloads == nil { + request.Payloads = make(map[string]interface{}) + } + request.Payloads[name] = payloadStr + } + } + + if len(request.Payloads) > 0 { + attackType := request.AttackType + if attackType == "" { + attackType = "batteringram" + } + var ok bool + request.attackType, ok = generators.StringToType[attackType] + if !ok { + return fmt.Errorf("invalid attack type provided: %s", attackType) } - r.attackType = generators.StringToType[attackType] // Resolve payload paths if they are files. - for name, payload := range r.Payloads { + for name, payload := range request.Payloads { payloadStr, ok := payload.(string) if ok { final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) if resolveErr != nil { return errors.Wrap(resolveErr, "could not read payload file") } - r.Payloads[name] = final + request.Payloads[name] = final } } - r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) if err != nil { return errors.Wrap(err, "could not parse payloads") } } - r.options = options - r.totalRequests = r.Requests() + request.options = options + request.totalRequests = request.Requests() return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { - if r.generator != nil { - payloadRequests := r.generator.NewIterator().Total() * len(r.Raw) +func (request *Request) Requests() int { + if request.generator != nil { + payloadRequests := request.generator.NewIterator().Total() * len(request.Raw) return payloadRequests } - if len(r.Raw) > 0 { - requests := len(r.Raw) - if requests == 1 && r.RaceNumberRequests != 0 { - requests *= r.RaceNumberRequests + if len(request.Raw) > 0 { + requests := len(request.Raw) + if requests == 1 && request.RaceNumberRequests != 0 { + requests *= request.RaceNumberRequests } return requests } - return len(r.Path) + return len(request.Path) } diff --git a/v2/pkg/protocols/http/operators.go b/v2/pkg/protocols/http/operators.go index 249bf3f3e..9ae939cf6 100644 --- a/v2/pkg/protocols/http/operators.go +++ b/v2/pkg/protocols/http/operators.go @@ -6,46 +6,57 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := getMatchPart(matcher.Part, data) if !ok { - return false + return false, []string{} } switch matcher.GetType() { case matchers.StatusMatcher: - statusCode, ok := data["status_code"] + statusCode, ok := getStatusCode(data) if !ok { - return false + return false, []string{} } - status, ok := statusCode.(int) - if !ok { - return false - } - return matcher.Result(matcher.MatchStatusCode(status)) + return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{responsehighlighter.CreateStatusCodeSnippet(data["response"].(string), statusCode)} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(item))) + return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, request.dynamicValues)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} +} + +func getStatusCode(data map[string]interface{}) (int, bool) { + statusCodeValue, ok := data["status_code"] + if !ok { + return 0, false + } + statusCode, ok := statusCodeValue.(int) + if !ok { + return 0, false + } + return statusCode, true } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { item, ok := getMatchPart(extractor.Part, data) if !ok { return nil @@ -86,8 +97,8 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) { } // responseToDSLMap converts an HTTP response to a map for use in DSL matching -func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { - data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) +func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent { + data := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { data[k] = v } @@ -107,42 +118,22 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["content_length"] = resp.ContentLength data["all_headers"] = headers data["duration"] = duration.Seconds() - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo + data["template-path"] = request.options.TemplatePath return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 && !wrapped.OperatorsResult.Matched { - return nil - } - - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), @@ -156,6 +147,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out IP: types.ToString(wrapped.InternalEvent["ip"]), Request: types.ToString(wrapped.InternalEvent["request"]), Response: types.ToString(wrapped.InternalEvent["response"]), + CURLCommand: types.ToString(wrapped.InternalEvent["curl-command"]), } return data } diff --git a/v2/pkg/protocols/http/operators_test.go b/v2/pkg/protocols/http/operators_test.go index 6c24f236a..c9d3b795d 100644 --- a/v2/pkg/protocols/http/operators_test.go +++ b/v2/pkg/protocols/http/operators_test.go @@ -84,8 +84,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -98,8 +99,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -111,8 +113,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } @@ -259,7 +262,7 @@ func TestHTTPMakeResult(t *testing.T) { event["ip"] = "192.169.1.1" finalEvent := &output.InternalWrappedEvent{InternalEvent: event} if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index ea23db738..2b3d44af5 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -28,9 +28,6 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { Headers: make(map[string]string), } if unsafe { - request = strings.ReplaceAll(request, "\\0", "\x00") - request = strings.ReplaceAll(request, "\\r", "\r") - request = strings.ReplaceAll(request, "\\n", "\n") rawRequest.UnsafeRawBytes = []byte(request) } reader := bufio.NewReader(strings.NewReader(request)) @@ -93,8 +90,10 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { return nil, fmt.Errorf("could not parse request URL: %s", parseErr) } - rawRequest.Path = parts[1] - rawRequest.Headers["Host"] = parsed.Host + rawRequest.Path = parsed.Path + if _, ok := rawRequest.Headers["Host"]; !ok { + rawRequest.Headers["Host"] = parsed.Host + } } else if len(parts) > 1 { rawRequest.Path = parts[1] } @@ -107,15 +106,17 @@ func Parse(request, baseURL string, unsafe bool) (*Request, error) { if strings.HasSuffix(parsedURL.Path, "/") && strings.HasPrefix(rawRequest.Path, "/") { parsedURL.Path = strings.TrimSuffix(parsedURL.Path, "/") } - rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path) + if parsedURL.Path != rawRequest.Path { + rawRequest.Path = fmt.Sprintf("%s%s", parsedURL.Path, rawRequest.Path) + } if strings.HasSuffix(rawRequest.Path, "//") { rawRequest.Path = strings.TrimSuffix(rawRequest.Path, "/") } rawRequest.FullURL = fmt.Sprintf("%s://%s%s", parsedURL.Scheme, strings.TrimSpace(hostURL), rawRequest.Path) - // If raw request doesn't have a Host header - // this will be generated from the parsed baseURL - if rawRequest.Headers["Host"] == "" { + // If raw request doesn't have a Host header and isn't marked unsafe, + // this will generate the Host header from the parsed baseURL + if !unsafe && rawRequest.Headers["Host"] == "" { rawRequest.Headers["Host"] = hostURL } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index afb52713d..1a619c179 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -15,11 +15,15 @@ import ( "github.com/pkg/errors" "github.com/remeh/sizedwaitgroup" "go.uber.org/multierr" + "moul.io/http2curl" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http/httpclientpool" @@ -30,52 +34,52 @@ import ( const defaultMaxWorkers = 150 // executeRaceRequest executes race condition request for a URL -func (r *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - var requests []*generatedRequest +func (request *Request) executeRaceRequest(reqURL string, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + var generatedRequests []*generatedRequest // Requests within race condition should be dumped once and the output prefilled to allow DSL language to work // This will introduce a delay and will populate in hacky way the field "request" of outputEvent - generator := r.newGenerator() + generator := request.newGenerator() requestForDump, err := generator.Make(reqURL, nil, "") if err != nil { return err } - r.setCustomHeaders(requestForDump) + request.setCustomHeaders(requestForDump) dumpedRequest, err := dump(requestForDump, reqURL) if err != nil { return err } - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) gologger.Print().Msgf("%s", string(dumpedRequest)) } previous["request"] = string(dumpedRequest) // Pre-Generate requests - for i := 0; i < r.RaceNumberRequests; i++ { - generator := r.newGenerator() - request, err := generator.Make(reqURL, nil, "") + for i := 0; i < request.RaceNumberRequests; i++ { + generator := request.newGenerator() + generatedRequest, err := generator.Make(reqURL, nil, "") if err != nil { return err } - requests = append(requests, request) + generatedRequests = append(generatedRequests, generatedRequest) } wg := sync.WaitGroup{} var requestErr error mutex := &sync.Mutex{} - for i := 0; i < r.RaceNumberRequests; i++ { + for i := 0; i < request.RaceNumberRequests; i++ { wg.Add(1) go func(httpRequest *generatedRequest) { defer wg.Done() - err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) } mutex.Unlock() - }(requests[i]) - r.options.Progress.IncrementRequests() + }(generatedRequests[i]) + request.options.Progress.IncrementRequests() } wg.Wait() @@ -83,47 +87,50 @@ func (r *Request) executeRaceRequest(reqURL string, previous output.InternalEven } // executeRaceRequest executes parallel requests for a template -func (r *Request) executeParallelHTTP(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error { - generator := r.newGenerator() +func (request *Request) executeParallelHTTP(reqURL string, dynamicValues output.InternalEvent, callback protocols.OutputEventCallback) error { + generator := request.newGenerator() // Workers that keeps enqueuing new requests - maxWorkers := r.Threads + maxWorkers := request.Threads swg := sizedwaitgroup.New(maxWorkers) var requestErr error mutex := &sync.Mutex{} for { - request, err := generator.Make(reqURL, dynamicValues, "") + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "") if err == io.EOF { break } + if reqURL == "" { + reqURL = generatedHttpRequest.URL() + } if err != nil { - r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() - r.options.RateLimiter.Take() + request.options.RateLimiter.Take() previous := make(map[string]interface{}) - err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) } mutex.Unlock() - }(request) - r.options.Progress.IncrementRequests() + }(generatedHttpRequest) + request.options.Progress.IncrementRequests() } swg.Wait() return requestErr } // executeTurboHTTP executes turbo http request for a URL -func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - generator := r.newGenerator() +func (request *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + generator := request.newGenerator() // need to extract the target from the url URL, err := url.Parse(reqURL) @@ -134,13 +141,13 @@ func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output pipeOptions := rawhttp.DefaultPipelineOptions pipeOptions.Host = URL.Host pipeOptions.MaxConnections = 1 - if r.PipelineConcurrentConnections > 0 { - pipeOptions.MaxConnections = r.PipelineConcurrentConnections + if request.PipelineConcurrentConnections > 0 { + pipeOptions.MaxConnections = request.PipelineConcurrentConnections } - if r.PipelineRequestsPerConnection > 0 { - pipeOptions.MaxPendingRequests = r.PipelineRequestsPerConnection + if request.PipelineRequestsPerConnection > 0 { + pipeOptions.MaxPendingRequests = request.PipelineRequestsPerConnection } - pipeclient := rawhttp.NewPipelineClient(pipeOptions) + pipeClient := rawhttp.NewPipelineClient(pipeOptions) // defaultMaxWorkers should be a sufficient value to keep queues always full maxWorkers := defaultMaxWorkers @@ -153,105 +160,116 @@ func (r *Request) executeTurboHTTP(reqURL string, dynamicValues, previous output var requestErr error mutex := &sync.Mutex{} for { - request, err := generator.Make(reqURL, dynamicValues, "") + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, "") if err == io.EOF { break } + if reqURL == "" { + reqURL = generatedHttpRequest.URL() + } if err != nil { - r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } - request.pipelinedClient = pipeclient + generatedHttpRequest.pipelinedClient = pipeClient swg.Add() go func(httpRequest *generatedRequest) { defer swg.Done() - err := r.executeRequest(reqURL, httpRequest, previous, false, callback, 0) + err := request.executeRequest(reqURL, httpRequest, previous, false, callback, 0) mutex.Lock() if err != nil { requestErr = multierr.Append(requestErr, err) } mutex.Unlock() - }(request) - r.options.Progress.IncrementRequests() + }(generatedHttpRequest) + request.options.Progress.IncrementRequests() } swg.Wait() return requestErr } // ExecuteWithResults executes the final request on a URL -func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) ExecuteWithResults(reqURL string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // verify if pipeline was requested - if r.Pipeline { - return r.executeTurboHTTP(reqURL, dynamicValues, previous, callback) + if request.Pipeline { + return request.executeTurboHTTP(reqURL, dynamicValues, previous, callback) } // verify if a basic race condition was requested - if r.Race && r.RaceNumberRequests > 0 { - return r.executeRaceRequest(reqURL, previous, callback) + if request.Race && request.RaceNumberRequests > 0 { + return request.executeRaceRequest(reqURL, previous, callback) } // verify if parallel elaboration was requested - if r.Threads > 0 { - return r.executeParallelHTTP(reqURL, dynamicValues, callback) + if request.Threads > 0 { + return request.executeParallelHTTP(reqURL, dynamicValues, callback) } - generator := r.newGenerator() + generator := request.newGenerator() requestCount := 1 var requestErr error for { - hasInteractMarkers := interactsh.HasMatchers(r.CompiledOperators) + hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) var interactURL string - if r.options.Interactsh != nil && hasInteractMarkers { - interactURL = r.options.Interactsh.URL() + if request.options.Interactsh != nil && hasInteractMarkers { + interactURL = request.options.Interactsh.URL() } - request, err := generator.Make(reqURL, dynamicValues, interactURL) + generatedHttpRequest, err := generator.Make(reqURL, dynamicValues, interactURL) if err == io.EOF { break } + if reqURL == "" { + reqURL = generatedHttpRequest.URL() + } if err != nil { - r.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) + request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total())) return err } + request.dynamicValues = generatedHttpRequest.dynamicValues // Check if hosts just keep erroring - if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.Check(reqURL) { + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.Check(reqURL) { break } var gotOutput bool - r.options.RateLimiter.Take() - err = r.executeRequest(reqURL, request, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) { + request.options.RateLimiter.Take() + err = request.executeRequest(reqURL, generatedHttpRequest, previous, hasInteractMarkers, func(event *output.InternalWrappedEvent) { // Add the extracts to the dynamic values if any. if event.OperatorsResult != nil { gotOutput = true dynamicValues = generators.MergeMaps(dynamicValues, event.OperatorsResult.DynamicValues) } - if hasInteractMarkers && r.options.Interactsh != nil { - r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ - MakeResultFunc: r.MakeResultEvent, + if hasInteractMarkers && request.options.Interactsh != nil { + request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ + MakeResultFunc: request.MakeResultEvent, Event: event, - Operators: r.CompiledOperators, - MatchFunc: r.Match, - ExtractFunc: r.Extract, + Operators: request.CompiledOperators, + MatchFunc: request.Match, + ExtractFunc: request.Extract, }) } else { callback(event) } }, requestCount) + // If a variable is unresolved, skip all further requests + if err == errStopExecution { + break + } if err != nil { - if r.options.HostErrorsCache != nil && r.options.HostErrorsCache.CheckError(err) { - r.options.HostErrorsCache.MarkFailed(reqURL) + if request.options.HostErrorsCache != nil && request.options.HostErrorsCache.CheckError(err) { + request.options.HostErrorsCache.MarkFailed(reqURL) } requestErr = err } requestCount++ - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() - if (request.original.options.Options.StopAtFirstMatch || r.StopAtFirstMatch) && gotOutput { - r.options.Progress.IncrementErrorsBy(int64(generator.Total())) + // If this was a match and we want to stop at first match, skip all further requests. + if (generatedHttpRequest.original.options.Options.StopAtFirstMatch || request.StopAtFirstMatch) && gotOutput { break } } @@ -260,89 +278,98 @@ func (r *Request) ExecuteWithResults(reqURL string, dynamicValues, previous outp const drainReqSize = int64(8 * 1024) +var errStopExecution = errors.New("stop execution due to unresolved variables") + // executeRequest executes the actual generated request and returns error if occurred -func (r *Request) executeRequest(reqURL string, request *generatedRequest, previous output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error { - r.setCustomHeaders(request) +func (request *Request) executeRequest(reqURL string, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMarkers bool, callback protocols.OutputEventCallback, requestCount int) error { + request.setCustomHeaders(generatedRequest) var ( resp *http.Response - fromcache bool + fromCache bool dumpedRequest []byte err error ) - var formedURL string - var hostname string - timeStart := time.Now() - if request.original.Pipeline { - if request.rawRequest != nil { - formedURL = request.rawRequest.FullURL - if parsed, parseErr := url.Parse(formedURL); parseErr == nil { - hostname = parsed.Host - } - resp, err = request.pipelinedClient.DoRaw(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data))) - } else if request.request != nil { - resp, err = request.pipelinedClient.Dor(request.request) - } - } else if request.original.Unsafe && request.rawRequest != nil { - formedURL = request.rawRequest.FullURL - if parsed, parseErr := url.Parse(formedURL); parseErr == nil { - hostname = parsed.Host - } - options := request.original.rawhttpClient.Options - options.FollowRedirects = r.Redirects - options.CustomRawBytes = request.rawRequest.UnsafeRawBytes - resp, err = request.original.rawhttpClient.DoRawWithOptions(request.rawRequest.Method, reqURL, request.rawRequest.Path, generators.ExpandMapValues(request.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(request.rawRequest.Data)), options) - } else { - hostname = request.request.URL.Host - formedURL = request.request.URL.String() - // if nuclei-project is available check if the request was already sent previously - if r.options.ProjectFile != nil { - // if unavailable fail silently - fromcache = true - resp, err = r.options.ProjectFile.Get(dumpedRequest) - if err != nil { - fromcache = false - } - } - if resp == nil { - resp, err = r.httpClient.Do(request.request) - } - } - // For race conditions we can't dump the request body at this point as it's already waiting the open-gate event, already handled with a similar code within the race function - if !request.original.Race { + if !generatedRequest.original.Race { var dumpError error - dumpedRequest, dumpError = dump(request, reqURL) + dumpedRequest, dumpError = dump(generatedRequest, reqURL) if dumpError != nil { return dumpError } + dumpedRequestString := string(dumpedRequest) - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", r.options.TemplateID, reqURL) - gologger.Print().Msgf("%s", string(dumpedRequest)) + // Check if are there any unresolved variables. If yes, skip unless overriden by user. + if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { + gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, reqURL, varErr) + return errStopExecution + } + + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped HTTP request for %s\n\n", request.options.TemplateID, reqURL) + gologger.Print().Msgf("%s", dumpedRequestString) + } + } + + var formedURL string + var hostname string + timeStart := time.Now() + if generatedRequest.original.Pipeline { + if generatedRequest.rawRequest != nil { + formedURL = generatedRequest.rawRequest.FullURL + if parsed, parseErr := url.Parse(formedURL); parseErr == nil { + hostname = parsed.Host + } + resp, err = generatedRequest.pipelinedClient.DoRaw(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data))) + } else if generatedRequest.request != nil { + resp, err = generatedRequest.pipelinedClient.Dor(generatedRequest.request) + } + } else if generatedRequest.original.Unsafe && generatedRequest.rawRequest != nil { + formedURL = generatedRequest.rawRequest.FullURL + if parsed, parseErr := url.Parse(formedURL); parseErr == nil { + hostname = parsed.Host + } + options := generatedRequest.original.rawhttpClient.Options + options.FollowRedirects = request.Redirects + options.CustomRawBytes = generatedRequest.rawRequest.UnsafeRawBytes + resp, err = generatedRequest.original.rawhttpClient.DoRawWithOptions(generatedRequest.rawRequest.Method, reqURL, generatedRequest.rawRequest.Path, generators.ExpandMapValues(generatedRequest.rawRequest.Headers), ioutil.NopCloser(strings.NewReader(generatedRequest.rawRequest.Data)), options) + } else { + hostname = generatedRequest.request.URL.Host + formedURL = generatedRequest.request.URL.String() + // if nuclei-project is available check if the request was already sent previously + if request.options.ProjectFile != nil { + // if unavailable fail silently + fromCache = true + resp, err = request.options.ProjectFile.Get(dumpedRequest) + if err != nil { + fromCache = false + } + } + if resp == nil { + resp, err = request.httpClient.Do(generatedRequest.request) } } if err != nil { // rawhttp doesn't support draining response bodies. - if resp != nil && resp.Body != nil && request.rawRequest == nil { + if resp != nil && resp.Body != nil && generatedRequest.rawRequest == nil { _, _ = io.CopyN(ioutil.Discard, resp.Body, drainReqSize) resp.Body.Close() } - r.options.Output.Request(r.options.TemplateID, formedURL, "http", err) - r.options.Progress.IncrementErrorsBy(1) + request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) + request.options.Progress.IncrementErrorsBy(1) // If we have interactsh markers and request times out, still send // a callback event so in case we receive an interaction, correlation is possible. if hasInteractMarkers { - outputEvent := r.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, request.meta) + outputEvent := request.responseToDSLMap(&http.Response{}, reqURL, formedURL, tostring.UnsafeToString(dumpedRequest), "", "", "", 0, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { + if request.CompiledOperators != nil { event.InternalEvent = outputEvent } callback(event) @@ -356,8 +383,18 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ resp.Body.Close() }() - gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", r.options.TemplateID, formedURL) - r.options.Output.Request(r.options.TemplateID, formedURL, "http", err) + var curlCommand string + if !request.Unsafe && resp != nil && generatedRequest.request != nil { + bodyBytes, _ := generatedRequest.request.BodyBytes() + resp.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) + command, _ := http2curl.GetCurlCommand(resp.Request) + if err == nil && command != nil { + curlCommand = command.String() + } + } + + gologger.Verbose().Msgf("[%s] Sent HTTP request to %s", request.options.TemplateID, formedURL) + request.options.Output.Request(request.options.TemplateID, formedURL, "http", err) duration := time.Since(timeStart) @@ -370,8 +407,8 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ // If the status code is HTTP 101, we should not proceed with reading body. if resp.StatusCode != http.StatusSwitchingProtocols { var bodyReader io.Reader - if r.MaxSize != 0 { - bodyReader = io.LimitReader(resp.Body, int64(r.MaxSize)) + if request.MaxSize != 0 { + bodyReader = io.LimitReader(resp.Body, int64(request.MaxSize)) } else { bodyReader = resp.Body } @@ -379,7 +416,7 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ if err != nil { // Ignore body read due to server misconfiguration errors if stringsutil.ContainsAny(err.Error(), "gzip: invalid header") { - gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", r.options.TemplateID, formedURL, err.Error()) + gologger.Warning().Msgf("[%s] Server sent an invalid gzip header and it was not possible to read the uncompressed body for %s: %s", request.options.TemplateID, formedURL, err.Error()) } else if !stringsutil.ContainsAny(err.Error(), "unexpected EOF", "user canceled") { // ignore EOF and random error return errors.Wrap(err, "could not read http body") } @@ -412,7 +449,8 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ redirectedResponse = bytes.ReplaceAll(redirectedResponse, dataOrig, data) // Decode gbk response content-types - if contentType := strings.ToLower(resp.Header.Get("Content-Type")); contentType != "" && (strings.Contains(contentType, "gbk") || strings.Contains(contentType, "gb2312")) { + // gb18030 supersedes gb2312 + if isContentTypeGbk(resp.Header.Get("Content-Type")) { dumpedResponse, err = decodegbk(dumpedResponse) if err != nil { return errors.Wrap(err, "could not gbk decode") @@ -421,68 +459,68 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ if err != nil { return errors.Wrap(err, "could not gbk decode") } - } - // Dump response - step 2 - replace gzip body with deflated one or with itself (NOP operation) - if r.options.Options.Debug || r.options.Options.DebugResponse { - gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", r.options.TemplateID, formedURL) - gologger.Print().Msgf("%s", string(redirectedResponse)) + // the uncompressed body needs to be decoded to standard utf8 + data, err = decodegbk(data) + if err != nil { + return errors.Wrap(err, "could not gbk decode") + } } // if nuclei-project is enabled store the response if not previously done - if r.options.ProjectFile != nil && !fromcache { - if err := r.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil { + if request.options.ProjectFile != nil && !fromCache { + if err := request.options.ProjectFile.Set(dumpedRequest, resp, data); err != nil { return errors.Wrap(err, "could not store in project file") } } matchedURL := reqURL - if request.rawRequest != nil && request.rawRequest.FullURL != "" { - matchedURL = request.rawRequest.FullURL + if generatedRequest.rawRequest != nil && generatedRequest.rawRequest.FullURL != "" { + matchedURL = generatedRequest.rawRequest.FullURL } - if request.request != nil { - matchedURL = request.request.URL.String() + if generatedRequest.request != nil { + matchedURL = generatedRequest.request.URL.String() } finalEvent := make(output.InternalEvent) - outputEvent := r.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, request.meta) + outputEvent := request.responseToDSLMap(resp, reqURL, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(data), headersToString(resp.Header), duration, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } + outputEvent["curl-command"] = curlCommand outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) outputEvent["redirect-chain"] = tostring.UnsafeToString(redirectedResponse) - for k, v := range previous { + for k, v := range previousEvent { finalEvent[k] = v } for k, v := range outputEvent { finalEvent[k] = v } // Add to history the current request number metadata if asked by the user. - if r.ReqCondition { + if request.ReqCondition { for k, v := range outputEvent { key := fmt.Sprintf("%s_%d", k, requestCount) - previous[key] = v + previousEvent[key] = v finalEvent[key] = v } } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - if r.CompiledOperators != nil { - var ok bool - event.OperatorsResult, ok = r.CompiledOperators.Execute(finalEvent, r.Match, r.Extract) - if ok && event.OperatorsResult != nil { - event.OperatorsResult.PayloadValues = request.meta - event.Results = r.MakeResultEvent(event) - } - event.InternalEvent = outputEvent + event := eventcreator.CreateEventWithAdditionalOptions(request, finalEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta + }) + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Info().Msgf("[%s] Dumped HTTP response for %s\n\n", request.options.TemplateID, formedURL) + gologger.Print().Msgf("%s", responsehighlighter.Highlight(event.OperatorsResult, string(redirectedResponse), request.options.Options.NoColor)) } + callback(event) return nil } // setCustomHeaders sets the custom headers for generated request -func (r *Request) setCustomHeaders(req *generatedRequest) { - for k, v := range r.customHeaders { +func (request *Request) setCustomHeaders(req *generatedRequest) { + for k, v := range request.customHeaders { if req.rawRequest != nil { req.rawRequest.Headers[k] = v } else { diff --git a/v2/pkg/protocols/http/request_generator.go b/v2/pkg/protocols/http/request_generator.go index 236225921..90c5d0fbd 100644 --- a/v2/pkg/protocols/http/request_generator.go +++ b/v2/pkg/protocols/http/request_generator.go @@ -19,11 +19,11 @@ type requestGenerator struct { } // newGenerator creates a new request generator instance -func (r *Request) newGenerator() *requestGenerator { - generator := &requestGenerator{request: r, options: r.options} +func (request *Request) newGenerator() *requestGenerator { + generator := &requestGenerator{request: request, options: request.options} - if len(r.Payloads) > 0 { - generator.payloadIterator = r.generator.NewIterator() + if len(request.Payloads) > 0 { + generator.payloadIterator = request.generator.NewIterator() } return generator } @@ -63,7 +63,6 @@ func (r *requestGenerator) nextValue() (value string, payloads map[string]interf } } - if len(r.request.Raw) > 0 && r.currentIndex < len(r.request.Raw) { if r.payloadIterator != nil { payload, ok := r.payloadIterator.Value() diff --git a/v2/pkg/protocols/http/utils.go b/v2/pkg/protocols/http/utils.go index bb12b5a66..89c35a233 100644 --- a/v2/pkg/protocols/http/utils.go +++ b/v2/pkg/protocols/http/utils.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" "github.com/projectdiscovery/rawhttp" + "github.com/projectdiscovery/stringsutil" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/transform" ) @@ -135,3 +136,9 @@ func decodegbk(s []byte) ([]byte, error) { } return d, nil } + +// isContentTypeGbk checks if the content-type header is gbk +func isContentTypeGbk(contentType string) bool { + contentType = strings.ToLower(contentType) + return stringsutil.ContainsAny(contentType, "gbk", "gb2312", "gb18030") +} diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 2763cf4be..a5274eff6 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -1,12 +1,14 @@ package network import ( + "fmt" "net" "strings" "github.com/pkg/errors" "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" @@ -33,13 +35,13 @@ type Request struct { // description: | // Attack is the type of payload combinations to perform. // - // Sniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates + // Batteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates // permutations and combinations for all payloads. // values: - // - "sniper" + // - "batteringram" // - "pitchfork" // - "clusterbomb" - AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=sniper,enum=pitchfork,enum=clusterbomb"` + AttackType string `yaml:"attack,omitempty" jsonschema:"title=attack is the payload combination,description=Attack is the type of payload combinations to perform,enum=batteringram,enum=pitchfork,enum=clusterbomb"` // description: | // Payloads contains any payloads for the current request. // @@ -59,6 +61,10 @@ type Request struct { // - value: "2048" ReadSize int `yaml:"read-size,omitempty" jsonschema:"title=size of network response to read,description=Size of response to read at the end. Default is 1024 bytes"` + // description: | + // SelfContained specifies if the request is self contained. + SelfContained bool `yaml:"-" json:"-"` + // Operators for the current request go here. operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` @@ -66,8 +72,9 @@ type Request struct { generator *generators.Generator attackType generators.Type // cache any variables that may be needed for operation. - dialer *fastdialer.Dialer - options *protocols.ExecuterOptions + dialer *fastdialer.Dialer + options *protocols.ExecuterOptions + dynamicValues map[string]interface{} } type addressKV struct { @@ -113,17 +120,17 @@ type Input struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { - return r.ID +func (request *Request) GetID() string { + return request.ID } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { var shouldUseTLS bool var err error - r.options = options - for _, address := range r.Address { + request.options = options + for _, address := range request.Address { // check if the connection should be encrypted if strings.HasPrefix(address, "tls://") { shouldUseTLS = true @@ -134,13 +141,13 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if portErr != nil { return errors.Wrap(portErr, "could not parse address") } - r.addresses = append(r.addresses, addressKV{ip: addressHost, port: addressPort, tls: shouldUseTLS}) + request.addresses = append(request.addresses, addressKV{ip: addressHost, port: addressPort, tls: shouldUseTLS}) } else { - r.addresses = append(r.addresses, addressKV{ip: address, tls: shouldUseTLS}) + request.addresses = append(request.addresses, addressKV{ip: address, tls: shouldUseTLS}) } } // Pre-compile any input dsl functions before executing the request. - for _, input := range r.Inputs { + for _, input := range request.Inputs { if input.Type != "" { continue } @@ -149,25 +156,51 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { } } - if len(r.Payloads) > 0 { - attackType := r.AttackType - if attackType == "" { - attackType = "sniper" + // Resolve payload paths from vars if they exists + for name, payload := range request.options.Options.VarsPayload() { + payloadStr, ok := payload.(string) + // check if inputs contains the payload + var hasPayloadName bool + for _, input := range request.Inputs { + if input.Type != "" { + continue + } + if expressions.ContainsVariablesWithNames(input.Data, map[string]interface{}{name: payload}) == nil { + hasPayloadName = true + break + } + } + if ok && hasPayloadName && fileutil.FileExists(payloadStr) { + if request.Payloads == nil { + request.Payloads = make(map[string]interface{}) + } + request.Payloads[name] = payloadStr + } + } + + if len(request.Payloads) > 0 { + attackType := request.AttackType + if attackType == "" { + attackType = "batteringram" + } + var ok bool + request.attackType, ok = generators.StringToType[attackType] + if !ok { + return fmt.Errorf("invalid attack type provided: %s", attackType) } - r.attackType = generators.StringToType[attackType] // Resolve payload paths if they are files. - for name, payload := range r.Payloads { + for name, payload := range request.Payloads { payloadStr, ok := payload.(string) if ok { final, resolveErr := options.Catalog.ResolvePath(payloadStr, options.TemplatePath) if resolveErr != nil { return errors.Wrap(resolveErr, "could not read payload file") } - r.Payloads[name] = final + request.Payloads[name] = final } } - r.generator, err = generators.New(r.Payloads, r.attackType, r.options.TemplatePath) + request.generator, err = generators.New(request.Payloads, request.attackType, request.options.TemplatePath) if err != nil { return errors.Wrap(err, "could not parse payloads") } @@ -178,19 +211,19 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { if err != nil { return errors.Wrap(err, "could not get network client") } - r.dialer = client + request.dialer = client - if len(r.Matchers) > 0 || len(r.Extractors) > 0 { - compiled := &r.Operators + if len(request.Matchers) > 0 || len(request.Extractors) > 0 { + compiled := &request.Operators if err := compiled.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.CompiledOperators = compiled + request.CompiledOperators = compiled } return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { - return len(r.Address) +func (request *Request) Requests() int { + return len(request.Address) } diff --git a/v2/pkg/protocols/network/operators.go b/v2/pkg/protocols/network/operators.go index 81b5fcffe..8e0330dc0 100644 --- a/v2/pkg/protocols/network/operators.go +++ b/v2/pkg/protocols/network/operators.go @@ -4,14 +4,16 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { partString := matcher.Part switch partString { case "body", "all", "": @@ -20,27 +22,27 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) item, ok := data[partString] if !ok { - return false + return false, []string{} } itemStr := types.ToString(item) switch matcher.GetType() { case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(itemStr))) + return matcher.Result(matcher.MatchSize(len(itemStr))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(itemStr, request.dynamicValues)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(itemStr)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(itemStr)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(itemStr)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { partString := extractor.Part switch partString { case "body", "all", "": @@ -62,51 +64,30 @@ func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Ext return nil } -// responseToDSLMap converts a DNS response to a map for use in DSL matching -func (r *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent { - data := make(output.InternalEvent, 6) - - // Some data regarding the request metadata - data["host"] = host - data["matched"] = matched - data["request"] = req - data["data"] = resp // Data is the last bytes read - data["raw"] = raw // Raw is the full transaction data for network - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath - return data +// responseToDSLMap converts a network response to a map for use in DSL matching +func (request *Request) responseToDSLMap(req, resp, raw, host, matched string) output.InternalEvent { + return output.InternalEvent{ + "host": host, + "matched": matched, + "request": req, + "data": resp, // Data is the last bytes read + "raw": raw, // Raw is the full transaction data for network + "template-id": request.options.TemplateID, + "template-info": request.options.TemplateInfo, + "template-path": request.options.TemplatePath, + } } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) GetCompiledOperators() []*operators.Operators { + return []*operators.Operators{request.CompiledOperators} +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/network/operators_test.go b/v2/pkg/protocols/network/operators_test.go index 4c8ea3028..bf577d7e7 100644 --- a/v2/pkg/protocols/network/operators_test.go +++ b/v2/pkg/protocols/network/operators_test.go @@ -70,8 +70,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -84,8 +85,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -97,8 +99,9 @@ func TestNetworkOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } @@ -189,7 +192,7 @@ func TestNetworkMakeResult(t *testing.T) { finalEvent := &output.InternalWrappedEvent{InternalEvent: event} event["ip"] = "192.168.1.1" if request.CompiledOperators != nil { - result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract) + result, ok := request.CompiledOperators.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 9d03a7cfb..acbbac9af 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -10,10 +10,14 @@ import ( "time" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/expressions" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" ) @@ -21,15 +25,22 @@ import ( var _ protocols.Request = &Request{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - address, err := getAddress(input) +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + var address string + var err error + + if request.SelfContained { + address = "" + } else { + address, err = getAddress(input) + } if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not get address from url") } - for _, kv := range r.addresses { + for _, kv := range request.addresses { actualAddress := replacer.Replace(kv.ip, map[string]interface{}{"Hostname": address}) if kv.port != "" { if strings.Contains(address, ":") { @@ -37,8 +48,11 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused } actualAddress = net.JoinHostPort(actualAddress, kv.port) } + if input != "" { + input = actualAddress + } - if err := r.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { + if err := request.executeAddress(actualAddress, address, input, kv.tls, previous, callback); err != nil { gologger.Verbose().Label("ERR").Msgf("Could not make network request for %s: %s\n", actualAddress, err) continue } @@ -47,70 +61,75 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused } // executeAddress executes the request for an address -func (r *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeAddress(actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if !strings.Contains(actualAddress, ":") { err := errors.New("no port provided in network protocol request") - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return err } - if r.generator != nil { - iterator := r.generator.NewIterator() + payloads := generators.BuildPayloadFromOptions(request.options.Options) + + if request.generator != nil { + iterator := request.generator.NewIterator() for { value, ok := iterator.Value() if !ok { break } - if err := r.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + value = generators.MergeMaps(value, payloads) + if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } } else { - value := make(map[string]interface{}) - if err := r.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { + value := generators.MergeMaps(map[string]interface{}{}, payloads) + if err := request.executeRequestWithPayloads(actualAddress, address, input, shouldUseTLS, value, previous, callback); err != nil { return err } } return nil } -func (r *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRequestWithPayloads(actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var ( hostname string conn net.Conn err error ) + request.dynamicValues = generators.MergeMaps(payloads, map[string]interface{}{"Hostname": address}) + if host, _, splitErr := net.SplitHostPort(actualAddress); splitErr == nil { hostname = host } if shouldUseTLS { - conn, err = r.dialer.DialTLS(context.Background(), "tcp", actualAddress) + conn, err = request.dialer.DialTLS(context.Background(), "tcp", actualAddress) } else { - conn, err = r.dialer.Dial(context.Background(), "tcp", actualAddress) + conn, err = request.dialer.Dial(context.Background(), "tcp", actualAddress) } if err != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not connect to server request") } defer conn.Close() - _ = conn.SetReadDeadline(time.Now().Add(time.Duration(r.options.Options.Timeout) * time.Second)) + _ = conn.SetReadDeadline(time.Now().Add(time.Duration(request.options.Options.Timeout) * time.Second)) - hasInteractMarkers := interactsh.HasMatchers(r.CompiledOperators) + hasInteractMarkers := interactsh.HasMatchers(request.CompiledOperators) var interactURL string - if r.options.Interactsh != nil && hasInteractMarkers { - interactURL = r.options.Interactsh.URL() + if request.options.Interactsh != nil && hasInteractMarkers { + interactURL = request.options.Interactsh.URL() } responseBuilder := &strings.Builder{} reqBuilder := &strings.Builder{} inputEvents := make(map[string]interface{}) - for _, input := range r.Inputs { + for _, input := range request.Inputs { var data []byte switch input.Type { @@ -118,28 +137,32 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin data, err = hex.DecodeString(input.Data) default: if interactURL != "" { - input.Data = r.options.Interactsh.ReplaceMarkers(input.Data, interactURL) + input.Data = request.options.Interactsh.ReplaceMarkers(input.Data, interactURL) } data = []byte(input.Data) } if err != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } reqBuilder.Grow(len(input.Data)) finalData, dataErr := expressions.EvaluateByte(data, payloads) if dataErr != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", dataErr) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", dataErr) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(dataErr, "could not evaluate template expressions") } reqBuilder.Write(finalData) + if varErr := expressions.ContainsUnresolvedVariables(string(finalData)); varErr != nil { + gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", request.options.TemplateID, actualAddress, varErr) + return nil + } if _, err := conn.Write(finalData); err != nil { - r.options.Output.Request(r.options.TemplateID, address, "network", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, address, "network", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not write request to server") } @@ -154,44 +177,40 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin } // Run any internal extractors for the request here and add found values to map. - if r.CompiledOperators != nil { - values := r.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{input.Name: bufferStr}, r.Extract) + if request.CompiledOperators != nil { + values := request.CompiledOperators.ExecuteInternalExtractors(map[string]interface{}{input.Name: bufferStr}, request.Extract) for k, v := range values { payloads[k] = v } } } } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() - if r.options.Options.Debug || r.options.Options.DebugRequests { + if request.options.Options.Debug || request.options.Options.DebugRequests { requestOutput := reqBuilder.String() - gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", r.options.TemplateID, actualAddress) + gologger.Info().Str("address", actualAddress).Msgf("[%s] Dumped Network request for %s", request.options.TemplateID, actualAddress) gologger.Print().Msgf("%s\nHex: %s", requestOutput, hex.EncodeToString([]byte(requestOutput))) } - r.options.Output.Request(r.options.TemplateID, actualAddress, "network", err) + request.options.Output.Request(request.options.TemplateID, actualAddress, "network", err) gologger.Verbose().Msgf("Sent TCP request to %s", actualAddress) bufferSize := 1024 - if r.ReadSize != 0 { - bufferSize = r.ReadSize + if request.ReadSize != 0 { + bufferSize = request.ReadSize } final := make([]byte, bufferSize) n, err := conn.Read(final) if err != nil && err != io.EOF { - r.options.Output.Request(r.options.TemplateID, address, "network", err) + request.options.Output.Request(request.options.TemplateID, address, "network", err) return errors.Wrap(err, "could not read from server") } responseBuilder.Write(final[:n]) - if r.options.Options.Debug || r.options.Options.DebugResponse { - responseOutput := responseBuilder.String() - gologger.Debug().Msgf("[%s] Dumped Network response for %s", r.options.TemplateID, actualAddress) - gologger.Print().Msgf("%s\nHex: %s", responseOutput, hex.EncodeToString([]byte(responseOutput))) - } - outputEvent := r.responseToDSLMap(reqBuilder.String(), string(final[:n]), responseBuilder.String(), input, actualAddress) - outputEvent["ip"] = r.dialer.GetDialedIP(hostname) + response := responseBuilder.String() + outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) + outputEvent["ip"] = request.dialer.GetDialedIP(hostname) for k, v := range previous { outputEvent[k] = v } @@ -202,26 +221,28 @@ func (r *Request) executeRequestWithPayloads(actualAddress, address, input strin outputEvent[k] = v } - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + var event *output.InternalWrappedEvent if interactURL == "" { - if r.CompiledOperators != nil { - result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) - if ok && result != nil { - event.OperatorsResult = result - event.OperatorsResult.PayloadValues = payloads - event.Results = r.MakeResultEvent(event) - } - } + event = eventcreator.CreateEventWithAdditionalOptions(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse, func(wrappedEvent *output.InternalWrappedEvent) { + wrappedEvent.OperatorsResult.PayloadValues = payloads + }) callback(event) - } else if r.options.Interactsh != nil { - r.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ - MakeResultFunc: r.MakeResultEvent, + } else if request.options.Interactsh != nil { + event = &output.InternalWrappedEvent{InternalEvent: outputEvent} + request.options.Interactsh.RequestEvent(interactURL, &interactsh.RequestData{ + MakeResultFunc: request.MakeResultEvent, Event: event, - Operators: r.CompiledOperators, - MatchFunc: r.Match, - ExtractFunc: r.Extract, + Operators: request.CompiledOperators, + MatchFunc: request.Match, + ExtractFunc: request.Extract, }) } + + if request.options.Options.Debug || request.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped Network response for %s", request.options.TemplateID, actualAddress) + gologger.Print().Msgf("%s\nHex: %s", response, responsehighlighter.Highlight(event.OperatorsResult, hex.EncodeToString([]byte(response)), request.options.Options.NoColor)) + } + return nil } diff --git a/v2/pkg/protocols/offlinehttp/find.go b/v2/pkg/protocols/offlinehttp/find.go index ee918ad57..2c876dab4 100644 --- a/v2/pkg/protocols/offlinehttp/find.go +++ b/v2/pkg/protocols/offlinehttp/find.go @@ -12,19 +12,19 @@ import ( // getInputPaths parses the specified input paths and returns a compiled // list of finished absolute paths to the files evaluating any allowlist, denylist, // glob, file or folders, etc. -func (r *Request) getInputPaths(target string, callback func(string)) error { +func (request *Request) getInputPaths(target string, callback func(string)) error { processed := make(map[string]struct{}) // Template input includes a wildcard if strings.Contains(target, "*") { - if err := r.findGlobPathMatches(target, processed, callback); err != nil { + if err := request.findGlobPathMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find glob matches") } return nil } // Template input is either a file or a directory - file, err := r.findFileMatches(target, processed, callback) + file, err := request.findFileMatches(target, processed, callback) if err != nil { return errors.Wrap(err, "could not find file") } @@ -34,14 +34,14 @@ func (r *Request) getInputPaths(target string, callback func(string)) error { // Recursively walk down the Templates directory and run all // the template file checks - if err := r.findDirectoryMatches(target, processed, callback); err != nil { + if err := request.findDirectoryMatches(target, processed, callback); err != nil { return errors.Wrap(err, "could not find directory matches") } return nil } // findGlobPathMatches returns the matched files from a glob path -func (r *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findGlobPathMatches(absPath string, processed map[string]struct{}, callback func(string)) error { matches, err := filepath.Glob(absPath) if err != nil { return errors.Errorf("wildcard found, but unable to glob: %s\n", err) @@ -60,7 +60,7 @@ func (r *Request) findGlobPathMatches(absPath string, processed map[string]struc // findFileMatches finds if a path is an absolute file. If the path // is a file, it returns true otherwise false with no errors. -func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { +func (request *Request) findFileMatches(absPath string, processed map[string]struct{}, callback func(string)) (bool, error) { info, err := os.Stat(absPath) if err != nil { return false, err @@ -79,7 +79,7 @@ func (r *Request) findFileMatches(absPath string, processed map[string]struct{}, } // findDirectoryMatches finds matches for templates from a directory -func (r *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { +func (request *Request) findDirectoryMatches(absPath string, processed map[string]struct{}, callback func(string)) error { err := godirwalk.Walk(absPath, &godirwalk.Options{ Unsorted: true, ErrorCallback: func(fsPath string, err error) godirwalk.ErrorAction { diff --git a/v2/pkg/protocols/offlinehttp/offlinehttp.go b/v2/pkg/protocols/offlinehttp/offlinehttp.go index c315c3372..85b669320 100644 --- a/v2/pkg/protocols/offlinehttp/offlinehttp.go +++ b/v2/pkg/protocols/offlinehttp/offlinehttp.go @@ -2,6 +2,7 @@ package offlinehttp import ( "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) @@ -13,23 +14,23 @@ type Request struct { } // GetID returns the unique ID of the request if any. -func (r *Request) GetID() string { +func (request *Request) GetID() string { return "" } // Compile compiles the protocol request for further execution. -func (r *Request) Compile(options *protocols.ExecuterOptions) error { +func (request *Request) Compile(options *protocols.ExecuterOptions) error { for _, operator := range options.Operators { if err := operator.Compile(); err != nil { return errors.Wrap(err, "could not compile operators") } - r.compiledOperators = append(r.compiledOperators, operator) + request.compiledOperators = append(request.compiledOperators, operator) } - r.options = options + request.options = options return nil } // Requests returns the total number of requests the YAML rule will perform -func (r *Request) Requests() int { +func (request *Request) Requests() int { return 1 } diff --git a/v2/pkg/protocols/offlinehttp/operators.go b/v2/pkg/protocols/offlinehttp/operators.go index 8aaf0663e..b5141037b 100644 --- a/v2/pkg/protocols/offlinehttp/operators.go +++ b/v2/pkg/protocols/offlinehttp/operators.go @@ -6,42 +6,57 @@ import ( "time" "github.com/projectdiscovery/nuclei/v2/pkg/model" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/responsehighlighter" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // Match matches a generic data response again a given matcher -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { +func (request *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { item, ok := getMatchPart(matcher.Part, data) if !ok { - return false + return false, []string{} } switch matcher.GetType() { case matchers.StatusMatcher: - statusCode, ok := data["status_code"] + statusCode, ok := getStatusCode(data) if !ok { - return false + return false, []string{} } - return matcher.Result(matcher.MatchStatusCode(statusCode.(int))) + return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{responsehighlighter.CreateStatusCodeSnippet(data["response"].(string), statusCode)} case matchers.SizeMatcher: - return matcher.Result(matcher.MatchSize(len(item))) + return matcher.Result(matcher.MatchSize(len(item))), []string{} case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchWords(item, nil)) case matchers.RegexMatcher: - return matcher.Result(matcher.MatchRegex(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchRegex(item)) case matchers.BinaryMatcher: - return matcher.Result(matcher.MatchBinary(item)) + return matcher.ResultWithMatchedSnippet(matcher.MatchBinary(item)) case matchers.DSLMatcher: - return matcher.Result(matcher.MatchDSL(data)) + return matcher.Result(matcher.MatchDSL(data)), []string{} } - return false + return false, []string{} +} + +func getStatusCode(data map[string]interface{}) (int, bool) { + statusCodeValue, ok := data["status_code"] + if !ok { + return 0, false + } + statusCode, ok := statusCodeValue.(int) + if !ok { + return 0, false + } + return statusCode, true } // Extract performs extracting operation for an extractor on model and returns true or false. -func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { +func (request *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { item, ok := getMatchPart(extractor.Part, data) if !ok { return nil @@ -78,11 +93,18 @@ func getMatchPart(part string, data output.InternalEvent) (string, bool) { } // responseToDSLMap converts an HTTP response to a map for use in DSL matching -func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) map[string]interface{} { - data := make(map[string]interface{}, len(extra)+8+len(resp.Header)+len(resp.Cookies())) +func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent { + data := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies())) for k, v := range extra { data[k] = v } + for _, cookie := range resp.Cookies() { + data[strings.ToLower(cookie.Name)] = cookie.Value + } + for k, v := range resp.Header { + k = strings.ToLower(strings.TrimSpace(k)) + data[k] = strings.Join(v, " ") + } data["path"] = host data["matched"] = matched @@ -91,50 +113,24 @@ func (r *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, r data["content_length"] = resp.ContentLength data["status_code"] = resp.StatusCode data["body"] = body - for _, cookie := range resp.Cookies() { - data[strings.ToLower(cookie.Name)] = cookie.Value - } - for k, v := range resp.Header { - k = strings.ToLower(strings.TrimSpace(k)) - data[k] = strings.Join(v, " ") - } data["all_headers"] = headers data["duration"] = duration.Seconds() - data["template-id"] = r.options.TemplateID - data["template-info"] = r.options.TemplateInfo - data["template-path"] = r.options.TemplatePath + data["template-id"] = request.options.TemplateID + data["template-info"] = request.options.TemplateInfo + data["template-path"] = request.options.TemplatePath return data } // MakeResultEvent creates a result event from internal wrapped event -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - if len(wrapped.OperatorsResult.DynamicValues) > 0 { - return nil - } - results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) - - // If we have multiple matchers with names, write each of them separately. - if len(wrapped.OperatorsResult.Matches) > 0 { - for k := range wrapped.OperatorsResult.Matches { - data := r.makeResultEventItem(wrapped) - data.MatcherName = k - results = append(results, data) - } - } else if len(wrapped.OperatorsResult.Extracts) > 0 { - for k, v := range wrapped.OperatorsResult.Extracts { - data := r.makeResultEventItem(wrapped) - data.ExtractedResults = v - data.ExtractorName = k - results = append(results, data) - } - } else { - data := r.makeResultEventItem(wrapped) - results = append(results, data) - } - return results +func (request *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + return protocols.MakeDefaultResultEvent(request, wrapped) } -func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { +func (request *Request) GetCompiledOperators() []*operators.Operators { + return request.compiledOperators +} + +func (request *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), diff --git a/v2/pkg/protocols/offlinehttp/operators_test.go b/v2/pkg/protocols/offlinehttp/operators_test.go index 56facee78..2f4b91fd5 100644 --- a/v2/pkg/protocols/offlinehttp/operators_test.go +++ b/v2/pkg/protocols/offlinehttp/operators_test.go @@ -76,8 +76,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err = matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid response") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid response") + require.Equal(t, matcher.Words, matched) }) t.Run("negative", func(t *testing.T) { @@ -90,8 +91,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile negative matcher") - matched := request.Match(event, matcher) - require.True(t, matched, "could not match valid negative response matcher") + isMatched, matched := request.Match(event, matcher) + require.True(t, isMatched, "could not match valid negative response matcher") + require.Equal(t, []string{}, matched) }) t.Run("invalid", func(t *testing.T) { @@ -103,8 +105,9 @@ func TestHTTPOperatorMatch(t *testing.T) { err := matcher.CompileMatchers() require.Nil(t, err, "could not compile matcher") - matched := request.Match(event, matcher) - require.False(t, matched, "could match invalid response matcher") + isMatched, matched := request.Match(event, matcher) + require.False(t, isMatched, "could match invalid response matcher") + require.Equal(t, []string{}, matched) }) } @@ -201,7 +204,7 @@ func TestHTTPMakeResult(t *testing.T) { event["ip"] = "192.169.1.1" finalEvent := &output.InternalWrappedEvent{InternalEvent: event} for _, operator := range request.compiledOperators { - result, ok := operator.Execute(event, request.Match, request.Extract) + result, ok := operator.Execute(event, request.Match, request.Extract, false) if ok && result != nil { finalEvent.OperatorsResult = result finalEvent.Results = request.MakeResultEvent(finalEvent) diff --git a/v2/pkg/protocols/offlinehttp/read_response.go b/v2/pkg/protocols/offlinehttp/read_response.go index 0869a5643..f864b5ab8 100644 --- a/v2/pkg/protocols/offlinehttp/read_response.go +++ b/v2/pkg/protocols/offlinehttp/read_response.go @@ -4,20 +4,41 @@ import ( "bufio" "errors" "net/http" + "regexp" "strings" ) +var noMinor = regexp.MustCompile(`HTTP\/([0-9]) `) + // readResponseFromString reads a raw http response from a string. func readResponseFromString(data string) (*http.Response, error) { var final string + if strings.HasPrefix(data, "HTTP/") { - final = data + final = addMinorVersionToHTTP(data) } else { lastIndex := strings.LastIndex(data, "HTTP/") if lastIndex == -1 { return nil, errors.New("malformed raw http response") } final = data[lastIndex:] // choose last http/ in case of it being later. + + final = addMinorVersionToHTTP(final) } return http.ReadResponse(bufio.NewReader(strings.NewReader(final)), nil) } + +// addMinorVersionToHTTP tries to add a minor version to http status header +// fixing the compatibility issue with go standard library. +func addMinorVersionToHTTP(data string) string { + matches := noMinor.FindAllStringSubmatch(data, 1) + if len(matches) == 0 { + return data + } + if len(matches[0]) < 2 { + return data + } + replacedVersion := strings.Replace(matches[0][0], matches[0][1], matches[0][1]+".0", 1) + data = strings.Replace(data, matches[0][0], replacedVersion, 1) + return data +} diff --git a/v2/pkg/protocols/offlinehttp/read_response_test.go b/v2/pkg/protocols/offlinehttp/read_response_test.go index cf29ba80b..4a75d7263 100644 --- a/v2/pkg/protocols/offlinehttp/read_response_test.go +++ b/v2/pkg/protocols/offlinehttp/read_response_test.go @@ -2,7 +2,10 @@ package offlinehttp import ( "io/ioutil" + "net/http" + "net/http/httputil" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -18,11 +21,15 @@ func TestReadResponseFromString(t *testing.T) {

What is the Firing Range?

- ` - t.Run("response", func(t *testing.T) { - data := `HTTP/1.1 200 OK + tests := []struct { + name string + data string + }{ + { + name: "response", + data: `HTTP/1.1 200 OK Age: 0 Cache-Control: public, max-age=600 Content-Type: text/html @@ -38,19 +45,51 @@ Server: Google Frontend

What is the Firing Range?

+`, + }, + { + name: "response-http2-without-minor-version", + data: `HTTP/2 200 OK +Age: 0 +Cache-Control: public, max-age=600 +Content-Type: text/html +Server: Google Frontend + + + + +Firing Range + + +

Version 0.48

+

What is the Firing Range?

+

-` - resp, err := readResponseFromString(data) - require.Nil(t, err, "could not read response from string") +`, + }, + { + name: "response-http2-with-minor-version", + data: `HTTP/2.0 200 OK +Age: 0 +Cache-Control: public, max-age=600 +Content-Type: text/html +Server: Google Frontend - respData, err := ioutil.ReadAll(resp.Body) - require.Nil(t, err, "could not read response body") - require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") - require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") - }) - - t.Run("request-response", func(t *testing.T) { - data := `GET http://public-firing-range.appspot.com/ HTTP/1.1 + + + +Firing Range + + +

Version 0.48

+

What is the Firing Range?

+

+ +`, + }, + { + name: "request-response", + data: `GET http://public-firing-range.appspot.com/ HTTP/1.1 Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 Accept-Encoding: gzip, deflate Upgrade-Insecure-Requests: 1 @@ -72,14 +111,67 @@ Server: Google Frontend

What is the Firing Range?

+`, + }, + { + name: "request-response-without-minor-version", + data: `GET http://public-firing-range.appspot.com/ HTTP/1.1 +Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 +Accept-Encoding: gzip, deflate +Upgrade-Insecure-Requests: 1 +User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 + +HTTP/2 200 OK +Age: 0 +Cache-Control: public, max-age=600 +Content-Type: text/html +Server: Google Frontend + + + + +Firing Range + + +

Version 0.48

+

What is the Firing Range?

+

-` - resp, err := readResponseFromString(data) +`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp, err := readResponseFromString(tt.data) + require.Nil(t, err, "could not read response from string") + + respData, err := ioutil.ReadAll(resp.Body) + require.Nil(t, err, "could not read response body") + require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") + require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") + }) + } + + t.Run("test-live-response-with-content-length", func(t *testing.T) { + client := &http.Client{ + Timeout: 3 * time.Second, + } + + data, err := client.Get("https://golang.org/doc/install") + require.Nil(t, err, "could not dial url") + defer data.Body.Close() + + b, err := httputil.DumpResponse(data, true) + require.Nil(t, err, "could not dump response") + + respData, err := readResponseFromString(string(b)) require.Nil(t, err, "could not read response from string") - respData, err := ioutil.ReadAll(resp.Body) + _, err = ioutil.ReadAll(respData.Body) require.Nil(t, err, "could not read response body") - require.Equal(t, expectedBody, string(respData), "could not get correct parsed body") - require.Equal(t, "Google Frontend", resp.Header.Get("Server"), "could not get correct headers") + + require.Equal(t, "Google Frontend", respData.Header.Get("Server"), "could not get correct headers") + }) } diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 27b10e9db..1eb4bfc57 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -8,11 +8,13 @@ import ( "strings" "github.com/pkg/errors" + "github.com/remeh/sizedwaitgroup" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring" - "github.com/remeh/sizedwaitgroup" ) var _ protocols.Request = &Request{} @@ -20,10 +22,10 @@ var _ protocols.Request = &Request{} const maxSize = 5 * 1024 * 1024 // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - wg := sizedwaitgroup.New(r.options.Options.BulkSize) +func (request *Request) ExecuteWithResults(input string, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + wg := sizedwaitgroup.New(request.options.Options.BulkSize) - err := r.getInputPaths(input, func(data string) { + err := request.getInputPaths(input, func(data string) { wg.Add() go func(data string) { @@ -59,11 +61,11 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused return } - if r.options.Options.Debug || r.options.Options.DebugRequests { - gologger.Info().Msgf("[%s] Dumped offline-http request for %s", r.options.TemplateID, data) + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Info().Msgf("[%s] Dumped offline-http request for %s", request.options.TemplateID, data) gologger.Print().Msgf("%s", dataStr) } - gologger.Verbose().Msgf("[%s] Sent OFFLINE-HTTP request to %s", r.options.TemplateID, data) + gologger.Verbose().Msgf("[%s] Sent OFFLINE-HTTP request to %s", request.options.TemplateID, data) dumpedResponse, err := httputil.DumpResponse(resp, true) if err != nil { @@ -77,31 +79,23 @@ func (r *Request) ExecuteWithResults(input string, metadata /*TODO review unused return } - outputEvent := r.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil) + outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), headersToString(resp.Header), 0, nil) outputEvent["ip"] = "" for k, v := range previous { outputEvent[k] = v } - for _, operator := range r.compiledOperators { - event := &output.InternalWrappedEvent{InternalEvent: outputEvent} - var ok bool - - event.OperatorsResult, ok = operator.Execute(outputEvent, r.Match, r.Extract) - if ok && event.OperatorsResult != nil { - event.Results = r.MakeResultEvent(event) - } - callback(event) - } + event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) + callback(event) }(data) }) wg.Wait() if err != nil { - r.options.Output.Request(r.options.TemplateID, input, "file", err) - r.options.Progress.IncrementFailedRequestsBy(1) + request.options.Output.Request(request.options.TemplateID, input, "file", err) + request.options.Progress.IncrementFailedRequestsBy(1) return errors.Wrap(err, "could not send file request") } - r.options.Progress.IncrementRequests() + request.options.Progress.IncrementRequests() return nil } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 9ca38333e..d77ba147e 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -74,13 +74,49 @@ type Request interface { // condition matching. So, two requests can be sent and their match can // be evaluated from the third request by using the IDs for both requests. GetID() string - // Match performs matching operation for a matcher on model and returns true or false. - Match(data map[string]interface{}, matcher *matchers.Matcher) bool + // Match performs matching operation for a matcher on model and returns: + // true and a list of matched snippets if the matcher type is supports it + // otherwise false and an empty string slice + Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) // Extract performs extracting operation for an extractor on model and returns true or false. Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} // ExecuteWithResults executes the protocol requests and returns results instead of writing them. ExecuteWithResults(input string, dynamicValues, previous output.InternalEvent, callback OutputEventCallback) error + // MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally + MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent + // MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data + MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent + // GetCompiledOperators returns a list of the compiled operators + GetCompiledOperators() []*operators.Operators } // OutputEventCallback is a callback event for any results found during scanning. type OutputEventCallback func(result *output.InternalWrappedEvent) + +func MakeDefaultResultEvent(request Request, wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + if len(wrapped.OperatorsResult.DynamicValues) > 0 && !wrapped.OperatorsResult.Matched { + return nil + } + + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for matcherNames := range wrapped.OperatorsResult.Matches { + data := request.MakeResultEventItem(wrapped) + data.MatcherName = matcherNames + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := request.MakeResultEventItem(wrapped) + data.ExtractorName = k + data.ExtractedResults = v + results = append(results, data) + } + } else { + data := request.MakeResultEventItem(wrapped) + results = append(results, data) + } + return results +} diff --git a/v2/pkg/reporting/exporters/es/elasticsearch.go b/v2/pkg/reporting/exporters/es/elasticsearch.go index e14f1a894..784e587d0 100644 --- a/v2/pkg/reporting/exporters/es/elasticsearch.go +++ b/v2/pkg/reporting/exporters/es/elasticsearch.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" "time" "encoding/base64" @@ -22,9 +23,9 @@ type Options struct { IP string `yaml:"ip"` // Port is the port of elasticsearch instance Port int `yaml:"port"` - // SSL enables ssl for elasticsearch connection + // SSL (optional) enables ssl for elasticsearch connection SSL bool `yaml:"ssl"` - // SSLVerification disables SSL verification for elasticsearch + // SSLVerification (optional) disables SSL verification for elasticsearch SSLVerification bool `yaml:"ssl-verification"` // Username for the elasticsearch instance Username string `yaml:"username"` @@ -49,6 +50,10 @@ type Exporter struct { // New creates and returns a new exporter for elasticsearch func New(option *Options) (*Exporter, error) { var ei *Exporter + err := validateOptions(option) + if err != nil { + return nil, err + } client := &http.Client{ Timeout: 5 * time.Second, @@ -81,7 +86,32 @@ func New(option *Options) (*Exporter, error) { return ei, nil } -// Export exports a passed result event to disk +func validateOptions(options *Options) error { + errs := []string{} + if options.IP == "" { + errs = append(errs, "IP") + } + if options.Port == 0 { + errs = append(errs, "Port") + } + if options.Username == "" { + errs = append(errs, "Username") + } + if options.Password == "" { + errs = append(errs, "Password") + } + if options.IndexName == "" { + errs = append(errs, "IndexName") + } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + + return nil +} + +// Export exports a passed result event to elasticsearch func (i *Exporter) Export(event *output.ResultEvent) error { // creating a request req, err := http.NewRequest(http.MethodPost, i.url, nil) @@ -105,9 +135,9 @@ func (i *Exporter) Export(event *output.ResultEvent) error { res, err := i.elasticsearch.Do(req) if err != nil { - return err + return err } - + b, err = ioutil.ReadAll(res.Body) if err != nil { return errors.New(err.Error() + "error thrown by elasticsearch " + string(b)) diff --git a/v2/pkg/reporting/exporters/disk/disk.go b/v2/pkg/reporting/exporters/markdown/markdown.go similarity index 81% rename from v2/pkg/reporting/exporters/disk/disk.go rename to v2/pkg/reporting/exporters/markdown/markdown.go index c1fb92a45..3e779ea1e 100644 --- a/v2/pkg/reporting/exporters/disk/disk.go +++ b/v2/pkg/reporting/exporters/markdown/markdown.go @@ -1,4 +1,4 @@ -package disk +package markdown import ( "bytes" @@ -22,7 +22,7 @@ type Options struct { Directory string `yaml:"directory"` } -// New creates a new disk exporter integration client based on options. +// New creates a new markdown exporter integration client based on options. func New(options *Options) (*Exporter, error) { directory := options.Directory if options.Directory == "" { @@ -36,7 +36,7 @@ func New(options *Options) (*Exporter, error) { return &Exporter{options: options, directory: directory}, nil } -// Export exports a passed result event to disk +// Export exports a passed result event to markdown func (i *Exporter) Export(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) @@ -45,6 +45,17 @@ func (i *Exporter) Export(event *output.ResultEvent) error { filenameBuilder.WriteString(event.TemplateID) filenameBuilder.WriteString("-") filenameBuilder.WriteString(strings.ReplaceAll(strings.ReplaceAll(event.Matched, "/", "_"), ":", "_")) + + var suffix string + if event.MatcherName != "" { + suffix = event.MatcherName + } else if event.ExtractorName != "" { + suffix = event.ExtractorName + } + if suffix != "" { + filenameBuilder.WriteRune('-') + filenameBuilder.WriteString(event.MatcherName) + } filenameBuilder.WriteString(".md") finalFilename := sanitizeFilename(filenameBuilder.String()) diff --git a/v2/pkg/reporting/exporters/sarif/sarif.go b/v2/pkg/reporting/exporters/sarif/sarif.go index 9c69dda3e..6395ee319 100644 --- a/v2/pkg/reporting/exporters/sarif/sarif.go +++ b/v2/pkg/reporting/exporters/sarif/sarif.go @@ -33,7 +33,7 @@ type Options struct { File string `yaml:"file"` } -// New creates a new disk exporter integration client based on options. +// New creates a new sarif exporter integration client based on options. func New(options *Options) (*Exporter, error) { report, err := sarif.New(sarif.Version210) if err != nil { diff --git a/v2/pkg/reporting/format/format.go b/v2/pkg/reporting/format/format.go index e1a0f8995..4f9d3e526 100644 --- a/v2/pkg/reporting/format/format.go +++ b/v2/pkg/reporting/format/format.go @@ -131,6 +131,13 @@ func MarkdownDescription(event *output.ResultEvent) string { // TODO remove the } } } + builder.WriteString("\n") + + if event.CURLCommand != "" { + builder.WriteString("\n**CURL Command**\n```\n") + builder.WriteString(event.CURLCommand) + builder.WriteString("\n```") + } builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei %s](https://github.com/projectdiscovery/nuclei)", config.Version)) data := builder.String() diff --git a/v2/pkg/reporting/reporting.go b/v2/pkg/reporting/reporting.go index e50ec90e0..92c162f8f 100644 --- a/v2/pkg/reporting/reporting.go +++ b/v2/pkg/reporting/reporting.go @@ -10,7 +10,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/model/types/stringslice" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/dedupe" - "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/markdown" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/es" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/sarif" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/trackers/github" @@ -30,8 +30,8 @@ type Options struct { Gitlab *gitlab.Options `yaml:"gitlab"` // Jira contains configuration options for Jira Issue Tracker Jira *jira.Options `yaml:"jira"` - // DiskExporter contains configuration options for Disk Exporter Module - DiskExporter *disk.Options `yaml:"disk"` + // MarkdownExporter contains configuration options for Markdown Exporter Module + MarkdownExporter *markdown.Options `yaml:"markdown"` // SarifExporter contains configuration options for Sarif Exporter Module SarifExporter *sarif.Options `yaml:"sarif"` // ElasticsearchExporter contains configuration options for Elasticsearch Exporter Module @@ -128,8 +128,8 @@ func New(options *Options, db string) (*Client, error) { } client.trackers = append(client.trackers, tracker) } - if options.DiskExporter != nil { - exporter, err := disk.New(options.DiskExporter) + if options.MarkdownExporter != nil { + exporter, err := markdown.New(options.MarkdownExporter) if err != nil { return nil, errors.Wrap(err, "could not create exporting client") } diff --git a/v2/pkg/reporting/trackers/github/github.go b/v2/pkg/reporting/trackers/github/github.go index 96c072c18..672f62ede 100644 --- a/v2/pkg/reporting/trackers/github/github.go +++ b/v2/pkg/reporting/trackers/github/github.go @@ -2,7 +2,9 @@ package github import ( "context" + "fmt" "net/url" + "strings" "golang.org/x/oauth2" @@ -21,22 +23,29 @@ type Integration struct { // Options contains the configuration options for github issue tracker client type Options struct { - // BaseURL is the optional self-hosted github application url + // BaseURL (optional) is the self-hosted github application url BaseURL string `yaml:"base-url"` // Username is the username of the github user Username string `yaml:"username"` - // Owner is the owner name of the repository for issues. + // Owner (manadatory) is the owner name of the repository for issues. Owner string `yaml:"owner"` // Token is the token for github account. Token string `yaml:"token"` // ProjectName is the name of the repository. ProjectName string `yaml:"project-name"` - // IssueLabel is the label of the created issue type + // IssueLabel (optional) is the label of the created issue type IssueLabel string `yaml:"issue-label"` + // SeverityAsLabel (optional) sends the severity as the label of the created + // issue. + SeverityAsLabel bool `yaml:"severity-as-label"` } // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { + err := validateOptions(options) + if err != nil { + return nil, err + } ctx := context.Background() ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: options.Token}, @@ -54,15 +63,45 @@ func New(options *Options) (*Integration, error) { return &Integration{client: client, options: options}, nil } +func validateOptions(options *Options) error { + errs := []string{} + if options.Username == "" { + errs = append(errs, "Username") + } + if options.Owner == "" { + errs = append(errs, "Owner") + } + if options.Token == "" { + errs = append(errs, "Token") + } + if options.ProjectName == "" { + errs = append(errs, "ProjectName") + } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + + return nil +} + // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) + labels := []string{} + severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) + if i.options.SeverityAsLabel && severityLabel != "" { + labels = append(labels, severityLabel) + } + if label := i.options.IssueLabel; label != "" { + labels = append(labels, label) + } req := &github.IssueRequest{ Title: &summary, Body: &description, - Labels: &[]string{i.options.IssueLabel}, + Labels: &labels, Assignees: &[]string{i.options.Username}, } _, _, err := i.client.Issues.Create(context.Background(), i.options.Owner, i.options.ProjectName, req) diff --git a/v2/pkg/reporting/trackers/gitlab/gitlab.go b/v2/pkg/reporting/trackers/gitlab/gitlab.go index d8ffe36e6..35922970b 100644 --- a/v2/pkg/reporting/trackers/gitlab/gitlab.go +++ b/v2/pkg/reporting/trackers/gitlab/gitlab.go @@ -1,10 +1,14 @@ package gitlab import ( - "github.com/xanzy/go-gitlab" + "fmt" + "strings" + + "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/format" + "github.com/xanzy/go-gitlab" ) // Integration is a client for an issue tracker integration @@ -16,7 +20,7 @@ type Integration struct { // Options contains the configuration options for gitlab issue tracker client type Options struct { - // BaseURL is the optional self-hosted gitlab application url + // BaseURL (optional) is the self-hosted gitlab application url BaseURL string `yaml:"base-url"` // Username is the username of the gitlab user Username string `yaml:"username"` @@ -26,10 +30,17 @@ type Options struct { ProjectName string `yaml:"project-name"` // IssueLabel is the label of the created issue type IssueLabel string `yaml:"issue-label"` + // SeverityAsLabel (optional) sends the severity as the label of the created + // issue. + SeverityAsLabel bool `yaml:"severity-as-label"` } // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { + err := validateOptions(options) + if err != nil { + return nil, err + } gitlabOpts := []gitlab.ClientOptionFunc{} if options.BaseURL != "" { gitlabOpts = append(gitlabOpts, gitlab.WithBaseURL(options.BaseURL)) @@ -45,16 +56,44 @@ func New(options *Options) (*Integration, error) { return &Integration{client: git, userID: user.ID, options: options}, nil } +func validateOptions(options *Options) error { + errs := []string{} + if options.Username == "" { + errs = append(errs, "Username") + } + if options.Token == "" { + errs = append(errs, "Token") + } + if options.ProjectName == "" { + errs = append(errs, "ProjectName") + } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + + return nil +} + // CreateIssue creates an issue in the tracker func (i *Integration) CreateIssue(event *output.ResultEvent) error { summary := format.Summary(event) description := format.MarkdownDescription(event) + labels := []string{} + severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) + if i.options.SeverityAsLabel && severityLabel != "" { + labels = append(labels, severityLabel) + } + if label := i.options.IssueLabel; label != "" { + labels = append(labels, label) + } _, _, err := i.client.Issues.CreateIssue(i.options.ProjectName, &gitlab.CreateIssueOptions{ Title: &summary, Description: &description, - Labels: gitlab.Labels{i.options.IssueLabel}, + Labels: labels, AssigneeIDs: []int{i.userID}, }) + return err } diff --git a/v2/pkg/reporting/trackers/jira/jira.go b/v2/pkg/reporting/trackers/jira/jira.go index 1bfe2edd5..3237e929b 100644 --- a/v2/pkg/reporting/trackers/jira/jira.go +++ b/v2/pkg/reporting/trackers/jira/jira.go @@ -2,6 +2,7 @@ package jira import ( "bytes" + "errors" "fmt" "io/ioutil" "strings" @@ -22,9 +23,9 @@ type Integration struct { // Options contains the configuration options for jira client type Options struct { - // Cloud value is set to true when Jira cloud is used + // Cloud value (optional) is set to true when Jira cloud is used Cloud bool `yaml:"cloud"` - // UpdateExisting value if true, the existing opened issue is updated + // UpdateExisting value (optional) if true, the existing opened issue is updated UpdateExisting bool `yaml:"update-existing"` // URL is the URL of the jira server URL string `yaml:"url"` @@ -36,12 +37,19 @@ type Options struct { Token string `yaml:"token"` // ProjectName is the name of the project. ProjectName string `yaml:"project-name"` - // IssueType is the name of the created issue type + // IssueType (optional) is the name of the created issue type IssueType string `yaml:"issue-type"` + // SeverityAsLabel (optional) sends the severity as the label of the created + // issue. + SeverityAsLabel bool `yaml:"severity-as-label"` } // New creates a new issue tracker integration client based on options. func New(options *Options) (*Integration, error) { + err := validateOptions(options) + if err != nil { + return nil, err + } username := options.Email if !options.Cloud { username = options.AccountID @@ -57,9 +65,42 @@ func New(options *Options) (*Integration, error) { return &Integration{jira: jiraClient, options: options}, nil } +func validateOptions(options *Options) error { + errs := []string{} + if options.URL == "" { + errs = append(errs, "URL") + } + if options.AccountID == "" { + errs = append(errs, "AccountID") + } + if options.Email == "" { + errs = append(errs, "Email") + } + if options.Token == "" { + errs = append(errs, "Token") + } + if options.ProjectName == "" { + errs = append(errs, "ProjectName") + } + + if len(errs) > 0 { + return errors.New("Mandatory reporting configuration fields are missing: " + strings.Join(errs, ",")) + } + + return nil +} + // CreateNewIssue creates a new issue in the tracker func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { summary := format.Summary(event) + labels := []string{} + severityLabel := fmt.Sprintf("Severity: %s", event.Info.SeverityHolder.Severity.String()) + if i.options.SeverityAsLabel && severityLabel != "" { + labels = append(labels, severityLabel) + } + if label := i.options.IssueType; label != "" { + labels = append(labels, label) + } fields := &jira.IssueFields{ Assignee: &jira.User{AccountID: i.options.AccountID}, @@ -68,6 +109,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, Summary: summary, + Labels: labels, } // On-prem version of Jira server does not use AccountID if !i.options.Cloud { @@ -77,6 +119,7 @@ func (i *Integration) CreateNewIssue(event *output.ResultEvent) error { Type: jira.IssueType{Name: i.options.IssueType}, Project: jira.Project{Key: i.options.ProjectName}, Summary: summary, + Labels: []string{severityLabel}, } } @@ -241,6 +284,13 @@ func jiraFormatDescription(event *output.ResultEvent) string { // TODO remove th } } } + builder.WriteString("\n") + + if event.CURLCommand != "" { + builder.WriteString("\n*CURL Command*\n{code}\n") + builder.WriteString(event.CURLCommand) + builder.WriteString("\n{code}") + } builder.WriteString(fmt.Sprintf("\n---\nGenerated by [Nuclei v%s](https://github.com/projectdiscovery/nuclei)", config.Version)) data := builder.String() return data diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index d7f545bc3..bf4e79b18 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -144,6 +144,21 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute } template.Path = filePath + template.parseSelfContainedRequests() + parsedTemplatesCache.Store(filePath, template, err) return template, nil } + +// parseSelfContainedRequests parses the self contained template requests. +func (t *Template) parseSelfContainedRequests() { + if !t.SelfContained { + return + } + for _, request := range t.RequestsHTTP { + request.SelfContained = true + } + for _, request := range t.RequestsNetwork { + request.SelfContained = true + } +} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index b6f2491ac..330e87bab 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -62,6 +62,10 @@ type Template struct { workflows.Workflow `yaml:",inline,omitempty" jsonschema:"title=workflows to run,description=Workflows to run for the template"` CompiledWorkflow *workflows.Workflow `yaml:"-" json:"-" jsonschema:"-"` + // description: | + // Self Contained marks Requests for the template as self-contained + SelfContained bool `yaml:"self-contained,omitempty" jsonschema:"title=mark requests as self-contained,description=Mark Requests for the template as self-contained"` + // TotalRequests is the total number of requests for the template. TotalRequests int `yaml:"-" json:"-"` // Executer is the actual template executor for running template requests diff --git a/v2/pkg/templates/templates_doc.go b/v2/pkg/templates/templates_doc.go index ce06e5b62..0b19e675d 100644 --- a/v2/pkg/templates/templates_doc.go +++ b/v2/pkg/templates/templates_doc.go @@ -31,7 +31,7 @@ func init() { TemplateDoc.Type = "Template" TemplateDoc.Comments[encoder.LineComment] = " Template is a YAML input file which defines all the requests and" TemplateDoc.Description = "Template is a YAML input file which defines all the requests and\n other metadata for a template." - TemplateDoc.Fields = make([]encoder.Doc, 8) + TemplateDoc.Fields = make([]encoder.Doc, 9) TemplateDoc.Fields[0].Name = "id" TemplateDoc.Fields[0].Type = "string" TemplateDoc.Fields[0].Note = "" @@ -84,6 +84,11 @@ func init() { TemplateDoc.Fields[7].Note = "" TemplateDoc.Fields[7].Description = "Workflows is a list of workflows to execute for a template." TemplateDoc.Fields[7].Comments[encoder.LineComment] = "Workflows is a list of workflows to execute for a template." + TemplateDoc.Fields[8].Name = "self-contained" + TemplateDoc.Fields[8].Type = "bool" + TemplateDoc.Fields[8].Note = "" + TemplateDoc.Fields[8].Description = "Self Contained marks Requests for the template as self-contained" + TemplateDoc.Fields[8].Comments[encoder.LineComment] = "Self Contained marks Requests for the template as self-contained" MODELInfoDoc.Type = "model.Info" MODELInfoDoc.Comments[encoder.LineComment] = " Info contains metadata information about a template" @@ -270,7 +275,7 @@ func init() { FieldName: "requests", }, } - HTTPRequestDoc.Fields = make([]encoder.Doc, 25) + HTTPRequestDoc.Fields = make([]encoder.Doc, 26) HTTPRequestDoc.Fields[0].Name = "matchers" HTTPRequestDoc.Fields[0].Type = "[]matchers.Matcher" HTTPRequestDoc.Fields[0].Note = "" @@ -317,10 +322,10 @@ func init() { HTTPRequestDoc.Fields[7].Name = "attack" HTTPRequestDoc.Fields[7].Type = "string" HTTPRequestDoc.Fields[7].Note = "" - HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + HTTPRequestDoc.Fields[7].Description = "Attack is the type of payload combinations to perform.\n\nbatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." HTTPRequestDoc.Fields[7].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." HTTPRequestDoc.Fields[7].Values = []string{ - "sniper", + "batteringram", "pitchfork", "clusterbomb", } @@ -437,6 +442,11 @@ func init() { HTTPRequestDoc.Fields[24].Note = "" HTTPRequestDoc.Fields[24].Description = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found." HTTPRequestDoc.Fields[24].Comments[encoder.LineComment] = "StopAtFirstMatch stops the execution of the requests and template as soon as a match is found." + HTTPRequestDoc.Fields[25].Name = "skip-variables-check" + HTTPRequestDoc.Fields[25].Type = "bool" + HTTPRequestDoc.Fields[25].Note = "" + HTTPRequestDoc.Fields[25].Description = "SkipVariablesCheck skips the check for unresolved variables in request" + HTTPRequestDoc.Fields[25].Comments[encoder.LineComment] = "SkipVariablesCheck skips the check for unresolved variables in request" MATCHERSMatcherDoc.Type = "matchers.Matcher" MATCHERSMatcherDoc.Comments[encoder.LineComment] = " Matcher is used to match a part in the output from a protocol." @@ -680,7 +690,7 @@ func init() { FieldName: "dns", }, } - DNSRequestDoc.Fields = make([]encoder.Doc, 9) + DNSRequestDoc.Fields = make([]encoder.Doc, 10) DNSRequestDoc.Fields[0].Name = "matchers" DNSRequestDoc.Fields[0].Type = "[]matchers.Matcher" DNSRequestDoc.Fields[0].Note = "" @@ -753,6 +763,11 @@ func init() { DNSRequestDoc.Fields[8].Note = "" DNSRequestDoc.Fields[8].Description = "Recursion determines if resolver should recurse all records to get fresh results." DNSRequestDoc.Fields[8].Comments[encoder.LineComment] = "Recursion determines if resolver should recurse all records to get fresh results." + DNSRequestDoc.Fields[9].Name = "resolvers" + DNSRequestDoc.Fields[9].Type = "[]string" + DNSRequestDoc.Fields[9].Note = "" + DNSRequestDoc.Fields[9].Description = "Resolvers to use for the dns requests" + DNSRequestDoc.Fields[9].Comments[encoder.LineComment] = " Resolvers to use for the dns requests" FILERequestDoc.Type = "file.Request" FILERequestDoc.Comments[encoder.LineComment] = " Request contains a File matching mechanism for local disk operations." @@ -844,10 +859,10 @@ func init() { NETWORKRequestDoc.Fields[2].Name = "attack" NETWORKRequestDoc.Fields[2].Type = "string" NETWORKRequestDoc.Fields[2].Note = "" - NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nSniper is each payload once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." + NETWORKRequestDoc.Fields[2].Description = "Attack is the type of payload combinations to perform.\n\nBatteringram is same payload into all of the defined payload positions at once, pitchfork combines multiple payload sets and clusterbomb generates\npermutations and combinations for all payloads." NETWORKRequestDoc.Fields[2].Comments[encoder.LineComment] = "Attack is the type of payload combinations to perform." NETWORKRequestDoc.Fields[2].Values = []string{ - "sniper", + "batteringram", "pitchfork", "clusterbomb", } diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index dafb2f3e7..d258f4245 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -23,8 +23,12 @@ type Options struct { CustomHeaders goflags.StringSlice // Vars is the list of custom global vars Vars goflags.RuntimeMap + // vars to use as iterative payload + varsPayload map[string]interface{} // Severities filters templates based on their severity and only run the matching ones. Severities severity.Severities + // ExcludeSeverities specifies severities to exclude + ExcludeSeverities severity.Severities // Author filters templates based on their author and only run the matching ones. Author goflags.NormalizedStringSlice // IncludeTags includes specified tags to be run even while being in denylist @@ -57,8 +61,8 @@ type Options struct { ReportingDB string // ReportingConfig is the config file for nuclei reporting module ReportingConfig string - // DiskExportDirectory is the directory to export reports in markdown on disk to - DiskExportDirectory string + // MarkdownExportDirectory is the directory to export reports in markdown format + MarkdownExportDirectory string // SarifExport is the file to export sarif output format to SarifExport string // ResolversFile is a file containing resolvers for nuclei. @@ -103,7 +107,9 @@ type Options struct { Headless bool // ShowBrowser specifies whether the show the browser in headless mode ShowBrowser bool - // SytemResolvers enables override of nuclei's DNS client opting to use system resolver stack. + // UseInstalledChrome skips chrome install and use local instance + UseInstalledChrome bool + // SystemResolvers enables override of nuclei's DNS client opting to use system resolver stack. SystemResolvers bool // Metrics enables display of metrics via an http endpoint Metrics bool @@ -140,6 +146,8 @@ type Options struct { Stdin bool // StopAtFirstMatch stops processing template at first full match (this may break chained requests) StopAtFirstMatch bool + // Stream the input without sorting + Stream bool // NoMeta disables display of metadata for the matches NoMeta bool // NoTimestamp disables display of timestamp for the matcher @@ -157,3 +165,15 @@ type Options struct { // EnvironmentVariables enables support for environment variables EnvironmentVariables bool } + +func (options *Options) AddVarPayload(key string, value interface{}) { + if options.varsPayload == nil { + options.varsPayload = make(map[string]interface{}) + } + + options.varsPayload[key] = value +} + +func (options *Options) VarsPayload() map[string]interface{} { + return options.varsPayload +} diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/workflows/execute_test.go index 26f7c2b99..6d9ab6a09 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/workflows/execute_test.go @@ -3,12 +3,13 @@ package workflows import ( "testing" + "github.com/stretchr/testify/require" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/stretchr/testify/require" ) func TestWorkflowsSimple(t *testing.T) { @@ -106,7 +107,7 @@ func TestWorkflowsSubtemplatesWithMatcher(t *testing.T) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{ - Matches: map[string]struct{}{"tomcat": {}}, + Matches: map[string][]string{"tomcat": {}}, Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}}, @@ -134,7 +135,7 @@ func TestWorkflowsSubtemplatesWithMatcherNoMatch(t *testing.T) { firstInput = input }, outputs: []*output.InternalWrappedEvent{ {OperatorsResult: &operators.Result{ - Matches: map[string]struct{}{"tomcat": {}}, + Matches: map[string][]string{"tomcat": {}}, Extracts: map[string][]string{}, }}, }}, Options: &protocols.ExecuterOptions{Progress: progressBar}},