diff --git a/integration_tests/http/self-contained-file-input.yaml b/integration_tests/http/self-contained-file-input.yaml new file mode 100644 index 000000000..a5ac4f78e --- /dev/null +++ b/integration_tests/http/self-contained-file-input.yaml @@ -0,0 +1,25 @@ +id: self-contained-file-input + +info: + name: Test Self Contained Template With File Input + author: pdteam + severity: info + +self-contained: true +requests: + - method: GET + path: + - "http://127.0.0.1:5431/{{test}}" + matchers: + - type: word + words: + - This is self-contained response + + - raw: + - | + GET http://127.0.0.1:5431/{{test}} HTTP/1.1 + Host: {{Hostname}} + matchers: + - type: word + words: + - This is self-contained response \ No newline at end of file diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index 8b0d066ca..15505fbb2 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "net/http/httputil" "os" + "path/filepath" "reflect" "strconv" "strings" @@ -18,7 +19,9 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/projectdiscovery/retryablehttp-go" + errorutil "github.com/projectdiscovery/utils/errors" logutil "github.com/projectdiscovery/utils/log" + sliceutil "github.com/projectdiscovery/utils/slice" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -47,7 +50,8 @@ var httpTestcases = map[string]testutils.TestCase{ "http/request-condition-new.yaml": &httpRequestCondition{}, "http/interactsh.yaml": &httpInteractshRequest{}, "http/interactsh-stop-at-first-match.yaml": &httpInteractshStopAtFirstMatchRequest{}, - "http/self-contained.yaml": &httpRequestSelContained{}, + "http/self-contained.yaml": &httpRequestSelfContained{}, + "http/self-contained-file-input.yaml": &httpRequestSelfContainedFileInput{}, "http/get-case-insensitive.yaml": &httpGetCaseInsensitive{}, "http/get.yaml,http/get-case-insensitive.yaml": &httpGetCaseInsensitiveCluster{}, "http/get-redirects-chain-headers.yaml": &httpGetRedirectsChainHeaders{}, @@ -782,10 +786,10 @@ func (h *httpRequestCondition) Execute(filePath string) error { return expectResultsCount(results, 1) } -type httpRequestSelContained struct{} +type httpRequestSelfContained struct{} // Execute executes a test case and returns an error if occurred -func (h *httpRequestSelContained) Execute(filePath string) error { +func (h *httpRequestSelfContained) Execute(filePath string) error { router := httprouter.New() router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { _, _ = w.Write([]byte("This is self-contained response")) @@ -807,6 +811,50 @@ func (h *httpRequestSelContained) Execute(filePath string) error { return expectResultsCount(results, 1) } +type httpRequestSelfContainedFileInput struct{} + +func (h *httpRequestSelfContainedFileInput) Execute(filePath string) error { + router := httprouter.New() + gotReqToEndpoints := []string{} + router.GET("/one", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + gotReqToEndpoints = append(gotReqToEndpoints, "/one") + _, _ = w.Write([]byte("This is self-contained response")) + }) + router.GET("/two", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + gotReqToEndpoints = append(gotReqToEndpoints, "/two") + _, _ = w.Write([]byte("This is self-contained response")) + }) + server := &http.Server{ + Addr: fmt.Sprintf("localhost:%d", defaultStaticPort), + Handler: router, + } + go func() { + _ = server.ListenAndServe() + }() + defer server.Close() + + // create temp file + FileLoc := filepath.Join(os.TempDir(), "httpselfcontained.yaml") + err := os.WriteFile(FileLoc, []byte("one\ntwo\n"), 0600) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to create temporary file").WithTag(filePath) + } + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-V", "test="+FileLoc) + if err != nil { + return err + } + + if err := expectResultsCount(results, 4); err != nil { + return err + } + + if !sliceutil.ElementsMatch(gotReqToEndpoints, []string{"/one", "/two", "/one", "/two"}) { + return errorutil.NewWithTag(filePath, "expected requests to be sent to `/one` and `/two` endpoints but were sent to `%v`", gotReqToEndpoints) + } + return nil +} + type httpGetCaseInsensitive struct{} // Execute executes a test case and returns an error if occurred diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index 49e25a316..fbe8a91ac 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -329,7 +329,7 @@ on extensive configurability, massive extensibility and ease of use.`) _ = flagSet.Parse() gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug) - + if options.LeaveDefaultPorts { http.LeaveDefaultPorts = true } @@ -339,7 +339,9 @@ on extensive configurability, massive extensibility and ease of use.`) configPath := filepath.Join(options.CustomConfigDir, "config.yaml") ignoreFile := filepath.Join(options.CustomConfigDir, ".nuclei-ignore") if !fileutil.FileExists(ignoreFile) { - _ = fileutil.CopyFile(originalIgnorePath, ignoreFile) + if err := fileutil.CopyFile(originalIgnorePath, ignoreFile); err != nil { + gologger.Error().Msgf("failed to copy .nuclei-ignore file in custom config directory got %v", err) + } } readConfigFile := func() error { if err := flagSet.MergeConfigFile(configPath); err != nil && !errors.Is(err, io.EOF) { diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 833d20c44..cf7e1f3a5 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -93,14 +93,6 @@ func ParseOptions(options *types.Options) { // Load the resolvers if user asked for them loadResolvers(options) - // removes all cli variables containing payloads and add them to the internal struct - for key, value := range options.Vars.AsMap() { - if fileutil.FileExists(value.(string)) { - _ = options.Vars.Del(key) - options.AddVarPayload(key, value) - } - } - err := protocolinit.Init(options) if err != nil { gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err) diff --git a/v2/internal/runner/update.go b/v2/internal/runner/update.go index 4564e2836..8cf463b7c 100644 --- a/v2/internal/runner/update.go +++ b/v2/internal/runner/update.go @@ -449,6 +449,11 @@ func writeUnZippedTemplateFile(templateAbsolutePath string, zipTemplateFile *zip func calculateTemplateAbsolutePath(zipFilePath, configuredTemplateDirectory string) (string, bool, error) { directory, fileName := filepath.Split(zipFilePath) + // overwrite .nuclei-ignore everytime nuclei-templates are downloaded + if fileName == ".nuclei-ignore" { + return config.GetIgnoreFilePath(), false, nil + } + if !strings.EqualFold(fileName, ".new-additions") { if strings.TrimSpace(fileName) == "" || strings.HasPrefix(fileName, ".") || strings.EqualFold(fileName, "README.md") { return "", true, nil diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 49ac08fcb..4f6241e4f 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -85,7 +85,7 @@ func (request *Request) GetID() string { func (request *Request) Compile(options *protocols.ExecuterOptions) error { // TODO: logic similar to network + http => probably can be refactored // Resolve payload paths from vars if they exists - for name, payload := range options.Options.VarsPayload() { + for name, payload := range options.Options.Vars.AsMap() { payloadStr, ok := payload.(string) // check if inputs contains the payload if ok && fileutil.FileExists(payloadStr) { diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index c84e436c9..3d1af6445 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -108,7 +108,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, allVars := generators.MergeMaps(dynamicValues, defaultReqVars, optionVars) // finalVars contains allVars and any generator/fuzzing specific payloads - // TODO: Review Override preference of below maps (before it was generator.MergeMaps(payloads,allVars)) + // payloads used in generator should be given the most preference finalVars := generators.MergeMaps(allVars, payloads) if vardump.EnableVarDump { @@ -164,9 +164,11 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st return nil, fmt.Errorf("malformed request supplied") } + // Note: Here the order of payloads matter since payloads passed through file ex: -V "test=numbers.txt" + // are stored in payloads and options vars should not override payloads in any case values := generators.MergeMaps( - payloads, generators.BuildPayloadFromOptions(r.request.options.Options), + payloads, ) parts[1] = replacer.Replace(parts[1], values) @@ -199,8 +201,9 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st return r.generateRawRequest(ctx, data, parsed, values, payloads) } values := generators.MergeMaps( - dynamicValues, generators.BuildPayloadFromOptions(r.request.options.Options), + dynamicValues, + payloads, // payloads should override other variables in case of duplicate vars ) // Evaluate (replace) variable with final values data, err := expressions.Evaluate(data, values) diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 19b49828d..c23634e76 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -302,7 +302,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } // Resolve payload paths from vars if they exists - for name, payload := range request.options.Options.VarsPayload() { + for name, payload := range request.options.Options.Vars.AsMap() { payloadStr, ok := payload.(string) // check if inputs contains the payload var hasPayloadName bool diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 2e206d6d5..18cddd763 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -353,7 +353,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa ctx := request.newContext(input) ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second) defer cancel() - generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue) if err != nil { if err == io.EOF { diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index f3189be5a..0ac06248d 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -162,7 +162,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { } // Resolve payload paths from vars if they exists - for name, payload := range request.options.Options.VarsPayload() { + for name, payload := range request.options.Options.Vars.AsMap() { payloadStr, ok := payload.(string) // check if inputs contains the payload var hasPayloadName bool diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 02723301e..d2b0edb63 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -35,8 +35,6 @@ type Options struct { CustomHeaders goflags.StringSlice // Vars is the list of custom global vars Vars goflags.RuntimeMap - // vars to use as iterative payload - varsPayload map[string]interface{} // Severities filters templates based on their severity and only run the matching ones. Severities severity.Severities // ExcludeSeverities specifies severities to exclude @@ -348,18 +346,6 @@ type Options struct { ScanStrategy string } -func (options *Options) AddVarPayload(key string, value interface{}) { - if options.varsPayload == nil { - options.varsPayload = make(map[string]interface{}) - } - - options.varsPayload[key] = value -} - -func (options *Options) VarsPayload() map[string]interface{} { - return options.varsPayload -} - // ShouldLoadResume resume file func (options *Options) ShouldLoadResume() bool { return options.Resume != "" && fileutil.FileExists(options.Resume)