fix file input in custom vars for self contained http template (#3385)

* fix file input in variables(-V)

* fix lint error

* fix nuclei-ignore file failures

integration_tests/http/self-contained-file-input.yaml

@@ -0,0 +1,25 @@
+id: self-contained-file-input
+
+info:
+  name: Test Self Contained Template With File Input
+  author: pdteam
+  severity: info
+
+self-contained: true
+requests:
+  - method: GET
+    path:
+      - "http://127.0.0.1:5431/{{test}}"
+    matchers:
+      - type: word
+        words:
+          - This is self-contained response
+      
+  - raw:
+      - |
+        GET http://127.0.0.1:5431/{{test}} HTTP/1.1
+        Host: {{Hostname}}
+    matchers:
+      - type: word
+        words:
+          - This is self-contained response

v2/cmd/integration-test/http.go

@@ -8,6 +8,7 @@ import (
 	"net/http/httptest"
 	"net/http/httputil"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
@@ -18,7 +19,9 @@ import (
 
 	"github.com/projectdiscovery/nuclei/v2/pkg/testutils"
 	"github.com/projectdiscovery/retryablehttp-go"
+	errorutil "github.com/projectdiscovery/utils/errors"
 	logutil "github.com/projectdiscovery/utils/log"
+	sliceutil "github.com/projectdiscovery/utils/slice"
 	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
@@ -47,7 +50,8 @@ var httpTestcases = map[string]testutils.TestCase{
 	"http/request-condition-new.yaml":               &httpRequestCondition{},
 	"http/interactsh.yaml":                          &httpInteractshRequest{},
 	"http/interactsh-stop-at-first-match.yaml":      &httpInteractshStopAtFirstMatchRequest{},
-	"http/self-contained.yaml":                      &httpRequestSelContained{},
+	"http/self-contained.yaml":                      &httpRequestSelfContained{},
+	"http/self-contained-file-input.yaml":           &httpRequestSelfContainedFileInput{},
 	"http/get-case-insensitive.yaml":                &httpGetCaseInsensitive{},
 	"http/get.yaml,http/get-case-insensitive.yaml":  &httpGetCaseInsensitiveCluster{},
 	"http/get-redirects-chain-headers.yaml":         &httpGetRedirectsChainHeaders{},
@@ -782,10 +786,10 @@ func (h *httpRequestCondition) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
-type httpRequestSelContained struct{}
+type httpRequestSelfContained struct{}
 
 // Execute executes a test case and returns an error if occurred
-func (h *httpRequestSelContained) Execute(filePath string) error {
+func (h *httpRequestSelfContained) Execute(filePath string) error {
 	router := httprouter.New()
 	router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		_, _ = w.Write([]byte("This is self-contained response"))
@@ -807,6 +811,50 @@ func (h *httpRequestSelContained) Execute(filePath string) error {
 	return expectResultsCount(results, 1)
 }
 
+type httpRequestSelfContainedFileInput struct{}
+
+func (h *httpRequestSelfContainedFileInput) Execute(filePath string) error {
+	router := httprouter.New()
+	gotReqToEndpoints := []string{}
+	router.GET("/one", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		gotReqToEndpoints = append(gotReqToEndpoints, "/one")
+		_, _ = w.Write([]byte("This is self-contained response"))
+	})
+	router.GET("/two", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		gotReqToEndpoints = append(gotReqToEndpoints, "/two")
+		_, _ = w.Write([]byte("This is self-contained response"))
+	})
+	server := &http.Server{
+		Addr:    fmt.Sprintf("localhost:%d", defaultStaticPort),
+		Handler: router,
+	}
+	go func() {
+		_ = server.ListenAndServe()
+	}()
+	defer server.Close()
+
+	// create temp file
+	FileLoc := filepath.Join(os.TempDir(), "httpselfcontained.yaml")
+	err := os.WriteFile(FileLoc, []byte("one\ntwo\n"), 0600)
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("failed to create temporary file").WithTag(filePath)
+	}
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "", debug, "-V", "test="+FileLoc)
+	if err != nil {
+		return err
+	}
+
+	if err := expectResultsCount(results, 4); err != nil {
+		return err
+	}
+
+	if !sliceutil.ElementsMatch(gotReqToEndpoints, []string{"/one", "/two", "/one", "/two"}) {
+		return errorutil.NewWithTag(filePath, "expected requests to be sent to `/one` and `/two` endpoints but were sent to `%v`", gotReqToEndpoints)
+	}
+	return nil
+}
+
 type httpGetCaseInsensitive struct{}
 
 // Execute executes a test case and returns an error if occurred

v2/cmd/nuclei/main.go

@@ -329,7 +329,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 	_ = flagSet.Parse()
 
 	gologger.DefaultLogger.SetTimestamp(options.Timestamp, levels.LevelDebug)
-
+	
 	if options.LeaveDefaultPorts {
 		http.LeaveDefaultPorts = true
 	}
@@ -339,7 +339,9 @@ on extensive configurability, massive extensibility and ease of use.`)
 		configPath := filepath.Join(options.CustomConfigDir, "config.yaml")
 		ignoreFile := filepath.Join(options.CustomConfigDir, ".nuclei-ignore")
 		if !fileutil.FileExists(ignoreFile) {
-			_ = fileutil.CopyFile(originalIgnorePath, ignoreFile)
+			if err := fileutil.CopyFile(originalIgnorePath, ignoreFile); err != nil {
+				gologger.Error().Msgf("failed to copy .nuclei-ignore file in custom config directory got %v", err)
+			}
 		}
 		readConfigFile := func() error {
 			if err := flagSet.MergeConfigFile(configPath); err != nil && !errors.Is(err, io.EOF) {

v2/internal/runner/options.go

@@ -93,14 +93,6 @@ func ParseOptions(options *types.Options) {
 	// Load the resolvers if user asked for them
 	loadResolvers(options)
 
-	// removes all cli variables containing payloads and add them to the internal struct
-	for key, value := range options.Vars.AsMap() {
-		if fileutil.FileExists(value.(string)) {
-			_ = options.Vars.Del(key)
-			options.AddVarPayload(key, value)
-		}
-	}
-
 	err := protocolinit.Init(options)
 	if err != nil {
 		gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)

v2/internal/runner/update.go

@@ -449,6 +449,11 @@ func writeUnZippedTemplateFile(templateAbsolutePath string, zipTemplateFile *zip
 func calculateTemplateAbsolutePath(zipFilePath, configuredTemplateDirectory string) (string, bool, error) {
 	directory, fileName := filepath.Split(zipFilePath)
 
+	// overwrite .nuclei-ignore everytime nuclei-templates are downloaded
+	if fileName == ".nuclei-ignore" {
+		return config.GetIgnoreFilePath(), false, nil
+	}
+
 	if !strings.EqualFold(fileName, ".new-additions") {
 		if strings.TrimSpace(fileName) == "" || strings.HasPrefix(fileName, ".") || strings.EqualFold(fileName, "README.md") {
 			return "", true, nil

v2/pkg/protocols/headless/headless.go

@@ -85,7 +85,7 @@ func (request *Request) GetID() string {
 func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	// TODO: logic similar to network + http => probably can be refactored
 	// Resolve payload paths from vars if they exists
-	for name, payload := range options.Options.VarsPayload() {
+	for name, payload := range options.Options.Vars.AsMap() {
 		payloadStr, ok := payload.(string)
 		// check if inputs contains the payload
 		if ok && fileutil.FileExists(payloadStr) {

v2/pkg/protocols/http/build_request.go

@@ -108,7 +108,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
 	allVars := generators.MergeMaps(dynamicValues, defaultReqVars, optionVars)
 
 	// finalVars contains allVars and any generator/fuzzing specific payloads
-	// TODO: Review Override preference of below maps (before it was generator.MergeMaps(payloads,allVars))
+	// payloads used in generator should be given the most preference
 	finalVars := generators.MergeMaps(allVars, payloads)
 
 	if vardump.EnableVarDump {
@@ -164,9 +164,11 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 			return nil, fmt.Errorf("malformed request supplied")
 		}
 
+		// Note: Here the order of payloads matter since payloads passed through file ex: -V "test=numbers.txt"
+		// are stored in payloads and options vars should not override payloads in any case
 		values := generators.MergeMaps(
-			payloads,
 			generators.BuildPayloadFromOptions(r.request.options.Options),
+			payloads,
 		)
 
 		parts[1] = replacer.Replace(parts[1], values)
@@ -199,8 +201,9 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
 		return r.generateRawRequest(ctx, data, parsed, values, payloads)
 	}
 	values := generators.MergeMaps(
-		dynamicValues,
 		generators.BuildPayloadFromOptions(r.request.options.Options),
+		dynamicValues,
+		payloads, // payloads should override other variables in case of duplicate vars
 	)
 	// Evaluate (replace) variable with final values
 	data, err := expressions.Evaluate(data, values)

v2/pkg/protocols/http/http.go

@@ -302,7 +302,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	}
 
 	// Resolve payload paths from vars if they exists
-	for name, payload := range request.options.Options.VarsPayload() {
+	for name, payload := range request.options.Options.Vars.AsMap() {
 		payloadStr, ok := payload.(string)
 		// check if inputs contains the payload
 		var hasPayloadName bool

v2/pkg/protocols/http/request.go

@@ -353,7 +353,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 			ctx := request.newContext(input)
 			ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(request.options.Options.Timeout)*time.Second)
 			defer cancel()
-
 			generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
 			if err != nil {
 				if err == io.EOF {

v2/pkg/protocols/network/network.go

@@ -162,7 +162,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error {
 	}
 
 	// Resolve payload paths from vars if they exists
-	for name, payload := range request.options.Options.VarsPayload() {
+	for name, payload := range request.options.Options.Vars.AsMap() {
 		payloadStr, ok := payload.(string)
 		// check if inputs contains the payload
 		var hasPayloadName bool

v2/pkg/types/types.go

@@ -35,8 +35,6 @@ type Options struct {
 	CustomHeaders goflags.StringSlice
 	// Vars is the list of custom global vars
 	Vars goflags.RuntimeMap
-	// vars to use as iterative payload
-	varsPayload map[string]interface{}
 	// Severities filters templates based on their severity and only run the matching ones.
 	Severities severity.Severities
 	// ExcludeSeverities specifies severities to exclude
@@ -348,18 +346,6 @@ type Options struct {
 	ScanStrategy string
 }
 
-func (options *Options) AddVarPayload(key string, value interface{}) {
-	if options.varsPayload == nil {
-		options.varsPayload = make(map[string]interface{})
-	}
-
-	options.varsPayload[key] = value
-}
-
-func (options *Options) VarsPayload() map[string]interface{} {
-	return options.varsPayload
-}
-
 // ShouldLoadResume resume file
 func (options *Options) ShouldLoadResume() bool {
 	return options.Resume != "" && fileutil.FileExists(options.Resume)