fix(http): pass dynamicValues to EvaluateWithInteractsh (#6685)

* fix(http): pass `dynamicValues` to `EvaluateWithInteractsh`

When `LazyEval` is true (triggered by `variables`
containing `BaseURL`, `Hostname`,
`interactsh-url`, etc.), variable expressions are not
eval'ed during YAML parsing & remain as raw exprs
like "{{rand_base(5)}}".

At request build time, `EvaluateWithInteractsh()`
checks if a variable already has a value in the
passed map before re-evaluating its expression.
But, `dynamicValues` (which contains the template
context with previously eval'ed values) was not
being passed, causing exprs like `rand_*` to be
re-evaluated on each request, producing different
values.

Fixes #6684 by including `dynamicValues` in the
map passed to `EvaluateWithInteractsh()`, so
variables evaluated in earlier requests retain
their values in subsequent requests.

Signed-off-by: Dwi Siswanto <git@dw1.io>

* chore(http): rm early eval in `(*Request).ExecuteWithResults()`

Signed-off-by: Dwi Siswanto <git@dw1.io>

* test: adds variables-threads-previous integration test

Signed-off-by: Dwi Siswanto <git@dw1.io>

* test: adds constants-with-threads integration test

Signed-off-by: Dwi Siswanto <git@dw1.io>

* test: adds race-with-variables integration test

Signed-off-by: Dwi Siswanto <git@dw1.io>

---------

Signed-off-by: Dwi Siswanto <git@dw1.io>

cmd/integration-test/http.go

@@ -62,9 +62,11 @@ var httpTestcases = []TestCaseInfo{
 	{Path: "protocols/http/dsl-functions.yaml", TestCase: &httpDSLFunctions{}},
 	{Path: "protocols/http/race-simple.yaml", TestCase: &httpRaceSimple{}},
 	{Path: "protocols/http/race-multiple.yaml", TestCase: &httpRaceMultiple{}},
+	{Path: "protocols/http/race-with-variables.yaml", TestCase: &httpRaceWithVariables{}},
 	{Path: "protocols/http/stop-at-first-match.yaml", TestCase: &httpStopAtFirstMatch{}},
 	{Path: "protocols/http/stop-at-first-match-with-extractors.yaml", TestCase: &httpStopAtFirstMatchWithExtractors{}},
 	{Path: "protocols/http/variables.yaml", TestCase: &httpVariables{}},
+	{Path: "protocols/http/variables-threads-previous.yaml", TestCase: &httpVariablesThreadsPrevious{}},
 	{Path: "protocols/http/variable-dsl-function.yaml", TestCase: &httpVariableDSLFunction{}},
 	{Path: "protocols/http/get-override-sni.yaml", TestCase: &httpSniAnnotation{}},
 	{Path: "protocols/http/get-sni.yaml", TestCase: &customCLISNI{}},
@@ -77,6 +79,7 @@ var httpTestcases = []TestCaseInfo{
 	{Path: "protocols/http/cl-body-without-header.yaml", TestCase: &httpCLBodyWithoutHeader{}},
 	{Path: "protocols/http/cl-body-with-header.yaml", TestCase: &httpCLBodyWithHeader{}},
 	{Path: "protocols/http/cli-with-constants.yaml", TestCase: &ConstantWithCliVar{}},
+	{Path: "protocols/http/constants-with-threads.yaml", TestCase: &constantsWithThreads{}},
 	{Path: "protocols/http/matcher-status.yaml", TestCase: &matcherStatusTest{}},
 	{Path: "protocols/http/disable-path-automerge.yaml", TestCase: &httpDisablePathAutomerge{}},
 	{Path: "protocols/http/http-preprocessor.yaml", TestCase: &httpPreprocessor{}},
@@ -1153,6 +1156,26 @@ func (h *httpRaceMultiple) Execute(filePath string) error {
 	return expectResultsCount(results, 5)
 }
 
+type httpRaceWithVariables struct{}
+
+// Execute tests that variables and constants are properly resolved in race mode.
+func (h *httpRaceWithVariables) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/race", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		// Echo back the API key header so we can match on it
+		_, _ = fmt.Fprint(w, r.Header.Get("X-API-Key"))
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 3)
+}
+
 type httpStopAtFirstMatch struct{}
 
 // Execute executes a test case and returns an error if occurred
@@ -1220,6 +1243,30 @@ func (h *httpVariables) Execute(filePath string) error {
 	return expectResultsCount(results, 0)
 }
 
+type httpVariablesThreadsPrevious struct{}
+
+// Execute tests that variables can reference data extracted from previous requests
+// when using threads mode (parallel execution).
+func (h *httpVariablesThreadsPrevious) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/login", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		_, _ = fmt.Fprint(w, "token=secret123")
+	})
+	router.GET("/api", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		// Echo back the Authorization header so we can match on it
+		_, _ = fmt.Fprint(w, r.Header.Get("Authorization"))
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
 type httpVariableDSLFunction struct{}
 
 // Execute executes a test case and returns an error if occurred
@@ -1466,6 +1513,26 @@ func (h *ConstantWithCliVar) Execute(filePath string) error {
 	return expectResultsCount(got, 1)
 }
 
+type constantsWithThreads struct{}
+
+// Execute tests that constants are properly resolved when using threads mode.
+func (h *constantsWithThreads) Execute(filePath string) error {
+	router := httprouter.New()
+	router.GET("/api/:version", func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+		// Echo back the API key header and version so we can match on them
+		_, _ = fmt.Fprintf(w, "%s %s", r.Header.Get("X-API-Key"), p.ByName("version"))
+	})
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug)
+	if err != nil {
+		return err
+	}
+
+	return expectResultsCount(results, 1)
+}
+
 type matcherStatusTest struct{}
 
 // Execute executes a test case and returns an error if occurred

integration_tests/protocols/http/constants-with-threads.yaml

@@ -0,0 +1,27 @@
+id: constants-with-threads
+
+info:
+  name: Constants with Threads
+  author: pdteam
+  severity: info
+  description: |
+    Test that constants are properly resolved when using threads mode.
+
+constants:
+  api_key: "supersecretkey123"
+  api_version: "v2"
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}/api/{{api_version}}"
+    threads: 5
+    headers:
+      X-API-Key: "{{api_key}}"
+
+    matchers:
+      - type: word
+        words:
+          - "supersecretkey123"
+          - "v2"
+        condition: and

integration_tests/protocols/http/race-with-variables.yaml

@@ -0,0 +1,30 @@
+id: race-with-variables
+
+info:
+  name: Race Condition with Variables
+  author: pdteam
+  severity: info
+  description: |
+    Test that variables and constants are properly resolved in race mode.
+
+variables:
+  random_id: "{{rand_base(8)}}"
+
+constants:
+  api_key: "racekey123"
+
+http:
+  - raw:
+      - |
+        GET /race HTTP/1.1
+        Host: {{Hostname}}
+        X-Request-Id: {{random_id}}
+        X-API-Key: {{api_key}}
+
+    race: true
+    race_count: 3
+
+    matchers:
+      - type: word
+        words:
+          - "racekey123"

integration_tests/protocols/http/variables-threads-previous.yaml

@@ -0,0 +1,38 @@
+id: variables-threads-previous
+
+info:
+  name: Variables with Threads and Previous Request Data
+  author: pdteam
+  severity: info
+  description: |
+    Test that variables can reference data extracted from previous requests
+    when using threads mode (parallel execution).
+
+variables:
+  auth_header: "Bearer {{extracted_token}}"
+
+http:
+  - method: GET
+    path:
+      - "{{BaseURL}}/login"
+
+    extractors:
+      - type: regex
+        name: extracted_token
+        part: body
+        regex:
+          - 'token=([a-z0-9]+)'
+        group: 1
+        internal: true
+
+  - method: GET
+    path:
+      - "{{BaseURL}}/api"
+    threads: 5
+    headers:
+      Authorization: "{{auth_header}}"
+
+    matchers:
+      - type: word
+        words:
+          - "Bearer secret123"

pkg/protocols/http/build_request.go

@@ -209,7 +209,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context,
 	// optionvars are vars passed from CLI or env variables
 	optionVars := generators.BuildPayloadFromOptions(r.request.options.Options)
 
-	variablesMap, interactURLs := r.options.Variables.EvaluateWithInteractsh(generators.MergeMaps(defaultReqVars, optionVars), r.options.Interactsh)
+	variablesMap, interactURLs := r.options.Variables.EvaluateWithInteractsh(generators.MergeMaps(dynamicValues, defaultReqVars, optionVars), r.options.Interactsh)
 	if len(interactURLs) > 0 {
 		r.interactshURLs = append(r.interactshURLs, interactURLs...)
 	}

pkg/protocols/http/request.go

@@ -486,10 +486,6 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 
 // ExecuteWithResults executes the final request on a URL
 func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
-	if request.Pipeline || request.Race && request.RaceNumberRequests > 0 || request.Threads > 0 {
-		variablesMap := request.options.Variables.Evaluate(generators.MergeMaps(dynamicValues, previous))
-		dynamicValues = generators.MergeMaps(variablesMap, dynamicValues, request.options.Constants)
-	}
 	// verify if pipeline was requested
 	if request.Pipeline {
 		return request.executeTurboHTTP(input, dynamicValues, previous, callback)