mirror of
https://github.com/projectdiscovery/nuclei.git
synced 2026-02-06 10:33:08 +08:00
bb79a061bc
`MergeMaps` accounts for 11.41% of allocs (13.8 GB) in clusterbomb mode. With 1,305 combinations per target, this function is called millions of times in the hot path. RCA: * Request generator calls `MergeMaps` with single arg on every payload combination, incurring variadic overhead. * Build request merges same maps multiple times per request. * `BuildPayloadFromOptions` recomputes static CLI options on every call. * Variables calls `MergeMaps` $$2×N$$ times per variable evaluation (once in loop, once in `evaluateVariableValue`) Changes: Core optimizations in maps.go: * Pre-size merged map to avoid rehashing (30-40% reduction) * Add `CopyMap` for efficient single-map copy without variadic overhead. * Add `MergeMapsInto` for in-place mutation when caller owns destination. Hot path fixes: * Replace `MergeMaps(r.currentPayloads)` with `CopyMap(r.currentPayloads)` to eliminates allocation on every combination iteration. * Pre-allocate combined map once, extend in-place during `ForEach` loop instead of creating new map per variable (eliminates $$2×N$$ allocations per request). Caching with concurrency safety: * Cache `BuildPayloadFromOptions` computation in `sync.Map` keyed by `types.Options` ptr, but return copy to prevent concurrent modification. * Cost: shallow copy of ~10-20 entries vs. full merge of vars + env (85-90% savings in typical case) * Clear cache in `closeInternal()` to prevent memory leaks when SDK instances are created or destroyed. Estimated impact: 40-60% reduction in `MergeMaps` allocations (5.5-8.3 GB savings from original 13.8 GB). Safe for concurrent execution and SDK usage with multiple instances. Signed-off-by: Dwi Siswanto <git@dw1.io>
93 lines
2.1 KiB
Go
93 lines
2.1 KiB
Go
package generators
|
|
|
|
import (
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/projectdiscovery/goflags"
|
|
"github.com/projectdiscovery/nuclei/v3/pkg/types"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestBuildPayloadFromOptionsConcurrency(t *testing.T) {
|
|
// Test that BuildPayloadFromOptions is safe for concurrent use
|
|
// and returns independent copies that can be modified without races
|
|
vars := goflags.RuntimeMap{}
|
|
_ = vars.Set("key=value")
|
|
|
|
opts := &types.Options{
|
|
Vars: vars,
|
|
}
|
|
|
|
const numGoroutines = 100
|
|
var wg sync.WaitGroup
|
|
wg.Add(numGoroutines)
|
|
|
|
// Each goroutine gets a map and modifies it
|
|
for i := 0; i < numGoroutines; i++ {
|
|
go func(id int) {
|
|
defer wg.Done()
|
|
|
|
// Get the map (should be a copy of cached data)
|
|
m := BuildPayloadFromOptions(opts)
|
|
|
|
// Modify it - this should not cause races
|
|
m["goroutine_id"] = id
|
|
m["test_key"] = "test_value"
|
|
|
|
// Verify original cached value is present
|
|
require.Equal(t, "value", m["key"])
|
|
}(i)
|
|
}
|
|
|
|
wg.Wait()
|
|
}
|
|
|
|
func TestBuildPayloadFromOptionsCaching(t *testing.T) {
|
|
// Test that caching actually works
|
|
vars := goflags.RuntimeMap{}
|
|
_ = vars.Set("cached=yes")
|
|
|
|
opts := &types.Options{
|
|
Vars: vars,
|
|
EnvironmentVariables: false,
|
|
}
|
|
|
|
// First call - builds and caches
|
|
m1 := BuildPayloadFromOptions(opts)
|
|
require.Equal(t, "yes", m1["cached"])
|
|
|
|
// Second call - should return copy of cached result
|
|
m2 := BuildPayloadFromOptions(opts)
|
|
require.Equal(t, "yes", m2["cached"])
|
|
|
|
// Modify m1 - should not affect m2 since they're copies
|
|
m1["modified"] = "in_m1"
|
|
require.NotContains(t, m2, "modified")
|
|
|
|
// Modify m2 - should not affect future calls
|
|
m2["modified"] = "in_m2"
|
|
m3 := BuildPayloadFromOptions(opts)
|
|
require.NotContains(t, m3, "modified")
|
|
}
|
|
|
|
func TestClearOptionsPayloadMap(t *testing.T) {
|
|
vars := goflags.RuntimeMap{}
|
|
_ = vars.Set("temp=data")
|
|
|
|
opts := &types.Options{
|
|
Vars: vars,
|
|
}
|
|
|
|
// Build and cache
|
|
m1 := BuildPayloadFromOptions(opts)
|
|
require.Equal(t, "data", m1["temp"])
|
|
|
|
// Clear the cache
|
|
ClearOptionsPayloadMap(opts)
|
|
|
|
// Verify it still works (rebuilds)
|
|
m2 := BuildPayloadFromOptions(opts)
|
|
require.Equal(t, "data", m2["temp"])
|
|
}
|