diff --git a/pkg/operators/extractors/compile.go b/pkg/operators/extractors/compile.go index bcfd37eeb..ecd5ec2ce 100644 --- a/pkg/operators/extractors/compile.go +++ b/pkg/operators/extractors/compile.go @@ -9,6 +9,7 @@ import ( "github.com/itchyny/gojq" "github.com/projectdiscovery/nuclei/v3/pkg/operators/cache" "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" ) // CompileExtractors performs the initial setup operation on an extractor @@ -21,6 +22,11 @@ func (e *Extractor) CompileExtractors() error { e.extractorType = computedType // Compile the regexes for _, regex := range e.Regex { + if varErr := expressions.ContainsUnresolvedVariables(regex); varErr != nil { + e.regexCompiled = append(e.regexCompiled, nil) + continue + } + if cached, err := cache.Regex().GetIFPresent(regex); err == nil && cached != nil { e.regexCompiled = append(e.regexCompiled, cached) continue @@ -37,6 +43,10 @@ func (e *Extractor) CompileExtractors() error { } for _, query := range e.JSON { + if varErr := expressions.ContainsUnresolvedVariables(query); varErr != nil { + e.jsonCompiled = append(e.jsonCompiled, nil) + continue + } query, err := gojq.Parse(query) if err != nil { return fmt.Errorf("could not parse json: %s", query) diff --git a/pkg/operators/extractors/extract.go b/pkg/operators/extractors/extract.go index 1a8ca63b6..5ef0b2aed 100644 --- a/pkg/operators/extractors/extract.go +++ b/pkg/operators/extractors/extract.go @@ -2,21 +2,38 @@ package extractors import ( "fmt" + "regexp" "strings" "github.com/antchfx/htmlquery" "github.com/antchfx/xmlquery" + "github.com/itchyny/gojq" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/nuclei/v3/pkg/utils/json" ) // ExtractRegex extracts text from a corpus and returns it -func (e *Extractor) ExtractRegex(corpus string) map[string]struct{} { +func (e *Extractor) ExtractRegex(corpus string, data map[string]interface{}) map[string]struct{} { results := make(map[string]struct{}) groupPlusOne := e.RegexGroup + 1 - for _, regex := range e.regexCompiled { + for i, regex := range e.regexCompiled { + if varErr := expressions.ContainsUnresolvedVariables(e.Regex[i]); varErr != nil { + regexStr, err := expressions.Evaluate(e.Regex[i], data) + if err != nil { + gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", e.Regex[i], err.Error()) + continue + } + regex, err = regexp.Compile(regexStr) + if err != nil { + gologger.Warning().Msgf("Could not compile regex: %s, error: %s", regexStr, err.Error()) + continue + } + } + // skip prefix short-circuit for case-insensitive patterns rstr := regex.String() if !strings.Contains(rstr, "(?i") { @@ -138,7 +155,7 @@ func (e *Extractor) ExtractXML(corpus string) map[string]struct{} { } // ExtractJSON extracts text from a corpus using JQ queries and returns it -func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} { +func (e *Extractor) ExtractJSON(corpus string, data map[string]interface{}) map[string]struct{} { results := make(map[string]struct{}) var jsonObj interface{} @@ -147,7 +164,25 @@ func (e *Extractor) ExtractJSON(corpus string) map[string]struct{} { return results } - for _, k := range e.jsonCompiled { + for i, k := range e.jsonCompiled { + if varErr := expressions.ContainsUnresolvedVariables(e.JSON[i]); varErr != nil { + jsonStr, err := expressions.Evaluate(e.JSON[i], data) + if err != nil { + gologger.Warning().Msgf("Could not evaluate expression: %s, error: %s", e.JSON[i], err.Error()) + continue + } + query, err := gojq.Parse(jsonStr) + if err != nil { + gologger.Warning().Msgf("Could not parse json: %s, error: %s", jsonStr, err.Error()) + continue + } + k, err = gojq.Compile(query) + if err != nil { + gologger.Warning().Msgf("Could not compile json: %s, error: %s", jsonStr, err.Error()) + continue + } + } + iter := k.Run(jsonObj) for { v, ok := iter.Next() diff --git a/pkg/operators/extractors/extract_test.go b/pkg/operators/extractors/extract_test.go index ab6d5f6ff..18c7b8fbe 100644 --- a/pkg/operators/extractors/extract_test.go +++ b/pkg/operators/extractors/extract_test.go @@ -11,10 +11,10 @@ func TestExtractor_ExtractRegex(t *testing.T) { err := e.CompileExtractors() require.Nil(t, err) - got := e.ExtractRegex("RegEx") + got := e.ExtractRegex("RegEx", map[string]interface{}{}) require.Equal(t, map[string]struct{}{"RegEx": {}}, got) - got = e.ExtractRegex("regex") + got = e.ExtractRegex("regex", map[string]interface{}{}) require.Equal(t, map[string]struct{}{}, got) } @@ -70,10 +70,10 @@ func TestExtractor_ExtractJSON(t *testing.T) { err := e.CompileExtractors() require.Nil(t, err) - got := e.ExtractJSON(`[{"id": 1}]`) + got := e.ExtractJSON(`[{"id": 1}]`, map[string]interface{}{}) require.Equal(t, map[string]struct{}{"1": {}}, got) - got = e.ExtractJSON(`{"id": 1}`) + got = e.ExtractJSON(`{"id": 1}`, map[string]interface{}{}) require.Equal(t, map[string]struct{}{}, got) } diff --git a/pkg/protocols/dns/operators.go b/pkg/protocols/dns/operators.go index fec229447..f5c6ad976 100644 --- a/pkg/protocols/dns/operators.go +++ b/pkg/protocols/dns/operators.go @@ -57,7 +57,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(types.ToString(item)) + return extractor.ExtractRegex(types.ToString(item), data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.DSLExtractor: diff --git a/pkg/protocols/file/operators.go b/pkg/protocols/file/operators.go index 5dab75bb9..cc3e1229c 100644 --- a/pkg/protocols/file/operators.go +++ b/pkg/protocols/file/operators.go @@ -45,11 +45,11 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) + return extractor.ExtractRegex(itemStr, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.JSONExtractor: - return extractor.ExtractJSON(itemStr) + return extractor.ExtractJSON(itemStr, data) case extractors.XPathExtractor: return extractor.ExtractXPath(itemStr) case extractors.DSLExtractor: diff --git a/pkg/protocols/headless/operators.go b/pkg/protocols/headless/operators.go index 0f97f438b..f5f995e0d 100644 --- a/pkg/protocols/headless/operators.go +++ b/pkg/protocols/headless/operators.go @@ -71,7 +71,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) + return extractor.ExtractRegex(itemStr, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.DSLExtractor: @@ -79,7 +79,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto case extractors.XPathExtractor: return extractor.ExtractXPath(itemStr) case extractors.JSONExtractor: - return extractor.ExtractJSON(itemStr) + return extractor.ExtractJSON(itemStr, data) } return nil } diff --git a/pkg/protocols/http/operators.go b/pkg/protocols/http/operators.go index 0ba0fc557..61b0a9517 100644 --- a/pkg/protocols/http/operators.go +++ b/pkg/protocols/http/operators.go @@ -68,13 +68,13 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto } switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(item) + return extractor.ExtractRegex(item, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.XPathExtractor: return extractor.ExtractXPath(item) case extractors.JSONExtractor: - return extractor.ExtractJSON(item) + return extractor.ExtractJSON(item, data) case extractors.DSLExtractor: return extractor.ExtractDSL(data) } diff --git a/pkg/protocols/network/operators.go b/pkg/protocols/network/operators.go index 2aa19e5b3..f5da94d50 100644 --- a/pkg/protocols/network/operators.go +++ b/pkg/protocols/network/operators.go @@ -46,7 +46,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) + return extractor.ExtractRegex(itemStr, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.DSLExtractor: diff --git a/pkg/protocols/offlinehttp/operators.go b/pkg/protocols/offlinehttp/operators.go index 3164c086a..969f88e69 100644 --- a/pkg/protocols/offlinehttp/operators.go +++ b/pkg/protocols/offlinehttp/operators.go @@ -67,7 +67,7 @@ func (request *Request) Extract(data map[string]interface{}, extractor *extracto } switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(item) + return extractor.ExtractRegex(item, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.DSLExtractor: diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 69a8b7bea..2b40d8c95 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -388,11 +388,11 @@ func MakeDefaultExtractFunc(data map[string]interface{}, extractor *extractors.E switch extractor.GetType() { case extractors.RegexExtractor: - return extractor.ExtractRegex(itemStr) + return extractor.ExtractRegex(itemStr, data) case extractors.KValExtractor: return extractor.ExtractKval(data) case extractors.JSONExtractor: - return extractor.ExtractJSON(itemStr) + return extractor.ExtractJSON(itemStr, data) case extractors.XPathExtractor: return extractor.ExtractXPath(itemStr) case extractors.DSLExtractor: