From e7d260ea4d2337715bcbb49ef9805e5f009a007a Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 22 Mar 2022 10:52:57 +0100 Subject: [PATCH 1/4] Fixing stats counter --- v2/pkg/protocols/file/request.go | 40 +++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index 2c3d2745b..f3411d60c 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -46,7 +46,6 @@ var emptyResultErr = errors.New("Empty result") func (request *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { wg := sizedwaitgroup.New(request.options.Options.BulkSize) err := request.getInputPaths(input, func(filePath string) { - request.options.Progress.AddToTotal(1) wg.Add() func(filePath string) { defer wg.Done() @@ -59,17 +58,25 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp if !request.validatePath("/", file.Name()) { return nil } + // every new file in the compressed multi-file archive counts 1 + request.options.Progress.AddToTotal(1) archiveFileName := filepath.Join(filePath, file.Name()) event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input, file.Size(), previous) if err != nil { if errors.Is(err, emptyResultErr) { + // no matches but one file elaborated + request.options.Progress.IncrementRequests() return nil } + gologger.Error().Msgf("%s\n", err) + // error while elaborating the file + request.options.Progress.IncrementFailedRequestsBy(1) return err } defer file.Close() dumpResponse(event, request.options, fileMatches, filePath) callback(event) + // file elaborated and matched request.options.Progress.IncrementRequests() return nil }) @@ -78,9 +85,13 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp return } case archiver.Decompressor: + // compressed archive - contains only one file => increments the counter by 1 + request.options.Progress.AddToTotal(1) file, err := os.Open(filePath) if err != nil { gologger.Error().Msgf("%s\n", err) + // error while elaborating the file + request.options.Progress.IncrementFailedRequestsBy(1) return } defer file.Close() @@ -88,12 +99,16 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp tmpFileOut, err := os.CreateTemp("", "") if err != nil { gologger.Error().Msgf("%s\n", err) + // error while elaborating the file + request.options.Progress.IncrementFailedRequestsBy(1) return } defer tmpFileOut.Close() defer os.RemoveAll(tmpFileOut.Name()) if err := archiveInstance.Decompress(file, tmpFileOut); err != nil { gologger.Error().Msgf("%s\n", err) + // error while elaborating the file + request.options.Progress.IncrementFailedRequestsBy(1) return } _ = tmpFileOut.Sync() @@ -101,26 +116,39 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp _, _ = tmpFileOut.Seek(0, 0) event, fileMatches, err := request.processReader(tmpFileOut, filePath, input, fileStat.Size(), previous) if err != nil { - if !errors.Is(err, emptyResultErr) { - gologger.Error().Msgf("%s\n", err) + if errors.Is(err, emptyResultErr) { + // no matches but one file elaborated + request.options.Progress.IncrementRequests() + return } + gologger.Error().Msgf("%s\n", err) + // error while elaborating the file + request.options.Progress.IncrementFailedRequestsBy(1) return } dumpResponse(event, request.options, fileMatches, filePath) callback(event) + // file elaborated and matched request.options.Progress.IncrementRequests() } default: - // normal file + // normal file - increments the counter by 1 + request.options.Progress.AddToTotal(1) event, fileMatches, err := request.processFile(filePath, input, previous) if err != nil { - if !errors.Is(err, emptyResultErr) { - gologger.Error().Msgf("%s\n", err) + if errors.Is(err, emptyResultErr) { + // no matches but one file elaborated + request.options.Progress.IncrementRequests() + return } + gologger.Error().Msgf("%s\n", err) + // error while elaborating the file + request.options.Progress.IncrementFailedRequestsBy(1) return } dumpResponse(event, request.options, fileMatches, filePath) callback(event) + // file elaborated and matched request.options.Progress.IncrementRequests() } }(filePath) From 838ddb63e7af0d73df80a50a778f1babec6f4e64 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 22 Mar 2022 12:35:11 +0100 Subject: [PATCH 2/4] adding mime type file support --- v2/go.mod | 1 + v2/go.sum | 2 ++ v2/pkg/protocols/file/file.go | 43 +++++++++++++++++++++++++++++------ v2/pkg/protocols/file/find.go | 38 ++++++++++++++++++++++++++++++- 4 files changed, 76 insertions(+), 8 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index e33e6015b..24ce3f36e 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -114,6 +114,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gosuri/uilive v0.0.4 // indirect github.com/gosuri/uiprogress v0.0.1 // indirect + github.com/h2non/filetype v1.1.3 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 35de73b5e..3a3772124 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -263,6 +263,8 @@ github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= +github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= +github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index 56dd931e9..c858f46c4 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/docker/go-units" + "github.com/h2non/filetype" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators" @@ -21,12 +22,12 @@ type Request struct { // Operators for the current request go here. operators.Operators `yaml:",inline"` // description: | - // Extensions is the list of extensions to perform matching on. + // Extensions is the list of extensions or mime types to perform matching on. // examples: // - value: '[]string{".txt", ".go", ".json"}' Extensions []string `yaml:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"` // description: | - // DenyList is the list of file, directories or extensions to deny during matching. + // DenyList is the list of file, directories, mime types or extensions to deny during matching. // // By default, it contains some non-interesting extensions that are hardcoded // in nuclei. @@ -55,9 +56,11 @@ type Request struct { CompiledOperators *operators.Operators `yaml:"-"` // cache any variables that may be needed for operation. - options *protocols.ExecuterOptions - extensions map[string]struct{} - denyList map[string]struct{} + options *protocols.ExecuterOptions + mimeTypesChecks []string + extensions map[string]struct{} + denyList map[string]struct{} + denyMimeTypesChecks []string // description: | // NoRecursive specifies whether to not do recursive checks if folders are provided. @@ -120,15 +123,20 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.denyList = make(map[string]struct{}) for _, extension := range request.Extensions { - if extension == "all" { + switch { + case extension == "all": request.allExtensions = true - } else { + case filetype.IsMIMESupported(extension): + continue + default: if !strings.HasPrefix(extension, ".") { extension = "." + extension } request.extensions[extension] = struct{}{} } } + request.mimeTypesChecks = extractMimeTypes(request.Extensions) + // process default denylist (extensions) var denyList []string if !request.Archive { @@ -147,9 +155,30 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { // also add a cleaned version as the exclusion path can be dirty (eg. /a/b/c, /a/b/c/, a///b///c/../d) request.denyList[filepath.Clean(excludeItem)] = struct{}{} } + request.denyMimeTypesChecks = extractMimeTypes(request.DenyList) return nil } +func matchAnyMimeTypes(data []byte, mimeTypes []string) bool { + for _, mimeType := range mimeTypes { + if filetype.Is(data, mimeType) { + return true + } + } + return false +} + +func extractMimeTypes(m []string) []string { + var mimeTypes []string + for _, mm := range m { + if !filetype.IsMIMESupported(mm) { + continue + } + mimeTypes = append(mimeTypes, mm) + } + return mimeTypes +} + // Requests returns the total number of requests the YAML rule will perform func (request *Request) Requests() int { return 0 diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go index 63b1597c8..3fb04957a 100644 --- a/v2/pkg/protocols/file/find.go +++ b/v2/pkg/protocols/file/find.go @@ -1,12 +1,14 @@ package file import ( + "io" "os" "path/filepath" "strings" "github.com/karrick/godirwalk" "github.com/pkg/errors" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/folderutil" "github.com/projectdiscovery/gologger" ) @@ -109,7 +111,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin // validatePath validates a file path for blacklist and whitelist options func (request *Request) validatePath(absPath, item string) bool { extension := filepath.Ext(item) - + // extension check if len(request.extensions) > 0 { if _, ok := request.extensions[extension]; ok { return true @@ -117,11 +119,30 @@ func (request *Request) validatePath(absPath, item string) bool { return false } } + + // mime type check + // read first bytes to infer runtime type + fileExists := fileutil.FileExists(item) + var dataChunk []byte + if fileExists { + dataChunk, _ = readChunk(item) + if len(request.mimeTypesChecks) > 0 && matchAnyMimeTypes(dataChunk, request.mimeTypesChecks) { + return true + } + } + if matchingRule, ok := request.isInDenyList(absPath, item); ok { gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, matchingRule) return false } + // denied mime type checks + if fileExists { + if len(request.denyMimeTypesChecks) > 0 && matchAnyMimeTypes(dataChunk, request.denyMimeTypesChecks) { + return false + } + } + return true } @@ -175,6 +196,21 @@ func (request *Request) isInDenyList(absPath, item string) (string, bool) { return "", false } +func readChunk(fileName string) ([]byte, error) { + r, err := os.Open(fileName) + if err != nil { + return nil, err + } + + defer r.Close() + + var buff [1024]byte + if _, err = io.ReadFull(r, buff[:]); err != nil { + return nil, err + } + return buff[:], nil +} + func (request *Request) isAnyChunkInDenyList(path string, splitWithUtils bool) (string, bool) { var paths []string From 3288c77692e18e1ff4dd81e412d0c6ec1e732b8c Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 22 Mar 2022 13:47:13 +0100 Subject: [PATCH 3/4] fixing headless test cases --- v2/pkg/protocols/headless/engine/page_actions_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index d624df653..fe01f4999 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -52,7 +52,7 @@ func TestActionScript(t *testing.T) { actions := []*Action{ {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, - {ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "window.test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "() => window.test"}}, } testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { @@ -64,10 +64,10 @@ func TestActionScript(t *testing.T) { t.Run("hook", func(t *testing.T) { actions := []*Action{ - {ActionType: ActionTypeHolder{ActionType: ActionScript}, Data: map[string]string{"code": "window.test = 'some-data';", "hook": "true"}}, + {ActionType: ActionTypeHolder{ActionType: ActionScript}, Data: map[string]string{"code": "() => window.test = 'some-data';", "hook": "true"}}, {ActionType: ActionTypeHolder{ActionType: ActionNavigate}, Data: map[string]string{"url": "{{BaseURL}}"}}, {ActionType: ActionTypeHolder{ActionType: ActionWaitLoad}}, - {ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "window.test"}}, + {ActionType: ActionTypeHolder{ActionType: ActionScript}, Name: "test", Data: map[string]string{"code": "() => window.test"}}, } testHeadlessSimpleResponse(t, response, actions, timeout, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") From 5cd25bd069dd1099e12a4adeec099cc9074a8334 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 22 Mar 2022 14:18:01 +0100 Subject: [PATCH 4/4] more checks + test cases fix --- .../headless/headless-extract-values.yaml | 2 +- v2/pkg/protocols/file/file.go | 6 +++- v2/pkg/protocols/file/find.go | 31 +++++++++++-------- v2/pkg/protocols/file/request.go | 2 +- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/integration_tests/headless/headless-extract-values.yaml b/integration_tests/headless/headless-extract-values.yaml index e780ac32c..b4a90909e 100644 --- a/integration_tests/headless/headless-extract-values.yaml +++ b/integration_tests/headless/headless-extract-values.yaml @@ -17,7 +17,7 @@ headless: name: extract args: code: | - '\n' + [...new Set(Array.from(document.querySelectorAll('[src], [href], [url], [action]')).map(i => i.src || i.href || i.url || i.action))].join('\r\n') + '\n' + () => '\n' + [...new Set(Array.from(document.querySelectorAll('[src], [href], [url], [action]')).map(i => i.src || i.href || i.url || i.action))].join('\r\n') + '\n' matchers: - type: word diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index c858f46c4..64c32746a 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -53,6 +53,10 @@ type Request struct { // elaborates archives Archive bool + // description: | + // enables mime types check + MimeType bool + CompiledOperators *operators.Operators `yaml:"-"` // cache any variables that may be needed for operation. @@ -126,7 +130,7 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { switch { case extension == "all": request.allExtensions = true - case filetype.IsMIMESupported(extension): + case request.MimeType && filetype.IsMIMESupported(extension): continue default: if !strings.HasPrefix(extension, ".") { diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go index 3fb04957a..6ad221392 100644 --- a/v2/pkg/protocols/file/find.go +++ b/v2/pkg/protocols/file/find.go @@ -53,7 +53,7 @@ func (request *Request) findGlobPathMatches(absPath string, processed map[string return errors.Errorf("wildcard found, but unable to glob: %s\n", err) } for _, match := range matches { - if !request.validatePath(absPath, match) { + if !request.validatePath(absPath, match, false) { continue } if _, ok := processed[match]; !ok { @@ -75,7 +75,7 @@ func (request *Request) findFileMatches(absPath string, processed map[string]str return false, nil } if _, ok := processed[absPath]; !ok { - if !request.validatePath(absPath, absPath) { + if !request.validatePath(absPath, absPath, false) { return false, nil } processed[absPath] = struct{}{} @@ -95,7 +95,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin if d.IsDir() { return nil } - if !request.validatePath(absPath, path) { + if !request.validatePath(absPath, path, false) { return nil } if _, ok := processed[path]; !ok { @@ -109,7 +109,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin } // validatePath validates a file path for blacklist and whitelist options -func (request *Request) validatePath(absPath, item string) bool { +func (request *Request) validatePath(absPath, item string, inArchive bool) bool { extension := filepath.Ext(item) // extension check if len(request.extensions) > 0 { @@ -120,14 +120,19 @@ func (request *Request) validatePath(absPath, item string) bool { } } - // mime type check - // read first bytes to infer runtime type - fileExists := fileutil.FileExists(item) - var dataChunk []byte - if fileExists { - dataChunk, _ = readChunk(item) - if len(request.mimeTypesChecks) > 0 && matchAnyMimeTypes(dataChunk, request.mimeTypesChecks) { - return true + var ( + fileExists bool + dataChunk []byte + ) + if !inArchive && request.MimeType { + // mime type check + // read first bytes to infer runtime type + fileExists = fileutil.FileExists(item) + if fileExists { + dataChunk, _ = readChunk(item) + if len(request.mimeTypesChecks) > 0 && matchAnyMimeTypes(dataChunk, request.mimeTypesChecks) { + return true + } } } @@ -137,7 +142,7 @@ func (request *Request) validatePath(absPath, item string) bool { } // denied mime type checks - if fileExists { + if !inArchive && request.MimeType && fileExists { if len(request.denyMimeTypesChecks) > 0 && matchAnyMimeTypes(dataChunk, request.denyMimeTypesChecks) { return false } diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index f3411d60c..62e6543df 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -55,7 +55,7 @@ func (request *Request) ExecuteWithResults(input string, metadata, previous outp switch archiveInstance := archiveReader.(type) { case archiver.Walker: err := archiveInstance.Walk(filePath, func(file archiver.File) error { - if !request.validatePath("/", file.Name()) { + if !request.validatePath("/", file.Name(), true) { return nil } // every new file in the compressed multi-file archive counts 1