diff --git a/v2/go.mod b/v2/go.mod index e33e6015b..24ce3f36e 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -114,6 +114,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gosuri/uilive v0.0.4 // indirect github.com/gosuri/uiprogress v0.0.1 // indirect + github.com/h2non/filetype v1.1.3 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 35de73b5e..3a3772124 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -263,6 +263,8 @@ github.com/gosuri/uilive v0.0.4 h1:hUEBpQDj8D8jXgtCdBu7sWsy5sbW/5GhuO8KBwJ2jyY= github.com/gosuri/uilive v0.0.4/go.mod h1:V/epo5LjjlDE5RJUcqx8dbw+zc93y5Ya3yg8tfZ74VI= github.com/gosuri/uiprogress v0.0.1 h1:0kpv/XY/qTmFWl/SkaJykZXrBBzwwadmW8fRb7RJSxw= github.com/gosuri/uiprogress v0.0.1/go.mod h1:C1RTYn4Sc7iEyf6j8ft5dyoZ4212h8G1ol9QQluh5+0= +github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= +github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= diff --git a/v2/pkg/protocols/file/file.go b/v2/pkg/protocols/file/file.go index 56dd931e9..c858f46c4 100644 --- a/v2/pkg/protocols/file/file.go +++ b/v2/pkg/protocols/file/file.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/docker/go-units" + "github.com/h2non/filetype" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators" @@ -21,12 +22,12 @@ type Request struct { // Operators for the current request go here. operators.Operators `yaml:",inline"` // description: | - // Extensions is the list of extensions to perform matching on. + // Extensions is the list of extensions or mime types to perform matching on. // examples: // - value: '[]string{".txt", ".go", ".json"}' Extensions []string `yaml:"extensions,omitempty" jsonschema:"title=extensions to match,description=List of extensions to perform matching on"` // description: | - // DenyList is the list of file, directories or extensions to deny during matching. + // DenyList is the list of file, directories, mime types or extensions to deny during matching. // // By default, it contains some non-interesting extensions that are hardcoded // in nuclei. @@ -55,9 +56,11 @@ type Request struct { CompiledOperators *operators.Operators `yaml:"-"` // cache any variables that may be needed for operation. - options *protocols.ExecuterOptions - extensions map[string]struct{} - denyList map[string]struct{} + options *protocols.ExecuterOptions + mimeTypesChecks []string + extensions map[string]struct{} + denyList map[string]struct{} + denyMimeTypesChecks []string // description: | // NoRecursive specifies whether to not do recursive checks if folders are provided. @@ -120,15 +123,20 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { request.denyList = make(map[string]struct{}) for _, extension := range request.Extensions { - if extension == "all" { + switch { + case extension == "all": request.allExtensions = true - } else { + case filetype.IsMIMESupported(extension): + continue + default: if !strings.HasPrefix(extension, ".") { extension = "." + extension } request.extensions[extension] = struct{}{} } } + request.mimeTypesChecks = extractMimeTypes(request.Extensions) + // process default denylist (extensions) var denyList []string if !request.Archive { @@ -147,9 +155,30 @@ func (request *Request) Compile(options *protocols.ExecuterOptions) error { // also add a cleaned version as the exclusion path can be dirty (eg. /a/b/c, /a/b/c/, a///b///c/../d) request.denyList[filepath.Clean(excludeItem)] = struct{}{} } + request.denyMimeTypesChecks = extractMimeTypes(request.DenyList) return nil } +func matchAnyMimeTypes(data []byte, mimeTypes []string) bool { + for _, mimeType := range mimeTypes { + if filetype.Is(data, mimeType) { + return true + } + } + return false +} + +func extractMimeTypes(m []string) []string { + var mimeTypes []string + for _, mm := range m { + if !filetype.IsMIMESupported(mm) { + continue + } + mimeTypes = append(mimeTypes, mm) + } + return mimeTypes +} + // Requests returns the total number of requests the YAML rule will perform func (request *Request) Requests() int { return 0 diff --git a/v2/pkg/protocols/file/find.go b/v2/pkg/protocols/file/find.go index 63b1597c8..3fb04957a 100644 --- a/v2/pkg/protocols/file/find.go +++ b/v2/pkg/protocols/file/find.go @@ -1,12 +1,14 @@ package file import ( + "io" "os" "path/filepath" "strings" "github.com/karrick/godirwalk" "github.com/pkg/errors" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/folderutil" "github.com/projectdiscovery/gologger" ) @@ -109,7 +111,7 @@ func (request *Request) findDirectoryMatches(absPath string, processed map[strin // validatePath validates a file path for blacklist and whitelist options func (request *Request) validatePath(absPath, item string) bool { extension := filepath.Ext(item) - + // extension check if len(request.extensions) > 0 { if _, ok := request.extensions[extension]; ok { return true @@ -117,11 +119,30 @@ func (request *Request) validatePath(absPath, item string) bool { return false } } + + // mime type check + // read first bytes to infer runtime type + fileExists := fileutil.FileExists(item) + var dataChunk []byte + if fileExists { + dataChunk, _ = readChunk(item) + if len(request.mimeTypesChecks) > 0 && matchAnyMimeTypes(dataChunk, request.mimeTypesChecks) { + return true + } + } + if matchingRule, ok := request.isInDenyList(absPath, item); ok { gologger.Verbose().Msgf("Ignoring path %s due to denylist item %s\n", item, matchingRule) return false } + // denied mime type checks + if fileExists { + if len(request.denyMimeTypesChecks) > 0 && matchAnyMimeTypes(dataChunk, request.denyMimeTypesChecks) { + return false + } + } + return true } @@ -175,6 +196,21 @@ func (request *Request) isInDenyList(absPath, item string) (string, bool) { return "", false } +func readChunk(fileName string) ([]byte, error) { + r, err := os.Open(fileName) + if err != nil { + return nil, err + } + + defer r.Close() + + var buff [1024]byte + if _, err = io.ReadFull(r, buff[:]); err != nil { + return nil, err + } + return buff[:], nil +} + func (request *Request) isAnyChunkInDenyList(path string, splitWithUtils bool) (string, bool) { var paths []string