From 6c56a2054465f636fae9e35afe3cd35b143a2e70 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 22 Jan 2023 10:38:50 +0100 Subject: [PATCH] Adding support for nmhe (#3219) * adding support for nmhe * updating docs --- README.md | 1 + README_CN.md | 1 + README_ID.md | 1 + README_KR.md | 1 + v2/cmd/nuclei/main.go | 1 + v2/internal/runner/options.go | 6 ++---- v2/internal/runner/runner.go | 15 +++++++------- .../common/hosterrorscache/hosterrorscache.go | 3 --- v2/pkg/types/types.go | 20 ++++++++++++++++++- 9 files changed, 34 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 56dce5252..293c3a745 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,7 @@ OPTIMIZATIONS: -retries int number of times to retry a failed request (default 1) -ldp, -leave-default-ports leave default HTTP/HTTPS ports (eg. host:80,host:443) -mhe, -max-host-error int max errors for a host before skipping from scan (default 30) + -nmhe, -no-mhe disable skipping host from scan based on errors -project use a project folder to avoid sending same request multiple times -project-path string set a specific project path -spm, -stop-at-first-match stop processing HTTP requests after the first match (may break template/workflow logic) diff --git a/README_CN.md b/README_CN.md index b8f8075dd..0764c88e5 100644 --- a/README_CN.md +++ b/README_CN.md @@ -189,6 +189,7 @@ Nuclei是一款注重于可配置性、可扩展性和易用性的基于模板 -retries int 重试次数（默认：1） -ldp, -leave-default-ports 指定HTTP/HTTPS默认端口（例如：host:80，host:443） -mhe, -max-host-error int 某主机扫描失败次数，跳过该主机（默认：30） + -nmhe, -no-mhe disable skipping host from scan based on errors -project 使用项目文件夹避免多次发送同一请求 -project-path string 设置特定的项目文件夹 -spm, -stop-at-first-path 得到一个结果后停止（或许会中断模板和工作流的逻辑） diff --git a/README_ID.md b/README_ID.md index 7fb972835..ca4d6a7f9 100644 --- a/README_ID.md +++ b/README_ID.md @@ -188,6 +188,7 @@ OPTIMIZATIONS: -retries int number of times to retry a failed request (default 1) -ldp, -leave-default-ports leave default HTTP/HTTPS ports (eg. host:80,host:443 -mhe, -max-host-error int max errors for a host before skipping from scan (default 30) + -nmhe, -no-mhe disable skipping host from scan based on errors -project use a project folder to avoid sending same request multiple times -project-path string set a specific project path -spm, -stop-at-first-path stop processing HTTP requests after the first match (may break template/workflow logic) diff --git a/README_KR.md b/README_KR.md index 0a80c97de..4daa7099c 100644 --- a/README_KR.md +++ b/README_KR.md @@ -178,6 +178,7 @@ OPTIMIZATIONS: -retries int 실패한 요청을 재시도하는 횟수 (기본 1) -ldp, -leave-default-ports leave default HTTP/HTTPS ports (eg. host:80,host:443 -mhe, -max-host-error int 스캔을 건너뛰기 전에 호스트에 대한 최대 오류 수 (기본 30) + -nmhe, -no-mhe disable skipping host from scan based on errors -project 프로젝트 폴더를 사용하여 동일한 요청을 여러 번 보내지 않음 -project-path string 특정 프로젝트 경로 설정 -spm, -stop-at-first-match 첫 번째 일치 후 HTTP 요청 처리 중지 (template/workflow 로직이 중단될 수 있음) diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index bae8cc9bf..ab27c70b3 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -243,6 +243,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"), flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default HTTP/HTTPS ports (eg. host:80,host:443)"), flagSet.IntVarP(&options.MaxHostError, "max-host-error", "mhe", 30, "max errors for a host before skipping from scan"), + flagSet.BoolVarP(&options.NoHostErrors, "no-mhe", "nmhe", false, "disable skipping host from scan based on errors"), flagSet.BoolVar(&options.Project, "project", false, "use a project folder to avoid sending same request multiple times"), flagSet.StringVar(&options.ProjectPath, "project-path", os.TempDir(), "set a specific project path"), flagSet.BoolVarP(&options.StopAtFirstMatch, "stop-at-first-match", "spm", false, "stop processing HTTP requests after the first match (may break template/workflow logic)"), diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 8cace994f..833d20c44 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -3,8 +3,6 @@ package runner import ( "bufio" "fmt" - "io" - "log" "os" "path/filepath" "strings" @@ -24,6 +22,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/types" "github.com/projectdiscovery/stringsutil" fileutil "github.com/projectdiscovery/utils/file" + logutil "github.com/projectdiscovery/utils/log" ) func ConfigureOptions() error { @@ -260,8 +259,7 @@ func configureOutput(options *types.Options) { } // disable standard logger (ref: https://github.com/golang/go/issues/19895) - log.SetFlags(0) - log.SetOutput(io.Discard) + logutil.DisableDefaultLogger() } // loadResolvers loads resolvers from both user provided flag and file diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 578c8a846..d3a109654 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -392,12 +392,6 @@ func (r *Runner) RunEnumeration() error { r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...) r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) } - var cache *hosterrorscache.Cache - if r.options.MaxHostError > 0 { - cache = hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount) - cache.SetVerbose(r.options.Verbose) - } - r.hostErrors = cache // Create the executer options which will be used throughout the execution // stage by the nuclei engine modules. @@ -411,12 +405,19 @@ func (r *Runner) RunEnumeration() error { Interactsh: r.interactsh, ProjectFile: r.projectFile, Browser: r.browser, - HostErrorsCache: cache, Colorizer: r.colorizer, ResumeCfg: r.resumeCfg, ExcludeMatchers: excludematchers.New(r.options.ExcludeMatchers), InputHelper: input.NewHelper(), } + + if r.options.ShouldUseHostError() { + cache := hosterrorscache.New(r.options.MaxHostError, hosterrorscache.DefaultMaxHostsCount) + cache.SetVerbose(r.options.Verbose) + r.hostErrors = cache + executerOpts.HostErrorsCache = cache + } + engine := core.New(r.options) engine.SetExecuterOptions(executerOpts) diff --git a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go index b6cded59e..d7e6610a1 100644 --- a/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go +++ b/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go @@ -89,9 +89,6 @@ func (c *Cache) normalizeCacheValue(value string) string { // - host type func (c *Cache) Check(value string) bool { finalValue := c.normalizeCacheValue(value) - if !c.failedTargets.Has(finalValue) { - return false - } existingCacheItem, err := c.failedTargets.GetIFPresent(finalValue) if err != nil { diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index b53a80fc0..02723301e 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -141,6 +141,8 @@ type Options struct { MetricsPort int // MaxHostError is the maximum number of errors allowed for a host MaxHostError int + // NoHostErrors disables host skipping after maximum number of errors + NoHostErrors bool // BulkSize is the of targets analyzed in parallel for each template BulkSize int // TemplateThreads is the number of templates executed in parallel @@ -389,5 +391,21 @@ func DefaultOptions() *Options { // HasCloudOptions returns true if cloud options have been specified func (options *Options) HasCloudOptions() bool { - return options.ScanList || options.DeleteScan != "" || options.ScanOutput != "" || options.ListDatasources || options.ListTargets || options.ListTemplates || options.RemoveDatasource != "" || options.AddTarget != "" || options.AddTemplate != "" || options.RemoveTarget != "" || options.RemoveTemplate != "" || options.GetTarget != "" || options.GetTemplate != "" + return options.ScanList || + options.DeleteScan != "" || + options.ScanOutput != "" || + options.ListDatasources || + options.ListTargets || + options.ListTemplates || + options.RemoveDatasource != "" || + options.AddTarget != "" || + options.AddTemplate != "" || + options.RemoveTarget != "" || + options.RemoveTemplate != "" || + options.GetTarget != "" || + options.GetTemplate != "" +} + +func (options *Options) ShouldUseHostError() bool { + return options.MaxHostError > 0 && !options.NoHostErrors }