diff --git a/README.md b/README.md index bfa68972e..3fc76ec7a 100644 --- a/README.md +++ b/README.md @@ -162,9 +162,8 @@ DEBUG: -debug show all requests and responses -debug-req show all sent requests -debug-resp show all received responses - -proxy, -proxy-url string URL of the HTTP proxy server - -proxy-socks-url string URL of the SOCKS proxy server - -tlog, -trace-log string file to write sent requests trace log + -proxy string List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list + -trace-log string file to write sent requests trace log -version show nuclei version -v, -verbose show verbose output -vv display extra verbose information diff --git a/README_CN.md b/README_CN.md index 04138acd2..0c4aa69c0 100644 --- a/README_CN.md +++ b/README_CN.md @@ -121,8 +121,7 @@ nuclei -h |templates-version|显示已安装的模板版本|nuclei -templates-version| |v|显示发送请求的详细信息|nuclei -v| |version|显示nuclei的版本号|nuclei -version| -|proxy-url|输入代理地址|nuclei -proxy-url hxxp://127.0.0.1:8080| -|proxy-socks-url|输入socks代理地址|nuclei -proxy-socks-url socks5://127.0.0.1:8080| +|proxy|输入代理地址|nuclei -proxy ./proxy.txt| |random-agent|使用随机的UA|nuclei -random-agent| |H|自定义请求头|nuclei -H “x-bug-bounty:hacker”| diff --git a/v2/cmd/nuclei/main.go b/v2/cmd/nuclei/main.go index d2f8ebb9f..f958ec169 100644 --- a/v2/cmd/nuclei/main.go +++ b/v2/cmd/nuclei/main.go @@ -134,11 +134,9 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "show all received responses"), - /* TODO why the separation? http://proxy:port vs socks5://proxy:port etc - TODO should auto-set the HTTP_PROXY variable for the process? */ - flagSet.StringVarP(&options.ProxyURL, "proxy-url", "proxy", "", "URL of the HTTP proxy server"), - flagSet.StringVar(&options.ProxySocksURL, "proxy-socks-url", "", "URL of the SOCKS proxy server"), - flagSet.StringVarP(&options.TraceLogFile, "trace-log", "tlog", "", "file to write sent requests trace log"), + /* TODO should auto-set the HTTP_PROXY variable for the process? */ + flagSet.NormalizedStringSliceVarP(&options.Proxy, "proxy", "p", []string{}, "List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list"), + flagSet.StringVar(&options.TraceLogFile, "trace-log", "", "file to write sent requests trace log"), flagSet.BoolVar(&options.Version, "version", false, "show nuclei version"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "show verbose output"), flagSet.BoolVar(&options.VerboseVerbose, "vv", false, "display templates loaded for scan"), diff --git a/v2/internal/runner/options.go b/v2/internal/runner/options.go index 59791fe63..1529466fb 100644 --- a/v2/internal/runner/options.go +++ b/v2/internal/runner/options.go @@ -3,7 +3,6 @@ package runner import ( "bufio" "errors" - "net/url" "os" "path/filepath" "strings" @@ -24,7 +23,6 @@ func ParseOptions(options *types.Options) { // Read the inputs and configure the logging configureOutput(options) - // Show the user the banner showBanner() @@ -89,15 +87,10 @@ func validateOptions(options *types.Options) error { if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } - - if err := validateProxyURL(options.ProxyURL, "invalid http proxy format (It should be http://username:password@host:port)"); err != nil { + //loading the proxy server list from file or cli and test the connectivity + if err := loadProxyServers(options); err != nil { return err } - - if err := validateProxyURL(options.ProxySocksURL, "invalid socks proxy format (It should be socks5://username:password@host:port)"); err != nil { - return err - } - if options.Validate { options.Headless = true // required for correct validation of headless templates validateTemplatePaths(options.TemplatesDirectory, options.Templates, options.Workflows) @@ -106,19 +99,6 @@ func validateOptions(options *types.Options) error { return nil } -func validateProxyURL(proxyURL, message string) error { - if proxyURL != "" && !isValidURL(proxyURL) { - return errors.New(message) - } - - return nil -} - -func isValidURL(urlString string) bool { - _, err := url.Parse(urlString) - return err == nil -} - // configureOutput configures the output logging levels to be displayed on the screen func configureOutput(options *types.Options) { // If the user desires verbose output, show verbose output @@ -164,7 +144,6 @@ func loadResolvers(options *types.Options) { func validateTemplatePaths(templatesDirectory string, templatePaths, workflowPaths []string) { allGivenTemplatePaths := append(templatePaths, workflowPaths...) - for _, templatePath := range allGivenTemplatePaths { if templatesDirectory != templatePath && filepath.IsAbs(templatePath) { fileInfo, err := os.Stat(templatePath) diff --git a/v2/internal/runner/proxy.go b/v2/internal/runner/proxy.go new file mode 100644 index 000000000..e254472a7 --- /dev/null +++ b/v2/internal/runner/proxy.go @@ -0,0 +1,123 @@ +package runner + +import ( + "bufio" + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "time" + + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +var proxyURLList []url.URL + +// loadProxyServers load list of proxy servers from file or comma seperated +func loadProxyServers(options *types.Options) error { + if len(options.Proxy) == 0 { + return nil + } + for _, p := range options.Proxy { + if proxyURL, err := validateProxyURL(p); err == nil { + proxyURLList = append(proxyURLList, proxyURL) + } else if fileutil.FileExists(p) { + file, err := os.Open(p) + if err != nil { + return fmt.Errorf("could not open proxy file: %s", err) + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + proxy := scanner.Text() + if strings.TrimSpace(proxy) == "" { + continue + } + if proxyURL, err := validateProxyURL(proxy); err != nil { + return err + } else { + proxyURLList = append(proxyURLList, proxyURL) + } + } + } else { + return fmt.Errorf("invalid proxy file or URL provided for %s", p) + } + } + return processProxyList(options) +} + +func processProxyList(options *types.Options) error { + if len(proxyURLList) == 0 { + return fmt.Errorf("could not find any valid proxy") + } else { + done := make(chan bool) + exitCounter := make(chan bool) + counter := 0 + for _, url := range proxyURLList { + go runProxyConnectivity(url, options, done, exitCounter) + } + for { + select { + case <-done: + { + close(done) + return nil + } + case <-exitCounter: + { + if counter += 1; counter == len(proxyURLList) { + return errors.New("no reachable proxy found") + } + } + } + } + } +} + +func runProxyConnectivity(proxyURL url.URL, options *types.Options, done chan bool, exitCounter chan bool) { + if err := testProxyConnection(proxyURL, options.Timeout); err == nil { + if types.ProxyURL == "" && types.ProxySocksURL == "" { + assignProxyURL(proxyURL, options) + done <- true + } + } + exitCounter <- true +} + +func testProxyConnection(proxyURL url.URL, timeoutDelay int) error { + timeout := time.Duration(timeoutDelay) * time.Second + _, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), timeout) + if err != nil { + return err + } + return nil +} + +func assignProxyURL(proxyURL url.URL, options *types.Options) { + os.Setenv(types.HTTP_PROXY_ENV, proxyURL.String()) + if proxyURL.Scheme == types.HTTP || proxyURL.Scheme == types.HTTPS { + types.ProxyURL = proxyURL.String() + types.ProxySocksURL = "" + gologger.Verbose().Msgf("Using %s as proxy server", proxyURL.String()) + } else if proxyURL.Scheme == types.SOCKS5 { + types.ProxyURL = "" + types.ProxySocksURL = proxyURL.String() + gologger.Verbose().Msgf("Using %s as socket proxy server", proxyURL.String()) + } +} + +func validateProxyURL(proxy string) (url.URL, error) { + if url, err := url.Parse(proxy); err == nil && isSupportedProtocol(url.Scheme) { + return *url, nil + } + return url.URL{}, errors.New("invalid proxy format (It should be http[s]/socks5://[username:password@]host:port)") +} + +//isSupportedProtocol checks given protocols are supported +func isSupportedProtocol(value string) bool { + return value == types.HTTP || value == types.HTTPS || value == types.SOCKS5 +} diff --git a/v2/internal/testutils/testutils.go b/v2/internal/testutils/testutils.go index d62b1d630..029f8bb73 100644 --- a/v2/internal/testutils/testutils.go +++ b/v2/internal/testutils/testutils.go @@ -51,8 +51,7 @@ var DefaultOptions = &types.Options{ Targets: []string{}, TargetsFilePath: "", Output: "", - ProxyURL: "", - ProxySocksURL: "", + Proxy: []string{}, TemplatesDirectory: "", TraceLogFile: "", Templates: []string{}, diff --git a/v2/pkg/protocols/headless/engine/engine.go b/v2/pkg/protocols/headless/engine/engine.go index a506cbda4..a42595801 100644 --- a/v2/pkg/protocols/headless/engine/engine.go +++ b/v2/pkg/protocols/headless/engine/engine.go @@ -63,8 +63,8 @@ func New(options *types.Options) (*Browser, error) { } else { chromeLauncher = chromeLauncher.Headless(true) } - if options.ProxyURL != "" { - chromeLauncher = chromeLauncher.Proxy(options.ProxyURL) + if types.ProxyURL != "" { + chromeLauncher = chromeLauncher.Proxy(types.ProxyURL) } launcherURL, err := chromeLauncher.Launch() if err != nil { diff --git a/v2/pkg/protocols/headless/engine/http_client.go b/v2/pkg/protocols/headless/engine/http_client.go index b4747d541..71a9421cc 100644 --- a/v2/pkg/protocols/headless/engine/http_client.go +++ b/v2/pkg/protocols/headless/engine/http_client.go @@ -23,9 +23,8 @@ func newhttpClient(options *types.Options) *http.Client { InsecureSkipVerify: true, }, } - - if options.ProxyURL != "" { - if proxyURL, err := url.Parse(options.ProxyURL); err == nil { + if types.ProxyURL != "" { + if proxyURL, err := url.Parse(types.ProxyURL); err == nil { transport.Proxy = http.ProxyURL(proxyURL) } } diff --git a/v2/pkg/protocols/http/httpclientpool/clientpool.go b/v2/pkg/protocols/http/httpclientpool/clientpool.go index a4faef07d..b3d953e7e 100644 --- a/v2/pkg/protocols/http/httpclientpool/clientpool.go +++ b/v2/pkg/protocols/http/httpclientpool/clientpool.go @@ -128,9 +128,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl return client, nil } poolMutex.RUnlock() - - if options.ProxyURL != "" { - proxyURL, err = url.Parse(options.ProxyURL) + if types.ProxyURL != "" { + proxyURL, err = url.Parse(types.ProxyURL) } if err != nil { return nil, err @@ -172,27 +171,24 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl }, DisableKeepAlives: disableKeepAlives, } - - // Attempts to overwrite the dial function with the socks proxied version - if options.ProxySocksURL != "" { - var proxyAuth *proxy.Auth - - socksURL, proxyErr := url.Parse(options.ProxySocksURL) - if proxyErr == nil { - proxyAuth = &proxy.Auth{} - proxyAuth.User = socksURL.User.Username() - proxyAuth.Password, _ = socksURL.User.Password() - } - dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", socksURL.Hostname(), socksURL.Port()), proxyAuth, proxy.Direct) - dc := dialer.(interface { - DialContext(ctx context.Context, network, addr string) (net.Conn, error) - }) - if proxyErr == nil { - transport.DialContext = dc.DialContext - } - } if proxyURL != nil { - transport.Proxy = http.ProxyURL(proxyURL) + // Attempts to overwrite the dial function with the socks proxied version + if proxyURL.Scheme == types.SOCKS5 { + var proxyAuth *proxy.Auth = &proxy.Auth{} + proxyAuth.User = proxyURL.User.Username() + proxyAuth.Password, _ = proxyURL.User.Password() + + dialer, proxyErr := proxy.SOCKS5("tcp", fmt.Sprintf("%s:%s", proxyURL.Hostname(), proxyURL.Port()), proxyAuth, proxy.Direct) + + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + if proxyErr == nil { + transport.DialContext = dc.DialContext + } + } else { + transport.Proxy = http.ProxyURL(proxyURL) + } } var jar *cookiejar.Jar diff --git a/v2/pkg/types/proxy.go b/v2/pkg/types/proxy.go new file mode 100644 index 000000000..a45b4eca6 --- /dev/null +++ b/v2/pkg/types/proxy.go @@ -0,0 +1,15 @@ +package types + +const ( + HTTP_PROXY_ENV = "HTTP_PROXY" + SOCKS5 = "socks5" + HTTP = "http" + HTTPS = "https" +) + +var ( + // ProxyURL is the URL for the proxy server + ProxyURL string + // ProxySocksURL is the URL for the proxy socks server + ProxySocksURL string +) diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index d258f4245..549482890 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -49,10 +49,8 @@ type Options struct { TargetsFilePath string // Output is the file to write found results to. Output string - // ProxyURL is the URL for the proxy server - ProxyURL string - // ProxySocksURL is the URL for the proxy socks server - ProxySocksURL string + // List of HTTP(s)/SOCKS5 proxy servers or path to file containing such list + Proxy goflags.NormalizedStringSlice // TemplatesDirectory is the directory to use for storing templates TemplatesDirectory string // TraceLogFile specifies a file to write with the trace of all requests