From c3750be38025156a30cb97efbf9be2e9d13b0b80 Mon Sep 17 00:00:00 2001 From: tvroi Date: Sun, 19 Oct 2025 15:33:31 +0700 Subject: [PATCH 1/8] feat(openapi/swagger): direct fuzzing using target url --- internal/runner/runner.go | 10 +- pkg/input/formats/formats.go | 9 + pkg/input/formats/openapi/downloader.go | 120 ++++++++ pkg/input/formats/openapi/downloader_test.go | 240 +++++++++++++++ pkg/input/formats/swagger/downloader.go | 148 +++++++++ pkg/input/formats/swagger/downloader_test.go | 306 +++++++++++++++++++ pkg/input/provider/interface.go | 59 +++- 7 files changed, 874 insertions(+), 18 deletions(-) create mode 100644 pkg/input/formats/openapi/downloader.go create mode 100644 pkg/input/formats/openapi/downloader_test.go create mode 100644 pkg/input/formats/swagger/downloader.go create mode 100644 pkg/input/formats/swagger/downloader_test.go diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 59910f824..1594f1e26 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -254,8 +254,12 @@ func New(options *types.Options) (*Runner, error) { os.Exit(0) } + if tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*"); err == nil { + runner.tmpDir = tmpDir + } + // create the input provider and load the inputs - inputProvider, err := provider.NewInputProvider(provider.InputOptions{Options: options}) + inputProvider, err := provider.NewInputProvider(provider.InputOptions{Options: options, TempDir: runner.tmpDir}) if err != nil { return nil, errors.Wrap(err, "could not create input provider") } @@ -386,10 +390,6 @@ func New(options *types.Options) (*Runner, error) { } runner.rateLimiter = utils.GetRateLimiter(context.Background(), options.RateLimit, options.RateLimitDuration) - if tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*"); err == nil { - runner.tmpDir = tmpDir - } - return runner, nil } diff --git a/pkg/input/formats/formats.go b/pkg/input/formats/formats.go index c7798286a..4cbd96a59 100644 --- a/pkg/input/formats/formats.go +++ b/pkg/input/formats/formats.go @@ -47,6 +47,15 @@ type Format interface { SetOptions(options InputFormatOptions) } +// SpecDownloader is an interface for downloading API specifications from URLs +type SpecDownloader interface { + // Download downloads the spec from the given URL and saves it to tmpDir + // Returns the path to the downloaded file + Download(url, tmpDir string) (string, error) + // SupportedExtensions returns the list of supported file extensions + SupportedExtensions() []string +} + var ( DefaultVarDumpFileName = "required_openapi_params.yaml" ErrNoVarsDumpFile = errors.New("no required params file found") diff --git a/pkg/input/formats/openapi/downloader.go b/pkg/input/formats/openapi/downloader.go new file mode 100644 index 000000000..22ca51387 --- /dev/null +++ b/pkg/input/formats/openapi/downloader.go @@ -0,0 +1,120 @@ +package openapi + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" +) + +// OpenAPIDownloader implements the SpecDownloader interface for OpenAPI 3.0 specs +type OpenAPIDownloader struct{} + +// NewDownloader creates a new OpenAPI downloader +func NewDownloader() formats.SpecDownloader { + return &OpenAPIDownloader{} +} + +// This function downloads an OpenAPI 3.0 spec from the given URL and saves it to tmpDir +func (d *OpenAPIDownloader) Download(urlStr, tmpDir string) (string, error) { + // Validate URL format, OpenAPI 3.0 specs are typically JSON + if !strings.HasSuffix(urlStr, ".json") && !strings.Contains(urlStr, "openapi") { + return "", fmt.Errorf("URL does not appear to be an OpenAPI JSON spec") + } + + client := &http.Client{Timeout: 30 * time.Second} + + resp, err := client.Get(urlStr) + if err != nil { + return "", errors.Wrap(err, "failed to download OpenAPI spec") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("HTTP %d when downloading OpenAPI spec", resp.StatusCode) + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", errors.Wrap(err, "failed to read response body") + } + + // Validate it's a valid JSON and has OpenAPI structure + var spec map[string]interface{} + if err := json.Unmarshal(bodyBytes, &spec); err != nil { + return "", fmt.Errorf("downloaded content is not valid JSON: %w", err) + } + + // Check if it's an OpenAPI 3.0 spec + if openapi, exists := spec["openapi"]; exists { + if openapiStr, ok := openapi.(string); ok && strings.HasPrefix(openapiStr, "3.") { + // Valid OpenAPI 3.0 spec + } else { + return "", fmt.Errorf("not a valid OpenAPI 3.0 spec (found version: %v)", openapi) + } + } else { + return "", fmt.Errorf("not an OpenAPI spec (missing 'openapi' field)") + } + + // Extract host from URL for server configuration + parsedURL, err := url.Parse(urlStr) + if err != nil { + return "", errors.Wrap(err, "failed to parse URL") + } + host := parsedURL.Host + + // Add servers section if missing or empty + servers, exists := spec["servers"] + if !exists || servers == nil { + spec["servers"] = []map[string]interface{}{ + {"url": "https://" + host}, + } + } else if serversList, ok := servers.([]interface{}); ok && len(serversList) == 0 { + spec["servers"] = []map[string]interface{}{ + {"url": "https://" + host}, + } + } + + // Marshal back to JSON + modifiedJSON, err := json.Marshal(spec) + if err != nil { + return "", errors.Wrap(err, "failed to marshal modified spec") + } + + // Create output directory + openapiDir := filepath.Join(tmpDir, "openapi") + if err := os.MkdirAll(openapiDir, 0755); err != nil { + return "", errors.Wrap(err, "failed to create openapi directory") + } + + // Generate filename + filename := fmt.Sprintf("openapi-spec-%d.json", time.Now().Unix()) + filePath := filepath.Join(openapiDir, filename) + + // Write file + file, err := os.Create(filePath) + if err != nil { + return "", fmt.Errorf("failed to create file: %w", err) + } + defer file.Close() + + if _, err := file.Write(modifiedJSON); err != nil { + os.Remove(filePath) + return "", errors.Wrap(err, "failed to write OpenAPI spec to file") + } + + return filePath, nil +} + +// SupportedExtensions returns the list of supported file extensions for OpenAPI +func (d *OpenAPIDownloader) SupportedExtensions() []string { + return []string{".json"} +} diff --git a/pkg/input/formats/openapi/downloader_test.go b/pkg/input/formats/openapi/downloader_test.go new file mode 100644 index 000000000..e5fc7784a --- /dev/null +++ b/pkg/input/formats/openapi/downloader_test.go @@ -0,0 +1,240 @@ +package openapi + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" +) + +func TestOpenAPIDownloader_SupportedExtensions(t *testing.T) { + downloader := &OpenAPIDownloader{} + extensions := downloader.SupportedExtensions() + + expected := []string{".json"} + if len(extensions) != len(expected) { + t.Errorf("Expected %d extensions, got %d", len(expected), len(extensions)) + } + + for i, ext := range extensions { + if ext != expected[i] { + t.Errorf("Expected extension %s, got %s", expected[i], ext) + } + } +} + +func TestOpenAPIDownloader_Download_Success(t *testing.T) { + // Create a mock OpenAPI spec + mockSpec := map[string]interface{}{ + "openapi": "3.0.0", + "info": map[string]interface{}{ + "title": "Test API", + "version": "1.0.0", + }, + "paths": map[string]interface{}{ + "/test": map[string]interface{}{ + "get": map[string]interface{}{ + "summary": "Test endpoint", + }, + }, + }, + } + + // Create mock server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(mockSpec) + })) + defer server.Close() + + // Create temp directory + tmpDir, err := os.MkdirTemp("", "openapi_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + // Test download + downloader := &OpenAPIDownloader{} + filePath, err := downloader.Download(server.URL+"/openapi.json", tmpDir) + if err != nil { + t.Fatalf("Download failed: %v", err) + } + + // Verify file exists + if !fileExists(filePath) { + t.Errorf("Downloaded file does not exist: %s", filePath) + } + + // Verify file content + content, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("Failed to read downloaded file: %v", err) + } + + var downloadedSpec map[string]interface{} + if err := json.Unmarshal(content, &downloadedSpec); err != nil { + t.Fatalf("Failed to parse downloaded JSON: %v", err) + } + + // Verify servers field was added + servers, exists := downloadedSpec["servers"] + if !exists { + t.Error("Servers field was not added to the spec") + } + + if serversList, ok := servers.([]interface{}); ok { + if len(serversList) == 0 { + t.Error("Servers list is empty") + } + } else { + t.Error("Servers field is not a list") + } +} + +func TestOpenAPIDownloader_Download_NonJSONURL(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "openapi_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + downloader := &OpenAPIDownloader{} + _, err = downloader.Download("http://example.com/spec.yaml", tmpDir) + if err == nil { + t.Error("Expected error for non-JSON URL, but got none") + } + + if !strings.Contains(err.Error(), "URL does not appear to be an OpenAPI JSON spec") { + t.Errorf("Unexpected error message: %v", err) + } +} + +func TestOpenAPIDownloader_Download_HTTPError(t *testing.T) { + // Create mock server that returns 404 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + tmpDir, err := os.MkdirTemp("", "openapi_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + downloader := &OpenAPIDownloader{} + _, err = downloader.Download(server.URL+"/openapi.json", tmpDir) + if err == nil { + t.Error("Expected error for HTTP 404, but got none") + } +} + +func TestOpenAPIDownloader_Download_InvalidJSON(t *testing.T) { + // Create mock server that returns invalid JSON + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte("invalid json")) + })) + defer server.Close() + + tmpDir, err := os.MkdirTemp("", "openapi_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + downloader := &OpenAPIDownloader{} + _, err = downloader.Download(server.URL+"/openapi.json", tmpDir) + if err == nil { + t.Error("Expected error for invalid JSON, but got none") + } +} + +func TestOpenAPIDownloader_Download_Timeout(t *testing.T) { + // Create mock server with delay + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(35 * time.Second) // Longer than 30 second timeout + json.NewEncoder(w).Encode(map[string]interface{}{"test": "data"}) + })) + defer server.Close() + + tmpDir, err := os.MkdirTemp("", "openapi_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + downloader := &OpenAPIDownloader{} + _, err = downloader.Download(server.URL+"/openapi.json", tmpDir) + if err == nil { + t.Error("Expected timeout error, but got none") + } +} + +func TestOpenAPIDownloader_Download_WithExistingServers(t *testing.T) { + // Create a mock OpenAPI spec with existing servers + mockSpec := map[string]interface{}{ + "openapi": "3.0.0", + "info": map[string]interface{}{ + "title": "Test API", + "version": "1.0.0", + }, + "servers": []interface{}{ + map[string]interface{}{ + "url": "https://existing-server.com", + }, + }, + "paths": map[string]interface{}{}, + } + + // Create mock server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(mockSpec) + })) + defer server.Close() + + tmpDir, err := os.MkdirTemp("", "openapi_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + downloader := &OpenAPIDownloader{} + filePath, err := downloader.Download(server.URL+"/openapi.json", tmpDir) + if err != nil { + t.Fatalf("Download failed: %v", err) + } + + // Verify existing servers are preserved + content, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("Failed to read downloaded file: %v", err) + } + + var downloadedSpec map[string]interface{} + if err := json.Unmarshal(content, &downloadedSpec); err != nil { + t.Fatalf("Failed to parse downloaded JSON: %v", err) + } + + servers, exists := downloadedSpec["servers"] + if !exists { + t.Error("Servers field was removed from the spec") + } + + if serversList, ok := servers.([]interface{}); ok { + if len(serversList) != 1 { + t.Errorf("Expected 1 server, got %d", len(serversList)) + } + } +} + +// Helper function to check if file exists +func fileExists(filename string) bool { + _, err := os.Stat(filename) + return !os.IsNotExist(err) +} diff --git a/pkg/input/formats/swagger/downloader.go b/pkg/input/formats/swagger/downloader.go new file mode 100644 index 000000000..010287f6f --- /dev/null +++ b/pkg/input/formats/swagger/downloader.go @@ -0,0 +1,148 @@ +package swagger + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" + "gopkg.in/yaml.v3" +) + +// SwaggerDownloader implements the SpecDownloader interface for Swagger 2.0 specs +type SwaggerDownloader struct{} + +// NewDownloader creates a new Swagger downloader +func NewDownloader() formats.SpecDownloader { + return &SwaggerDownloader{} +} + +// This function downloads a Swagger 2.0 spec from the given URL and saves it to tmpDir +func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { + // Swagger can be JSON or YAML + supportedExts := []string{".json", ".yaml", ".yml"} + isSupported := false + for _, ext := range supportedExts { + if strings.HasSuffix(urlStr, ext) { + isSupported = true + break + } + } + if !isSupported && !strings.Contains(urlStr, "swagger") { + return "", fmt.Errorf("URL does not appear to be a Swagger spec (supported: %v)", supportedExts) + } + + client := &http.Client{Timeout: 30 * time.Second} + + resp, err := client.Get(urlStr) + if err != nil { + return "", errors.Wrap(err, "failed to download Swagger spec") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("HTTP %d when downloading Swagger spec", resp.StatusCode) + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", errors.Wrap(err, "failed to read response body") + } + + // Determine format and parse + var spec map[string]interface{} + var isYAML bool + + // Try JSON first + if err := json.Unmarshal(bodyBytes, &spec); err != nil { + // Then try YAML + if err := yaml.Unmarshal(bodyBytes, &spec); err != nil { + return "", fmt.Errorf("downloaded content is neither valid JSON nor YAML: %w", err) + } + isYAML = true + } + + // Validate it's a Swagger 2.0 spec + if swagger, exists := spec["swagger"]; exists { + if swaggerStr, ok := swagger.(string); ok && strings.HasPrefix(swaggerStr, "2.") { + // Valid Swagger 2.0 spec + } else { + return "", fmt.Errorf("not a valid Swagger 2.0 spec (found version: %v)", swagger) + } + } else { + return "", fmt.Errorf("not a Swagger spec (missing 'swagger' field)") + } + + // Extract host from URL for host configuration + parsedURL, err := url.Parse(urlStr) + if err != nil { + return "", errors.Wrap(err, "failed to parse URL") + } + host := parsedURL.Host + + // Add host if missing + if _, exists := spec["host"]; !exists { + spec["host"] = host + } + + // Add schemes if missing + if _, exists := spec["schemes"]; !exists { + scheme := parsedURL.Scheme + if scheme == "" { + scheme = "https" + } + spec["schemes"] = []string{scheme} + } + + // Create output directory + swaggerDir := filepath.Join(tmpDir, "swagger") + if err := os.MkdirAll(swaggerDir, 0755); err != nil { + return "", errors.Wrap(err, "failed to create swagger directory") + } + + // Generate filename and content based on original format + var filename string + var content []byte + + if isYAML { + filename = fmt.Sprintf("swagger-spec-%d.yaml", time.Now().Unix()) + content, err = yaml.Marshal(spec) + if err != nil { + return "", errors.Wrap(err, "failed to marshal modified YAML spec") + } + } else { + filename = fmt.Sprintf("swagger-spec-%d.json", time.Now().Unix()) + content, err = json.Marshal(spec) + if err != nil { + return "", errors.Wrap(err, "failed to marshal modified JSON spec") + } + } + + filePath := filepath.Join(swaggerDir, filename) + + // Write file + file, err := os.Create(filePath) + if err != nil { + return "", errors.Wrap(err, "failed to create file") + } + defer file.Close() + + if _, err := file.Write(content); err != nil { + os.Remove(filePath) + return "", errors.Wrap(err, "failed to write file") + } + + return filePath, nil +} + +// SupportedExtensions returns the list of supported file extensions for Swagger +func (d *SwaggerDownloader) SupportedExtensions() []string { + return []string{".json", ".yaml", ".yml"} +} diff --git a/pkg/input/formats/swagger/downloader_test.go b/pkg/input/formats/swagger/downloader_test.go new file mode 100644 index 000000000..abc45dfbb --- /dev/null +++ b/pkg/input/formats/swagger/downloader_test.go @@ -0,0 +1,306 @@ +package swagger + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "gopkg.in/yaml.v3" +) + +func TestSwaggerDownloader_SupportedExtensions(t *testing.T) { + downloader := &SwaggerDownloader{} + extensions := downloader.SupportedExtensions() + + expected := []string{".json", ".yaml", ".yml"} + if len(extensions) != len(expected) { + t.Errorf("Expected %d extensions, got %d", len(expected), len(extensions)) + } + + for i, ext := range extensions { + if ext != expected[i] { + t.Errorf("Expected extension %s, got %s", expected[i], ext) + } + } +} + +func TestSwaggerDownloader_Download_JSON_Success(t *testing.T) { + // Create a mock Swagger spec (JSON) + mockSpec := map[string]interface{}{ + "swagger": "2.0", + "info": map[string]interface{}{ + "title": "Test API", + "version": "1.0.0", + }, + "paths": map[string]interface{}{ + "/test": map[string]interface{}{ + "get": map[string]interface{}{ + "summary": "Test endpoint", + }, + }, + }, + } + + // Create mock server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(mockSpec) + })) + defer server.Close() + + // Create temp directory + tmpDir, err := os.MkdirTemp("", "swagger_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + // Test download + downloader := &SwaggerDownloader{} + filePath, err := downloader.Download(server.URL+"/swagger.json", tmpDir) + if err != nil { + t.Fatalf("Download failed: %v", err) + } + + // Verify file exists + if !fileExists(filePath) { + t.Errorf("Downloaded file does not exist: %s", filePath) + } + + // Verify file content + content, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("Failed to read downloaded file: %v", err) + } + + var downloadedSpec map[string]interface{} + if err := json.Unmarshal(content, &downloadedSpec); err != nil { + t.Fatalf("Failed to parse downloaded JSON: %v", err) + } + + // Verify host field was added + _, exists := downloadedSpec["host"] + if !exists { + t.Error("Host field was not added to the spec") + } +} + +func TestSwaggerDownloader_Download_YAML_Success(t *testing.T) { + // Create a mock Swagger spec (YAML) + mockSpecYAML := ` +swagger: "2.0" +info: + title: "Test API" + version: "1.0.0" +paths: + /test: + get: + summary: "Test endpoint" +` + + // Create mock server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/yaml") + w.Write([]byte(mockSpecYAML)) + })) + defer server.Close() + + // Create temp directory + tmpDir, err := os.MkdirTemp("", "swagger_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + // Test download + downloader := &SwaggerDownloader{} + filePath, err := downloader.Download(server.URL+"/swagger.yaml", tmpDir) + if err != nil { + t.Fatalf("Download failed: %v", err) + } + + // Verify file exists + if !fileExists(filePath) { + t.Errorf("Downloaded file does not exist: %s", filePath) + } + + // Verify file content + content, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("Failed to read downloaded file: %v", err) + } + + var downloadedSpec map[string]interface{} + if err := yaml.Unmarshal(content, &downloadedSpec); err != nil { + t.Fatalf("Failed to parse downloaded YAML: %v", err) + } + + // Verify host field was added + _, exists := downloadedSpec["host"] + if !exists { + t.Error("Host field was not added to the spec") + } +} + +func TestSwaggerDownloader_Download_UnsupportedExtension(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "swagger_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + downloader := &SwaggerDownloader{} + _, err = downloader.Download("http://example.com/spec.xml", tmpDir) + if err == nil { + t.Error("Expected error for unsupported extension, but got none") + } + + if !strings.Contains(err.Error(), "URL does not appear to be a Swagger spec") { + t.Errorf("Unexpected error message: %v", err) + } +} + +func TestSwaggerDownloader_Download_HTTPError(t *testing.T) { + // Create mock server that returns 404 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + tmpDir, err := os.MkdirTemp("", "swagger_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + downloader := &SwaggerDownloader{} + _, err = downloader.Download(server.URL+"/swagger.json", tmpDir) + if err == nil { + t.Error("Expected error for HTTP 404, but got none") + } +} + +func TestSwaggerDownloader_Download_InvalidJSON(t *testing.T) { + // Create mock server that returns invalid JSON + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte("invalid json")) + })) + defer server.Close() + + tmpDir, err := os.MkdirTemp("", "swagger_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + downloader := &SwaggerDownloader{} + _, err = downloader.Download(server.URL+"/swagger.json", tmpDir) + if err == nil { + t.Error("Expected error for invalid JSON, but got none") + } +} + +func TestSwaggerDownloader_Download_InvalidYAML(t *testing.T) { + // Create mock server that returns invalid YAML + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/yaml") + w.Write([]byte("invalid: yaml: content: [")) + })) + defer server.Close() + + tmpDir, err := os.MkdirTemp("", "swagger_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + downloader := &SwaggerDownloader{} + _, err = downloader.Download(server.URL+"/swagger.yaml", tmpDir) + if err == nil { + t.Error("Expected error for invalid YAML, but got none") + } +} + +func TestSwaggerDownloader_Download_Timeout(t *testing.T) { + // Create mock server with delay + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(35 * time.Second) // Longer than 30 second timeout + json.NewEncoder(w).Encode(map[string]interface{}{"test": "data"}) + })) + defer server.Close() + + tmpDir, err := os.MkdirTemp("", "swagger_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + downloader := &SwaggerDownloader{} + _, err = downloader.Download(server.URL+"/swagger.json", tmpDir) + if err == nil { + t.Error("Expected timeout error, but got none") + } +} + +func TestSwaggerDownloader_Download_WithExistingHost(t *testing.T) { + // Create a mock Swagger spec with existing host + mockSpec := map[string]interface{}{ + "swagger": "2.0", + "info": map[string]interface{}{ + "title": "Test API", + "version": "1.0.0", + }, + "host": "existing-host.com", + "paths": map[string]interface{}{}, + } + + // Create mock server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(mockSpec) + })) + defer server.Close() + + tmpDir, err := os.MkdirTemp("", "swagger_test") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + downloader := &SwaggerDownloader{} + filePath, err := downloader.Download(server.URL+"/swagger.json", tmpDir) + if err != nil { + t.Fatalf("Download failed: %v", err) + } + + // Verify existing host is preserved + content, err := os.ReadFile(filePath) + if err != nil { + t.Fatalf("Failed to read downloaded file: %v", err) + } + + var downloadedSpec map[string]interface{} + if err := json.Unmarshal(content, &downloadedSpec); err != nil { + t.Fatalf("Failed to parse downloaded JSON: %v", err) + } + + host, exists := downloadedSpec["host"] + if !exists { + t.Error("Host field was removed from the spec") + } + + if hostStr, ok := host.(string); !ok || hostStr != "existing-host.com" { + t.Errorf("Expected host 'existing-host.com', got '%v'", host) + } +} + +// Helper function to check if file exists +func fileExists(filename string) bool { + _, err := os.Stat(filename) + return !os.IsNotExist(err) +} diff --git a/pkg/input/provider/interface.go b/pkg/input/provider/interface.go index 9e1d09ab2..881f59aa3 100644 --- a/pkg/input/provider/interface.go +++ b/pkg/input/provider/interface.go @@ -7,6 +7,8 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/openapi" + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/swagger" "github.com/projectdiscovery/nuclei/v3/pkg/input/provider/http" "github.com/projectdiscovery/nuclei/v3/pkg/input/provider/list" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" @@ -74,6 +76,8 @@ type InputProvider interface { type InputOptions struct { // Options for global config Options *configTypes.Options + // TempDir is the temporary directory for storing files + TempDir string // NotFoundCallback is the callback to call when input is not found // only supported in list input provider NotFoundCallback func(template string) bool @@ -107,20 +111,49 @@ func NewInputProvider(opts InputOptions) (InputProvider, error) { Options: opts.Options, NotFoundCallback: opts.NotFoundCallback, }) - } else { - // use HttpInputProvider - return http.NewHttpInputProvider(&http.HttpMultiFormatOptions{ - InputFile: opts.Options.TargetsFilePath, - InputMode: opts.Options.InputFileMode, - Options: formats.InputFormatOptions{ - Variables: generators.MergeMaps(extraVars, opts.Options.Vars.AsMap()), - SkipFormatValidation: opts.Options.SkipFormatValidation, - RequiredOnly: opts.Options.FormatUseRequiredOnly, - VarsTextTemplating: opts.Options.VarsTextTemplating, - VarsFilePaths: opts.Options.VarsFilePaths, - }, - }) + } else if len(opts.Options.Targets) > 0 && + (strings.EqualFold(opts.Options.InputFileMode, "openapi") || strings.EqualFold(opts.Options.InputFileMode, "swagger")) { + + if len(opts.Options.Targets) > 1 { + return nil, fmt.Errorf("only one target URL is supported in %s input mode", opts.Options.InputFileMode) + } + + target := opts.Options.Targets[0] + if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") { + var downloader formats.SpecDownloader + var tempFile string + var err error + + switch strings.ToLower(opts.Options.InputFileMode) { + case "openapi": + downloader = openapi.NewDownloader() + tempFile, err = downloader.Download(target, opts.TempDir) + case "swagger": + downloader = swagger.NewDownloader() + tempFile, err = downloader.Download(target, opts.TempDir) + default: + return nil, fmt.Errorf("unsupported input mode: %s", opts.Options.InputFileMode) + } + + if err != nil { + return nil, fmt.Errorf("failed to download %s spec from url %s: %w", opts.Options.InputFileMode, target, err) + } + + opts.Options.TargetsFilePath = tempFile + } } + + return http.NewHttpInputProvider(&http.HttpMultiFormatOptions{ + InputFile: opts.Options.TargetsFilePath, + InputMode: opts.Options.InputFileMode, + Options: formats.InputFormatOptions{ + Variables: generators.MergeMaps(extraVars, opts.Options.Vars.AsMap()), + SkipFormatValidation: opts.Options.SkipFormatValidation, + RequiredOnly: opts.Options.FormatUseRequiredOnly, + VarsTextTemplating: opts.Options.VarsTextTemplating, + VarsFilePaths: opts.Options.VarsFilePaths, + }, + }) } // SupportedInputFormats returns all supported input formats of nuclei From 1684f4143e8268d57a52ab033bc38ac331b39439 Mon Sep 17 00:00:00 2001 From: tvroi Date: Mon, 20 Oct 2025 18:36:17 +0700 Subject: [PATCH 2/8] fix (openapi/swagger): improve error handling and tmpDir cleanup --- internal/runner/runner.go | 17 ++++- pkg/input/formats/openapi/downloader.go | 38 +++++++---- pkg/input/formats/openapi/downloader_test.go | 58 +++++++++++++--- pkg/input/formats/swagger/downloader.go | 26 ++++++-- pkg/input/formats/swagger/downloader_test.go | 69 ++++++++++++++++---- 5 files changed, 168 insertions(+), 40 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 1594f1e26..9000bacb0 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -254,9 +254,20 @@ func New(options *types.Options) (*Runner, error) { os.Exit(0) } - if tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*"); err == nil { - runner.tmpDir = tmpDir + tmpDir, err := os.MkdirTemp("", "nuclei-tmp-*") + if err != nil { + return nil, errors.Wrap(err, "could not create temporary directory") } + runner.tmpDir = tmpDir + + // Cleanup tmpDir only if initialization fails + // On successful initialization, Close() method will handle cleanup + cleanupOnError := true + defer func() { + if cleanupOnError && runner.tmpDir != "" { + _ = os.RemoveAll(runner.tmpDir) + } + }() // create the input provider and load the inputs inputProvider, err := provider.NewInputProvider(provider.InputOptions{Options: options, TempDir: runner.tmpDir}) @@ -390,6 +401,8 @@ func New(options *types.Options) (*Runner, error) { } runner.rateLimiter = utils.GetRateLimiter(context.Background(), options.RateLimit, options.RateLimitDuration) + // Initialization successful, disable cleanup on error + cleanupOnError = false return runner, nil } diff --git a/pkg/input/formats/openapi/downloader.go b/pkg/input/formats/openapi/downloader.go index 22ca51387..a1d9c8042 100644 --- a/pkg/input/formats/openapi/downloader.go +++ b/pkg/input/formats/openapi/downloader.go @@ -30,19 +30,26 @@ func (d *OpenAPIDownloader) Download(urlStr, tmpDir string) (string, error) { return "", fmt.Errorf("URL does not appear to be an OpenAPI JSON spec") } - client := &http.Client{Timeout: 30 * time.Second} + var httpTimeout = 30 * time.Second + const maxSpecSizeBytes = 10 * 1024 * 1024 // 10MB + client := &http.Client{Timeout: httpTimeout} resp, err := client.Get(urlStr) if err != nil { return "", errors.Wrap(err, "failed to download OpenAPI spec") } - defer resp.Body.Close() + + defer func() { + if err := resp.Body.Close(); err != nil { + errors.Wrap(err, "failed to close response body") + } + }() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("HTTP %d when downloading OpenAPI spec", resp.StatusCode) } - bodyBytes, err := io.ReadAll(resp.Body) + bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxSpecSizeBytes)) if err != nil { return "", errors.Wrap(err, "failed to read response body") } @@ -70,17 +77,17 @@ func (d *OpenAPIDownloader) Download(urlStr, tmpDir string) (string, error) { return "", errors.Wrap(err, "failed to parse URL") } host := parsedURL.Host + scheme := parsedURL.Scheme + if scheme == "" { + scheme = "https" + } // Add servers section if missing or empty servers, exists := spec["servers"] if !exists || servers == nil { - spec["servers"] = []map[string]interface{}{ - {"url": "https://" + host}, - } - } else if serversList, ok := servers.([]interface{}); ok && len(serversList) == 0 { - spec["servers"] = []map[string]interface{}{ - {"url": "https://" + host}, - } + spec["servers"] = []map[string]interface{}{{"url": scheme + "://" + host}} + } else if serverList, ok := servers.([]interface{}); ok && len(serverList) == 0 { + spec["servers"] = []map[string]interface{}{{"url": scheme + "://" + host}} } // Marshal back to JSON @@ -92,6 +99,7 @@ func (d *OpenAPIDownloader) Download(urlStr, tmpDir string) (string, error) { // Create output directory openapiDir := filepath.Join(tmpDir, "openapi") if err := os.MkdirAll(openapiDir, 0755); err != nil { + return "", errors.Wrap(err, "failed to create openapi directory") } @@ -104,10 +112,16 @@ func (d *OpenAPIDownloader) Download(urlStr, tmpDir string) (string, error) { if err != nil { return "", fmt.Errorf("failed to create file: %w", err) } - defer file.Close() + + defer func() { + if err := file.Close(); err != nil { + errors.Wrap(err, "failed to close file") + } + }() if _, err := file.Write(modifiedJSON); err != nil { - os.Remove(filePath) + _ = os.Remove(filePath) + return "", errors.Wrap(err, "failed to write OpenAPI spec to file") } diff --git a/pkg/input/formats/openapi/downloader_test.go b/pkg/input/formats/openapi/downloader_test.go index e5fc7784a..2add3d4f0 100644 --- a/pkg/input/formats/openapi/downloader_test.go +++ b/pkg/input/formats/openapi/downloader_test.go @@ -46,7 +46,9 @@ func TestOpenAPIDownloader_Download_Success(t *testing.T) { // Create mock server server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(mockSpec) + if err := json.NewEncoder(w).Encode(mockSpec); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + } })) defer server.Close() @@ -55,7 +57,12 @@ func TestOpenAPIDownloader_Download_Success(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() // Test download downloader := &OpenAPIDownloader{} @@ -100,7 +107,12 @@ func TestOpenAPIDownloader_Download_NonJSONURL(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() downloader := &OpenAPIDownloader{} _, err = downloader.Download("http://example.com/spec.yaml", tmpDir) @@ -124,7 +136,12 @@ func TestOpenAPIDownloader_Download_HTTPError(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() downloader := &OpenAPIDownloader{} _, err = downloader.Download(server.URL+"/openapi.json", tmpDir) @@ -137,7 +154,9 @@ func TestOpenAPIDownloader_Download_InvalidJSON(t *testing.T) { // Create mock server that returns invalid JSON server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - w.Write([]byte("invalid json")) + if _, err := w.Write([]byte("invalid json")); err != nil { + http.Error(w, "failed to write response", http.StatusInternalServerError) + } })) defer server.Close() @@ -145,7 +164,12 @@ func TestOpenAPIDownloader_Download_InvalidJSON(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() downloader := &OpenAPIDownloader{} _, err = downloader.Download(server.URL+"/openapi.json", tmpDir) @@ -158,7 +182,9 @@ func TestOpenAPIDownloader_Download_Timeout(t *testing.T) { // Create mock server with delay server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(35 * time.Second) // Longer than 30 second timeout - json.NewEncoder(w).Encode(map[string]interface{}{"test": "data"}) + if err := json.NewEncoder(w).Encode(map[string]interface{}{"test": "data"}); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + } })) defer server.Close() @@ -166,7 +192,12 @@ func TestOpenAPIDownloader_Download_Timeout(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() downloader := &OpenAPIDownloader{} _, err = downloader.Download(server.URL+"/openapi.json", tmpDir) @@ -194,7 +225,9 @@ func TestOpenAPIDownloader_Download_WithExistingServers(t *testing.T) { // Create mock server server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(mockSpec) + if err := json.NewEncoder(w).Encode(mockSpec); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + } })) defer server.Close() @@ -202,7 +235,12 @@ func TestOpenAPIDownloader_Download_WithExistingServers(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() downloader := &OpenAPIDownloader{} filePath, err := downloader.Download(server.URL+"/openapi.json", tmpDir) diff --git a/pkg/input/formats/swagger/downloader.go b/pkg/input/formats/swagger/downloader.go index 010287f6f..f78f77f7d 100644 --- a/pkg/input/formats/swagger/downloader.go +++ b/pkg/input/formats/swagger/downloader.go @@ -39,19 +39,26 @@ func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { return "", fmt.Errorf("URL does not appear to be a Swagger spec (supported: %v)", supportedExts) } - client := &http.Client{Timeout: 30 * time.Second} + var httpTimeout = 30 * time.Second + const maxSpecSizeBytes = 10 * 1024 * 1024 // 10MB + client := &http.Client{Timeout: httpTimeout} resp, err := client.Get(urlStr) if err != nil { return "", errors.Wrap(err, "failed to download Swagger spec") } - defer resp.Body.Close() + + defer func() { + if err := resp.Body.Close(); err != nil { + errors.Wrap(err, "failed to close response body") + } + }() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("HTTP %d when downloading Swagger spec", resp.StatusCode) } - bodyBytes, err := io.ReadAll(resp.Body) + bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxSpecSizeBytes)) if err != nil { return "", errors.Wrap(err, "failed to read response body") } @@ -132,10 +139,19 @@ func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { if err != nil { return "", errors.Wrap(err, "failed to create file") } - defer file.Close() + + defer func() { + if err := file.Close(); err != nil { + errors.Wrap(err, "failed to close file") + } + }() if _, err := file.Write(content); err != nil { - os.Remove(filePath) + err := os.Remove(filePath) + if err != nil { + errors.Wrap(err, "failed to remove incomplete file") + } + return "", errors.Wrap(err, "failed to write file") } diff --git a/pkg/input/formats/swagger/downloader_test.go b/pkg/input/formats/swagger/downloader_test.go index abc45dfbb..7d85a276a 100644 --- a/pkg/input/formats/swagger/downloader_test.go +++ b/pkg/input/formats/swagger/downloader_test.go @@ -48,7 +48,9 @@ func TestSwaggerDownloader_Download_JSON_Success(t *testing.T) { // Create mock server server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(mockSpec) + if err := json.NewEncoder(w).Encode(mockSpec); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + } })) defer server.Close() @@ -57,7 +59,12 @@ func TestSwaggerDownloader_Download_JSON_Success(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() // Test download downloader := &SwaggerDownloader{} @@ -107,6 +114,7 @@ paths: w.Header().Set("Content-Type", "application/yaml") w.Write([]byte(mockSpecYAML)) })) + defer server.Close() // Create temp directory @@ -114,7 +122,12 @@ paths: if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() // Test download downloader := &SwaggerDownloader{} @@ -151,7 +164,12 @@ func TestSwaggerDownloader_Download_UnsupportedExtension(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() downloader := &SwaggerDownloader{} _, err = downloader.Download("http://example.com/spec.xml", tmpDir) @@ -175,7 +193,12 @@ func TestSwaggerDownloader_Download_HTTPError(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() downloader := &SwaggerDownloader{} _, err = downloader.Download(server.URL+"/swagger.json", tmpDir) @@ -188,7 +211,9 @@ func TestSwaggerDownloader_Download_InvalidJSON(t *testing.T) { // Create mock server that returns invalid JSON server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - w.Write([]byte("invalid json")) + if _, err := w.Write([]byte("invalid json")); err != nil { + http.Error(w, "failed to write response", http.StatusInternalServerError) + } })) defer server.Close() @@ -196,7 +221,12 @@ func TestSwaggerDownloader_Download_InvalidJSON(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() downloader := &SwaggerDownloader{} _, err = downloader.Download(server.URL+"/swagger.json", tmpDir) @@ -217,7 +247,12 @@ func TestSwaggerDownloader_Download_InvalidYAML(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() downloader := &SwaggerDownloader{} _, err = downloader.Download(server.URL+"/swagger.yaml", tmpDir) @@ -230,7 +265,9 @@ func TestSwaggerDownloader_Download_Timeout(t *testing.T) { // Create mock server with delay server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(35 * time.Second) // Longer than 30 second timeout - json.NewEncoder(w).Encode(map[string]interface{}{"test": "data"}) + if err := json.NewEncoder(w).Encode(map[string]interface{}{"test": "data"}); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + } })) defer server.Close() @@ -238,7 +275,12 @@ func TestSwaggerDownloader_Download_Timeout(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() downloader := &SwaggerDownloader{} _, err = downloader.Download(server.URL+"/swagger.json", tmpDir) @@ -270,7 +312,12 @@ func TestSwaggerDownloader_Download_WithExistingHost(t *testing.T) { if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(tmpDir) + + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to remove temp dir: %v", err) + } + }() downloader := &SwaggerDownloader{} filePath, err := downloader.Download(server.URL+"/swagger.json", tmpDir) From f0429aa4b732e75078796dc9aae81ebf23f4e9f0 Mon Sep 17 00:00:00 2001 From: tvroi Date: Mon, 20 Oct 2025 18:49:06 +0700 Subject: [PATCH 3/8] fix(openapi/swagger): err shadowing on write failure --- pkg/input/formats/openapi/downloader.go | 5 ++--- pkg/input/formats/swagger/downloader.go | 10 +++------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/input/formats/openapi/downloader.go b/pkg/input/formats/openapi/downloader.go index a1d9c8042..72c7edb4d 100644 --- a/pkg/input/formats/openapi/downloader.go +++ b/pkg/input/formats/openapi/downloader.go @@ -119,10 +119,9 @@ func (d *OpenAPIDownloader) Download(urlStr, tmpDir string) (string, error) { } }() - if _, err := file.Write(modifiedJSON); err != nil { + if _, writeErr := file.Write(modifiedJSON); writeErr != nil { _ = os.Remove(filePath) - - return "", errors.Wrap(err, "failed to write OpenAPI spec to file") + return "", errors.Wrap(writeErr, "failed to write OpenAPI spec to file") } return filePath, nil diff --git a/pkg/input/formats/swagger/downloader.go b/pkg/input/formats/swagger/downloader.go index f78f77f7d..de30079bd 100644 --- a/pkg/input/formats/swagger/downloader.go +++ b/pkg/input/formats/swagger/downloader.go @@ -146,13 +146,9 @@ func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { } }() - if _, err := file.Write(content); err != nil { - err := os.Remove(filePath) - if err != nil { - errors.Wrap(err, "failed to remove incomplete file") - } - - return "", errors.Wrap(err, "failed to write file") + if _, writeErr := file.Write(content); writeErr != nil { + _ = os.Remove(filePath) + return "", errors.Wrap(writeErr, "failed to write file") } return filePath, nil From 89cfb75bb67ea44782ef039605d42b44b4062728 Mon Sep 17 00:00:00 2001 From: tvroi Date: Mon, 20 Oct 2025 18:56:47 +0700 Subject: [PATCH 4/8] fix(openapi/swagger): remove discarded error in defer --- pkg/input/formats/openapi/downloader.go | 9 ++------- pkg/input/formats/swagger/downloader.go | 8 ++------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/pkg/input/formats/openapi/downloader.go b/pkg/input/formats/openapi/downloader.go index 72c7edb4d..3c50cc9fc 100644 --- a/pkg/input/formats/openapi/downloader.go +++ b/pkg/input/formats/openapi/downloader.go @@ -40,9 +40,7 @@ func (d *OpenAPIDownloader) Download(urlStr, tmpDir string) (string, error) { } defer func() { - if err := resp.Body.Close(); err != nil { - errors.Wrap(err, "failed to close response body") - } + _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { @@ -99,7 +97,6 @@ func (d *OpenAPIDownloader) Download(urlStr, tmpDir string) (string, error) { // Create output directory openapiDir := filepath.Join(tmpDir, "openapi") if err := os.MkdirAll(openapiDir, 0755); err != nil { - return "", errors.Wrap(err, "failed to create openapi directory") } @@ -114,9 +111,7 @@ func (d *OpenAPIDownloader) Download(urlStr, tmpDir string) (string, error) { } defer func() { - if err := file.Close(); err != nil { - errors.Wrap(err, "failed to close file") - } + _ = file.Close() }() if _, writeErr := file.Write(modifiedJSON); writeErr != nil { diff --git a/pkg/input/formats/swagger/downloader.go b/pkg/input/formats/swagger/downloader.go index de30079bd..3a770e307 100644 --- a/pkg/input/formats/swagger/downloader.go +++ b/pkg/input/formats/swagger/downloader.go @@ -49,9 +49,7 @@ func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { } defer func() { - if err := resp.Body.Close(); err != nil { - errors.Wrap(err, "failed to close response body") - } + _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { @@ -141,9 +139,7 @@ func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { } defer func() { - if err := file.Close(); err != nil { - errors.Wrap(err, "failed to close file") - } + _ = file.Close() }() if _, writeErr := file.Write(content); writeErr != nil { From f57bd8c8eea4ddba2d21e4f439ca0c187201b46a Mon Sep 17 00:00:00 2001 From: tvroi Date: Tue, 21 Oct 2025 20:16:05 +0700 Subject: [PATCH 5/8] fix(openapi/swagger): linter and url validation --- pkg/input/formats/openapi/downloader.go | 2 +- pkg/input/formats/swagger/downloader.go | 11 ++++++----- pkg/input/formats/swagger/downloader_test.go | 12 +++++++++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/pkg/input/formats/openapi/downloader.go b/pkg/input/formats/openapi/downloader.go index 3c50cc9fc..1089642a9 100644 --- a/pkg/input/formats/openapi/downloader.go +++ b/pkg/input/formats/openapi/downloader.go @@ -26,7 +26,7 @@ func NewDownloader() formats.SpecDownloader { // This function downloads an OpenAPI 3.0 spec from the given URL and saves it to tmpDir func (d *OpenAPIDownloader) Download(urlStr, tmpDir string) (string, error) { // Validate URL format, OpenAPI 3.0 specs are typically JSON - if !strings.HasSuffix(urlStr, ".json") && !strings.Contains(urlStr, "openapi") { + if !strings.HasSuffix(urlStr, ".json") { return "", fmt.Errorf("URL does not appear to be an OpenAPI JSON spec") } diff --git a/pkg/input/formats/swagger/downloader.go b/pkg/input/formats/swagger/downloader.go index 3a770e307..10c3ba25a 100644 --- a/pkg/input/formats/swagger/downloader.go +++ b/pkg/input/formats/swagger/downloader.go @@ -35,7 +35,7 @@ func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { break } } - if !isSupported && !strings.Contains(urlStr, "swagger") { + if !isSupported { return "", fmt.Errorf("URL does not appear to be a Swagger spec (supported: %v)", supportedExts) } @@ -90,7 +90,12 @@ func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { if err != nil { return "", errors.Wrap(err, "failed to parse URL") } + host := parsedURL.Host + scheme := parsedURL.Scheme + if scheme == "" { + scheme = "https" + } // Add host if missing if _, exists := spec["host"]; !exists { @@ -99,10 +104,6 @@ func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { // Add schemes if missing if _, exists := spec["schemes"]; !exists { - scheme := parsedURL.Scheme - if scheme == "" { - scheme = "https" - } spec["schemes"] = []string{scheme} } diff --git a/pkg/input/formats/swagger/downloader_test.go b/pkg/input/formats/swagger/downloader_test.go index 7d85a276a..41d9958d2 100644 --- a/pkg/input/formats/swagger/downloader_test.go +++ b/pkg/input/formats/swagger/downloader_test.go @@ -112,7 +112,9 @@ paths: // Create mock server server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/yaml") - w.Write([]byte(mockSpecYAML)) + if _, err := w.Write([]byte(mockSpecYAML)); err != nil { + http.Error(w, "failed to write response", http.StatusInternalServerError) + } })) defer server.Close() @@ -239,7 +241,9 @@ func TestSwaggerDownloader_Download_InvalidYAML(t *testing.T) { // Create mock server that returns invalid YAML server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/yaml") - w.Write([]byte("invalid: yaml: content: [")) + if _, err := w.Write([]byte("invalid: yaml: content: [")); err != nil { + http.Error(w, "failed to write response", http.StatusInternalServerError) + } })) defer server.Close() @@ -304,7 +308,9 @@ func TestSwaggerDownloader_Download_WithExistingHost(t *testing.T) { // Create mock server server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(mockSpec) + if err := json.NewEncoder(w).Encode(mockSpec); err != nil { + http.Error(w, "failed to encode response", http.StatusInternalServerError) + } })) defer server.Close() From e168f8dbfaf26ef52ab8a67704e12f992d13ea3e Mon Sep 17 00:00:00 2001 From: tvroi Date: Tue, 21 Oct 2025 20:27:33 +0700 Subject: [PATCH 6/8] fix(openapi/swagger): remove code duplication --- pkg/input/formats/swagger/downloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/input/formats/swagger/downloader.go b/pkg/input/formats/swagger/downloader.go index 10c3ba25a..21bc3945d 100644 --- a/pkg/input/formats/swagger/downloader.go +++ b/pkg/input/formats/swagger/downloader.go @@ -27,7 +27,7 @@ func NewDownloader() formats.SpecDownloader { // This function downloads a Swagger 2.0 spec from the given URL and saves it to tmpDir func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { // Swagger can be JSON or YAML - supportedExts := []string{".json", ".yaml", ".yml"} + supportedExts := d.SupportedExtensions() isSupported := false for _, ext := range supportedExts { if strings.HasSuffix(urlStr, ext) { From 6f59472f78a87c9c174130805185369a2c17610a Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 29 Oct 2025 19:03:59 +0400 Subject: [PATCH 7/8] reusing dialer --- pkg/input/formats/formats.go | 4 +++- pkg/input/formats/openapi/downloader.go | 16 +++++++++++++--- pkg/input/formats/openapi/downloader_test.go | 12 ++++++------ pkg/input/formats/swagger/downloader.go | 16 +++++++++++++--- pkg/input/formats/swagger/downloader_test.go | 16 ++++++++-------- pkg/input/provider/interface.go | 15 +++++++++++++-- 6 files changed, 56 insertions(+), 23 deletions(-) diff --git a/pkg/input/formats/formats.go b/pkg/input/formats/formats.go index 4cbd96a59..9de4d0d01 100644 --- a/pkg/input/formats/formats.go +++ b/pkg/input/formats/formats.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + "github.com/projectdiscovery/retryablehttp-go" fileutil "github.com/projectdiscovery/utils/file" "gopkg.in/yaml.v3" ) @@ -51,7 +52,8 @@ type Format interface { type SpecDownloader interface { // Download downloads the spec from the given URL and saves it to tmpDir // Returns the path to the downloaded file - Download(url, tmpDir string) (string, error) + // httpClient is a retryablehttp.Client instance (can be nil for fallback) + Download(url, tmpDir string, httpClient *retryablehttp.Client) (string, error) // SupportedExtensions returns the list of supported file extensions SupportedExtensions() []string } diff --git a/pkg/input/formats/openapi/downloader.go b/pkg/input/formats/openapi/downloader.go index 1089642a9..b7c363aad 100644 --- a/pkg/input/formats/openapi/downloader.go +++ b/pkg/input/formats/openapi/downloader.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "log" "net/http" "net/url" "os" @@ -13,6 +14,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" + "github.com/projectdiscovery/retryablehttp-go" ) // OpenAPIDownloader implements the SpecDownloader interface for OpenAPI 3.0 specs @@ -24,15 +26,23 @@ func NewDownloader() formats.SpecDownloader { } // This function downloads an OpenAPI 3.0 spec from the given URL and saves it to tmpDir -func (d *OpenAPIDownloader) Download(urlStr, tmpDir string) (string, error) { +func (d *OpenAPIDownloader) Download(urlStr, tmpDir string, httpClient *retryablehttp.Client) (string, error) { // Validate URL format, OpenAPI 3.0 specs are typically JSON if !strings.HasSuffix(urlStr, ".json") { return "", fmt.Errorf("URL does not appear to be an OpenAPI JSON spec") } - var httpTimeout = 30 * time.Second const maxSpecSizeBytes = 10 * 1024 * 1024 // 10MB - client := &http.Client{Timeout: httpTimeout} + + // Use provided httpClient or create a fallback + var client *http.Client + if httpClient != nil { + client = httpClient.HTTPClient + } else { + // Fallback to simple client if no httpClient provided + log.Fatal("no httpClient provided") + client = &http.Client{Timeout: 30 * time.Second} + } resp, err := client.Get(urlStr) if err != nil { diff --git a/pkg/input/formats/openapi/downloader_test.go b/pkg/input/formats/openapi/downloader_test.go index 2add3d4f0..10ee93817 100644 --- a/pkg/input/formats/openapi/downloader_test.go +++ b/pkg/input/formats/openapi/downloader_test.go @@ -66,7 +66,7 @@ func TestOpenAPIDownloader_Download_Success(t *testing.T) { // Test download downloader := &OpenAPIDownloader{} - filePath, err := downloader.Download(server.URL+"/openapi.json", tmpDir) + filePath, err := downloader.Download(server.URL+"/openapi.json", tmpDir, nil) if err != nil { t.Fatalf("Download failed: %v", err) } @@ -115,7 +115,7 @@ func TestOpenAPIDownloader_Download_NonJSONURL(t *testing.T) { }() downloader := &OpenAPIDownloader{} - _, err = downloader.Download("http://example.com/spec.yaml", tmpDir) + _, err = downloader.Download("http://example.com/spec.yaml", tmpDir, nil) if err == nil { t.Error("Expected error for non-JSON URL, but got none") } @@ -144,7 +144,7 @@ func TestOpenAPIDownloader_Download_HTTPError(t *testing.T) { }() downloader := &OpenAPIDownloader{} - _, err = downloader.Download(server.URL+"/openapi.json", tmpDir) + _, err = downloader.Download(server.URL+"/openapi.json", tmpDir, nil) if err == nil { t.Error("Expected error for HTTP 404, but got none") } @@ -172,7 +172,7 @@ func TestOpenAPIDownloader_Download_InvalidJSON(t *testing.T) { }() downloader := &OpenAPIDownloader{} - _, err = downloader.Download(server.URL+"/openapi.json", tmpDir) + _, err = downloader.Download(server.URL+"/openapi.json", tmpDir, nil) if err == nil { t.Error("Expected error for invalid JSON, but got none") } @@ -200,7 +200,7 @@ func TestOpenAPIDownloader_Download_Timeout(t *testing.T) { }() downloader := &OpenAPIDownloader{} - _, err = downloader.Download(server.URL+"/openapi.json", tmpDir) + _, err = downloader.Download(server.URL+"/openapi.json", tmpDir, nil) if err == nil { t.Error("Expected timeout error, but got none") } @@ -243,7 +243,7 @@ func TestOpenAPIDownloader_Download_WithExistingServers(t *testing.T) { }() downloader := &OpenAPIDownloader{} - filePath, err := downloader.Download(server.URL+"/openapi.json", tmpDir) + filePath, err := downloader.Download(server.URL+"/openapi.json", tmpDir, nil) if err != nil { t.Fatalf("Download failed: %v", err) } diff --git a/pkg/input/formats/swagger/downloader.go b/pkg/input/formats/swagger/downloader.go index 21bc3945d..40f0a2727 100644 --- a/pkg/input/formats/swagger/downloader.go +++ b/pkg/input/formats/swagger/downloader.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "log" "net/http" "net/url" "os" @@ -13,6 +14,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" + "github.com/projectdiscovery/retryablehttp-go" "gopkg.in/yaml.v3" ) @@ -25,7 +27,7 @@ func NewDownloader() formats.SpecDownloader { } // This function downloads a Swagger 2.0 spec from the given URL and saves it to tmpDir -func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { +func (d *SwaggerDownloader) Download(urlStr, tmpDir string, httpClient *retryablehttp.Client) (string, error) { // Swagger can be JSON or YAML supportedExts := d.SupportedExtensions() isSupported := false @@ -39,9 +41,17 @@ func (d *SwaggerDownloader) Download(urlStr, tmpDir string) (string, error) { return "", fmt.Errorf("URL does not appear to be a Swagger spec (supported: %v)", supportedExts) } - var httpTimeout = 30 * time.Second const maxSpecSizeBytes = 10 * 1024 * 1024 // 10MB - client := &http.Client{Timeout: httpTimeout} + + // Use provided httpClient or create a fallback + var client *http.Client + if httpClient != nil { + client = httpClient.HTTPClient + } else { + // Fallback to simple client if no httpClient provided + log.Fatal("no httpClient provided") + client = &http.Client{Timeout: 30 * time.Second} + } resp, err := client.Get(urlStr) if err != nil { diff --git a/pkg/input/formats/swagger/downloader_test.go b/pkg/input/formats/swagger/downloader_test.go index 41d9958d2..d55b57395 100644 --- a/pkg/input/formats/swagger/downloader_test.go +++ b/pkg/input/formats/swagger/downloader_test.go @@ -68,7 +68,7 @@ func TestSwaggerDownloader_Download_JSON_Success(t *testing.T) { // Test download downloader := &SwaggerDownloader{} - filePath, err := downloader.Download(server.URL+"/swagger.json", tmpDir) + filePath, err := downloader.Download(server.URL+"/swagger.json", tmpDir, nil) if err != nil { t.Fatalf("Download failed: %v", err) } @@ -133,7 +133,7 @@ paths: // Test download downloader := &SwaggerDownloader{} - filePath, err := downloader.Download(server.URL+"/swagger.yaml", tmpDir) + filePath, err := downloader.Download(server.URL+"/swagger.yaml", tmpDir, nil) if err != nil { t.Fatalf("Download failed: %v", err) } @@ -174,7 +174,7 @@ func TestSwaggerDownloader_Download_UnsupportedExtension(t *testing.T) { }() downloader := &SwaggerDownloader{} - _, err = downloader.Download("http://example.com/spec.xml", tmpDir) + _, err = downloader.Download("http://example.com/spec.xml", tmpDir, nil) if err == nil { t.Error("Expected error for unsupported extension, but got none") } @@ -203,7 +203,7 @@ func TestSwaggerDownloader_Download_HTTPError(t *testing.T) { }() downloader := &SwaggerDownloader{} - _, err = downloader.Download(server.URL+"/swagger.json", tmpDir) + _, err = downloader.Download(server.URL+"/swagger.json", tmpDir, nil) if err == nil { t.Error("Expected error for HTTP 404, but got none") } @@ -231,7 +231,7 @@ func TestSwaggerDownloader_Download_InvalidJSON(t *testing.T) { }() downloader := &SwaggerDownloader{} - _, err = downloader.Download(server.URL+"/swagger.json", tmpDir) + _, err = downloader.Download(server.URL+"/swagger.json", tmpDir, nil) if err == nil { t.Error("Expected error for invalid JSON, but got none") } @@ -259,7 +259,7 @@ func TestSwaggerDownloader_Download_InvalidYAML(t *testing.T) { }() downloader := &SwaggerDownloader{} - _, err = downloader.Download(server.URL+"/swagger.yaml", tmpDir) + _, err = downloader.Download(server.URL+"/swagger.yaml", tmpDir, nil) if err == nil { t.Error("Expected error for invalid YAML, but got none") } @@ -287,7 +287,7 @@ func TestSwaggerDownloader_Download_Timeout(t *testing.T) { }() downloader := &SwaggerDownloader{} - _, err = downloader.Download(server.URL+"/swagger.json", tmpDir) + _, err = downloader.Download(server.URL+"/swagger.json", tmpDir, nil) if err == nil { t.Error("Expected timeout error, but got none") } @@ -326,7 +326,7 @@ func TestSwaggerDownloader_Download_WithExistingHost(t *testing.T) { }() downloader := &SwaggerDownloader{} - filePath, err := downloader.Download(server.URL+"/swagger.json", tmpDir) + filePath, err := downloader.Download(server.URL+"/swagger.json", tmpDir, nil) if err != nil { t.Fatalf("Download failed: %v", err) } diff --git a/pkg/input/provider/interface.go b/pkg/input/provider/interface.go index 881f59aa3..33cfbee7f 100644 --- a/pkg/input/provider/interface.go +++ b/pkg/input/provider/interface.go @@ -14,7 +14,9 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/input/types" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" configTypes "github.com/projectdiscovery/nuclei/v3/pkg/types" + "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/utils/errkit" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -124,13 +126,22 @@ func NewInputProvider(opts InputOptions) (InputProvider, error) { var tempFile string var err error + // Get HttpClient from protocolstate if available + var httpClient *retryablehttp.Client + if opts.Options.ExecutionId != "" { + dialers := protocolstate.GetDialersWithId(opts.Options.ExecutionId) + if dialers != nil { + httpClient = dialers.DefaultHTTPClient + } + } + switch strings.ToLower(opts.Options.InputFileMode) { case "openapi": downloader = openapi.NewDownloader() - tempFile, err = downloader.Download(target, opts.TempDir) + tempFile, err = downloader.Download(target, opts.TempDir, httpClient) case "swagger": downloader = swagger.NewDownloader() - tempFile, err = downloader.Download(target, opts.TempDir) + tempFile, err = downloader.Download(target, opts.TempDir, httpClient) default: return nil, fmt.Errorf("unsupported input mode: %s", opts.Options.InputFileMode) } From c814128ee2c924d4d8b62c5aef0a198eb6465d7d Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 29 Oct 2025 19:54:51 +0400 Subject: [PATCH 8/8] removing debug log --- pkg/input/formats/openapi/downloader.go | 2 -- pkg/input/formats/swagger/downloader.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/pkg/input/formats/openapi/downloader.go b/pkg/input/formats/openapi/downloader.go index b7c363aad..955fdc50c 100644 --- a/pkg/input/formats/openapi/downloader.go +++ b/pkg/input/formats/openapi/downloader.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "io" - "log" "net/http" "net/url" "os" @@ -40,7 +39,6 @@ func (d *OpenAPIDownloader) Download(urlStr, tmpDir string, httpClient *retryabl client = httpClient.HTTPClient } else { // Fallback to simple client if no httpClient provided - log.Fatal("no httpClient provided") client = &http.Client{Timeout: 30 * time.Second} } diff --git a/pkg/input/formats/swagger/downloader.go b/pkg/input/formats/swagger/downloader.go index 40f0a2727..b6b5a333f 100644 --- a/pkg/input/formats/swagger/downloader.go +++ b/pkg/input/formats/swagger/downloader.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "io" - "log" "net/http" "net/url" "os" @@ -49,7 +48,6 @@ func (d *SwaggerDownloader) Download(urlStr, tmpDir string, httpClient *retryabl client = httpClient.HTTPClient } else { // Fallback to simple client if no httpClient provided - log.Fatal("no httpClient provided") client = &http.Client{Timeout: 30 * time.Second} }