From e542633ad3605c57f274ae89c87969208c3f96fe Mon Sep 17 00:00:00 2001 From: yyhuni Date: Wed, 14 Jan 2026 18:04:16 +0800 Subject: [PATCH] refactor(backend): consolidate migration files and restructure host port entities - Remove seed data generation command (cmd/seed/main.go) - Consolidate database migrations into single init schema file - Rename ip_address DTO to host_port for consistency - Add host_port_snapshot DTO and model for snapshot tracking - Rename host_port handler and repository files for clarity - Implement host_port_snapshot service layer with CRUD operations - Update website_snapshot service to work with new host_port structure - Enhance terminal login UI with focus state tracking and Tab key navigation - Update docker-compose configuration for development environment - Refactor directory and website snapshot DTOs for improved data structure - Add comprehensive test coverage for model and handler changes - Simplify database schema by consolidating related migrations into single initialization file --- frontend/components/ui/terminal-login.tsx | 18 + go-backend/cmd/seed/main.go | 1081 ----------------- go-backend/cmd/server/main.go | 26 +- .../migrations/000001_init_schema.up.sql | 35 +- .../000002_add_gin_indexes.down.sql | 11 - .../migrations/000002_add_gin_indexes.up.sql | 23 - ...000003_drop_directory_words_lines.down.sql | 3 - .../000003_drop_directory_words_lines.up.sql | 3 - .../000004_add_vulnerability_review.down.sql | 5 - .../000004_add_vulnerability_review.up.sql | 7 - ...05_add_vuln_target_reviewed_index.down.sql | 2 - ...0005_add_vuln_target_reviewed_index.up.sql | 3 - go-backend/docker-compose.dev.yml | 8 +- go-backend/go.mod | 2 +- go-backend/internal/dto/directory.go | 8 +- go-backend/internal/dto/directory_snapshot.go | 19 +- go-backend/internal/dto/host_port.go | 44 + go-backend/internal/dto/host_port_snapshot.go | 30 + go-backend/internal/dto/ip_address.go | 44 - go-backend/internal/dto/website_snapshot.go | 19 +- go-backend/internal/handler/directory.go | 4 +- .../internal/handler/directory_snapshot.go | 20 +- .../handler/{ip_address.go => host_port.go} | 58 +- .../internal/handler/host_port_snapshot.go | 147 +++ .../internal/handler/website_snapshot.go | 2 +- .../internal/handler/website_snapshot_test.go | 18 +- go-backend/internal/handler/wordlist.go | 4 +- go-backend/internal/model/directory.go | 4 +- .../internal/model/directory_snapshot.go | 6 +- .../{host_port_mapping.go => host_port.go} | 8 +- ...ping_snapshot.go => host_port_snapshot.go} | 8 +- go-backend/internal/model/model_test.go | 4 +- go-backend/internal/model/website_snapshot.go | 2 +- .../{ip_address.go => host_port.go} | 50 +- .../internal/repository/host_port_snapshot.go | 99 ++ .../repository/website_snapshot_test.go | 73 -- .../internal/service/directory_snapshot.go | 2 - .../service/{ip_address.go => host_port.go} | 40 +- .../internal/service/host_port_snapshot.go | 181 +++ .../internal/service/website_snapshot.go | 32 +- .../internal/service/website_snapshot_test.go | 16 +- 41 files changed, 684 insertions(+), 1485 deletions(-) delete mode 100644 go-backend/cmd/seed/main.go delete mode 100644 go-backend/cmd/server/migrations/000002_add_gin_indexes.down.sql delete mode 100644 go-backend/cmd/server/migrations/000002_add_gin_indexes.up.sql delete mode 100644 go-backend/cmd/server/migrations/000003_drop_directory_words_lines.down.sql delete mode 100644 go-backend/cmd/server/migrations/000003_drop_directory_words_lines.up.sql delete mode 100644 go-backend/cmd/server/migrations/000004_add_vulnerability_review.down.sql delete mode 100644 go-backend/cmd/server/migrations/000004_add_vulnerability_review.up.sql delete mode 100644 go-backend/cmd/server/migrations/000005_add_vuln_target_reviewed_index.down.sql delete mode 100644 go-backend/cmd/server/migrations/000005_add_vuln_target_reviewed_index.up.sql create mode 100644 go-backend/internal/dto/host_port.go create mode 100644 go-backend/internal/dto/host_port_snapshot.go delete mode 100644 go-backend/internal/dto/ip_address.go rename go-backend/internal/handler/{ip_address.go => host_port.go} (65%) create mode 100644 go-backend/internal/handler/host_port_snapshot.go rename go-backend/internal/model/{host_port_mapping.go => host_port.go} (83%) rename go-backend/internal/model/{host_port_mapping_snapshot.go => host_port_snapshot.go} (80%) rename go-backend/internal/repository/{ip_address.go => host_port.go} (64%) create mode 100644 go-backend/internal/repository/host_port_snapshot.go rename go-backend/internal/service/{ip_address.go => host_port.go} (64%) create mode 100644 go-backend/internal/service/host_port_snapshot.go diff --git a/frontend/components/ui/terminal-login.tsx b/frontend/components/ui/terminal-login.tsx index 0a0bbd5c..c3360b55 100644 --- a/frontend/components/ui/terminal-login.tsx +++ b/frontend/components/ui/terminal-login.tsx @@ -50,6 +50,7 @@ export function TerminalLogin({ const [password, setPassword] = React.useState("") const [lines, setLines] = React.useState([]) const [cursorPosition, setCursorPosition] = React.useState(0) + const [isFocused, setIsFocused] = React.useState(false) const inputRef = React.useRef(null) const containerRef = React.useRef(null) @@ -137,6 +138,17 @@ export function TerminalLogin({ return } + // Tab - Move to next field (username -> password) + if (e.key === "Tab" && step === "username") { + e.preventDefault() + if (!username.trim()) return + addLine({ text: `> ${t.usernamePrompt}: `, type: "prompt" }) + addLine({ text: username, type: "input" }) + setStep("password") + setCursorPosition(0) + return + } + // Enter - Submit if (e.key === "Enter") { if (step === "username") { @@ -208,6 +220,10 @@ export function TerminalLogin({ const after = displayValue.slice(cursorPosition) const cursorChar = after[0] || "" + if (!isFocused) { + return {displayValue} + } + return ( <> {before} @@ -359,6 +375,8 @@ export function TerminalLogin({ onChange={handleInputChange} onKeyDown={handleKeyDown} onSelect={handleSelect} + onFocus={() => setIsFocused(true)} + onBlur={() => setIsFocused(false)} disabled={isInputDisabled} className="absolute opacity-0 pointer-events-none" autoComplete={step === "username" ? "username" : "current-password"} diff --git a/go-backend/cmd/seed/main.go b/go-backend/cmd/seed/main.go deleted file mode 100644 index d0db1dc6..00000000 --- a/go-backend/cmd/seed/main.go +++ /dev/null @@ -1,1081 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "fmt" - "math/rand" - "os" - "strings" - "time" - - "github.com/lib/pq" - "github.com/shopspring/decimal" - "github.com/xingrin/go-backend/internal/config" - "github.com/xingrin/go-backend/internal/database" - "github.com/xingrin/go-backend/internal/model" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -var ( - clear = flag.Bool("clear", false, "Clear existing data before generating") - orgCount = flag.Int("orgs", 15, "Number of organizations to generate") -) - -func main() { - flag.Parse() - - // Calculate counts based on org count - // 15 orgs Ɨ 15 targets/org Ɨ 15 assets/target - targetsPerOrg := 15 - assetsPerTarget := 15 - targetCount := *orgCount * targetsPerOrg - - // Load config - cfg, err := config.Load() - if err != nil { - fmt.Printf("āŒ Failed to load config: %v\n", err) - os.Exit(1) - } - - // Connect to database - db, err := database.NewDatabase(&cfg.Database) - if err != nil { - fmt.Printf("āŒ Failed to connect to database: %v\n", err) - os.Exit(1) - } - - fmt.Println("šŸš€ Starting test data generation...") - fmt.Printf(" Organizations: %d\n", *orgCount) - fmt.Printf(" Targets: %d (%d per org)\n", targetCount, targetsPerOrg) - fmt.Printf(" Assets per target: %d (websites, endpoints, directories, subdomains for domains)\n", assetsPerTarget) - fmt.Println() - - if *clear { - fmt.Println("šŸ—‘ļø Clearing existing data...") - if err := clearData(db); err != nil { - fmt.Printf("āŒ Failed to clear data: %v\n", err) - os.Exit(1) - } - fmt.Println(" āœ“ Data cleared") - } - - // Generate data - orgIDs, err := createOrganizations(db, *orgCount) - if err != nil { - fmt.Printf("āŒ Failed to create organizations: %v\n", err) - os.Exit(1) - } - - targets, err := createTargets(db, targetCount) - if err != nil { - fmt.Printf("āŒ Failed to create targets: %v\n", err) - os.Exit(1) - } - - // Extract target IDs - targetIDs := make([]int, len(targets)) - for i, t := range targets { - targetIDs[i] = t.ID - } - - // Link targets to organizations (20 per org) - if err := linkTargetsToOrganizations(db, targetIDs, orgIDs); err != nil { - fmt.Printf("āŒ Failed to link targets to organizations: %v\n", err) - os.Exit(1) - } - - // Create websites for targets (20 per target) - if err := createWebsites(db, targets, assetsPerTarget); err != nil { - fmt.Printf("āŒ Failed to create websites: %v\n", err) - os.Exit(1) - } - - // Create subdomains for domain-type targets only (20 per target) - if err := createSubdomains(db, targets, assetsPerTarget); err != nil { - fmt.Printf("āŒ Failed to create subdomains: %v\n", err) - os.Exit(1) - } - - // Create endpoints for targets (20 per target) - if err := createEndpoints(db, targets, assetsPerTarget); err != nil { - fmt.Printf("āŒ Failed to create endpoints: %v\n", err) - os.Exit(1) - } - - // Create directories for targets (20 per target) - if err := createDirectories(db, targets, assetsPerTarget); err != nil { - fmt.Printf("āŒ Failed to create directories: %v\n", err) - os.Exit(1) - } - - // Create host port mappings for targets (20 per target) - if err := createHostPortMappings(db, targets, assetsPerTarget); err != nil { - fmt.Printf("āŒ Failed to create host port mappings: %v\n", err) - os.Exit(1) - } - - // Create screenshots for targets (no actual image data, just metadata) - // Note: Real screenshots would be generated by scanners with actual WebP image data - if err := createScreenshots(db, targets, 0); err != nil { - fmt.Printf("āŒ Failed to create screenshots: %v\n", err) - os.Exit(1) - } - - // Create vulnerabilities for targets - if err := createVulnerabilities(db, targets, assetsPerTarget); err != nil { - fmt.Printf("āŒ Failed to create vulnerabilities: %v\n", err) - os.Exit(1) - } - - // Create scans for targets - if err := createScans(db, targets, 3); err != nil { - fmt.Printf("āŒ Failed to create scans: %v\n", err) - os.Exit(1) - } - - fmt.Println("\nāœ… Test data generation completed!") -} - -func clearData(db *gorm.DB) error { - // Delete in order to respect foreign key constraints - tables := []string{ - "scan_log", - "scan_input_target", - "scan", - "vulnerability", - "screenshot", - "host_port_mapping", - "directory", - "endpoint", - "subdomain", - "website", - "organization_target", - "target", - "organization", - } - - for _, table := range tables { - if err := db.Exec(fmt.Sprintf("DELETE FROM %s", table)).Error; err != nil { - return fmt.Errorf("failed to clear %s: %w", table, err) - } - } - return nil -} - -func createOrganizations(db *gorm.DB, count int) ([]int, error) { - fmt.Printf("šŸ¢ Creating %d organizations...\n", count) - - orgNames := []string{ - "Acme Corporation", "TechStart Labs", "Global Finance", "HealthCare Plus", - "E-Commerce Platform", "Smart City Systems", "Educational Tech", "Green Energy", - "CyberSec Defense", "CloudNative Systems", "DataFlow Analytics", "MobileFirst Tech", - "Quantum Research", "Autonomous Vehicles", "Biotech Innovations", "Space Technology", - "AI Research Lab", "Blockchain Solutions", "IoT Platform", "DevOps Enterprise", - "Security Operations", "Data Science Hub", "Machine Learning Co", "Network Solutions", - "Infrastructure Corp", "Platform Services", "Digital Transformation", "Innovation Hub", - "Tech Consulting", "Software Factory", - } - - divisions := []string{ - "Global", "Asia Pacific", "EMEA", "Americas", "R&D", "Cloud Services", - "Security Team", "Innovation Lab", "Enterprise", "Consumer Products", - } - - descriptions := []string{ - "A leading technology company specializing in enterprise software solutions and cloud computing services.", - "Innovative research lab focused on artificial intelligence and machine learning applications.", - "Global financial services provider offering digital banking and payment solutions.", - "Healthcare technology company developing electronic health records and telemedicine platforms.", - "E-commerce platform serving millions of customers with B2B and B2C solutions.", - "Smart city infrastructure provider specializing in IoT and urban management systems.", - "Educational technology company providing online learning platforms and courses.", - "Renewable energy management company focused on solar and wind power optimization.", - "Cybersecurity firm offering penetration testing and security consulting services.", - "Cloud-native systems developer specializing in Kubernetes and microservices.", - } - - suffix := rand.Intn(9000) + 1000 - - // Build all organizations in memory first - orgs := make([]model.Organization, 0, count) - for i := 0; i < count; i++ { - name := fmt.Sprintf("%s - %s (%d-%d)", - orgNames[i%len(orgNames)], - divisions[rand.Intn(len(divisions))], - suffix, i) - - desc := descriptions[rand.Intn(len(descriptions))] - daysAgo := rand.Intn(365) - createdAt := time.Now().AddDate(0, 0, -daysAgo) - - orgs = append(orgs, model.Organization{ - Name: name, - Description: desc, - CreatedAt: createdAt, - }) - } - - // Batch insert - if err := db.CreateInBatches(orgs, 100).Error; err != nil { - return nil, err - } - - // Extract IDs - ids := make([]int, len(orgs)) - for i := range orgs { - ids[i] = orgs[i].ID - } - - fmt.Printf(" āœ“ Created %d organizations\n", len(ids)) - return ids, nil -} - -func createTargets(db *gorm.DB, count int) ([]model.Target, error) { - fmt.Printf("šŸŽÆ Creating %d targets...\n", count) - - // Domain templates - envs := []string{"prod", "staging", "dev", "test", "api", "app", "www", "admin", "portal", "dashboard"} - companies := []string{"acme", "techstart", "globalfinance", "healthcare", "ecommerce", "smartcity", "cybersec", "cloudnative", "dataflow", "mobilefirst"} - tlds := []string{".com", ".io", ".net", ".org", ".dev", ".app", ".cloud", ".tech"} - - suffix := rand.Intn(9000) + 1000 - usedNames := make(map[string]bool) - - // Build all targets in memory first - targets := make([]model.Target, 0, count) - - // Generate domains (70%) - domainCount := count * 70 / 100 - for i := 0; i < domainCount; i++ { - var name string - for { - env := envs[rand.Intn(len(envs))] - company := companies[rand.Intn(len(companies))] - tld := tlds[rand.Intn(len(tlds))] - name = fmt.Sprintf("%s.%s-%d%s", env, company, suffix+i, tld) - if !usedNames[name] { - usedNames[name] = true - break - } - } - - targets = append(targets, model.Target{ - Name: name, - Type: "domain", - CreatedAt: time.Now().AddDate(0, 0, -rand.Intn(365)), - }) - } - - // Generate IPs (20%) - ipCount := count * 20 / 100 - actualIPCount := 0 - for i := 0; i < ipCount; i++ { - name := fmt.Sprintf("%d.%d.%d.%d", - rand.Intn(223)+1, - rand.Intn(256), - rand.Intn(256), - rand.Intn(254)+1) - - if usedNames[name] { - continue - } - usedNames[name] = true - - targets = append(targets, model.Target{ - Name: name, - Type: "ip", - CreatedAt: time.Now().AddDate(0, 0, -rand.Intn(365)), - }) - actualIPCount++ - } - - // Generate CIDRs (10%) - cidrCount := count * 10 / 100 - actualCIDRCount := 0 - for i := 0; i < cidrCount; i++ { - masks := []int{8, 16, 24} - mask := masks[rand.Intn(len(masks))] - name := fmt.Sprintf("%d.%d.%d.0/%d", - rand.Intn(223)+1, - rand.Intn(256), - rand.Intn(256), - mask) - - if usedNames[name] { - continue - } - usedNames[name] = true - - targets = append(targets, model.Target{ - Name: name, - Type: "cidr", - CreatedAt: time.Now().AddDate(0, 0, -rand.Intn(365)), - }) - actualCIDRCount++ - } - - // Batch insert - if err := db.CreateInBatches(targets, 100).Error; err != nil { - return nil, err - } - - fmt.Printf(" āœ“ Created %d targets (domains: %d, IPs: %d, CIDRs: %d)\n", - len(targets), domainCount, actualIPCount, actualCIDRCount) - return targets, nil -} - -func linkTargetsToOrganizations(db *gorm.DB, targetIDs, orgIDs []int) error { - fmt.Println("šŸ”— Linking targets to organizations...") - - if len(orgIDs) == 0 || len(targetIDs) == 0 { - return nil - } - - // Each organization gets exactly targetsPerOrg targets (evenly distributed) - targetsPerOrg := len(targetIDs) / len(orgIDs) - - // Build all values for batch insert - var values []string - var args []interface{} - - for orgIdx, orgID := range orgIDs { - startIdx := orgIdx * targetsPerOrg - endIdx := startIdx + targetsPerOrg - if orgIdx == len(orgIDs)-1 { - endIdx = len(targetIDs) // Last org gets remaining targets - } - - for i := startIdx; i < endIdx; i++ { - values = append(values, "(?, ?)") - args = append(args, orgID, targetIDs[i]) - } - } - - // Batch insert in chunks of 500 - chunkSize := 500 - for i := 0; i < len(values); i += chunkSize { - end := i + chunkSize - if end > len(values) { - end = len(values) - } - - query := "INSERT INTO organization_target (organization_id, target_id) VALUES " + - strings.Join(values[i:end], ", ") + " ON CONFLICT DO NOTHING" - - // Calculate args range: each value has 2 args - argsStart := i * 2 - argsEnd := end * 2 - if err := db.Exec(query, args[argsStart:argsEnd]...).Error; err != nil { - return err - } - } - - fmt.Printf(" āœ“ Created %d target-organization links (%d per org)\n", len(values), targetsPerOrg) - return nil -} - -func createWebsites(db *gorm.DB, targets []model.Target, websitesPerTarget int) error { - totalCount := len(targets) * websitesPerTarget - fmt.Printf("šŸŒ Creating %d websites (%d per target)...\n", totalCount, websitesPerTarget) - - if len(targets) == 0 { - return nil - } - - // Website data templates - protocols := []string{"https://", "http://"} - subdomains := []string{"www", "api", "app", "admin", "portal", "dashboard", "dev", "staging", "test", "cdn", "static", "assets", "mail", "blog", "docs", "support", "auth", "login", "shop", "store"} - paths := []string{"", "/", "/api", "/v1", "/v2", "/login", "/dashboard", "/admin", "/app", "/docs"} - ports := []string{"", ":8080", ":8443", ":3000", ":443"} - - titles := []string{ - "Welcome - Dashboard", "Admin Panel", "API Documentation", "Login Portal", - "Home Page", "User Dashboard", "Settings", "Analytics", "Reports", - "Management Console", "Control Panel", "Service Status", "Developer Portal", - } - - webservers := []string{ - "nginx/1.24.0", "Apache/2.4.57", "cloudflare", "Microsoft-IIS/10.0", - "nginx", "Apache", "LiteSpeed", "Caddy", "Traefik", - } - - contentTypes := []string{ - "text/html; charset=utf-8", "text/html", "application/json", - "text/html; charset=UTF-8", "application/xhtml+xml", - } - - techStacks := [][]string{ - {"nginx", "PHP", "MySQL"}, - {"Apache", "Python", "PostgreSQL"}, - {"nginx", "Node.js", "MongoDB"}, - {"cloudflare", "React", "GraphQL"}, - {"nginx", "Vue.js", "Redis"}, - {"Apache", "Java", "Oracle"}, - {"nginx", "Go", "PostgreSQL"}, - {"cloudflare", "Next.js", "Vercel"}, - {"nginx", "Django", "PostgreSQL"}, - {"Apache", "Laravel", "MySQL"}, - {"nginx", "Express", "MongoDB"}, - {"cloudflare", "Angular", "Firebase"}, - } - - statusCodes := []int{200, 200, 200, 200, 200, 301, 302, 403, 404, 500} - - // Build all websites in memory first - websites := make([]model.Website, 0, totalCount) - - // Each target gets exactly websitesPerTarget websites - for targetIdx, target := range targets { - for i := 0; i < websitesPerTarget; i++ { - // Generate URL based on target type - var url string - protocol := protocols[i%len(protocols)] - subdomain := subdomains[i%len(subdomains)] - port := ports[i%len(ports)] - path := paths[i%len(paths)] - - switch target.Type { - case "domain": - url = fmt.Sprintf("%s%s.%s%s%s", protocol, subdomain, target.Name, port, path) - case "ip": - url = fmt.Sprintf("%s%s%s%s", protocol, target.Name, port, path) - case "cidr": - // Generate IP within CIDR range (simplified: use base IP) - baseIP := target.Name[:len(target.Name)-3] // Remove /XX - url = fmt.Sprintf("%s%s%s%s", protocol, baseIP, port, path) - default: - url = fmt.Sprintf("%s%s.target%d-site%d.com%s%s", protocol, subdomain, targetIdx, i, port, path) - } - - // Extract host from URL - host := extractHost(url) - - // Deterministic but varied values - statusCode := statusCodes[i%len(statusCodes)] - contentLength := 1000 + (i * 100) - tech := pq.StringArray(techStacks[i%len(techStacks)]) - vhost := i%5 == 0 // 20% are vhost - - websites = append(websites, model.Website{ - TargetID: target.ID, - URL: url, - Host: host, - Title: titles[i%len(titles)], - StatusCode: &statusCode, - ContentLength: &contentLength, - ContentType: contentTypes[i%len(contentTypes)], - Webserver: webservers[i%len(webservers)], - Tech: tech, - Vhost: &vhost, - Location: "", - CreatedAt: time.Now().AddDate(0, 0, -i), - }) - } - } - - // Batch insert with ON CONFLICT DO NOTHING - if err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(websites, 100).Error; err != nil { - return err - } - - fmt.Printf(" āœ“ Created %d websites\n", len(websites)) - return nil -} - -func extractHost(rawURL string) string { - // Simple host extraction - url := rawURL - // Remove protocol - if idx := len("https://"); len(url) > idx && url[:idx] == "https://" { - url = url[idx:] - } else if idx := len("http://"); len(url) > idx && url[:idx] == "http://" { - url = url[idx:] - } - // Remove path - if idx := findIndex(url, "/"); idx != -1 { - url = url[:idx] - } - return url -} - -func findIndex(s string, substr string) int { - for i := 0; i < len(s); i++ { - if s[i] == substr[0] { - return i - } - } - return -1 -} - -func createSubdomains(db *gorm.DB, targets []model.Target, subdomainsPerTarget int) error { - // Only create subdomains for domain-type targets - domainTargets := make([]model.Target, 0) - for _, t := range targets { - if t.Type == "domain" { - domainTargets = append(domainTargets, t) - } - } - - totalCount := len(domainTargets) * subdomainsPerTarget - fmt.Printf("šŸ“ Creating %d subdomains (%d per domain target, %d domain targets)...\n", - totalCount, subdomainsPerTarget, len(domainTargets)) - - if len(domainTargets) == 0 { - fmt.Println(" āš  No domain targets found, skipping subdomains") - return nil - } - - // Subdomain prefixes - prefixes := []string{ - "www", "api", "app", "admin", "portal", "dashboard", "dev", "staging", - "test", "cdn", "static", "assets", "mail", "blog", "docs", "support", - "auth", "login", "shop", "store", - } - - // Build all subdomains in memory first - subdomains := make([]model.Subdomain, 0, totalCount) - - for _, target := range domainTargets { - for i := 0; i < subdomainsPerTarget; i++ { - prefix := prefixes[i%len(prefixes)] - name := fmt.Sprintf("%s.%s", prefix, target.Name) - - subdomains = append(subdomains, model.Subdomain{ - TargetID: target.ID, - Name: name, - CreatedAt: time.Now().AddDate(0, 0, -i), - }) - } - } - - // Batch insert with ON CONFLICT DO NOTHING - if err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(subdomains, 100).Error; err != nil { - return err - } - - fmt.Printf(" āœ“ Created %d subdomains\n", len(subdomains)) - return nil -} - -func createEndpoints(db *gorm.DB, targets []model.Target, endpointsPerTarget int) error { - totalCount := len(targets) * endpointsPerTarget - fmt.Printf("šŸ”— Creating %d endpoints (%d per target)...\n", totalCount, endpointsPerTarget) - - if len(targets) == 0 { - return nil - } - - // Endpoint data templates - protocols := []string{"https://", "http://"} - subdomains := []string{"www", "api", "app", "admin", "portal", "dashboard", "dev", "staging", "test", "cdn"} - paths := []string{ - "/api/v1/users", "/api/v1/products", "/api/v2/orders", "/login", "/dashboard", - "/admin/settings", "/app/config", "/docs/api", "/health", "/metrics", - "/api/auth/login", "/api/auth/logout", "/api/data/export", "/api/search", - "/graphql", "/ws/connect", "/api/upload", "/api/download", "/status", "/version", - } - - titles := []string{ - "API Endpoint", "User Service", "Product API", "Authentication", - "Dashboard API", "Admin Panel", "Configuration", "Documentation", - "Health Check", "Metrics Endpoint", "GraphQL API", "WebSocket", - } - - webservers := []string{"nginx/1.24.0", "Apache/2.4.57", "cloudflare", "nginx", "Apache"} - contentTypes := []string{"application/json", "text/html", "application/xml", "text/plain"} - techStacks := [][]string{ - {"nginx", "Node.js", "Express"}, - {"Apache", "Python", "FastAPI"}, - {"nginx", "Go", "Gin"}, - {"cloudflare", "Rust", "Actix"}, - } - gfPatterns := [][]string{ - {"debug-pages", "potential-takeover"}, - {"cors", "ssrf"}, - {"sqli", "xss"}, - {"lfi", "rce"}, - {}, - } - statusCodes := []int{200, 200, 200, 201, 301, 400, 401, 403, 404, 500} - - // Build all endpoints in memory first - endpoints := make([]model.Endpoint, 0, totalCount) - - for _, target := range targets { - for i := 0; i < endpointsPerTarget; i++ { - var url string - protocol := protocols[i%len(protocols)] - subdomain := subdomains[i%len(subdomains)] - path := paths[i%len(paths)] - - switch target.Type { - case "domain": - url = fmt.Sprintf("%s%s.%s%s", protocol, subdomain, target.Name, path) - case "ip": - url = fmt.Sprintf("%s%s%s", protocol, target.Name, path) - case "cidr": - baseIP := target.Name[:len(target.Name)-3] - url = fmt.Sprintf("%s%s%s", protocol, baseIP, path) - default: - continue - } - - host := extractHost(url) - statusCode := statusCodes[i%len(statusCodes)] - contentLength := 500 + (i * 50) - tech := pq.StringArray(techStacks[i%len(techStacks)]) - matchedGF := pq.StringArray(gfPatterns[i%len(gfPatterns)]) - vhost := i%10 == 0 - - endpoints = append(endpoints, model.Endpoint{ - TargetID: target.ID, - URL: url, - Host: host, - Title: titles[i%len(titles)], - StatusCode: &statusCode, - ContentLength: &contentLength, - ContentType: contentTypes[i%len(contentTypes)], - Webserver: webservers[i%len(webservers)], - Tech: tech, - MatchedGFPatterns: matchedGF, - Vhost: &vhost, - CreatedAt: time.Now().AddDate(0, 0, -i), - }) - } - } - - // Batch insert with ON CONFLICT DO NOTHING - if err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(endpoints, 100).Error; err != nil { - return err - } - - fmt.Printf(" āœ“ Created %d endpoints\n", len(endpoints)) - return nil -} - -func createDirectories(db *gorm.DB, targets []model.Target, directoriesPerTarget int) error { - totalCount := len(targets) * directoriesPerTarget - fmt.Printf("šŸ“ Creating %d directories (%d per target)...\n", totalCount, directoriesPerTarget) - - if len(targets) == 0 { - return nil - } - - // Directory data templates - protocols := []string{"https://", "http://"} - subdomains := []string{"www", "api", "app", "admin", "portal", "dashboard", "dev", "staging", "test", "cdn"} - directories := []string{ - "/admin/", "/backup/", "/config/", "/data/", "/debug/", - "/files/", "/images/", "/js/", "/css/", "/uploads/", - "/api/", "/docs/", "/logs/", "/temp/", "/cache/", - "/static/", "/assets/", "/media/", "/public/", "/private/", - } - - contentTypes := []string{"text/html", "application/json", "text/plain", "application/xml"} - statusCodes := []int{200, 200, 200, 301, 302, 403, 404} - - // Build all directories in memory first - dirs := make([]model.Directory, 0, totalCount) - - for _, target := range targets { - for i := 0; i < directoriesPerTarget; i++ { - var url string - protocol := protocols[i%len(protocols)] - subdomain := subdomains[i%len(subdomains)] - dir := directories[i%len(directories)] - - switch target.Type { - case "domain": - url = fmt.Sprintf("%s%s.%s%s", protocol, subdomain, target.Name, dir) - case "ip": - url = fmt.Sprintf("%s%s%s", protocol, target.Name, dir) - case "cidr": - baseIP := target.Name[:len(target.Name)-3] - url = fmt.Sprintf("%s%s%s", protocol, baseIP, dir) - default: - continue - } - - status := statusCodes[i%len(statusCodes)] - contentLength := int64(1000 + (i * 100)) - duration := int64(50 + (i * 5)) - - dirs = append(dirs, model.Directory{ - TargetID: target.ID, - URL: url, - Status: &status, - ContentLength: &contentLength, - ContentType: contentTypes[i%len(contentTypes)], - Duration: &duration, - CreatedAt: time.Now().AddDate(0, 0, -i), - }) - } - } - - // Batch insert with ON CONFLICT DO NOTHING - if err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(dirs, 100).Error; err != nil { - return err - } - - fmt.Printf(" āœ“ Created %d directories\n", len(dirs)) - return nil -} - -func createHostPortMappings(db *gorm.DB, targets []model.Target, mappingsPerTarget int) error { - // Increase mappings to ensure pagination (100 per target = more IPs) - actualMappingsPerTarget := mappingsPerTarget * 5 // 20 * 5 = 100 mappings per target - totalCount := len(targets) * actualMappingsPerTarget - fmt.Printf("šŸ”Œ Creating %d host port mappings (%d per target)...\n", totalCount, actualMappingsPerTarget) - - if len(targets) == 0 { - return nil - } - - // Common ports - ports := []int{22, 80, 443, 8080, 8443, 3000, 3306, 5432, 6379, 27017, 9200, 9300, 5000, 8000, 8888, 9000, 9090, 10000, 11211, 25} - - // Subdomain prefixes for hosts - subdomains := []string{"www", "api", "app", "admin", "portal", "dashboard", "dev", "staging", "test", "cdn", "mail", "ftp", "db", "cache", "search", "auth", "login", "shop", "store", "blog"} - - // Build all mappings in memory first - mappings := make([]model.HostPortMapping, 0, totalCount) - - for _, target := range targets { - // Generate base IP for this target - baseIP1 := rand.Intn(223) + 1 - baseIP2 := rand.Intn(256) - baseIP3 := rand.Intn(256) - - // Generate more unique IPs per target (10-15 IPs with 6-10 ports each) - numIPs := 10 + rand.Intn(6) // 10-15 unique IPs - portsPerIP := actualMappingsPerTarget / numIPs - - for ipIdx := 0; ipIdx < numIPs; ipIdx++ { - // Generate unique IP - ip := fmt.Sprintf("%d.%d.%d.%d", baseIP1, baseIP2, baseIP3, ipIdx+1) - - // Generate multiple hosts for this IP - numHosts := 3 + rand.Intn(4) // 3-6 hosts per IP - - for hostIdx := 0; hostIdx < numHosts; hostIdx++ { - var host string - - // Generate host based on target type - switch target.Type { - case "domain": - subdomain := subdomains[(ipIdx*numHosts+hostIdx)%len(subdomains)] - host = fmt.Sprintf("%s.%s", subdomain, target.Name) - case "ip": - host = target.Name - case "cidr": - baseIP := target.Name[:len(target.Name)-3] - host = baseIP - default: - continue - } - - // Generate multiple ports for this host-IP combination - numPorts := portsPerIP / numHosts - if numPorts < 1 { - numPorts = 1 - } - - for portIdx := 0; portIdx < numPorts; portIdx++ { - port := ports[(ipIdx*numHosts*numPorts+hostIdx*numPorts+portIdx)%len(ports)] - - mappings = append(mappings, model.HostPortMapping{ - TargetID: target.ID, - Host: host, - IP: ip, - Port: port, - CreatedAt: time.Now().AddDate(0, 0, -(ipIdx*numHosts*numPorts + hostIdx*numPorts + portIdx)), - }) - } - } - } - } - - // Batch insert with ON CONFLICT DO NOTHING - if err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(mappings, 100).Error; err != nil { - return err - } - - fmt.Printf(" āœ“ Created %d host port mappings\n", len(mappings)) - return nil -} - -func createScreenshots(db *gorm.DB, targets []model.Target, screenshotsPerTarget int) error { - // Screenshots are typically generated by scanners with actual WebP image data - // For seed data, we create empty records (no image data) to demonstrate the structure - // In production, scanners would populate the image field with actual WebP binary data - - if screenshotsPerTarget == 0 { - fmt.Println("šŸ“ø Skipping screenshots (typically generated by scanners)") - return nil - } - - totalCount := len(targets) * screenshotsPerTarget - fmt.Printf("šŸ“ø Creating %d screenshots (%d per target)...\n", totalCount, screenshotsPerTarget) - - if len(targets) == 0 { - return nil - } - - // Screenshot data templates - protocols := []string{"https://", "http://"} - subdomains := []string{"www", "api", "app", "admin", "portal", "dashboard", "dev", "staging", "test", "cdn"} - paths := []string{"", "/", "/login", "/dashboard", "/admin", "/app", "/docs", "/api"} - statusCodes := []int{200, 200, 200, 200, 301, 302, 403, 404} - - // Build all screenshots in memory first - screenshots := make([]model.Screenshot, 0, totalCount) - - for _, target := range targets { - for i := 0; i < screenshotsPerTarget; i++ { - var url string - protocol := protocols[i%len(protocols)] - subdomain := subdomains[i%len(subdomains)] - path := paths[i%len(paths)] - - switch target.Type { - case "domain": - url = fmt.Sprintf("%s%s.%s%s", protocol, subdomain, target.Name, path) - case "ip": - url = fmt.Sprintf("%s%s%s", protocol, target.Name, path) - case "cidr": - baseIP := target.Name[:len(target.Name)-3] - url = fmt.Sprintf("%s%s%s", protocol, baseIP, path) - default: - continue - } - - statusCode := int16(statusCodes[i%len(statusCodes)]) - - screenshots = append(screenshots, model.Screenshot{ - TargetID: target.ID, - URL: url, - StatusCode: &statusCode, - Image: nil, // No actual image data in seed data - CreatedAt: time.Now().AddDate(0, 0, -i), - }) - } - } - - // Batch insert with ON CONFLICT DO NOTHING - if err := db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(screenshots, 100).Error; err != nil { - return err - } - - fmt.Printf(" āœ“ Created %d screenshots (without image data)\n", len(screenshots)) - return nil -} - -func createVulnerabilities(db *gorm.DB, targets []model.Target, vulnsPerTarget int) error { - totalCount := len(targets) * vulnsPerTarget - fmt.Printf("šŸ”“ Creating %d vulnerabilities (%d per target)...\n", totalCount, vulnsPerTarget) - - if len(targets) == 0 { - return nil - } - - // Vulnerability data templates - vulnTypes := []string{ - "SQL Injection", "Cross-Site Scripting (XSS)", "Remote Code Execution", - "Server-Side Request Forgery (SSRF)", "Local File Inclusion (LFI)", - "XML External Entity (XXE)", "Insecure Deserialization", "Command Injection", - "Path Traversal", "Open Redirect", "CRLF Injection", "CORS Misconfiguration", - "Information Disclosure", "Authentication Bypass", "Privilege Escalation", - } - - severities := []string{"critical", "high", "high", "medium", "medium", "medium", "low", "low", "info"} - - sources := []string{"nuclei", "dalfox", "sqlmap", "burpsuite", "manual"} - - descriptions := []string{ - "A SQL injection vulnerability was found that allows an attacker to execute arbitrary SQL queries.", - "A reflected XSS vulnerability exists that could allow attackers to inject malicious scripts.", - "Remote code execution is possible through unsafe deserialization of user input.", - "SSRF vulnerability allows an attacker to make requests to internal services.", - "Local file inclusion vulnerability could expose sensitive files on the server.", - "XXE vulnerability found that could lead to information disclosure or SSRF.", - "Insecure deserialization could lead to remote code execution.", - "OS command injection vulnerability found in user-controlled input.", - "Path traversal vulnerability allows access to files outside the web root.", - "Open redirect vulnerability could be used for phishing attacks.", - } - - paths := []string{ - "/login", "/api/v1/users", "/api/v1/search", "/admin/config", - "/api/export", "/upload", "/api/v2/data", "/graphql", - "/api/auth", "/dashboard", "/api/profile", "/settings", - } - - cvssScores := []float64{9.8, 9.1, 8.6, 7.5, 6.5, 5.4, 4.3, 3.1, 2.0} - - // Build all vulnerabilities in memory first - vulns := make([]model.Vulnerability, 0, totalCount) - - for _, target := range targets { - for i := 0; i < vulnsPerTarget; i++ { - var url string - path := paths[i%len(paths)] - - switch target.Type { - case "domain": - url = fmt.Sprintf("https://www.%s%s", target.Name, path) - case "ip": - url = fmt.Sprintf("https://%s%s", target.Name, path) - case "cidr": - baseIP := target.Name[:len(target.Name)-3] - url = fmt.Sprintf("https://%s%s", baseIP, path) - default: - continue - } - - cvss := decimal.NewFromFloat(cvssScores[i%len(cvssScores)]) - - vulns = append(vulns, model.Vulnerability{ - TargetID: target.ID, - URL: url, - VulnType: vulnTypes[i%len(vulnTypes)], - Severity: severities[i%len(severities)], - Source: sources[i%len(sources)], - CVSSScore: &cvss, - Description: descriptions[i%len(descriptions)], - CreatedAt: time.Now().AddDate(0, 0, -i), - }) - } - } - - // Batch insert (100 records per batch) - if err := db.CreateInBatches(vulns, 100).Error; err != nil { - return err - } - - fmt.Printf(" āœ“ Created %d vulnerabilities\n", len(vulns)) - return nil -} - -func createScans(db *gorm.DB, targets []model.Target, scansPerTarget int) error { - totalCount := len(targets) * scansPerTarget - fmt.Printf("šŸ” Creating %d scans (%d per target)...\n", totalCount, scansPerTarget) - - if len(targets) == 0 { - return nil - } - - // Scan data templates - statuses := []string{"completed", "completed", "completed", "running", "failed"} - scanModes := []string{"full", "full", "quick"} - stages := []string{"subdomain_discovery", "port_scan", "web_discovery", "vulnerability_scan", ""} - - engineNames := [][]string{ - {"Default Engine"}, - {"Fast Scanner", "Deep Scanner"}, - {"Nuclei Engine"}, - {"Full Recon"}, - } - - errorMessages := []string{ - "", - "Connection timeout to target", - "Worker node unavailable", - "Rate limit exceeded", - } - - // Build all scans in memory first - scans := make([]model.Scan, 0, totalCount) - - for _, target := range targets { - for i := 0; i < scansPerTarget; i++ { - status := statuses[i%len(statuses)] - scanMode := scanModes[i%len(scanModes)] - currentStage := stages[i%len(stages)] - - // Calculate progress based on status - var progress int - switch status { - case "completed": - progress = 100 - case "running": - progress = 30 + rand.Intn(60) - case "failed": - progress = rand.Intn(50) - default: - progress = 0 - } - - // Engine IDs and names - engineIDs := pq.Int64Array{1} - names := engineNames[i%len(engineNames)] - namesJSON, _ := json.Marshal(names) - - // Error message (only for failed scans) - var errorMsg string - if status == "failed" { - errorMsg = errorMessages[1+rand.Intn(len(errorMessages)-1)] - } - - // Created time - daysAgo := i * 7 // Each scan is 7 days apart - createdAt := time.Now().AddDate(0, 0, -daysAgo) - - // Stopped time (for completed/failed scans) - var stoppedAt *time.Time - if status == "completed" || status == "failed" { - t := createdAt.Add(time.Duration(30+rand.Intn(60)) * time.Minute) - stoppedAt = &t - } - - // Cached stats (random values for demonstration) - subdomainsCount := rand.Intn(50) - websitesCount := rand.Intn(30) - endpointsCount := rand.Intn(100) - ipsCount := rand.Intn(20) - directoriesCount := rand.Intn(40) - screenshotsCount := rand.Intn(20) - vulnsCritical := rand.Intn(3) - vulnsHigh := rand.Intn(5) - vulnsMedium := rand.Intn(10) - vulnsLow := rand.Intn(15) - vulnsTotal := vulnsCritical + vulnsHigh + vulnsMedium + vulnsLow - - scan := model.Scan{ - TargetID: target.ID, - EngineIDs: engineIDs, - EngineNames: namesJSON, - YamlConfiguration: "# Default scan configuration\nthreads: 10\ntimeout: 30s", - ScanMode: scanMode, - Status: status, - Progress: progress, - CurrentStage: currentStage, - ErrorMessage: errorMsg, - CreatedAt: createdAt, - StoppedAt: stoppedAt, - CachedSubdomainsCount: subdomainsCount, - CachedWebsitesCount: websitesCount, - CachedEndpointsCount: endpointsCount, - CachedIPsCount: ipsCount, - CachedDirectoriesCount: directoriesCount, - CachedScreenshotsCount: screenshotsCount, - CachedVulnsTotal: vulnsTotal, - CachedVulnsCritical: vulnsCritical, - CachedVulnsHigh: vulnsHigh, - CachedVulnsMedium: vulnsMedium, - CachedVulnsLow: vulnsLow, - } - - scans = append(scans, scan) - } - } - - // Batch insert - if err := db.CreateInBatches(scans, 100).Error; err != nil { - return err - } - - fmt.Printf(" āœ“ Created %d scans\n", len(scans)) - return nil -} diff --git a/go-backend/cmd/server/main.go b/go-backend/cmd/server/main.go index 6609d7a2..e788d965 100644 --- a/go-backend/cmd/server/main.go +++ b/go-backend/cmd/server/main.go @@ -136,7 +136,7 @@ func main() { subdomainRepo := repository.NewSubdomainRepository(db) endpointRepo := repository.NewEndpointRepository(db) directoryRepo := repository.NewDirectoryRepository(db) - ipAddressRepo := repository.NewIPAddressRepository(db) + hostPortRepo := repository.NewHostPortRepository(db) screenshotRepo := repository.NewScreenshotRepository(db) vulnerabilityRepo := repository.NewVulnerabilityRepository(db) scanRepo := repository.NewScanRepository(db) @@ -145,6 +145,7 @@ func main() { subdomainSnapshotRepo := repository.NewSubdomainSnapshotRepository(db) endpointSnapshotRepo := repository.NewEndpointSnapshotRepository(db) directorySnapshotRepo := repository.NewDirectorySnapshotRepository(db) + hostPortSnapshotRepo := repository.NewHostPortSnapshotRepository(db) // Create services userSvc := service.NewUserService(userRepo) @@ -156,7 +157,7 @@ func main() { subdomainSvc := service.NewSubdomainService(subdomainRepo, targetRepo) endpointSvc := service.NewEndpointService(endpointRepo, targetRepo) directorySvc := service.NewDirectoryService(directoryRepo, targetRepo) - ipAddressSvc := service.NewIPAddressService(ipAddressRepo, targetRepo) + hostPortSvc := service.NewHostPortService(hostPortRepo, targetRepo) screenshotSvc := service.NewScreenshotService(screenshotRepo, targetRepo) vulnerabilitySvc := service.NewVulnerabilityService(vulnerabilityRepo, targetRepo) scanSvc := service.NewScanService(scanRepo, scanLogRepo, targetRepo, orgRepo) @@ -165,6 +166,7 @@ func main() { subdomainSnapshotSvc := service.NewSubdomainSnapshotService(subdomainSnapshotRepo, scanRepo, subdomainSvc) endpointSnapshotSvc := service.NewEndpointSnapshotService(endpointSnapshotRepo, scanRepo, endpointSvc) directorySnapshotSvc := service.NewDirectorySnapshotService(directorySnapshotRepo, scanRepo, directorySvc) + hostPortSnapshotSvc := service.NewHostPortSnapshotService(hostPortSnapshotRepo, scanRepo, hostPortSvc) // Create handlers healthHandler := handler.NewHealthHandler(db, redisClient) @@ -178,7 +180,7 @@ func main() { subdomainHandler := handler.NewSubdomainHandler(subdomainSvc) endpointHandler := handler.NewEndpointHandler(endpointSvc) directoryHandler := handler.NewDirectoryHandler(directorySvc) - ipAddressHandler := handler.NewIPAddressHandler(ipAddressSvc) + hostPortHandler := handler.NewHostPortHandler(hostPortSvc) screenshotHandler := handler.NewScreenshotHandler(screenshotSvc) vulnerabilityHandler := handler.NewVulnerabilityHandler(vulnerabilitySvc) scanHandler := handler.NewScanHandler(scanSvc) @@ -187,6 +189,7 @@ func main() { subdomainSnapshotHandler := handler.NewSubdomainSnapshotHandler(subdomainSnapshotSvc) endpointSnapshotHandler := handler.NewEndpointSnapshotHandler(endpointSnapshotSvc) directorySnapshotHandler := handler.NewDirectorySnapshotHandler(directorySnapshotSvc) + hostPortSnapshotHandler := handler.NewHostPortSnapshotHandler(hostPortSnapshotSvc) // Register health routes router.GET("/health", healthHandler.Check) @@ -273,13 +276,13 @@ func main() { // Directories (standalone) protected.POST("/directories/bulk-delete", directoryHandler.BulkDelete) - // IP Addresses (nested under targets) - protected.GET("/targets/:id/ip-addresses", ipAddressHandler.List) - protected.GET("/targets/:id/ip-addresses/export", ipAddressHandler.Export) - protected.POST("/targets/:id/ip-addresses/bulk-upsert", ipAddressHandler.BulkUpsert) + // Host Ports (nested under targets) + protected.GET("/targets/:id/host-ports", hostPortHandler.List) + protected.GET("/targets/:id/host-ports/export", hostPortHandler.Export) + protected.POST("/targets/:id/host-ports/bulk-upsert", hostPortHandler.BulkUpsert) - // IP Addresses (standalone) - protected.POST("/ip-addresses/bulk-delete", ipAddressHandler.BulkDelete) + // Host Ports (standalone) + protected.POST("/host-ports/bulk-delete", hostPortHandler.BulkDelete) // Screenshots (nested under targets) protected.GET("/targets/:id/screenshots", screenshotHandler.ListByTargetID) @@ -355,6 +358,11 @@ func main() { protected.POST("/scans/:id/directories/bulk-upsert", directorySnapshotHandler.BulkUpsert) protected.GET("/scans/:id/directories", directorySnapshotHandler.List) protected.GET("/scans/:id/directories/export", directorySnapshotHandler.Export) + + // HostPort Snapshots (nested under scans) + protected.POST("/scans/:id/host-ports/bulk-upsert", hostPortSnapshotHandler.BulkUpsert) + protected.GET("/scans/:id/host-ports", hostPortSnapshotHandler.List) + protected.GET("/scans/:id/host-ports/export", hostPortSnapshotHandler.Export) } } diff --git a/go-backend/cmd/server/migrations/000001_init_schema.up.sql b/go-backend/cmd/server/migrations/000001_init_schema.up.sql index a8d88154..03cfb9de 100644 --- a/go-backend/cmd/server/migrations/000001_init_schema.up.sql +++ b/go-backend/cmd/server/migrations/000001_init_schema.up.sql @@ -332,11 +332,9 @@ CREATE TABLE IF NOT EXISTS directory ( target_id INTEGER NOT NULL REFERENCES target(id) ON DELETE CASCADE, url VARCHAR(2000) NOT NULL, status INTEGER, - content_length BIGINT, - words INTEGER, - lines INTEGER, + content_length INTEGER, content_type VARCHAR(200) NOT NULL DEFAULT '', - duration BIGINT, + duration INTEGER, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX IF NOT EXISTS idx_directory_target ON directory(target_id); @@ -370,6 +368,7 @@ CREATE TABLE IF NOT EXISTS vulnerability ( cvss_score DECIMAL(3,1) NOT NULL DEFAULT 0.0, description TEXT NOT NULL DEFAULT '', raw_output JSONB NOT NULL DEFAULT '{}', + reviewed BOOLEAN NOT NULL DEFAULT FALSE, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX IF NOT EXISTS idx_vuln_target ON vulnerability(target_id); @@ -378,6 +377,7 @@ CREATE INDEX IF NOT EXISTS idx_vuln_type ON vulnerability(vuln_type); CREATE INDEX IF NOT EXISTS idx_vuln_severity ON vulnerability(severity); CREATE INDEX IF NOT EXISTS idx_vuln_source ON vulnerability(source); CREATE INDEX IF NOT EXISTS idx_vuln_created_at ON vulnerability(created_at); +CREATE INDEX IF NOT EXISTS idx_vuln_target_reviewed ON vulnerability(target_id, reviewed); -- ============================================ -- Snapshot tables (depends on scan) @@ -419,7 +419,7 @@ CREATE TABLE IF NOT EXISTS website_snapshot ( host VARCHAR(253) NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', status_code INTEGER, - content_length BIGINT, + content_length INTEGER, location TEXT NOT NULL DEFAULT '', webserver TEXT NOT NULL DEFAULT '', content_type TEXT NOT NULL DEFAULT '', @@ -471,11 +471,9 @@ CREATE TABLE IF NOT EXISTS directory_snapshot ( scan_id INTEGER NOT NULL REFERENCES scan(id) ON DELETE CASCADE, url VARCHAR(2000) NOT NULL, status INTEGER, - content_length BIGINT, - words INTEGER, - lines INTEGER, + content_length INTEGER, content_type VARCHAR(200) NOT NULL DEFAULT '', - duration BIGINT, + duration INTEGER, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX IF NOT EXISTS idx_directory_snap_scan ON directory_snapshot(scan_id); @@ -574,3 +572,22 @@ CREATE INDEX IF NOT EXISTS idx_notification_category ON notification(category); CREATE INDEX IF NOT EXISTS idx_notification_level ON notification(level); CREATE INDEX IF NOT EXISTS idx_notification_created_at ON notification(created_at); CREATE INDEX IF NOT EXISTS idx_notification_is_read ON notification(is_read); + +-- ============================================ +-- GIN Indexes for array fields +-- ============================================ + +-- GIN index for website.tech array +CREATE INDEX IF NOT EXISTS idx_website_tech_gin ON website USING GIN (tech); + +-- GIN index for endpoint.tech array +CREATE INDEX IF NOT EXISTS idx_endpoint_tech_gin ON endpoint USING GIN (tech); + +-- GIN index for endpoint.matched_gf_patterns array +CREATE INDEX IF NOT EXISTS idx_endpoint_matched_gf_patterns_gin ON endpoint USING GIN (matched_gf_patterns); + +-- GIN index for scan.engine_ids array +CREATE INDEX IF NOT EXISTS idx_scan_engine_ids_gin ON scan USING GIN (engine_ids); + +-- GIN index for scan.container_ids array +CREATE INDEX IF NOT EXISTS idx_scan_container_ids_gin ON scan USING GIN (container_ids); diff --git a/go-backend/cmd/server/migrations/000002_add_gin_indexes.down.sql b/go-backend/cmd/server/migrations/000002_add_gin_indexes.down.sql deleted file mode 100644 index 9c9e6122..00000000 --- a/go-backend/cmd/server/migrations/000002_add_gin_indexes.down.sql +++ /dev/null @@ -1,11 +0,0 @@ --- Remove GIN indexes - -DROP INDEX IF EXISTS idx_scheduled_scan_engine_ids_gin; -DROP INDEX IF EXISTS idx_scan_container_ids_gin; -DROP INDEX IF EXISTS idx_scan_engine_ids_gin; -DROP INDEX IF EXISTS idx_endpoint_snap_matched_gf_patterns_gin; -DROP INDEX IF EXISTS idx_endpoint_snap_tech_gin; -DROP INDEX IF EXISTS idx_website_snap_tech_gin; -DROP INDEX IF EXISTS idx_endpoint_matched_gf_patterns_gin; -DROP INDEX IF EXISTS idx_endpoint_tech_gin; -DROP INDEX IF EXISTS idx_website_tech_gin; diff --git a/go-backend/cmd/server/migrations/000002_add_gin_indexes.up.sql b/go-backend/cmd/server/migrations/000002_add_gin_indexes.up.sql deleted file mode 100644 index 227fe56a..00000000 --- a/go-backend/cmd/server/migrations/000002_add_gin_indexes.up.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Add GIN indexes for PostgreSQL array fields --- GIN indexes enable efficient queries on array columns (e.g., @>, &&, etc.) - --- Website tech array -CREATE INDEX IF NOT EXISTS idx_website_tech_gin ON website USING GIN (tech); - --- Endpoint arrays -CREATE INDEX IF NOT EXISTS idx_endpoint_tech_gin ON endpoint USING GIN (tech); -CREATE INDEX IF NOT EXISTS idx_endpoint_matched_gf_patterns_gin ON endpoint USING GIN (matched_gf_patterns); - --- Website snapshot tech array -CREATE INDEX IF NOT EXISTS idx_website_snap_tech_gin ON website_snapshot USING GIN (tech); - --- Endpoint snapshot arrays -CREATE INDEX IF NOT EXISTS idx_endpoint_snap_tech_gin ON endpoint_snapshot USING GIN (tech); -CREATE INDEX IF NOT EXISTS idx_endpoint_snap_matched_gf_patterns_gin ON endpoint_snapshot USING GIN (matched_gf_patterns); - --- Scan arrays -CREATE INDEX IF NOT EXISTS idx_scan_engine_ids_gin ON scan USING GIN (engine_ids); -CREATE INDEX IF NOT EXISTS idx_scan_container_ids_gin ON scan USING GIN (container_ids); - --- Scheduled scan arrays -CREATE INDEX IF NOT EXISTS idx_scheduled_scan_engine_ids_gin ON scheduled_scan USING GIN (engine_ids); diff --git a/go-backend/cmd/server/migrations/000003_drop_directory_words_lines.down.sql b/go-backend/cmd/server/migrations/000003_drop_directory_words_lines.down.sql deleted file mode 100644 index 335893d6..00000000 --- a/go-backend/cmd/server/migrations/000003_drop_directory_words_lines.down.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Restore words and lines columns to directory table -ALTER TABLE directory ADD COLUMN IF NOT EXISTS words INTEGER; -ALTER TABLE directory ADD COLUMN IF NOT EXISTS lines INTEGER; diff --git a/go-backend/cmd/server/migrations/000003_drop_directory_words_lines.up.sql b/go-backend/cmd/server/migrations/000003_drop_directory_words_lines.up.sql deleted file mode 100644 index 922711f3..00000000 --- a/go-backend/cmd/server/migrations/000003_drop_directory_words_lines.up.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Drop words and lines columns from directory table -ALTER TABLE directory DROP COLUMN IF EXISTS words; -ALTER TABLE directory DROP COLUMN IF EXISTS lines; diff --git a/go-backend/cmd/server/migrations/000004_add_vulnerability_review.down.sql b/go-backend/cmd/server/migrations/000004_add_vulnerability_review.down.sql deleted file mode 100644 index 2b8b1ad9..00000000 --- a/go-backend/cmd/server/migrations/000004_add_vulnerability_review.down.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Remove review status fields from vulnerability table -DROP INDEX IF EXISTS idx_vuln_is_reviewed; -ALTER TABLE vulnerability -DROP COLUMN IF EXISTS reviewed_at, -DROP COLUMN IF EXISTS is_reviewed; diff --git a/go-backend/cmd/server/migrations/000004_add_vulnerability_review.up.sql b/go-backend/cmd/server/migrations/000004_add_vulnerability_review.up.sql deleted file mode 100644 index 4a03253a..00000000 --- a/go-backend/cmd/server/migrations/000004_add_vulnerability_review.up.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Add review status fields to vulnerability table -ALTER TABLE vulnerability -ADD COLUMN is_reviewed BOOLEAN NOT NULL DEFAULT FALSE, -ADD COLUMN reviewed_at TIMESTAMP NULL; - --- Create index for filtering by review status -CREATE INDEX idx_vuln_is_reviewed ON vulnerability(is_reviewed); diff --git a/go-backend/cmd/server/migrations/000005_add_vuln_target_reviewed_index.down.sql b/go-backend/cmd/server/migrations/000005_add_vuln_target_reviewed_index.down.sql deleted file mode 100644 index 56332d2a..00000000 --- a/go-backend/cmd/server/migrations/000005_add_vuln_target_reviewed_index.down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Remove composite index -DROP INDEX IF EXISTS idx_vuln_target_reviewed; diff --git a/go-backend/cmd/server/migrations/000005_add_vuln_target_reviewed_index.up.sql b/go-backend/cmd/server/migrations/000005_add_vuln_target_reviewed_index.up.sql deleted file mode 100644 index 711dfea7..00000000 --- a/go-backend/cmd/server/migrations/000005_add_vuln_target_reviewed_index.up.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Add composite index for target_id + is_reviewed queries --- Optimizes: COUNT pending vulnerabilities by target, filter by target + review status -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_vuln_target_reviewed ON vulnerability(target_id, is_reviewed); diff --git a/go-backend/docker-compose.dev.yml b/go-backend/docker-compose.dev.yml index 26c86a8d..004bc879 100644 --- a/go-backend/docker-compose.dev.yml +++ b/go-backend/docker-compose.dev.yml @@ -11,11 +11,11 @@ services: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres ports: - - "5432:5432" + - 5432:5432 volumes: - postgres_data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + test: [CMD-SHELL, pg_isready -U postgres] interval: 5s timeout: 5s retries: 5 @@ -24,9 +24,9 @@ services: image: redis:7-alpine restart: unless-stopped ports: - - "6379:6379" + - 6379:6379 healthcheck: - test: ["CMD", "redis-cli", "ping"] + test: [CMD, redis-cli, ping] interval: 5s timeout: 5s retries: 5 diff --git a/go-backend/go.mod b/go-backend/go.mod index d6d8bc4d..b6375c57 100644 --- a/go-backend/go.mod +++ b/go-backend/go.mod @@ -2,7 +2,7 @@ module github.com/xingrin/go-backend go 1.24.0 -toolchain go1.24.6 +toolchain go1.24.5 require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 diff --git a/go-backend/internal/dto/directory.go b/go-backend/internal/dto/directory.go index e0ec1fb7..7b28bb74 100644 --- a/go-backend/internal/dto/directory.go +++ b/go-backend/internal/dto/directory.go @@ -14,9 +14,9 @@ type DirectoryResponse struct { TargetID int `json:"targetId"` URL string `json:"url"` Status *int `json:"status"` - ContentLength *int64 `json:"contentLength"` + ContentLength *int `json:"contentLength"` ContentType string `json:"contentType"` - Duration *int64 `json:"duration"` + Duration *int `json:"duration"` CreatedAt time.Time `json:"createdAt"` } @@ -34,9 +34,9 @@ type BulkCreateDirectoriesResponse struct { type DirectoryUpsertItem struct { URL string `json:"url" binding:"required,url"` Status *int `json:"status"` - ContentLength *int64 `json:"contentLength"` + ContentLength *int `json:"contentLength"` ContentType string `json:"contentType"` - Duration *int64 `json:"duration"` + Duration *int `json:"duration"` } // BulkUpsertDirectoriesRequest represents bulk upsert directories request diff --git a/go-backend/internal/dto/directory_snapshot.go b/go-backend/internal/dto/directory_snapshot.go index 58e995b9..77480781 100644 --- a/go-backend/internal/dto/directory_snapshot.go +++ b/go-backend/internal/dto/directory_snapshot.go @@ -2,16 +2,9 @@ package dto import "time" -// DirectorySnapshotItem represents a single directory snapshot data for bulk upsert -type DirectorySnapshotItem struct { - URL string `json:"url" binding:"required,url"` - Status *int `json:"status"` - ContentLength *int64 `json:"contentLength"` - Words *int `json:"words"` - Lines *int `json:"lines"` - ContentType string `json:"contentType"` - Duration *int64 `json:"duration"` -} +// DirectorySnapshotItem is an alias for DirectoryUpsertItem +// Snapshot items and asset items have identical fields +type DirectorySnapshotItem = DirectoryUpsertItem // BulkUpsertDirectorySnapshotsRequest represents bulk upsert directory snapshots request type BulkUpsertDirectorySnapshotsRequest struct { @@ -37,10 +30,8 @@ type DirectorySnapshotResponse struct { ScanID int `json:"scanId"` URL string `json:"url"` Status *int `json:"status"` - ContentLength *int64 `json:"contentLength"` - Words *int `json:"words"` - Lines *int `json:"lines"` + ContentLength *int `json:"contentLength"` ContentType string `json:"contentType"` - Duration *int64 `json:"duration"` + Duration *int `json:"duration"` CreatedAt time.Time `json:"createdAt"` } diff --git a/go-backend/internal/dto/host_port.go b/go-backend/internal/dto/host_port.go new file mode 100644 index 00000000..bbafb026 --- /dev/null +++ b/go-backend/internal/dto/host_port.go @@ -0,0 +1,44 @@ +package dto + +import "time" + +// HostPortListQuery represents host-port list query parameters +type HostPortListQuery struct { + PaginationQuery + Filter string `form:"filter"` +} + +// HostPortResponse represents aggregated host-port response (grouped by IP) +type HostPortResponse struct { + IP string `json:"ip"` + Hosts []string `json:"hosts"` + Ports []int `json:"ports"` + CreatedAt time.Time `json:"createdAt"` +} + +// HostPortItem represents a single host-port mapping for bulk operations +type HostPortItem struct { + Host string `json:"host" binding:"required"` + IP string `json:"ip" binding:"required,ip"` + Port int `json:"port" binding:"required,min=1,max=65535"` +} + +// BulkUpsertHostPortsRequest represents bulk upsert request (for scanner import) +type BulkUpsertHostPortsRequest struct { + Mappings []HostPortItem `json:"mappings" binding:"required,min=1,max=5000,dive"` +} + +// BulkUpsertHostPortsResponse represents bulk upsert response +type BulkUpsertHostPortsResponse struct { + UpsertedCount int `json:"upsertedCount"` +} + +// BulkDeleteHostPortsRequest represents bulk delete request (by IP list) +type BulkDeleteHostPortsRequest struct { + IPs []string `json:"ips" binding:"required,min=1"` +} + +// BulkDeleteHostPortsResponse represents bulk delete response +type BulkDeleteHostPortsResponse struct { + DeletedCount int64 `json:"deletedCount"` +} diff --git a/go-backend/internal/dto/host_port_snapshot.go b/go-backend/internal/dto/host_port_snapshot.go new file mode 100644 index 00000000..5bb88bfd --- /dev/null +++ b/go-backend/internal/dto/host_port_snapshot.go @@ -0,0 +1,30 @@ +package dto + +import "time" + +// HostPortSnapshotItem is an alias for HostPortItem used in snapshot operations +type HostPortSnapshotItem = HostPortItem + +type BulkUpsertHostPortSnapshotsRequest struct { + TargetID int `json:"targetId" binding:"required"` + HostPorts []HostPortSnapshotItem `json:"hostPorts" binding:"required,min=1,max=5000,dive"` +} + +type BulkUpsertHostPortSnapshotsResponse struct { + SnapshotCount int `json:"snapshotCount"` + AssetCount int `json:"assetCount"` +} + +type HostPortSnapshotListQuery struct { + PaginationQuery + Filter string `form:"filter"` +} + +type HostPortSnapshotResponse struct { + ID int `json:"id"` + ScanID int `json:"scanId"` + Host string `json:"host"` + IP string `json:"ip"` + Port int `json:"port"` + CreatedAt time.Time `json:"createdAt"` +} diff --git a/go-backend/internal/dto/ip_address.go b/go-backend/internal/dto/ip_address.go deleted file mode 100644 index 17e765b4..00000000 --- a/go-backend/internal/dto/ip_address.go +++ /dev/null @@ -1,44 +0,0 @@ -package dto - -import "time" - -// IPAddressListQuery represents IP address list query parameters -type IPAddressListQuery struct { - PaginationQuery - Filter string `form:"filter"` -} - -// IPAddressResponse represents aggregated IP address response (grouped by IP) -type IPAddressResponse struct { - IP string `json:"ip"` - Hosts []string `json:"hosts"` - Ports []int `json:"ports"` - CreatedAt time.Time `json:"createdAt"` -} - -// IPAddressItem represents a single IP address mapping for bulk operations -type IPAddressItem struct { - Host string `json:"host" binding:"required"` - IP string `json:"ip" binding:"required,ip"` - Port int `json:"port" binding:"required,min=1,max=65535"` -} - -// BulkUpsertIPAddressesRequest represents bulk upsert request (for scanner import) -type BulkUpsertIPAddressesRequest struct { - Mappings []IPAddressItem `json:"mappings" binding:"required,min=1,max=5000,dive"` -} - -// BulkUpsertIPAddressesResponse represents bulk upsert response -type BulkUpsertIPAddressesResponse struct { - UpsertedCount int `json:"upsertedCount"` -} - -// BulkDeleteIPAddressesRequest represents bulk delete request (by IP list) -type BulkDeleteIPAddressesRequest struct { - IPs []string `json:"ips" binding:"required,min=1"` -} - -// BulkDeleteIPAddressesResponse represents bulk delete response -type BulkDeleteIPAddressesResponse struct { - DeletedCount int64 `json:"deletedCount"` -} diff --git a/go-backend/internal/dto/website_snapshot.go b/go-backend/internal/dto/website_snapshot.go index b0e8e614..ca926a97 100644 --- a/go-backend/internal/dto/website_snapshot.go +++ b/go-backend/internal/dto/website_snapshot.go @@ -2,21 +2,8 @@ package dto import "time" -// WebsiteSnapshotItem represents a single website snapshot data for bulk upsert -type WebsiteSnapshotItem struct { - URL string `json:"url" binding:"required,url"` - Host string `json:"host"` - Title string `json:"title"` - StatusCode *int `json:"statusCode"` - ContentLength *int64 `json:"contentLength"` - Location string `json:"location"` - Webserver string `json:"webserver"` - ContentType string `json:"contentType"` - Tech []string `json:"tech"` - ResponseBody string `json:"responseBody"` - Vhost *bool `json:"vhost"` - ResponseHeaders string `json:"responseHeaders"` -} +// WebsiteSnapshotItem is an alias for WebsiteUpsertItem used in snapshot operations +type WebsiteSnapshotItem = WebsiteUpsertItem // BulkUpsertWebsiteSnapshotsRequest represents bulk upsert website snapshots request type BulkUpsertWebsiteSnapshotsRequest struct { @@ -44,7 +31,7 @@ type WebsiteSnapshotResponse struct { Host string `json:"host"` Title string `json:"title"` StatusCode *int `json:"statusCode"` - ContentLength *int64 `json:"contentLength"` + ContentLength *int `json:"contentLength"` Location string `json:"location"` Webserver string `json:"webserver"` ContentType string `json:"contentType"` diff --git a/go-backend/internal/handler/directory.go b/go-backend/internal/handler/directory.go index 8aa10fc8..7f7d3403 100644 --- a/go-backend/internal/handler/directory.go +++ b/go-backend/internal/handler/directory.go @@ -148,12 +148,12 @@ func (h *DirectoryHandler) Export(c *gin.Context) { contentLength := "" if directory.ContentLength != nil { - contentLength = strconv.FormatInt(*directory.ContentLength, 10) + contentLength = strconv.Itoa(*directory.ContentLength) } duration := "" if directory.Duration != nil { - duration = strconv.FormatInt(*directory.Duration, 10) + duration = strconv.Itoa(*directory.Duration) } return []string{ diff --git a/go-backend/internal/handler/directory_snapshot.go b/go-backend/internal/handler/directory_snapshot.go index ae50dfb7..1634159b 100644 --- a/go-backend/internal/handler/directory_snapshot.go +++ b/go-backend/internal/handler/directory_snapshot.go @@ -112,7 +112,7 @@ func (h *DirectorySnapshotHandler) Export(c *gin.Context) { headers := []string{ "id", "scan_id", "url", "status", "content_length", - "words", "lines", "content_type", "duration", "created_at", + "content_type", "duration", "created_at", } filename := fmt.Sprintf("scan-%d-directories.csv", scanID) @@ -129,22 +129,12 @@ func (h *DirectorySnapshotHandler) Export(c *gin.Context) { contentLength := "" if snapshot.ContentLength != nil { - contentLength = strconv.FormatInt(*snapshot.ContentLength, 10) - } - - words := "" - if snapshot.Words != nil { - words = strconv.Itoa(*snapshot.Words) - } - - lines := "" - if snapshot.Lines != nil { - lines = strconv.Itoa(*snapshot.Lines) + contentLength = strconv.Itoa(*snapshot.ContentLength) } duration := "" if snapshot.Duration != nil { - duration = strconv.FormatInt(*snapshot.Duration, 10) + duration = strconv.Itoa(*snapshot.Duration) } return []string{ @@ -153,8 +143,6 @@ func (h *DirectorySnapshotHandler) Export(c *gin.Context) { snapshot.URL, status, contentLength, - words, - lines, snapshot.ContentType, duration, snapshot.CreatedAt.Format("2006-01-02 15:04:05"), @@ -173,8 +161,6 @@ func toDirectorySnapshotResponse(s *model.DirectorySnapshot) dto.DirectorySnapsh URL: s.URL, Status: s.Status, ContentLength: s.ContentLength, - Words: s.Words, - Lines: s.Lines, ContentType: s.ContentType, Duration: s.Duration, CreatedAt: s.CreatedAt, diff --git a/go-backend/internal/handler/ip_address.go b/go-backend/internal/handler/host_port.go similarity index 65% rename from go-backend/internal/handler/ip_address.go rename to go-backend/internal/handler/host_port.go index 833d0310..1bb9cb91 100644 --- a/go-backend/internal/handler/ip_address.go +++ b/go-backend/internal/handler/host_port.go @@ -13,26 +13,26 @@ import ( "github.com/xingrin/go-backend/internal/service" ) -// IPAddressHandler handles IP address endpoints -type IPAddressHandler struct { - svc *service.IPAddressService +// HostPortHandler handles host-port endpoints +type HostPortHandler struct { + svc *service.HostPortService } -// NewIPAddressHandler creates a new IP address handler -func NewIPAddressHandler(svc *service.IPAddressService) *IPAddressHandler { - return &IPAddressHandler{svc: svc} +// NewHostPortHandler creates a new host-port handler +func NewHostPortHandler(svc *service.HostPortService) *HostPortHandler { + return &HostPortHandler{svc: svc} } -// List returns paginated IP addresses aggregated by IP -// GET /api/targets/:id/ip-addresses -func (h *IPAddressHandler) List(c *gin.Context) { +// List returns paginated host-ports aggregated by IP +// GET /api/targets/:id/host-ports +func (h *HostPortHandler) List(c *gin.Context) { targetID, err := strconv.Atoi(c.Param("id")) if err != nil { dto.BadRequest(c, "Invalid target ID") return } - var query dto.IPAddressListQuery + var query dto.HostPortListQuery if !dto.BindQuery(c, &query) { return } @@ -43,7 +43,7 @@ func (h *IPAddressHandler) List(c *gin.Context) { dto.NotFound(c, "Target not found") return } - dto.InternalError(c, "Failed to list IP addresses") + dto.InternalError(c, "Failed to list host-ports") return } @@ -60,10 +60,10 @@ func (h *IPAddressHandler) List(c *gin.Context) { dto.Paginated(c, results, total, query.GetPage(), query.GetPageSize()) } -// Export exports IP addresses as CSV (raw format) -// GET /api/targets/:id/ip-addresses/export +// Export exports host-ports as CSV (raw format) +// GET /api/targets/:id/host-ports/export // Query params: ips (optional, comma-separated IP list for filtering) -func (h *IPAddressHandler) Export(c *gin.Context) { +func (h *HostPortHandler) Export(c *gin.Context) { targetID, err := strconv.Atoi(c.Param("id")) if err != nil { dto.BadRequest(c, "Invalid target ID") @@ -91,7 +91,7 @@ func (h *IPAddressHandler) Export(c *gin.Context) { dto.NotFound(c, "Target not found") return } - dto.InternalError(c, "Failed to export IP addresses") + dto.InternalError(c, "Failed to export host-ports") return } rows, err = h.svc.StreamByTarget(targetID) @@ -102,12 +102,12 @@ func (h *IPAddressHandler) Export(c *gin.Context) { dto.NotFound(c, "Target not found") return } - dto.InternalError(c, "Failed to export IP addresses") + dto.InternalError(c, "Failed to export host-ports") return } headers := []string{"ip", "host", "port", "created_at"} - filename := fmt.Sprintf("target-%d-ip-addresses.csv", targetID) + filename := fmt.Sprintf("target-%d-host-ports.csv", targetID) mapper := func(rows *sql.Rows) ([]string, error) { mapping, err := h.svc.ScanRow(rows) @@ -128,16 +128,16 @@ func (h *IPAddressHandler) Export(c *gin.Context) { } } -// BulkUpsert creates multiple IP address mappings (ignores duplicates) -// POST /api/targets/:id/ip-addresses/bulk-upsert -func (h *IPAddressHandler) BulkUpsert(c *gin.Context) { +// BulkUpsert creates multiple host-port mappings (ignores duplicates) +// POST /api/targets/:id/host-ports/bulk-upsert +func (h *HostPortHandler) BulkUpsert(c *gin.Context) { targetID, err := strconv.Atoi(c.Param("id")) if err != nil { dto.BadRequest(c, "Invalid target ID") return } - var req dto.BulkUpsertIPAddressesRequest + var req dto.BulkUpsertHostPortsRequest if !dto.BindJSON(c, &req) { return } @@ -148,30 +148,30 @@ func (h *IPAddressHandler) BulkUpsert(c *gin.Context) { dto.NotFound(c, "Target not found") return } - dto.InternalError(c, "Failed to upsert IP addresses") + dto.InternalError(c, "Failed to upsert host-ports") return } - dto.Success(c, dto.BulkUpsertIPAddressesResponse{ + dto.Success(c, dto.BulkUpsertHostPortsResponse{ UpsertedCount: int(upsertedCount), }) } -// BulkDelete deletes IP address mappings by IP list -// POST /api/ip-addresses/bulk-delete -func (h *IPAddressHandler) BulkDelete(c *gin.Context) { - var req dto.BulkDeleteIPAddressesRequest +// BulkDelete deletes host-port mappings by IP list +// POST /api/host-ports/bulk-delete +func (h *HostPortHandler) BulkDelete(c *gin.Context) { + var req dto.BulkDeleteHostPortsRequest if !dto.BindJSON(c, &req) { return } deletedCount, err := h.svc.BulkDeleteByIPs(req.IPs) if err != nil { - dto.InternalError(c, "Failed to delete IP addresses") + dto.InternalError(c, "Failed to delete host-ports") return } - dto.Success(c, dto.BulkDeleteIPAddressesResponse{ + dto.Success(c, dto.BulkDeleteHostPortsResponse{ DeletedCount: deletedCount, }) } diff --git a/go-backend/internal/handler/host_port_snapshot.go b/go-backend/internal/handler/host_port_snapshot.go new file mode 100644 index 00000000..b8365878 --- /dev/null +++ b/go-backend/internal/handler/host_port_snapshot.go @@ -0,0 +1,147 @@ +package handler + +import ( + "database/sql" + "errors" + "fmt" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/xingrin/go-backend/internal/dto" + "github.com/xingrin/go-backend/internal/model" + "github.com/xingrin/go-backend/internal/pkg/csv" + "github.com/xingrin/go-backend/internal/service" +) + +// HostPortSnapshotHandler handles host-port snapshot endpoints +type HostPortSnapshotHandler struct { + svc *service.HostPortSnapshotService +} + +// NewHostPortSnapshotHandler creates a new host-port snapshot handler +func NewHostPortSnapshotHandler(svc *service.HostPortSnapshotService) *HostPortSnapshotHandler { + return &HostPortSnapshotHandler{svc: svc} +} + +// BulkUpsert creates host-port snapshots and syncs to asset table +// POST /api/scans/:id/host-ports/bulk-upsert +func (h *HostPortSnapshotHandler) BulkUpsert(c *gin.Context) { + scanID, err := strconv.Atoi(c.Param("id")) + if err != nil { + dto.BadRequest(c, "Invalid scan ID") + return + } + + var req dto.BulkUpsertHostPortSnapshotsRequest + if !dto.BindJSON(c, &req) { + return + } + + snapshotCount, assetCount, err := h.svc.SaveAndSync(scanID, req.TargetID, req.HostPorts) + if err != nil { + if errors.Is(err, service.ErrScanNotFoundForSnapshot) { + dto.NotFound(c, "Scan not found") + return + } + dto.InternalError(c, "Failed to save snapshots") + return + } + + dto.Success(c, dto.BulkUpsertHostPortSnapshotsResponse{ + SnapshotCount: int(snapshotCount), + AssetCount: int(assetCount), + }) +} + +// List returns paginated host-port snapshots for a scan +// GET /api/scans/:id/host-ports +func (h *HostPortSnapshotHandler) List(c *gin.Context) { + scanID, err := strconv.Atoi(c.Param("id")) + if err != nil { + dto.BadRequest(c, "Invalid scan ID") + return + } + + var query dto.HostPortSnapshotListQuery + if !dto.BindQuery(c, &query) { + return + } + + snapshots, total, err := h.svc.ListByScan(scanID, &query) + if err != nil { + if errors.Is(err, service.ErrScanNotFoundForSnapshot) { + dto.NotFound(c, "Scan not found") + return + } + dto.InternalError(c, "Failed to list snapshots") + return + } + + var resp []dto.HostPortSnapshotResponse + for _, s := range snapshots { + resp = append(resp, toHostPortSnapshotResponse(&s)) + } + + dto.Paginated(c, resp, total, query.GetPage(), query.GetPageSize()) +} + +// Export exports host-port snapshots as CSV +// GET /api/scans/:id/host-ports/export +func (h *HostPortSnapshotHandler) Export(c *gin.Context) { + scanID, err := strconv.Atoi(c.Param("id")) + if err != nil { + dto.BadRequest(c, "Invalid scan ID") + return + } + + count, err := h.svc.CountByScan(scanID) + if err != nil { + if errors.Is(err, service.ErrScanNotFoundForSnapshot) { + dto.NotFound(c, "Scan not found") + return + } + dto.InternalError(c, "Failed to export snapshots") + return + } + + rows, err := h.svc.StreamByScan(scanID) + if err != nil { + dto.InternalError(c, "Failed to export snapshots") + return + } + + headers := []string{"id", "scan_id", "host", "ip", "port", "created_at"} + filename := fmt.Sprintf("scan-%d-host-ports.csv", scanID) + + mapper := func(rows *sql.Rows) ([]string, error) { + snapshot, err := h.svc.ScanRow(rows) + if err != nil { + return nil, err + } + + return []string{ + strconv.Itoa(snapshot.ID), + strconv.Itoa(snapshot.ScanID), + snapshot.Host, + snapshot.IP, + strconv.Itoa(snapshot.Port), + snapshot.CreatedAt.Format("2006-01-02 15:04:05"), + }, nil + } + + if err := csv.StreamCSV(c, rows, headers, filename, mapper, count); err != nil { + return + } +} + +// toHostPortSnapshotResponse converts model to response DTO +func toHostPortSnapshotResponse(s *model.HostPortSnapshot) dto.HostPortSnapshotResponse { + return dto.HostPortSnapshotResponse{ + ID: s.ID, + ScanID: s.ScanID, + Host: s.Host, + IP: s.IP, + Port: s.Port, + CreatedAt: s.CreatedAt, + } +} diff --git a/go-backend/internal/handler/website_snapshot.go b/go-backend/internal/handler/website_snapshot.go index cc5e0cd4..7a2a1363 100644 --- a/go-backend/internal/handler/website_snapshot.go +++ b/go-backend/internal/handler/website_snapshot.go @@ -134,7 +134,7 @@ func (h *WebsiteSnapshotHandler) Export(c *gin.Context) { contentLength := "" if snapshot.ContentLength != nil { - contentLength = strconv.FormatInt(*snapshot.ContentLength, 10) + contentLength = strconv.Itoa(*snapshot.ContentLength) } vhost := "" diff --git a/go-backend/internal/handler/website_snapshot_test.go b/go-backend/internal/handler/website_snapshot_test.go index c1f66871..5a238951 100644 --- a/go-backend/internal/handler/website_snapshot_test.go +++ b/go-backend/internal/handler/website_snapshot_test.go @@ -235,19 +235,7 @@ func TestListHandler(t *testing.T) { } }, }, - { - name: "list with ordering", - scanID: "1", - queryParams: "?ordering=-url", - mockFunc: func(scanID int, query *dto.WebsiteSnapshotListQuery) ([]model.WebsiteSnapshot, int64, error) { - if query.Ordering != "-url" { - t.Errorf("expected ordering '-url', got %q", query.Ordering) - } - return mockSnapshots, 3, nil - }, - expectedStatus: http.StatusOK, - checkResponse: nil, - }, + { name: "scan not found", scanID: "999", @@ -373,8 +361,8 @@ func TestFilterProperties(t *testing.T) { router := gin.New() router.GET("/api/scans/:scan_id/websites", func(c *gin.Context) { var query dto.WebsiteSnapshotListQuery - c.ShouldBindQuery(&query) - mockSvc.ListByScan(1, &query) + _ = c.ShouldBindQuery(&query) + _, _, _ = mockSvc.ListByScan(1, &query) dto.Paginated(c, []dto.WebsiteSnapshotResponse{}, 0, 1, 20) }) diff --git a/go-backend/internal/handler/wordlist.go b/go-backend/internal/handler/wordlist.go index 1c7dabd6..b84fe260 100644 --- a/go-backend/internal/handler/wordlist.go +++ b/go-backend/internal/handler/wordlist.go @@ -38,7 +38,9 @@ func (h *WordlistHandler) Create(c *gin.Context) { dto.InternalError(c, "Failed to read uploaded file") return } - defer src.Close() + defer func() { + _ = src.Close() // Ignore close error in defer + }() wordlist, err := h.svc.Create(name, description, file.Filename, src) if err != nil { diff --git a/go-backend/internal/model/directory.go b/go-backend/internal/model/directory.go index 21a0ca9e..c79588cc 100644 --- a/go-backend/internal/model/directory.go +++ b/go-backend/internal/model/directory.go @@ -10,9 +10,9 @@ type Directory struct { TargetID int `gorm:"column:target_id;not null;index:idx_directory_target;uniqueIndex:unique_directory_url_target,priority:1" json:"targetId"` URL string `gorm:"column:url;size:2000;not null;index:idx_directory_url;uniqueIndex:unique_directory_url_target,priority:2" json:"url"` Status *int `gorm:"column:status;index:idx_directory_status" json:"status"` - ContentLength *int64 `gorm:"column:content_length" json:"contentLength"` + ContentLength *int `gorm:"column:content_length" json:"contentLength"` ContentType string `gorm:"column:content_type;size:200" json:"contentType"` - Duration *int64 `gorm:"column:duration" json:"duration"` + Duration *int `gorm:"column:duration" json:"duration"` CreatedAt time.Time `gorm:"column:created_at;autoCreateTime;index:idx_directory_created_at" json:"createdAt"` // Relationships diff --git a/go-backend/internal/model/directory_snapshot.go b/go-backend/internal/model/directory_snapshot.go index 015e3e71..4550478f 100644 --- a/go-backend/internal/model/directory_snapshot.go +++ b/go-backend/internal/model/directory_snapshot.go @@ -10,11 +10,9 @@ type DirectorySnapshot struct { ScanID int `gorm:"column:scan_id;not null;index:idx_directory_snap_scan;uniqueIndex:unique_directory_per_scan_snapshot,priority:1" json:"scanId"` URL string `gorm:"column:url;size:2000;index:idx_directory_snap_url;uniqueIndex:unique_directory_per_scan_snapshot,priority:2" json:"url"` Status *int `gorm:"column:status;index:idx_directory_snap_status" json:"status"` - ContentLength *int64 `gorm:"column:content_length" json:"contentLength"` - Words *int `gorm:"column:words" json:"words"` - Lines *int `gorm:"column:lines" json:"lines"` + ContentLength *int `gorm:"column:content_length" json:"contentLength"` ContentType string `gorm:"column:content_type;size:200;index:idx_directory_snap_content_type" json:"contentType"` - Duration *int64 `gorm:"column:duration" json:"duration"` + Duration *int `gorm:"column:duration" json:"duration"` CreatedAt time.Time `gorm:"column:created_at;autoCreateTime;index:idx_directory_snap_created_at" json:"createdAt"` // Relationships diff --git a/go-backend/internal/model/host_port_mapping.go b/go-backend/internal/model/host_port.go similarity index 83% rename from go-backend/internal/model/host_port_mapping.go rename to go-backend/internal/model/host_port.go index 51900a4c..5a2faaa5 100644 --- a/go-backend/internal/model/host_port_mapping.go +++ b/go-backend/internal/model/host_port.go @@ -4,8 +4,8 @@ import ( "time" ) -// HostPortMapping represents a host-port mapping -type HostPortMapping struct { +// HostPort represents a host-port mapping +type HostPort struct { ID int `gorm:"primaryKey;autoIncrement" json:"id"` TargetID int `gorm:"column:target_id;not null;index:idx_hpm_target;uniqueIndex:unique_target_host_ip_port,priority:1" json:"targetId"` Host string `gorm:"column:host;size:1000;not null;index:idx_hpm_host;uniqueIndex:unique_target_host_ip_port,priority:2" json:"host"` @@ -17,7 +17,7 @@ type HostPortMapping struct { Target *Target `gorm:"foreignKey:TargetID" json:"target,omitempty"` } -// TableName returns the table name for HostPortMapping -func (HostPortMapping) TableName() string { +// TableName returns the table name for HostPort +func (HostPort) TableName() string { return "host_port_mapping" } diff --git a/go-backend/internal/model/host_port_mapping_snapshot.go b/go-backend/internal/model/host_port_snapshot.go similarity index 80% rename from go-backend/internal/model/host_port_mapping_snapshot.go rename to go-backend/internal/model/host_port_snapshot.go index ecec9b65..3e9407dc 100644 --- a/go-backend/internal/model/host_port_mapping_snapshot.go +++ b/go-backend/internal/model/host_port_snapshot.go @@ -4,8 +4,8 @@ import ( "time" ) -// HostPortMappingSnapshot represents a host-port mapping snapshot -type HostPortMappingSnapshot struct { +// HostPortSnapshot represents a host-port snapshot +type HostPortSnapshot struct { ID int `gorm:"primaryKey;autoIncrement" json:"id"` ScanID int `gorm:"column:scan_id;not null;index:idx_hpm_snap_scan;uniqueIndex:unique_scan_host_ip_port_snapshot,priority:1" json:"scanId"` Host string `gorm:"column:host;size:1000;not null;index:idx_hpm_snap_host;uniqueIndex:unique_scan_host_ip_port_snapshot,priority:2" json:"host"` @@ -17,7 +17,7 @@ type HostPortMappingSnapshot struct { Scan *Scan `gorm:"foreignKey:ScanID" json:"scan,omitempty"` } -// TableName returns the table name for HostPortMappingSnapshot -func (HostPortMappingSnapshot) TableName() string { +// TableName returns the table name for HostPortSnapshot +func (HostPortSnapshot) TableName() string { return "host_port_mapping_snapshot" } diff --git a/go-backend/internal/model/model_test.go b/go-backend/internal/model/model_test.go index b9760754..638ddc6c 100644 --- a/go-backend/internal/model/model_test.go +++ b/go-backend/internal/model/model_test.go @@ -33,7 +33,7 @@ func TestTableNames(t *testing.T) { // Asset models {Endpoint{}, "endpoint"}, {Directory{}, "directory"}, - {HostPortMapping{}, "host_port_mapping"}, + {HostPort{}, "host_port_mapping"}, {Vulnerability{}, "vulnerability"}, {Screenshot{}, "screenshot"}, // Snapshot models @@ -41,7 +41,7 @@ func TestTableNames(t *testing.T) { {WebsiteSnapshot{}, "website_snapshot"}, {EndpointSnapshot{}, "endpoint_snapshot"}, {DirectorySnapshot{}, "directory_snapshot"}, - {HostPortMappingSnapshot{}, "host_port_mapping_snapshot"}, + {HostPortSnapshot{}, "host_port_mapping_snapshot"}, {VulnerabilitySnapshot{}, "vulnerability_snapshot"}, {ScreenshotSnapshot{}, "screenshot_snapshot"}, // Scan-related models diff --git a/go-backend/internal/model/website_snapshot.go b/go-backend/internal/model/website_snapshot.go index 2b45cf24..9a40940b 100644 --- a/go-backend/internal/model/website_snapshot.go +++ b/go-backend/internal/model/website_snapshot.go @@ -14,7 +14,7 @@ type WebsiteSnapshot struct { Host string `gorm:"column:host;size:253;index:idx_website_snap_host" json:"host"` Title string `gorm:"column:title;type:text;index:idx_website_snap_title" json:"title"` StatusCode *int `gorm:"column:status_code" json:"statusCode"` - ContentLength *int64 `gorm:"column:content_length" json:"contentLength"` + ContentLength *int `gorm:"column:content_length" json:"contentLength"` Location string `gorm:"column:location;type:text" json:"location"` Webserver string `gorm:"column:webserver;type:text" json:"webserver"` ContentType string `gorm:"column:content_type;type:text" json:"contentType"` diff --git a/go-backend/internal/repository/ip_address.go b/go-backend/internal/repository/host_port.go similarity index 64% rename from go-backend/internal/repository/ip_address.go rename to go-backend/internal/repository/host_port.go index 79ff115b..270951b4 100644 --- a/go-backend/internal/repository/ip_address.go +++ b/go-backend/internal/repository/host_port.go @@ -11,18 +11,18 @@ import ( "gorm.io/gorm/clause" ) -// IPAddressRepository handles IP address (host_port_mapping) database operations -type IPAddressRepository struct { +// HostPortRepository handles host-port mapping (host_port_mapping) database operations +type HostPortRepository struct { db *gorm.DB } -// NewIPAddressRepository creates a new IP address repository -func NewIPAddressRepository(db *gorm.DB) *IPAddressRepository { - return &IPAddressRepository{db: db} +// NewHostPortRepository creates a new host-port repository +func NewHostPortRepository(db *gorm.DB) *HostPortRepository { + return &HostPortRepository{db: db} } -// IPAddressFilterMapping defines field mapping for filtering -var IPAddressFilterMapping = scope.FilterMapping{ +// HostPortFilterMapping defines field mapping for filtering +var HostPortFilterMapping = scope.FilterMapping{ "host": {Column: "host"}, "ip": {Column: "ip", NeedsCast: true}, "port": {Column: "port", IsNumeric: true}, @@ -35,12 +35,12 @@ type IPAggregationRow struct { } // GetIPAggregation returns IPs with their earliest created_at, ordered by created_at DESC -func (r *IPAddressRepository) GetIPAggregation(targetID int, filter string) ([]IPAggregationRow, int64, error) { +func (r *HostPortRepository) GetIPAggregation(targetID int, filter string) ([]IPAggregationRow, int64, error) { // Build base query - baseQuery := r.db.Model(&model.HostPortMapping{}).Where("target_id = ?", targetID) + baseQuery := r.db.Model(&model.HostPort{}).Where("target_id = ?", targetID) // Apply filter - baseQuery = baseQuery.Scopes(scope.WithFilter(filter, IPAddressFilterMapping)) + baseQuery = baseQuery.Scopes(scope.WithFilter(filter, HostPortFilterMapping)) // Get distinct IPs with MIN(created_at) var results []IPAggregationRow @@ -57,12 +57,12 @@ func (r *IPAddressRepository) GetIPAggregation(targetID int, filter string) ([]I } // GetHostsAndPortsByIP returns hosts and ports for a specific IP -func (r *IPAddressRepository) GetHostsAndPortsByIP(targetID int, ip string, filter string) ([]string, []int, error) { - baseQuery := r.db.Model(&model.HostPortMapping{}). +func (r *HostPortRepository) GetHostsAndPortsByIP(targetID int, ip string, filter string) ([]string, []int, error) { + baseQuery := r.db.Model(&model.HostPort{}). Where("target_id = ? AND ip = ?", targetID, ip) // Apply filter - baseQuery = baseQuery.Scopes(scope.WithFilter(filter, IPAddressFilterMapping)) + baseQuery = baseQuery.Scopes(scope.WithFilter(filter, HostPortFilterMapping)) // Get distinct host and port combinations var mappings []struct { @@ -101,34 +101,34 @@ func (r *IPAddressRepository) GetHostsAndPortsByIP(targetID int, ip string, filt } // StreamByTargetID returns a sql.Rows cursor for streaming export (raw format) -func (r *IPAddressRepository) StreamByTargetID(targetID int) (*sql.Rows, error) { - return r.db.Model(&model.HostPortMapping{}). +func (r *HostPortRepository) StreamByTargetID(targetID int) (*sql.Rows, error) { + return r.db.Model(&model.HostPort{}). Where("target_id = ?", targetID). Order("ip, host, port"). Rows() } // StreamByTargetIDAndIPs returns a sql.Rows cursor for streaming export filtered by IPs -func (r *IPAddressRepository) StreamByTargetIDAndIPs(targetID int, ips []string) (*sql.Rows, error) { - return r.db.Model(&model.HostPortMapping{}). +func (r *HostPortRepository) StreamByTargetIDAndIPs(targetID int, ips []string) (*sql.Rows, error) { + return r.db.Model(&model.HostPort{}). Where("target_id = ? AND ip IN ?", targetID, ips). Order("ip, host, port"). Rows() } // CountByTargetID returns the count of unique IPs for a target -func (r *IPAddressRepository) CountByTargetID(targetID int) (int64, error) { +func (r *HostPortRepository) CountByTargetID(targetID int) (int64, error) { var count int64 - err := r.db.Model(&model.HostPortMapping{}). + err := r.db.Model(&model.HostPort{}). Where("target_id = ?", targetID). Distinct("ip"). Count(&count).Error return count, err } -// ScanRow scans a single row into HostPortMapping model -func (r *IPAddressRepository) ScanRow(rows *sql.Rows) (*model.HostPortMapping, error) { - var mapping model.HostPortMapping +// ScanRow scans a single row into HostPort model +func (r *HostPortRepository) ScanRow(rows *sql.Rows) (*model.HostPort, error) { + var mapping model.HostPort if err := r.db.ScanRows(rows, &mapping); err != nil { return nil, err } @@ -136,7 +136,7 @@ func (r *IPAddressRepository) ScanRow(rows *sql.Rows) (*model.HostPortMapping, e } // BulkUpsert creates multiple mappings, ignoring duplicates (ON CONFLICT DO NOTHING) -func (r *IPAddressRepository) BulkUpsert(mappings []model.HostPortMapping) (int64, error) { +func (r *HostPortRepository) BulkUpsert(mappings []model.HostPort) (int64, error) { if len(mappings) == 0 { return 0, nil } @@ -161,10 +161,10 @@ func (r *IPAddressRepository) BulkUpsert(mappings []model.HostPortMapping) (int6 } // DeleteByIPs deletes all mappings for the given IPs -func (r *IPAddressRepository) DeleteByIPs(ips []string) (int64, error) { +func (r *HostPortRepository) DeleteByIPs(ips []string) (int64, error) { if len(ips) == 0 { return 0, nil } - result := r.db.Where("ip IN ?", ips).Delete(&model.HostPortMapping{}) + result := r.db.Where("ip IN ?", ips).Delete(&model.HostPort{}) return result.RowsAffected, result.Error } diff --git a/go-backend/internal/repository/host_port_snapshot.go b/go-backend/internal/repository/host_port_snapshot.go new file mode 100644 index 00000000..a91821ca --- /dev/null +++ b/go-backend/internal/repository/host_port_snapshot.go @@ -0,0 +1,99 @@ +package repository + +import ( + "database/sql" + + "github.com/xingrin/go-backend/internal/model" + "github.com/xingrin/go-backend/internal/pkg/scope" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// HostPortSnapshotRepository handles host-port mapping snapshot database operations +type HostPortSnapshotRepository struct { + db *gorm.DB +} + +// NewHostPortSnapshotRepository creates a new host-port mapping snapshot repository +func NewHostPortSnapshotRepository(db *gorm.DB) *HostPortSnapshotRepository { + return &HostPortSnapshotRepository{db: db} +} + +// HostPortSnapshotFilterMapping defines field mapping for host-port snapshot filtering +var HostPortSnapshotFilterMapping = scope.FilterMapping{ + "host": {Column: "host"}, + "ip": {Column: "ip"}, + "port": {Column: "port"}, +} + +// BulkCreate creates multiple host-port snapshots, ignoring duplicates +func (r *HostPortSnapshotRepository) BulkCreate(snapshots []model.HostPortSnapshot) (int64, error) { + if len(snapshots) == 0 { + return 0, nil + } + + var totalAffected int64 + + // Batch to avoid PostgreSQL parameter limit (65535) + // 5 fields per record: scan_id, host, ip, port, created_at + batchSize := 500 + for i := 0; i < len(snapshots); i += batchSize { + end := i + batchSize + if end > len(snapshots) { + end = len(snapshots) + } + batch := snapshots[i:end] + + result := r.db.Clauses(clause.OnConflict{DoNothing: true}).Create(&batch) + if result.Error != nil { + return totalAffected, result.Error + } + totalAffected += result.RowsAffected + } + + return totalAffected, nil +} + +// FindByScanID finds host-port snapshots by scan ID with pagination and filter +func (r *HostPortSnapshotRepository) FindByScanID(scanID int, page, pageSize int, filter string) ([]model.HostPortSnapshot, int64, error) { + var snapshots []model.HostPortSnapshot + var total int64 + + baseQuery := r.db.Model(&model.HostPortSnapshot{}).Where("scan_id = ?", scanID) + baseQuery = baseQuery.Scopes(scope.WithFilter(filter, HostPortSnapshotFilterMapping)) + + if err := baseQuery.Count(&total).Error; err != nil { + return nil, 0, err + } + + err := baseQuery.Scopes( + scope.WithPagination(page, pageSize), + scope.OrderByCreatedAtDesc(), + ).Find(&snapshots).Error + + return snapshots, total, err +} + +// StreamByScanID returns a sql.Rows cursor for streaming export +func (r *HostPortSnapshotRepository) StreamByScanID(scanID int) (*sql.Rows, error) { + return r.db.Model(&model.HostPortSnapshot{}). + Where("scan_id = ?", scanID). + Order("created_at DESC"). + Rows() +} + +// CountByScanID returns the count of host-port snapshots for a scan +func (r *HostPortSnapshotRepository) CountByScanID(scanID int) (int64, error) { + var count int64 + err := r.db.Model(&model.HostPortSnapshot{}).Where("scan_id = ?", scanID).Count(&count).Error + return count, err +} + +// ScanRow scans a single row into HostPortSnapshot model +func (r *HostPortSnapshotRepository) ScanRow(rows *sql.Rows) (*model.HostPortSnapshot, error) { + var snapshot model.HostPortSnapshot + if err := r.db.ScanRows(rows, &snapshot); err != nil { + return nil, err + } + return &snapshot, nil +} diff --git a/go-backend/internal/repository/website_snapshot_test.go b/go-backend/internal/repository/website_snapshot_test.go index f61fb634..98bf65e3 100644 --- a/go-backend/internal/repository/website_snapshot_test.go +++ b/go-backend/internal/repository/website_snapshot_test.go @@ -6,79 +6,6 @@ import ( "github.com/xingrin/go-backend/internal/model" ) -// TestApplyOrdering tests the ordering function -func TestApplyOrdering(t *testing.T) { - tests := []struct { - name string - ordering string - wantDesc bool - wantCol string - }{ - { - name: "ascending url", - ordering: "url", - wantDesc: false, - wantCol: "url", - }, - { - name: "descending url", - ordering: "-url", - wantDesc: true, - wantCol: "url", - }, - { - name: "ascending createdAt", - ordering: "createdAt", - wantDesc: false, - wantCol: "created_at", - }, - { - name: "descending createdAt", - ordering: "-createdAt", - wantDesc: true, - wantCol: "created_at", - }, - { - name: "statusCode ascending", - ordering: "statusCode", - wantDesc: false, - wantCol: "status_code", - }, - { - name: "statusCode descending", - ordering: "-statusCode", - wantDesc: true, - wantCol: "status_code", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - desc := false - field := tt.ordering - - if len(tt.ordering) > 0 && tt.ordering[0] == '-' { - desc = true - field = tt.ordering[1:] - } - - column, ok := orderingFieldMapping[field] - if !ok { - t.Errorf("field %s not found in orderingFieldMapping", field) - return - } - - if desc != tt.wantDesc { - t.Errorf("desc = %v, want %v", desc, tt.wantDesc) - } - - if column != tt.wantCol { - t.Errorf("column = %v, want %v", column, tt.wantCol) - } - }) - } -} - // TestWebsiteSnapshotFilterMapping tests the filter mapping configuration func TestWebsiteSnapshotFilterMapping(t *testing.T) { expectedFields := []string{"url", "host", "title", "status", "webserver", "tech"} diff --git a/go-backend/internal/service/directory_snapshot.go b/go-backend/internal/service/directory_snapshot.go index a21ad07f..62d1e8b4 100644 --- a/go-backend/internal/service/directory_snapshot.go +++ b/go-backend/internal/service/directory_snapshot.go @@ -70,8 +70,6 @@ func (s *DirectorySnapshotService) SaveAndSync(scanID int, targetID int, items [ URL: item.URL, Status: item.Status, ContentLength: item.ContentLength, - Words: item.Words, - Lines: item.Lines, ContentType: item.ContentType, Duration: item.Duration, }) diff --git a/go-backend/internal/service/ip_address.go b/go-backend/internal/service/host_port.go similarity index 64% rename from go-backend/internal/service/ip_address.go rename to go-backend/internal/service/host_port.go index 9ff4e8f1..62fe28a2 100644 --- a/go-backend/internal/service/ip_address.go +++ b/go-backend/internal/service/host_port.go @@ -10,19 +10,19 @@ import ( "gorm.io/gorm" ) -// IPAddressService handles IP address business logic -type IPAddressService struct { - repo *repository.IPAddressRepository +// HostPortService handles host-port business logic +type HostPortService struct { + repo *repository.HostPortRepository targetRepo *repository.TargetRepository } -// NewIPAddressService creates a new IP address service -func NewIPAddressService(repo *repository.IPAddressRepository, targetRepo *repository.TargetRepository) *IPAddressService { - return &IPAddressService{repo: repo, targetRepo: targetRepo} +// NewHostPortService creates a new host-port service +func NewHostPortService(repo *repository.HostPortRepository, targetRepo *repository.TargetRepository) *HostPortService { + return &HostPortService{repo: repo, targetRepo: targetRepo} } -// ListByTarget returns paginated IP addresses aggregated by IP -func (s *IPAddressService) ListByTarget(targetID int, query *dto.IPAddressListQuery) ([]dto.IPAddressResponse, int64, error) { +// ListByTarget returns paginated host-ports aggregated by IP +func (s *HostPortService) ListByTarget(targetID int, query *dto.HostPortListQuery) ([]dto.HostPortResponse, int64, error) { // Get IP aggregation (all IPs with their earliest created_at) ipRows, total, err := s.repo.GetIPAggregation(targetID, query.Filter) if err != nil { @@ -36,7 +36,7 @@ func (s *IPAddressService) ListByTarget(targetID int, query *dto.IPAddressListQu end := start + pageSize if start >= len(ipRows) { - return []dto.IPAddressResponse{}, total, nil + return []dto.HostPortResponse{}, total, nil } if end > len(ipRows) { end = len(ipRows) @@ -45,14 +45,14 @@ func (s *IPAddressService) ListByTarget(targetID int, query *dto.IPAddressListQu pagedIPs := ipRows[start:end] // For each IP, get its hosts and ports - results := make([]dto.IPAddressResponse, 0, len(pagedIPs)) + results := make([]dto.HostPortResponse, 0, len(pagedIPs)) for _, row := range pagedIPs { hosts, ports, err := s.repo.GetHostsAndPortsByIP(targetID, row.IP, query.Filter) if err != nil { return nil, 0, err } - results = append(results, dto.IPAddressResponse{ + results = append(results, dto.HostPortResponse{ IP: row.IP, Hosts: hosts, Ports: ports, @@ -64,7 +64,7 @@ func (s *IPAddressService) ListByTarget(targetID int, query *dto.IPAddressListQu } // StreamByTarget returns a cursor for streaming export (raw format) -func (s *IPAddressService) StreamByTarget(targetID int) (*sql.Rows, error) { +func (s *HostPortService) StreamByTarget(targetID int) (*sql.Rows, error) { _, err := s.targetRepo.FindByID(targetID) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { @@ -76,7 +76,7 @@ func (s *IPAddressService) StreamByTarget(targetID int) (*sql.Rows, error) { } // StreamByTargetAndIPs returns a cursor for streaming export filtered by IPs -func (s *IPAddressService) StreamByTargetAndIPs(targetID int, ips []string) (*sql.Rows, error) { +func (s *HostPortService) StreamByTargetAndIPs(targetID int, ips []string) (*sql.Rows, error) { _, err := s.targetRepo.FindByID(targetID) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { @@ -88,7 +88,7 @@ func (s *IPAddressService) StreamByTargetAndIPs(targetID int, ips []string) (*sq } // CountByTarget returns the count of unique IPs for a target -func (s *IPAddressService) CountByTarget(targetID int) (int64, error) { +func (s *HostPortService) CountByTarget(targetID int) (int64, error) { _, err := s.targetRepo.FindByID(targetID) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { @@ -99,13 +99,13 @@ func (s *IPAddressService) CountByTarget(targetID int) (int64, error) { return s.repo.CountByTargetID(targetID) } -// ScanRow scans a row into HostPortMapping model -func (s *IPAddressService) ScanRow(rows *sql.Rows) (*model.HostPortMapping, error) { +// ScanRow scans a row into HostPort model +func (s *HostPortService) ScanRow(rows *sql.Rows) (*model.HostPort, error) { return s.repo.ScanRow(rows) } // BulkUpsert creates multiple mappings for a target (ignores duplicates) -func (s *IPAddressService) BulkUpsert(targetID int, items []dto.IPAddressItem) (int64, error) { +func (s *HostPortService) BulkUpsert(targetID int, items []dto.HostPortItem) (int64, error) { _, err := s.targetRepo.FindByID(targetID) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { @@ -115,9 +115,9 @@ func (s *IPAddressService) BulkUpsert(targetID int, items []dto.IPAddressItem) ( } // Convert DTOs to models - mappings := make([]model.HostPortMapping, 0, len(items)) + mappings := make([]model.HostPort, 0, len(items)) for _, item := range items { - mappings = append(mappings, model.HostPortMapping{ + mappings = append(mappings, model.HostPort{ TargetID: targetID, Host: item.Host, IP: item.IP, @@ -133,7 +133,7 @@ func (s *IPAddressService) BulkUpsert(targetID int, items []dto.IPAddressItem) ( } // BulkDeleteByIPs deletes all mappings for the given IPs -func (s *IPAddressService) BulkDeleteByIPs(ips []string) (int64, error) { +func (s *HostPortService) BulkDeleteByIPs(ips []string) (int64, error) { if len(ips) == 0 { return 0, nil } diff --git a/go-backend/internal/service/host_port_snapshot.go b/go-backend/internal/service/host_port_snapshot.go new file mode 100644 index 00000000..c03dde9e --- /dev/null +++ b/go-backend/internal/service/host_port_snapshot.go @@ -0,0 +1,181 @@ +package service + +import ( + "database/sql" + "errors" + "fmt" + "net" + "strings" + + "github.com/xingrin/go-backend/internal/dto" + "github.com/xingrin/go-backend/internal/model" + "github.com/xingrin/go-backend/internal/pkg/validator" + "github.com/xingrin/go-backend/internal/repository" + "gorm.io/gorm" +) + +// HostPortSnapshotService handles host-port snapshot business logic +type HostPortSnapshotService struct { + snapshotRepo *repository.HostPortSnapshotRepository + scanRepo *repository.ScanRepository + hostPortService *HostPortService +} + +// NewHostPortSnapshotService creates a new host-port snapshot service +func NewHostPortSnapshotService( + snapshotRepo *repository.HostPortSnapshotRepository, + scanRepo *repository.ScanRepository, + hostPortService *HostPortService, +) *HostPortSnapshotService { + return &HostPortSnapshotService{ + snapshotRepo: snapshotRepo, + scanRepo: scanRepo, + hostPortService: hostPortService, + } +} + +// SaveAndSync saves host-port snapshots and syncs to asset table +// 1. Validates scan exists and is not soft-deleted +// 2. Validates host/ip match target (filters invalid items) +// 3. Saves to host_port_mapping_snapshot table +// 4. TODO: Sync to host_port_mapping table (when asset service is implemented) +func (s *HostPortSnapshotService) SaveAndSync(scanID int, targetID int, items []dto.HostPortSnapshotItem) (snapshotCount int64, assetCount int64, err error) { + if len(items) == 0 { + return 0, 0, nil + } + + // Validate scan exists + scan, err := s.scanRepo.FindByID(scanID) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return 0, 0, ErrScanNotFoundForSnapshot + } + return 0, 0, err + } + + if scan.TargetID != targetID { + return 0, 0, errors.New("target_id does not match scan's target") + } + + // Get target for validation + target, err := s.scanRepo.GetTargetByScanID(scanID) + if err != nil { + return 0, 0, err + } + + // Filter valid host-port mappings + snapshots := make([]model.HostPortSnapshot, 0, len(items)) + + for _, item := range items { + if isHostPortMatchTarget(item.Host, item.IP, target.Name, target.Type) { + snapshots = append(snapshots, model.HostPortSnapshot{ + ScanID: scanID, + Host: item.Host, + IP: item.IP, + Port: item.Port, + }) + } + } + + if len(snapshots) == 0 { + return 0, 0, nil + } + + // Save to snapshot table + snapshotCount, err = s.snapshotRepo.BulkCreate(snapshots) + if err != nil { + return 0, 0, fmt.Errorf("failed to bulk create snapshots: %w", err) + } + + // Sync to asset table (HostPortSnapshotItem is an alias of HostPortItem, no conversion needed) + assetCount, err = s.hostPortService.BulkUpsert(targetID, items) + if err != nil { + return snapshotCount, 0, fmt.Errorf("failed to sync to asset table: %w", err) + } + + return snapshotCount, assetCount, nil +} + +// ListByScan returns paginated host-port snapshots for a scan +func (s *HostPortSnapshotService) ListByScan(scanID int, query *dto.HostPortSnapshotListQuery) ([]model.HostPortSnapshot, int64, error) { + _, err := s.scanRepo.FindByID(scanID) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, 0, ErrScanNotFoundForSnapshot + } + return nil, 0, err + } + + return s.snapshotRepo.FindByScanID(scanID, query.GetPage(), query.GetPageSize(), query.Filter) +} + +// StreamByScan returns a cursor for streaming export +func (s *HostPortSnapshotService) StreamByScan(scanID int) (*sql.Rows, error) { + _, err := s.scanRepo.FindByID(scanID) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrScanNotFoundForSnapshot + } + return nil, err + } + + return s.snapshotRepo.StreamByScanID(scanID) +} + +// CountByScan returns the count of host-port snapshots for a scan +func (s *HostPortSnapshotService) CountByScan(scanID int) (int64, error) { + _, err := s.scanRepo.FindByID(scanID) + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return 0, ErrScanNotFoundForSnapshot + } + return 0, err + } + + return s.snapshotRepo.CountByScanID(scanID) +} + +// ScanRow scans a row into HostPortSnapshot model +func (s *HostPortSnapshotService) ScanRow(rows *sql.Rows) (*model.HostPortSnapshot, error) { + return s.snapshotRepo.ScanRow(rows) +} + +// isHostPortMatchTarget checks if host/ip belongs to target +// Matching rules by target type: +// - domain: host equals target or ends with .target +// - ip: ip must exactly equal target +// - cidr: ip must be within the CIDR range +func isHostPortMatchTarget(host, ip, targetName, targetType string) bool { + host = strings.ToLower(strings.TrimSpace(host)) + ip = strings.TrimSpace(ip) + targetName = strings.ToLower(strings.TrimSpace(targetName)) + + if host == "" || ip == "" || targetName == "" { + return false + } + + switch targetType { + case validator.TargetTypeDomain: + // Check if host matches target domain + return host == targetName || strings.HasSuffix(host, "."+targetName) + + case validator.TargetTypeIP: + // IP must exactly match target + return ip == targetName + + case validator.TargetTypeCIDR: + // IP must be within CIDR range + ipAddr := net.ParseIP(ip) + if ipAddr == nil { + return false + } + _, network, err := net.ParseCIDR(targetName) + if err != nil { + return false + } + return network.Contains(ipAddr) + + default: + return false + } +} diff --git a/go-backend/internal/service/website_snapshot.go b/go-backend/internal/service/website_snapshot.go index 4b8653b5..161b79b1 100644 --- a/go-backend/internal/service/website_snapshot.go +++ b/go-backend/internal/service/website_snapshot.go @@ -109,27 +109,8 @@ func (s *WebsiteSnapshotService) SaveAndSync(scanID int, targetID int, items []d return 0, 0, err } - // Step 5: Sync to asset table via WebsiteService - // Note: WebsiteService.BulkUpsert also validates, but we already filtered - assetItems := make([]dto.WebsiteUpsertItem, 0, len(validItems)) - for _, item := range validItems { - assetItems = append(assetItems, dto.WebsiteUpsertItem{ - URL: item.URL, - Host: item.Host, - Title: item.Title, - StatusCode: item.StatusCode, - ContentLength: intPtrToIntPtr(item.ContentLength), - Location: item.Location, - Webserver: item.Webserver, - ContentType: item.ContentType, - Tech: item.Tech, - ResponseBody: item.ResponseBody, - Vhost: item.Vhost, - ResponseHeaders: item.ResponseHeaders, - }) - } - - assetCount, err = s.websiteService.BulkUpsert(targetID, assetItems) + // Step 5: Sync to asset table (WebsiteSnapshotItem is an alias of WebsiteUpsertItem, no conversion needed) + assetCount, err = s.websiteService.BulkUpsert(targetID, validItems) if err != nil { // Log error but don't fail - snapshot is already saved // In production, consider using a transaction or compensation logic @@ -139,15 +120,6 @@ func (s *WebsiteSnapshotService) SaveAndSync(scanID int, targetID int, items []d return snapshotCount, assetCount, nil } -// intPtrToIntPtr converts *int64 to *int -func intPtrToIntPtr(v *int64) *int { - if v == nil { - return nil - } - i := int(*v) - return &i -} - // ListByScan returns paginated website snapshots for a scan func (s *WebsiteSnapshotService) ListByScan(scanID int, query *dto.WebsiteSnapshotListQuery) ([]model.WebsiteSnapshot, int64, error) { // Validate scan exists diff --git a/go-backend/internal/service/website_snapshot_test.go b/go-backend/internal/service/website_snapshot_test.go index a8439062..de9d1549 100644 --- a/go-backend/internal/service/website_snapshot_test.go +++ b/go-backend/internal/service/website_snapshot_test.go @@ -34,7 +34,7 @@ func TestSaveAndSyncDataConsistency(t *testing.T) { Host: "test.com", Title: "Test Page", StatusCode: intPtr(200), - ContentLength: int64Ptr(1024), + ContentLength: intPtr(1024), Location: "https://test.com/redirect", Webserver: "nginx", ContentType: "text/html", @@ -61,7 +61,7 @@ func TestSaveAndSyncDataConsistency(t *testing.T) { Host: tt.snapshot.Host, Title: tt.snapshot.Title, StatusCode: tt.snapshot.StatusCode, - ContentLength: intPtrFromInt64Ptr(tt.snapshot.ContentLength), + ContentLength: tt.snapshot.ContentLength, Location: tt.snapshot.Location, Webserver: tt.snapshot.Webserver, ContentType: tt.snapshot.ContentType, @@ -102,22 +102,10 @@ func intPtr(v int) *int { return &v } -func int64Ptr(v int64) *int64 { - return &v -} - func boolPtr(v bool) *bool { return &v } -func intPtrFromInt64Ptr(v *int64) *int { - if v == nil { - return nil - } - i := int(*v) - return &i -} - func intPtrEqual(a, b *int) bool { if a == nil && b == nil { return true