mirror of
https://github.com/yyhuni/xingrin.git
synced 2026-01-31 11:46:16 +08:00
chore: add server/.env to .gitignore and remove from git tracking
This commit is contained in:
4
.gitignore
vendored
4
.gitignore
vendored
@@ -90,6 +90,10 @@ backend/go.work.sum
|
||||
# Go 依赖管理
|
||||
backend/vendor/
|
||||
|
||||
# Go Server 环境变量
|
||||
server/.env
|
||||
server/.env.local
|
||||
|
||||
# ============================
|
||||
# IDE 和编辑器相关
|
||||
# ============================
|
||||
|
||||
65
WARP.md
Normal file
65
WARP.md
Normal file
@@ -0,0 +1,65 @@
|
||||
# WARP.md
|
||||
|
||||
This file provides guidance to WARP (warp.dev) when working with code in this repository.
|
||||
|
||||
## Common commands
|
||||
|
||||
### Repo scripts (Docker-based deployment)
|
||||
- `./install.sh` (prod) / `./install.sh --dev` (build local dev images) / `./install.sh --no-frontend`
|
||||
- `./start.sh` / `./start.sh --dev`
|
||||
- `./stop.sh`
|
||||
- `./restart.sh`
|
||||
- `./update.sh`
|
||||
- `./uninstall.sh`
|
||||
|
||||
### Docker Compose (full stack with Django backend)
|
||||
- Dev: `docker compose -f docker/docker-compose.dev.yml up -d`
|
||||
- Dev + local Postgres: `docker compose -f docker/docker-compose.dev.yml --profile local-db up -d`
|
||||
- Down: `docker compose -f docker/docker-compose.dev.yml down`
|
||||
|
||||
### Frontend (Next.js in `frontend/`)
|
||||
- `cd frontend`
|
||||
- `pnpm install`
|
||||
- `pnpm dev` (or `pnpm dev:mock`, `pnpm dev:noauth`)
|
||||
- `pnpm build`
|
||||
- `pnpm start`
|
||||
- `pnpm lint`
|
||||
|
||||
### Go server rewrite (Gin/GORM in `server/`)
|
||||
- `cd server`
|
||||
- `make run` / `make build` / `make test` / `make lint`
|
||||
- Single test: `go test ./internal/... -run TestName`
|
||||
- Dev deps only (Postgres/Redis): `docker compose -f docker-compose.dev.yml up -d`
|
||||
|
||||
### Go worker (scan executor in `worker/`)
|
||||
- `cd worker`
|
||||
- `make run` / `make build` / `make test`
|
||||
- Single test: `go test ./internal/... -run TestName`
|
||||
|
||||
### Django backend (production server in `backend/`)
|
||||
- `cd backend`
|
||||
- `pytest`
|
||||
- Single test: `pytest apps/<app>/... -k "TestName or test_name"`
|
||||
|
||||
### Seed data generator (API-based, `tools/seed-api/`)
|
||||
- `cd tools/seed-api`
|
||||
- `pip install -r requirements.txt`
|
||||
- `python seed_generator.py` (see `tools/seed-api/README.md` for options)
|
||||
- Tests: `pytest` (integration requires a running backend)
|
||||
|
||||
## Architecture overview (big picture)
|
||||
- **Monorepo services**:
|
||||
- **Django backend** in `backend/` (current production server, runs via `docker/server/start.sh` with migrations + `uvicorn`).
|
||||
- **Go backend rewrite** in `server/` (Gin/GORM; incomplete per `server/README.md`).
|
||||
- **Next.js frontend** in `frontend/` (App Router; containerized via `docker/frontend/Dockerfile`).
|
||||
- **Go worker** in `worker/` plus **agent** (heartbeat/monitor) in `docker/agent/Dockerfile`.
|
||||
- **Deployment topology**: `docker/docker-compose.yml` (prod) and `docker/docker-compose.dev.yml` (dev) orchestrate Postgres, Redis, Django server, agent, frontend, and nginx. Nginx terminates HTTPS on `8083` and proxies to backend `8888`.
|
||||
- **Versioning**: `VERSION` is the single source of release version. `IMAGE_TAG` in `docker/.env` pins all images to the same tag; `./update.sh` refreshes it (see `docs/version-management.md`).
|
||||
- **Scan pipeline**: stages and toolchain are documented in `docs/scan-flow-architecture.md`. Stage ordering is defined in `backend/apps/scan/configs/command_templates.py`.
|
||||
- **Templates & wordlists**:
|
||||
- Nuclei templates: server-side sync and worker-side checkout are documented in `docs/nuclei-template-architecture.md`.
|
||||
- Wordlists: upload on server, hash-based cache + download on worker (see `docs/wordlist-architecture.md`).
|
||||
- **Backend domain layout**: Django apps under `backend/apps/` (e.g., `scan`, `engine`, `asset`, `targets`, `common`). Worker/agent deployment helpers live in `backend/scripts/worker-deploy/`.
|
||||
- **Go server layout**: `server/internal/` is layered (`handler` → `service` → `repository` → `model`, plus `dto`, `middleware`, `config`, `database`).
|
||||
- **Go worker layout**: `worker/internal/` splits workflow orchestration (`workflow`, `activity`) from runtime/server glue (`server`, `config`, `pkg`).
|
||||
- **Config files**: `.env` templates live in `docker/.env.example` (Django stack), `server/.env.example` (Go server), and `worker/.env.example` (Go worker).
|
||||
@@ -40,8 +40,8 @@ export function ChangePasswordDialog({ open, onOpenChange }: ChangePasswordDialo
|
||||
return
|
||||
}
|
||||
|
||||
if (newPassword.length < 4) {
|
||||
setError(t("passwordTooShort", { min: 4 }))
|
||||
if (newPassword.length < 6) {
|
||||
setError(t("passwordTooShort", { min: 6 }))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ export interface OrganizationTranslations {
|
||||
selectRow: string
|
||||
}
|
||||
tooltips: {
|
||||
targetSummary: string
|
||||
organizationDetails: string
|
||||
initiateScan: string
|
||||
}
|
||||
}
|
||||
@@ -240,7 +240,7 @@ export const createOrganizationColumns = ({
|
||||
</Button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top">
|
||||
<p className="text-xs">{t.tooltips.targetSummary}</p>
|
||||
<p className="text-xs">{t.tooltips.organizationDetails}</p>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</TooltipProvider>
|
||||
|
||||
@@ -78,7 +78,7 @@ export function OrganizationList() {
|
||||
selectRow: tCommon("actions.selectRow"),
|
||||
},
|
||||
tooltips: {
|
||||
targetSummary: tTooltips("targetSummary"),
|
||||
organizationDetails: tTooltips("organizationDetails"),
|
||||
initiateScan: tTooltips("initiateScan"),
|
||||
},
|
||||
}), [tColumns, tCommon, tTooltips])
|
||||
|
||||
@@ -351,7 +351,11 @@ export const getErrorMessage = (error: unknown): string => {
|
||||
// Type guard: Check if it's an error object
|
||||
const err = error as {
|
||||
code?: string;
|
||||
response?: { data?: { message?: string; error?: string; detail?: string } };
|
||||
response?: { data?: {
|
||||
message?: string;
|
||||
error?: string | { code?: string; message?: string; details?: Array<{ field?: string; message?: string }> };
|
||||
detail?: string
|
||||
} };
|
||||
message?: string
|
||||
}
|
||||
|
||||
@@ -361,8 +365,19 @@ export const getErrorMessage = (error: unknown): string => {
|
||||
}
|
||||
|
||||
// Backend returned error message (supports multiple formats)
|
||||
if (err.response?.data?.error) {
|
||||
return err.response.data.error;
|
||||
const errorData = err.response?.data?.error;
|
||||
if (errorData) {
|
||||
// New format: { error: { code, message, details } }
|
||||
if (typeof errorData === 'object') {
|
||||
// If has validation details, return first detail message
|
||||
if (errorData.details && errorData.details.length > 0) {
|
||||
const detail = errorData.details[0];
|
||||
return detail.message || errorData.message || 'Validation error';
|
||||
}
|
||||
return errorData.message || 'Unknown error';
|
||||
}
|
||||
// Old format: { error: "string" }
|
||||
return errorData;
|
||||
}
|
||||
if (err.response?.data?.message) {
|
||||
return err.response.data.message;
|
||||
|
||||
@@ -117,6 +117,7 @@
|
||||
"targetDetails": "Target Details",
|
||||
"viewProgress": "Click to view progress details",
|
||||
"targetSummary": "Target Summary",
|
||||
"organizationDetails": "Organization Details",
|
||||
"initiateScan": "Initiate Scan",
|
||||
"scheduleScan": "Schedule Scan",
|
||||
"editEngine": "Edit Engine",
|
||||
|
||||
@@ -11,16 +11,16 @@
|
||||
"url": "URL"
|
||||
},
|
||||
"scanHistory": {
|
||||
"target": "Target",
|
||||
"summary": "Summary",
|
||||
"engineName": "Engine Name",
|
||||
"workerName": "Worker Node",
|
||||
"progress": "Progress",
|
||||
"subdomains": "Subdomains",
|
||||
"websites": "Websites",
|
||||
"ipAddresses": "IP Addresses",
|
||||
"endpoints": "Endpoints",
|
||||
"vulnerabilities": "Vulnerabilities"
|
||||
"target": "目标",
|
||||
"summary": "摘要",
|
||||
"engineName": "引擎名称",
|
||||
"workerName": "扫描节点",
|
||||
"progress": "进度",
|
||||
"subdomains": "子域名",
|
||||
"websites": "站点",
|
||||
"ipAddresses": "IP 地址",
|
||||
"endpoints": "URL",
|
||||
"vulnerabilities": "漏洞"
|
||||
},
|
||||
"vulnerability": {
|
||||
"severity": "严重程度",
|
||||
@@ -85,16 +85,16 @@
|
||||
"createdAt": "创建时间"
|
||||
},
|
||||
"engine": {
|
||||
"engineName": "Engine Name",
|
||||
"subdomainDiscovery": "Subdomain Discovery",
|
||||
"portScan": "Port Scan",
|
||||
"siteScan": "Site Scan",
|
||||
"directoryScan": "Directory Scan",
|
||||
"urlFetch": "URL Fetch",
|
||||
"engineName": "引擎名称",
|
||||
"subdomainDiscovery": "子域名发现",
|
||||
"portScan": "端口扫描",
|
||||
"siteScan": "站点扫描",
|
||||
"directoryScan": "目录扫描",
|
||||
"urlFetch": "URL 抓取",
|
||||
"osint": "OSINT",
|
||||
"vulnerabilityScan": "Vulnerability Scan",
|
||||
"wafDetection": "WAF Detection",
|
||||
"screenshot": "Screenshot"
|
||||
"vulnerabilityScan": "漏洞扫描",
|
||||
"wafDetection": "WAF 检测",
|
||||
"screenshot": "截图"
|
||||
},
|
||||
"scheduledScan": {
|
||||
"taskName": "任务名称",
|
||||
@@ -107,30 +107,31 @@
|
||||
"lastRun": "上次执行"
|
||||
},
|
||||
"fingerprint": {
|
||||
"name": "Name",
|
||||
"cats": "Categories",
|
||||
"rules": "Rules",
|
||||
"implies": "Implies",
|
||||
"website": "Website",
|
||||
"name": "名称",
|
||||
"cats": "分类",
|
||||
"rules": "规则",
|
||||
"implies": "依赖",
|
||||
"website": "官网",
|
||||
"cpe": "CPE",
|
||||
"created": "Created",
|
||||
"logic": "Logic",
|
||||
"ruleDetails": "Rule Details",
|
||||
"created": "创建时间",
|
||||
"logic": "逻辑",
|
||||
"ruleDetails": "规则详情",
|
||||
"cms": "CMS",
|
||||
"method": "Method",
|
||||
"keyword": "Keyword",
|
||||
"type": "Type",
|
||||
"important": "Important"
|
||||
"method": "方法",
|
||||
"keyword": "关键字",
|
||||
"type": "类型",
|
||||
"important": "重要"
|
||||
},
|
||||
"command": {
|
||||
"tool": "Tool",
|
||||
"commandTemplate": "Command Template"
|
||||
"tool": "工具",
|
||||
"commandTemplate": "命令模板"
|
||||
}
|
||||
},
|
||||
"tooltips": {
|
||||
"targetDetails": "目标详情",
|
||||
"viewProgress": "点击查看进度详情",
|
||||
"targetSummary": "目标摘要",
|
||||
"organizationDetails": "组织详情",
|
||||
"initiateScan": "发起扫描",
|
||||
"scheduleScan": "计划扫描",
|
||||
"editEngine": "编辑引擎",
|
||||
|
||||
@@ -91,6 +91,6 @@ export async function changePassword(data: ChangePasswordRequest): Promise<Chang
|
||||
await mockDelay()
|
||||
return { message: 'Password changed successfully' }
|
||||
}
|
||||
const res = await api.post<ChangePasswordResponse>('/auth/change-password/', data)
|
||||
const res = await api.put<ChangePasswordResponse>('/users/me/password/', data)
|
||||
return res.data
|
||||
}
|
||||
|
||||
@@ -86,7 +86,7 @@ export async function batchDeleteTargets(
|
||||
export async function batchCreateTargets(
|
||||
data: BatchCreateTargetsRequest
|
||||
): Promise<BatchCreateTargetsResponse> {
|
||||
const response = await api.post<BatchCreateTargetsResponse>('/targets/batch_create/', data)
|
||||
const response = await api.post<BatchCreateTargetsResponse>('/targets/bulk-create/', data)
|
||||
// Handle 204 No Content response - return default success response
|
||||
if (response.status === 204 || !response.data) {
|
||||
return {
|
||||
|
||||
158
resources/wordlists/dir_default.txt
Normal file
158
resources/wordlists/dir_default.txt
Normal file
@@ -0,0 +1,158 @@
|
||||
/api/admin/v1/users/all
|
||||
/registerSuccess.do
|
||||
/getALLUsers
|
||||
/swagger/static/index.html
|
||||
/swagger-ui/index.html
|
||||
/dubbo-provider/distv2/index.html
|
||||
/user/swagger-ui.html
|
||||
/swagger-dubbo/api-docs
|
||||
/distv2/index.html
|
||||
/static/swagger.json
|
||||
/api/index.html
|
||||
/v2/swagger.json
|
||||
/actuator/hystrix.stream
|
||||
/intergrationgraph
|
||||
/druid
|
||||
/swagger-ui/html
|
||||
/api/v2/api-docs
|
||||
/swagger/codes
|
||||
/template/swagger-ui.html
|
||||
/spring-security-rest/api/swagger-ui.html
|
||||
/spring-security-oauth-resource/swagger-ui.html
|
||||
/v2/api-docs
|
||||
/api.html
|
||||
/sw/swagger-ui.html
|
||||
/~www
|
||||
/~xfs
|
||||
/~uucp
|
||||
/~user5
|
||||
/~web
|
||||
/~user4
|
||||
/~user3
|
||||
/~user
|
||||
/~user1
|
||||
/~toor
|
||||
/~user2
|
||||
/~sync
|
||||
/~system
|
||||
/~testuser
|
||||
/~test
|
||||
/~shutdown
|
||||
/~sql
|
||||
/~root
|
||||
/~staff
|
||||
/~rpc
|
||||
/~reception
|
||||
/~rpcuser
|
||||
/~operator
|
||||
/~office
|
||||
/~nscd
|
||||
/~postmaster
|
||||
/~pop
|
||||
/~nobody
|
||||
/~news
|
||||
/~mailnull
|
||||
/~mail
|
||||
/~lp
|
||||
/~ident
|
||||
/~http
|
||||
/~helpdesk
|
||||
/~help
|
||||
/~halt
|
||||
/~gdm
|
||||
/~gopher
|
||||
/~guest
|
||||
/~games
|
||||
/~fwuser
|
||||
/~fwadmin
|
||||
/~firewall
|
||||
/~fw
|
||||
/~db
|
||||
/~database
|
||||
/~backup
|
||||
/~ftp
|
||||
/~data
|
||||
/~daemon
|
||||
/~bin
|
||||
/~apache
|
||||
/~admin/
|
||||
/~/
|
||||
/~anonymous
|
||||
/~administrator
|
||||
/zipkin/
|
||||
/~admin
|
||||
/~adm
|
||||
/zimbra
|
||||
/zf_backend.php
|
||||
/zone-h.php
|
||||
/zimbra/
|
||||
/yonetim.php
|
||||
/zeroclipboard.swf
|
||||
/zebra.conf
|
||||
/zehir.php
|
||||
/zabbix/
|
||||
/yum.log
|
||||
/yonetim.html
|
||||
/yonetici
|
||||
/ylwrap
|
||||
/yonetici.html
|
||||
/yonetici.php
|
||||
/yonetim
|
||||
/yarn.lock
|
||||
/yarn-error.log
|
||||
/yarn-debug.log
|
||||
/yaml_cron.log
|
||||
/xw.php
|
||||
/xx.php
|
||||
/yaml.log
|
||||
/xw1.php
|
||||
/xsql/lib/XSQLConfig.xml
|
||||
/xslt/
|
||||
/xsl/common.xsl
|
||||
/xsl/
|
||||
/xsl/_common.xsl
|
||||
/xsql/
|
||||
/xshell.php
|
||||
/xmlrpc.php
|
||||
/xphpMyAdmin/
|
||||
/xprober.php
|
||||
/xmlrpc_server.php
|
||||
/xphperrors.log
|
||||
/xmlrpc
|
||||
/xml/_common.xml
|
||||
/xml/common.xml
|
||||
/xml/
|
||||
/xls/
|
||||
/xml
|
||||
/xcuserdata/
|
||||
/xlogin/
|
||||
/xiaoma.php
|
||||
/xferlog
|
||||
/xd.php
|
||||
/xampp/phpmyadmin/scripts/setup.php
|
||||
/xampp/phpmyadmin/index.php
|
||||
/xampp/phpmyadmin/
|
||||
/xampp/
|
||||
/x.php
|
||||
/wwwstats.htm
|
||||
/wwwstat
|
||||
/wwwroot.zip
|
||||
/wwwroot.tgz
|
||||
/wwwroot.tar.bz2
|
||||
/wwwroot.tar.gz
|
||||
/wwwlog
|
||||
/wwwroot.tar
|
||||
/wwwroot.7z
|
||||
/wwwroot.sql
|
||||
/wwwroot.rar
|
||||
/wwwboard/passwd.txt
|
||||
/wwwboard/
|
||||
/www.zip
|
||||
/www/phpMyAdmin/index.php
|
||||
/www-error.log
|
||||
/www.tgz
|
||||
/www.tar.bz2
|
||||
/www-test/
|
||||
/www.tar
|
||||
/www.rar
|
||||
/www.tar.gz
|
||||
42
resources/wordlists/resolvers.txt
Normal file
42
resources/wordlists/resolvers.txt
Normal file
@@ -0,0 +1,42 @@
|
||||
# DNS Resolvers for subdomain enumeration
|
||||
# Public DNS servers with good reliability
|
||||
|
||||
# Google
|
||||
8.8.8.8
|
||||
8.8.4.4
|
||||
|
||||
# Cloudflare
|
||||
1.1.1.1
|
||||
1.0.0.1
|
||||
|
||||
# Quad9
|
||||
9.9.9.9
|
||||
149.112.112.112
|
||||
|
||||
# OpenDNS
|
||||
208.67.222.222
|
||||
208.67.220.220
|
||||
|
||||
# Level3
|
||||
4.2.2.1
|
||||
4.2.2.2
|
||||
|
||||
# Verisign
|
||||
64.6.64.6
|
||||
64.6.65.6
|
||||
|
||||
# Comodo
|
||||
8.26.56.26
|
||||
8.20.247.20
|
||||
|
||||
# DNS.Watch
|
||||
84.200.69.80
|
||||
84.200.70.40
|
||||
|
||||
# Yandex
|
||||
77.88.8.8
|
||||
77.88.8.1
|
||||
|
||||
# AdGuard
|
||||
94.140.14.14
|
||||
94.140.15.15
|
||||
114442
resources/wordlists/subdomains-top1million-110000.txt
Normal file
114442
resources/wordlists/subdomains-top1million-110000.txt
Normal file
File diff suppressed because it is too large
Load Diff
35
server/.env
35
server/.env
@@ -1,35 +0,0 @@
|
||||
# Server Configuration
|
||||
SERVER_PORT=8888
|
||||
GIN_MODE=debug
|
||||
|
||||
# Database Configuration (PostgreSQL)
|
||||
DB_HOST=localhost
|
||||
DB_PORT=5432
|
||||
DB_USER=postgres
|
||||
DB_PASSWORD=postgres
|
||||
DB_NAME=orbit
|
||||
DB_SSLMODE=disable
|
||||
DB_MAX_OPEN_CONNS=25
|
||||
DB_MAX_IDLE_CONNS=5
|
||||
DB_CONN_MAX_LIFETIME=300
|
||||
|
||||
# Redis Configuration
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=6379
|
||||
REDIS_PASSWORD=
|
||||
REDIS_DB=0
|
||||
|
||||
# Logging Configuration
|
||||
LOG_LEVEL=debug
|
||||
LOG_FORMAT=json
|
||||
|
||||
# JWT Configuration
|
||||
JWT_SECRET=dev-secret-key-change-in-production
|
||||
JWT_ACCESS_EXPIRE=15m
|
||||
JWT_REFRESH_EXPIRE=168h
|
||||
|
||||
# Worker Configuration
|
||||
WORKER_TOKEN=dev-worker-token
|
||||
|
||||
# Storage Configuration
|
||||
WORDLISTS_BASE_PATH=/opt/orbit/wordlists
|
||||
@@ -9,8 +9,8 @@ DB_USER=postgres
|
||||
DB_PASSWORD=your_password
|
||||
DB_NAME=orbit
|
||||
DB_SSLMODE=disable
|
||||
DB_MAX_OPEN_CONNS=25
|
||||
DB_MAX_IDLE_CONNS=5
|
||||
DB_MAX_OPEN_CONNS=50
|
||||
DB_MAX_IDLE_CONNS=10
|
||||
DB_CONN_MAX_LIFETIME=300
|
||||
|
||||
# Redis Configuration
|
||||
|
||||
@@ -141,6 +141,7 @@ func main() {
|
||||
vulnerabilityRepo := repository.NewVulnerabilityRepository(db)
|
||||
scanRepo := repository.NewScanRepository(db)
|
||||
scanLogRepo := repository.NewScanLogRepository(db)
|
||||
subfinderProviderSettingsRepo := repository.NewSubfinderProviderSettingsRepository(db)
|
||||
websiteSnapshotRepo := repository.NewWebsiteSnapshotRepository(db)
|
||||
subdomainSnapshotRepo := repository.NewSubdomainSnapshotRepository(db)
|
||||
endpointSnapshotRepo := repository.NewEndpointSnapshotRepository(db)
|
||||
@@ -164,7 +165,8 @@ func main() {
|
||||
vulnerabilitySvc := service.NewVulnerabilityService(vulnerabilityRepo, targetRepo)
|
||||
scanSvc := service.NewScanService(scanRepo, scanLogRepo, targetRepo, orgRepo)
|
||||
scanLogSvc := service.NewScanLogService(scanLogRepo, scanRepo)
|
||||
scanInputSvc := service.NewScanInputService(scanRepo, subdomainRepo, websiteRepo, hostPortRepo)
|
||||
workerSvc := service.NewWorkerService(scanRepo, subfinderProviderSettingsRepo)
|
||||
agentSvc := service.NewAgentService(scanRepo)
|
||||
websiteSnapshotSvc := service.NewWebsiteSnapshotService(websiteSnapshotRepo, scanRepo, websiteSvc)
|
||||
subdomainSnapshotSvc := service.NewSubdomainSnapshotService(subdomainSnapshotRepo, scanRepo, subdomainSvc)
|
||||
endpointSnapshotSvc := service.NewEndpointSnapshotService(endpointSnapshotRepo, scanRepo, endpointSvc)
|
||||
@@ -190,7 +192,8 @@ func main() {
|
||||
vulnerabilityHandler := handler.NewVulnerabilityHandler(vulnerabilitySvc)
|
||||
scanHandler := handler.NewScanHandler(scanSvc)
|
||||
scanLogHandler := handler.NewScanLogHandler(scanLogSvc)
|
||||
scanInputHandler := handler.NewScanInputHandler(scanInputSvc)
|
||||
workerHandler := handler.NewWorkerHandler(workerSvc)
|
||||
agentHandler := handler.NewAgentHandler(agentSvc)
|
||||
websiteSnapshotHandler := handler.NewWebsiteSnapshotHandler(websiteSnapshotSvc)
|
||||
subdomainSnapshotHandler := handler.NewSubdomainSnapshotHandler(subdomainSnapshotSvc)
|
||||
endpointSnapshotHandler := handler.NewEndpointSnapshotHandler(endpointSnapshotSvc)
|
||||
@@ -218,16 +221,25 @@ func main() {
|
||||
api.GET("/screenshots/:id/image", screenshotHandler.GetImage)
|
||||
api.GET("/scans/:id/screenshots/:snapshotId/image", screenshotSnapshotHandler.GetImage)
|
||||
|
||||
// Worker API routes (token auth)
|
||||
// Worker API routes (token auth) - for Worker to fetch scan data and save results
|
||||
workerAPI := api.Group("/worker")
|
||||
workerAPI.Use(middleware.WorkerAuthMiddleware(cfg.Worker.Token))
|
||||
{
|
||||
workerAPI.GET("/scans/:id/domains", scanInputHandler.GetDomains)
|
||||
workerAPI.GET("/scans/:id/subdomains", scanInputHandler.GetSubdomains)
|
||||
workerAPI.GET("/scans/:id/websites", scanInputHandler.GetWebsites)
|
||||
workerAPI.GET("/scans/:id/hosts", scanInputHandler.GetHosts)
|
||||
workerAPI.GET("/scans/:id/provider-config", scanInputHandler.GetProviderConfig)
|
||||
workerAPI.PATCH("/scans/:id/status", scanInputHandler.UpdateStatus)
|
||||
workerAPI.GET("/scans/:id/target-name", workerHandler.GetTargetName)
|
||||
workerAPI.GET("/scans/:id/provider-config", workerHandler.GetProviderConfig)
|
||||
workerAPI.GET("/wordlists/:name", wordlistHandler.GetByName)
|
||||
workerAPI.GET("/wordlists/:name/download", wordlistHandler.DownloadByName)
|
||||
// Batch upsert endpoints - reuse existing handlers
|
||||
workerAPI.POST("/scans/:id/subdomains/bulk-upsert", subdomainSnapshotHandler.BulkUpsert)
|
||||
workerAPI.POST("/scans/:id/websites/bulk-upsert", websiteSnapshotHandler.BulkUpsert)
|
||||
workerAPI.POST("/scans/:id/endpoints/bulk-upsert", endpointSnapshotHandler.BulkUpsert)
|
||||
}
|
||||
|
||||
// Agent API routes (token auth) - for Agent to manage scan status
|
||||
agentAPI := api.Group("/agent")
|
||||
agentAPI.Use(middleware.WorkerAuthMiddleware(cfg.Worker.Token))
|
||||
{
|
||||
agentAPI.PATCH("/scans/:id/status", agentHandler.UpdateStatus)
|
||||
}
|
||||
|
||||
// Protected routes
|
||||
@@ -240,7 +252,7 @@ func main() {
|
||||
// Users
|
||||
protected.POST("/users", userHandler.Create)
|
||||
protected.GET("/users", userHandler.List)
|
||||
protected.PUT("/users/password", userHandler.UpdatePassword)
|
||||
protected.PUT("/users/me/password", userHandler.UpdatePassword)
|
||||
|
||||
// Organizations
|
||||
protected.POST("/organizations", orgHandler.Create)
|
||||
@@ -255,7 +267,7 @@ func main() {
|
||||
|
||||
// Targets
|
||||
protected.POST("/targets", targetHandler.Create)
|
||||
protected.POST("/targets/batch_create", targetHandler.BatchCreate)
|
||||
protected.POST("/targets/bulk-create", targetHandler.BatchCreate)
|
||||
protected.POST("/targets/bulk-delete", targetHandler.BulkDelete)
|
||||
protected.GET("/targets", targetHandler.List)
|
||||
protected.GET("/targets/:id", targetHandler.GetByID)
|
||||
@@ -343,19 +355,19 @@ func main() {
|
||||
// Wordlists
|
||||
protected.POST("/wordlists", wordlistHandler.Create)
|
||||
protected.GET("/wordlists", wordlistHandler.List)
|
||||
protected.GET("/wordlists/:id", wordlistHandler.Get)
|
||||
protected.GET("/wordlists/:id/download", wordlistHandler.DownloadByID)
|
||||
protected.DELETE("/wordlists/:id", wordlistHandler.Delete)
|
||||
protected.GET("/wordlists/download", wordlistHandler.Download)
|
||||
protected.GET("/wordlists/:id/content", wordlistHandler.GetContent)
|
||||
protected.PUT("/wordlists/:id/content", wordlistHandler.UpdateContent)
|
||||
|
||||
// Scans
|
||||
protected.GET("/scans", scanHandler.List)
|
||||
protected.POST("/scans", scanHandler.Create)
|
||||
protected.GET("/scans/statistics", scanHandler.Statistics)
|
||||
protected.GET("/scans/:id", scanHandler.GetByID)
|
||||
protected.DELETE("/scans/:id", scanHandler.Delete)
|
||||
protected.POST("/scans/:id/stop", scanHandler.Stop)
|
||||
protected.POST("/scans/initiate", scanHandler.Initiate)
|
||||
protected.POST("/scans/quick", scanHandler.Quick)
|
||||
protected.POST("/scans/bulk-delete", scanHandler.BulkDelete)
|
||||
|
||||
// Scan Logs (nested under scans)
|
||||
|
||||
@@ -601,3 +601,25 @@ CREATE INDEX IF NOT EXISTS idx_scan_container_ids_gin ON scan USING GIN (contain
|
||||
INSERT INTO auth_user (username, password, is_superuser, is_staff, is_active, date_joined)
|
||||
VALUES ('admin', '$2b$12$.4wL49eZfJuwVjP85Qxa7.xFb7HE3TDer4wcF9Z7c.oTOo7fExlgq', TRUE, TRUE, TRUE, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (username) DO NOTHING;
|
||||
|
||||
-- ============================================
|
||||
-- Subfinder Provider Settings (singleton)
|
||||
-- ============================================
|
||||
CREATE TABLE IF NOT EXISTS subfinder_provider_settings (
|
||||
id INTEGER PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
providers JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Insert default row with all providers disabled
|
||||
INSERT INTO subfinder_provider_settings (id, providers) VALUES (1, '{
|
||||
"fofa": {"enabled": false, "email": "", "api_key": ""},
|
||||
"hunter": {"enabled": false, "api_key": ""},
|
||||
"shodan": {"enabled": false, "api_key": ""},
|
||||
"censys": {"enabled": false, "api_id": "", "api_secret": ""},
|
||||
"zoomeye": {"enabled": false, "api_key": ""},
|
||||
"securitytrails": {"enabled": false, "api_key": ""},
|
||||
"threatbook": {"enabled": false, "api_key": ""},
|
||||
"quake": {"enabled": false, "api_key": ""}
|
||||
}'::jsonb) ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
42
server/configs/engines/subdomain_discovery.yaml
Normal file
42
server/configs/engines/subdomain_discovery.yaml
Normal file
@@ -0,0 +1,42 @@
|
||||
# Subdomain Discovery Engine Configuration
|
||||
#
|
||||
# Parameters use kebab-case (e.g., rate-limit, timeout)
|
||||
# timeout: timeout in seconds (default: 86400 = 24 hours)
|
||||
|
||||
# Stage 1: Passive Collection (parallel execution)
|
||||
# Each tool needs "enabled: true" to run
|
||||
passive-tools:
|
||||
subfinder:
|
||||
enabled: true
|
||||
timeout: 3600 # 1 hour
|
||||
# threads: 10
|
||||
|
||||
sublist3r:
|
||||
enabled: true
|
||||
timeout: 3600
|
||||
|
||||
assetfinder:
|
||||
enabled: true
|
||||
timeout: 3600
|
||||
|
||||
# Stage 2: Dictionary Bruteforce (optional)
|
||||
# Stage-level "enabled" controls whether this stage runs
|
||||
bruteforce:
|
||||
enabled: false
|
||||
subdomain-bruteforce:
|
||||
timeout: 86400 # 24 hours
|
||||
wordlist-name: subdomains-top1million-110000.txt
|
||||
|
||||
# Stage 3: Permutation + Resolve (optional)
|
||||
# Requires output from previous stages
|
||||
permutation:
|
||||
enabled: true
|
||||
subdomain-permutation-resolve:
|
||||
timeout: 86400
|
||||
|
||||
# Stage 4: DNS Resolution Validation (optional)
|
||||
# Requires output from previous stages
|
||||
resolve:
|
||||
enabled: true
|
||||
subdomain-resolve:
|
||||
timeout: 86400
|
||||
@@ -6,10 +6,10 @@ toolchain go1.24.5
|
||||
|
||||
require (
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2
|
||||
github.com/gin-gonic/gin v1.9.1
|
||||
github.com/gin-gonic/gin v1.11.0
|
||||
github.com/go-playground/locales v0.14.1
|
||||
github.com/go-playground/universal-translator v0.18.1
|
||||
github.com/go-playground/validator/v10 v10.14.0
|
||||
github.com/go-playground/validator/v10 v10.28.0
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0
|
||||
github.com/golang-migrate/migrate/v4 v4.19.1
|
||||
@@ -21,21 +21,26 @@ require (
|
||||
github.com/spf13/viper v1.18.2
|
||||
go.uber.org/zap v1.26.0
|
||||
golang.org/x/crypto v0.46.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
gorm.io/datatypes v1.2.0
|
||||
gorm.io/driver/postgres v1.5.4
|
||||
gorm.io/gorm v1.25.5
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/bytedance/sonic v1.9.1 // indirect
|
||||
github.com/bytedance/gopkg v0.1.3 // indirect
|
||||
github.com/bytedance/sonic v1.14.1 // indirect
|
||||
github.com/bytedance/sonic/loader v0.3.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
|
||||
github.com/cloudwego/base64x v0.1.6 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.10 // indirect
|
||||
github.com/gin-contrib/gzip v1.2.5 // indirect
|
||||
github.com/gin-contrib/sse v1.1.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.7.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
|
||||
@@ -43,14 +48,16 @@ require (
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/leodido/go-urn v1.2.4 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
github.com/quic-go/quic-go v0.55.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
@@ -59,16 +66,18 @@ require (
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.11 // indirect
|
||||
github.com/ugorji/go/codec v1.3.0 // indirect
|
||||
go.uber.org/mock v0.6.0 // indirect
|
||||
go.uber.org/multierr v1.10.0 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/arch v0.22.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
|
||||
golang.org/x/mod v0.30.0 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
google.golang.org/protobuf v1.36.7 // indirect
|
||||
golang.org/x/tools v0.39.0 // indirect
|
||||
google.golang.org/protobuf v1.36.10 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
gorm.io/driver/mysql v1.4.7 // indirect
|
||||
)
|
||||
|
||||
@@ -51,19 +51,21 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
|
||||
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
|
||||
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
|
||||
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
|
||||
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
|
||||
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
|
||||
github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w=
|
||||
github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc=
|
||||
github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
|
||||
github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
|
||||
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
|
||||
@@ -105,13 +107,15 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
||||
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
|
||||
github.com/gabriel-vasile/mimetype v1.4.10 h1:zyueNbySn/z8mJZHLt6IPw0KoZsiQNszIpU+bX4+ZK0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.10/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
|
||||
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
|
||||
github.com/gin-contrib/gzip v1.2.5 h1:fIZs0S+l17pIu1P5XRJOo/YNqfIuPCrZZ3TWB7pjckI=
|
||||
github.com/gin-contrib/gzip v1.2.5/go.mod h1:aomRgR7ftdZV3uWY0gW/m8rChfxau0n8YVvwlOHONzw=
|
||||
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
|
||||
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
|
||||
github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
|
||||
github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
@@ -125,14 +129,16 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
|
||||
github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
|
||||
github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688=
|
||||
github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
|
||||
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
|
||||
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
@@ -187,8 +193,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
@@ -258,9 +264,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
@@ -271,8 +276,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4=
|
||||
github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c=
|
||||
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
||||
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
|
||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
|
||||
@@ -280,8 +285,8 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V
|
||||
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/microsoft/go-mssqldb v1.0.0 h1:k2p2uuG8T5T/7Hp7/e3vMGTnnR0sU4h8d1CcC71iLHU=
|
||||
@@ -324,8 +329,8 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ
|
||||
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
|
||||
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
|
||||
github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@@ -335,6 +340,10 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
|
||||
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
|
||||
github.com/quic-go/quic-go v0.55.0 h1:zccPQIqYCXDt5NmcEabyYvOnomjs8Tlwl7tISjJh9Mk=
|
||||
github.com/quic-go/quic-go v0.55.0/go.mod h1:DR51ilwU1uE164KuWXhinFcKWGlEjzys2l8zUl5Ss1U=
|
||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
@@ -384,17 +393,15 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
|
||||
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA=
|
||||
github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
@@ -424,15 +431,16 @@ go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXe
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
|
||||
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
|
||||
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
|
||||
go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
|
||||
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
||||
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
|
||||
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI=
|
||||
golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
@@ -484,6 +492,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
|
||||
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -597,7 +607,6 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -677,6 +686,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
|
||||
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
|
||||
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -783,8 +794,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A=
|
||||
google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
|
||||
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
|
||||
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
@@ -825,6 +836,5 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
|
||||
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
|
||||
12
server/internal/dto/agent.go
Normal file
12
server/internal/dto/agent.go
Normal file
@@ -0,0 +1,12 @@
|
||||
package dto
|
||||
|
||||
// AgentUpdateStatusRequest represents update scan status request from agent
|
||||
type AgentUpdateStatusRequest struct {
|
||||
Status string `json:"status" binding:"required,oneof=scheduled running completed failed cancelled"`
|
||||
ErrorMessage string `json:"errorMessage" binding:"omitempty"`
|
||||
}
|
||||
|
||||
// AgentUpdateStatusResponse represents update scan status response
|
||||
type AgentUpdateStatusResponse struct {
|
||||
Success bool `json:"success"`
|
||||
}
|
||||
@@ -58,7 +58,7 @@ type ScanDetailResponse struct {
|
||||
StageProgress map[string]interface{} `json:"stageProgress,omitempty"`
|
||||
}
|
||||
|
||||
// InitiateScanRequest represents initiate scan request
|
||||
// InitiateScanRequest represents initiate scan request (deprecated, use CreateScanRequest)
|
||||
type InitiateScanRequest struct {
|
||||
OrganizationID *int `json:"organizationId" binding:"omitempty"`
|
||||
TargetID *int `json:"targetId" binding:"omitempty"`
|
||||
@@ -67,7 +67,7 @@ type InitiateScanRequest struct {
|
||||
Configuration string `json:"configuration" binding:"required"`
|
||||
}
|
||||
|
||||
// QuickScanRequest represents quick scan request
|
||||
// QuickScanRequest represents quick scan request (deprecated, use CreateScanRequest)
|
||||
type QuickScanRequest struct {
|
||||
Targets []QuickScanTarget `json:"targets" binding:"required,min=1"`
|
||||
EngineIDs []int `json:"engineIds"`
|
||||
@@ -75,6 +75,24 @@ type QuickScanRequest struct {
|
||||
Configuration string `json:"configuration" binding:"required"`
|
||||
}
|
||||
|
||||
// CreateScanRequest represents unified scan creation request
|
||||
// POST /api/scans
|
||||
type CreateScanRequest struct {
|
||||
// Mode: "normal" (default) or "quick"
|
||||
Mode string `json:"mode" binding:"omitempty,oneof=normal quick"`
|
||||
|
||||
// For mode=normal: target ID (required)
|
||||
TargetID int `json:"targetId" binding:"omitempty"`
|
||||
|
||||
// For mode=quick: raw targets (required)
|
||||
Targets []string `json:"targets" binding:"omitempty"`
|
||||
|
||||
// Common fields
|
||||
EngineIDs []int `json:"engineIds" binding:"omitempty"`
|
||||
EngineNames []string `json:"engineNames" binding:"omitempty"`
|
||||
Configuration string `json:"configuration" binding:"omitempty"`
|
||||
}
|
||||
|
||||
// QuickScanTarget represents a target in quick scan
|
||||
type QuickScanTarget struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
|
||||
@@ -2,6 +2,11 @@ package dto
|
||||
|
||||
import "time"
|
||||
|
||||
// UpdateWordlistContentRequest represents update wordlist content request
|
||||
type UpdateWordlistContentRequest struct {
|
||||
Content string `json:"content" binding:"required"`
|
||||
}
|
||||
|
||||
// WordlistResponse represents wordlist response
|
||||
type WordlistResponse struct {
|
||||
ID int `json:"id"`
|
||||
@@ -15,11 +20,6 @@ type WordlistResponse struct {
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
// UpdateWordlistContentRequest represents update wordlist content request
|
||||
type UpdateWordlistContentRequest struct {
|
||||
Content string `json:"content" binding:"required"`
|
||||
}
|
||||
|
||||
// WordlistContentResponse represents wordlist content response
|
||||
type WordlistContentResponse struct {
|
||||
Content string `json:"content"`
|
||||
|
||||
12
server/internal/dto/worker.go
Normal file
12
server/internal/dto/worker.go
Normal file
@@ -0,0 +1,12 @@
|
||||
package dto
|
||||
|
||||
// WorkerTargetNameResponse is the response for GetTargetName
|
||||
type WorkerTargetNameResponse struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
// WorkerProviderConfigResponse is the response for GetProviderConfig
|
||||
type WorkerProviderConfigResponse struct {
|
||||
Content string `json:"content"`
|
||||
}
|
||||
50
server/internal/handler/agent.go
Normal file
50
server/internal/handler/agent.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/orbit/server/internal/dto"
|
||||
"github.com/orbit/server/internal/service"
|
||||
)
|
||||
|
||||
// AgentHandler handles agent API endpoints
|
||||
type AgentHandler struct {
|
||||
svc *service.AgentService
|
||||
}
|
||||
|
||||
// NewAgentHandler creates a new agent handler
|
||||
func NewAgentHandler(svc *service.AgentService) *AgentHandler {
|
||||
return &AgentHandler{svc: svc}
|
||||
}
|
||||
|
||||
// UpdateStatus updates scan status (called by Agent based on Worker exit code)
|
||||
// PATCH /api/agent/scans/:id/status
|
||||
func (h *AgentHandler) UpdateStatus(c *gin.Context) {
|
||||
scanID, err := strconv.Atoi(c.Param("id"))
|
||||
if err != nil {
|
||||
dto.BadRequest(c, "Invalid scan ID")
|
||||
return
|
||||
}
|
||||
|
||||
var req dto.AgentUpdateStatusRequest
|
||||
if !dto.BindJSON(c, &req) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.svc.UpdateStatus(scanID, req.Status, req.ErrorMessage); err != nil {
|
||||
if errors.Is(err, service.ErrAgentScanNotFound) {
|
||||
dto.NotFound(c, "Scan not found")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, service.ErrAgentInvalidTransition) {
|
||||
dto.BadRequest(c, "Invalid status transition")
|
||||
return
|
||||
}
|
||||
dto.InternalError(c, "Failed to update status")
|
||||
return
|
||||
}
|
||||
|
||||
dto.Success(c, dto.AgentUpdateStatusResponse{Success: true})
|
||||
}
|
||||
@@ -151,16 +151,49 @@ func (h *ScanHandler) Stop(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// Initiate starts a new scan
|
||||
// POST /api/scans/initiate
|
||||
func (h *ScanHandler) Initiate(c *gin.Context) {
|
||||
// TODO: Implement when worker integration is ready
|
||||
dto.Error(c, http.StatusNotImplemented, "NOT_IMPLEMENTED", "Scan initiation is not yet implemented")
|
||||
}
|
||||
// Create starts a new scan
|
||||
// POST /api/scans
|
||||
//
|
||||
// Request body:
|
||||
//
|
||||
// {
|
||||
// "mode": "normal" | "quick", // scan mode (default: "normal")
|
||||
// "targetId": 123, // required for mode=normal
|
||||
// "targets": ["example.com"], // required for mode=quick (raw targets)
|
||||
// "engineIds": [1, 2], // engine IDs to run
|
||||
// "config": {} // optional scan configuration
|
||||
// }
|
||||
func (h *ScanHandler) Create(c *gin.Context) {
|
||||
var req dto.CreateScanRequest
|
||||
if !dto.BindJSON(c, &req) {
|
||||
return
|
||||
}
|
||||
|
||||
// Quick starts a quick scan with raw targets
|
||||
// POST /api/scans/quick
|
||||
func (h *ScanHandler) Quick(c *gin.Context) {
|
||||
// TODO: Implement when worker integration is ready
|
||||
dto.Error(c, http.StatusNotImplemented, "NOT_IMPLEMENTED", "Quick scan is not yet implemented")
|
||||
// Default mode is "normal"
|
||||
if req.Mode == "" {
|
||||
req.Mode = "normal"
|
||||
}
|
||||
|
||||
switch req.Mode {
|
||||
case "normal":
|
||||
// Normal scan: requires targetId
|
||||
if req.TargetID == 0 {
|
||||
dto.BadRequest(c, "targetId is required for normal mode")
|
||||
return
|
||||
}
|
||||
// TODO: Implement when worker integration is ready
|
||||
dto.Error(c, http.StatusNotImplemented, "NOT_IMPLEMENTED", "Normal scan is not yet implemented")
|
||||
|
||||
case "quick":
|
||||
// Quick scan: requires targets list
|
||||
if len(req.Targets) == 0 {
|
||||
dto.BadRequest(c, "targets is required for quick mode")
|
||||
return
|
||||
}
|
||||
// TODO: Implement when worker integration is ready
|
||||
dto.Error(c, http.StatusNotImplemented, "NOT_IMPLEMENTED", "Quick scan is not yet implemented")
|
||||
|
||||
default:
|
||||
dto.BadRequest(c, "Invalid mode, must be 'normal' or 'quick'")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,8 +78,8 @@ func (h *UserHandler) List(c *gin.Context) {
|
||||
dto.Paginated(c, resp, total, query.GetPage(), query.GetPageSize())
|
||||
}
|
||||
|
||||
// UpdatePassword updates user password
|
||||
// PUT /api/users/:id/password
|
||||
// UpdatePassword updates current user's password
|
||||
// PUT /api/users/me/password
|
||||
func (h *UserHandler) UpdatePassword(c *gin.Context) {
|
||||
// Get current user from context
|
||||
claims, ok := middleware.GetUserClaims(c)
|
||||
|
||||
@@ -2,6 +2,7 @@ package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
@@ -10,46 +11,215 @@ import (
|
||||
"github.com/orbit/server/internal/service"
|
||||
)
|
||||
|
||||
// WordlistHandler handles wordlist endpoints
|
||||
// WordlistHandler handles wordlist API requests
|
||||
type WordlistHandler struct {
|
||||
svc *service.WordlistService
|
||||
}
|
||||
|
||||
// NewWordlistHandler creates a new wordlist handler
|
||||
func NewWordlistHandler(svc *service.WordlistService) *WordlistHandler {
|
||||
return &WordlistHandler{svc: svc}
|
||||
return &WordlistHandler{
|
||||
svc: svc,
|
||||
}
|
||||
}
|
||||
|
||||
// Create uploads and creates a new wordlist
|
||||
// POST /api/wordlists
|
||||
// List returns all wordlists
|
||||
// GET /api/wordlists/
|
||||
func (h *WordlistHandler) List(c *gin.Context) {
|
||||
wordlists, err := h.svc.ListAll()
|
||||
if err != nil {
|
||||
dto.InternalError(c, "Failed to list wordlists")
|
||||
return
|
||||
}
|
||||
|
||||
// Initialize empty slice to return [] instead of null
|
||||
resp := make([]dto.WordlistResponse, 0, len(wordlists))
|
||||
for _, w := range wordlists {
|
||||
resp = append(resp, dto.WordlistResponse{
|
||||
ID: w.ID,
|
||||
Name: w.Name,
|
||||
Description: w.Description,
|
||||
FilePath: w.FilePath,
|
||||
FileSize: w.FileSize,
|
||||
LineCount: w.LineCount,
|
||||
FileHash: w.FileHash,
|
||||
CreatedAt: w.CreatedAt,
|
||||
UpdatedAt: w.UpdatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
dto.Success(c, resp)
|
||||
}
|
||||
|
||||
// Get returns a wordlist by ID
|
||||
// GET /api/wordlists/:id
|
||||
func (h *WordlistHandler) Get(c *gin.Context) {
|
||||
id, err := strconv.Atoi(c.Param("id"))
|
||||
if err != nil {
|
||||
dto.BadRequest(c, "Invalid wordlist ID")
|
||||
return
|
||||
}
|
||||
|
||||
wordlist, err := h.svc.GetByID(id)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWordlistNotFound) {
|
||||
dto.NotFound(c, "Wordlist not found")
|
||||
return
|
||||
}
|
||||
dto.InternalError(c, "Failed to get wordlist")
|
||||
return
|
||||
}
|
||||
|
||||
dto.Success(c, dto.WordlistResponse{
|
||||
ID: wordlist.ID,
|
||||
Name: wordlist.Name,
|
||||
Description: wordlist.Description,
|
||||
FilePath: wordlist.FilePath,
|
||||
FileSize: wordlist.FileSize,
|
||||
LineCount: wordlist.LineCount,
|
||||
FileHash: wordlist.FileHash,
|
||||
CreatedAt: wordlist.CreatedAt,
|
||||
UpdatedAt: wordlist.UpdatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
// GetByName returns a wordlist by name (for worker API)
|
||||
// GET /api/worker/wordlists/:name
|
||||
func (h *WordlistHandler) GetByName(c *gin.Context) {
|
||||
name := c.Param("name")
|
||||
if name == "" {
|
||||
dto.BadRequest(c, "Name is required")
|
||||
return
|
||||
}
|
||||
|
||||
wordlist, err := h.svc.GetByName(name)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWordlistNotFound) {
|
||||
dto.NotFound(c, "Wordlist not found")
|
||||
return
|
||||
}
|
||||
dto.InternalError(c, "Failed to get wordlist")
|
||||
return
|
||||
}
|
||||
|
||||
dto.Success(c, dto.WordlistResponse{
|
||||
ID: wordlist.ID,
|
||||
Name: wordlist.Name,
|
||||
Description: wordlist.Description,
|
||||
FilePath: wordlist.FilePath,
|
||||
FileSize: wordlist.FileSize,
|
||||
LineCount: wordlist.LineCount,
|
||||
FileHash: wordlist.FileHash,
|
||||
CreatedAt: wordlist.CreatedAt,
|
||||
UpdatedAt: wordlist.UpdatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
// DownloadByID serves the wordlist file by ID (RESTful)
|
||||
// GET /api/wordlists/:id/download
|
||||
func (h *WordlistHandler) DownloadByID(c *gin.Context) {
|
||||
id, err := strconv.Atoi(c.Param("id"))
|
||||
if err != nil {
|
||||
dto.BadRequest(c, "Invalid wordlist ID")
|
||||
return
|
||||
}
|
||||
|
||||
wordlist, err := h.svc.GetByID(id)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWordlistNotFound) {
|
||||
dto.NotFound(c, "Wordlist not found")
|
||||
return
|
||||
}
|
||||
dto.InternalError(c, "Failed to get wordlist")
|
||||
return
|
||||
}
|
||||
|
||||
h.serveWordlistFile(c, wordlist.Name)
|
||||
}
|
||||
|
||||
// DownloadByName serves the wordlist file (path parameter style - RESTful)
|
||||
// GET /api/worker/wordlists/:name/download
|
||||
func (h *WordlistHandler) DownloadByName(c *gin.Context) {
|
||||
name := c.Param("name")
|
||||
if name == "" {
|
||||
dto.BadRequest(c, "Name is required")
|
||||
return
|
||||
}
|
||||
|
||||
h.serveWordlistFile(c, name)
|
||||
}
|
||||
|
||||
// serveWordlistFile is a helper to serve wordlist file by name
|
||||
func (h *WordlistHandler) serveWordlistFile(c *gin.Context, name string) {
|
||||
filePath, err := h.svc.GetFilePath(name)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWordlistNotFound) {
|
||||
dto.NotFound(c, "Wordlist not found")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, service.ErrFileNotFound) {
|
||||
dto.NotFound(c, "Wordlist file not found on server")
|
||||
return
|
||||
}
|
||||
dto.InternalError(c, "Failed to get wordlist")
|
||||
return
|
||||
}
|
||||
|
||||
// Check if file exists
|
||||
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
||||
dto.NotFound(c, "Wordlist file not found on server")
|
||||
return
|
||||
}
|
||||
|
||||
// Serve file
|
||||
c.Header("Content-Disposition", "attachment; filename="+filepath.Base(filePath))
|
||||
c.Header("Content-Type", "application/octet-stream")
|
||||
c.File(filePath)
|
||||
}
|
||||
|
||||
// Create creates a new wordlist with file upload
|
||||
// POST /api/wordlists/
|
||||
// Content-Type: multipart/form-data
|
||||
// Fields: name (required), description (optional), file (required)
|
||||
func (h *WordlistHandler) Create(c *gin.Context) {
|
||||
// Get form fields
|
||||
name := c.PostForm("name")
|
||||
if name == "" {
|
||||
dto.BadRequest(c, "Name is required")
|
||||
return
|
||||
}
|
||||
|
||||
description := c.PostForm("description")
|
||||
|
||||
file, err := c.FormFile("file")
|
||||
// Get uploaded file (streaming, not loaded into memory)
|
||||
file, header, err := c.Request.FormFile("file")
|
||||
if err != nil {
|
||||
dto.BadRequest(c, "Missing wordlist file")
|
||||
dto.BadRequest(c, "File is required")
|
||||
return
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
// Open the uploaded file
|
||||
src, err := file.Open()
|
||||
if err != nil {
|
||||
dto.InternalError(c, "Failed to read uploaded file")
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
_ = src.Close() // Ignore close error in defer
|
||||
}()
|
||||
|
||||
wordlist, err := h.svc.Create(name, description, file.Filename, src)
|
||||
// Create wordlist with streamed file content
|
||||
wordlist, err := h.svc.Create(name, description, header.Filename, file)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWordlistExists) {
|
||||
dto.BadRequest(c, "Wordlist name already exists")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, service.ErrEmptyName) {
|
||||
dto.BadRequest(c, "Wordlist name cannot be empty")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, service.ErrWordlistExists) {
|
||||
dto.BadRequest(c, "Wordlist name already exists")
|
||||
if errors.Is(err, service.ErrNameTooLong) {
|
||||
dto.BadRequest(c, "Wordlist name too long (max 200 characters)")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, service.ErrInvalidName) {
|
||||
dto.BadRequest(c, "Wordlist name contains invalid characters (newlines, tabs, etc.)")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, service.ErrInvalidFileType) {
|
||||
dto.BadRequest(c, "File appears to be binary, only text files are allowed")
|
||||
return
|
||||
}
|
||||
dto.InternalError(c, "Failed to create wordlist")
|
||||
@@ -69,38 +239,6 @@ func (h *WordlistHandler) Create(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// List returns paginated wordlists
|
||||
// GET /api/wordlists
|
||||
func (h *WordlistHandler) List(c *gin.Context) {
|
||||
var query dto.PaginationQuery
|
||||
if !dto.BindQuery(c, &query) {
|
||||
return
|
||||
}
|
||||
|
||||
wordlists, total, err := h.svc.List(&query)
|
||||
if err != nil {
|
||||
dto.InternalError(c, "Failed to list wordlists")
|
||||
return
|
||||
}
|
||||
|
||||
var resp []dto.WordlistResponse
|
||||
for _, w := range wordlists {
|
||||
resp = append(resp, dto.WordlistResponse{
|
||||
ID: w.ID,
|
||||
Name: w.Name,
|
||||
Description: w.Description,
|
||||
FilePath: w.FilePath,
|
||||
FileSize: w.FileSize,
|
||||
LineCount: w.LineCount,
|
||||
FileHash: w.FileHash,
|
||||
CreatedAt: w.CreatedAt,
|
||||
UpdatedAt: w.UpdatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
dto.Paginated(c, resp, total, query.GetPage(), query.GetPageSize())
|
||||
}
|
||||
|
||||
// Delete deletes a wordlist
|
||||
// DELETE /api/wordlists/:id
|
||||
func (h *WordlistHandler) Delete(c *gin.Context) {
|
||||
@@ -110,8 +248,7 @@ func (h *WordlistHandler) Delete(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
err = h.svc.Delete(id)
|
||||
if err != nil {
|
||||
if err := h.svc.Delete(id); err != nil {
|
||||
if errors.Is(err, service.ErrWordlistNotFound) {
|
||||
dto.NotFound(c, "Wordlist not found")
|
||||
return
|
||||
@@ -123,27 +260,8 @@ func (h *WordlistHandler) Delete(c *gin.Context) {
|
||||
dto.NoContent(c)
|
||||
}
|
||||
|
||||
// Download downloads a wordlist file by name
|
||||
// GET /api/wordlists/download?wordlist=xxx
|
||||
func (h *WordlistHandler) Download(c *gin.Context) {
|
||||
name := c.Query("wordlist")
|
||||
if name == "" {
|
||||
dto.BadRequest(c, "Missing parameter: wordlist")
|
||||
return
|
||||
}
|
||||
|
||||
filePath, err := h.svc.GetFilePath(name)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWordlistNotFound) || errors.Is(err, service.ErrFileNotFound) {
|
||||
dto.NotFound(c, "Wordlist not found")
|
||||
return
|
||||
}
|
||||
dto.InternalError(c, "Failed to get wordlist")
|
||||
return
|
||||
}
|
||||
|
||||
c.FileAttachment(filePath, filepath.Base(filePath))
|
||||
}
|
||||
// maxEditableSize is the maximum file size allowed for online editing (10MB)
|
||||
const maxEditableSize = 10 * 1024 * 1024
|
||||
|
||||
// GetContent returns the content of a wordlist file
|
||||
// GET /api/wordlists/:id/content
|
||||
@@ -154,6 +272,22 @@ func (h *WordlistHandler) GetContent(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check file size first
|
||||
wordlist, err := h.svc.GetByID(id)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWordlistNotFound) {
|
||||
dto.NotFound(c, "Wordlist not found")
|
||||
return
|
||||
}
|
||||
dto.InternalError(c, "Failed to get wordlist")
|
||||
return
|
||||
}
|
||||
|
||||
if wordlist.FileSize > maxEditableSize {
|
||||
dto.BadRequest(c, "File too large for online editing (max 10MB), please download and edit locally")
|
||||
return
|
||||
}
|
||||
|
||||
content, err := h.svc.GetContent(id)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWordlistNotFound) {
|
||||
@@ -180,12 +314,34 @@ func (h *WordlistHandler) UpdateContent(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// Check file size first
|
||||
wordlist, err := h.svc.GetByID(id)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWordlistNotFound) {
|
||||
dto.NotFound(c, "Wordlist not found")
|
||||
return
|
||||
}
|
||||
dto.InternalError(c, "Failed to get wordlist")
|
||||
return
|
||||
}
|
||||
|
||||
if wordlist.FileSize > maxEditableSize {
|
||||
dto.BadRequest(c, "File too large for online editing (max 10MB), please re-upload the file")
|
||||
return
|
||||
}
|
||||
|
||||
var req dto.UpdateWordlistContentRequest
|
||||
if !dto.BindJSON(c, &req) {
|
||||
return
|
||||
}
|
||||
|
||||
wordlist, err := h.svc.UpdateContent(id, req.Content)
|
||||
// Also check the new content size
|
||||
if int64(len(req.Content)) > maxEditableSize {
|
||||
dto.BadRequest(c, "Content too large (max 10MB)")
|
||||
return
|
||||
}
|
||||
|
||||
wordlist, err = h.svc.UpdateContent(id, req.Content)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWordlistNotFound) {
|
||||
dto.NotFound(c, "Wordlist not found")
|
||||
|
||||
72
server/internal/handler/worker.go
Normal file
72
server/internal/handler/worker.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/orbit/server/internal/dto"
|
||||
"github.com/orbit/server/internal/service"
|
||||
)
|
||||
|
||||
// WorkerHandler handles worker API endpoints
|
||||
type WorkerHandler struct {
|
||||
svc *service.WorkerService
|
||||
}
|
||||
|
||||
// NewWorkerHandler creates a new worker handler
|
||||
func NewWorkerHandler(svc *service.WorkerService) *WorkerHandler {
|
||||
return &WorkerHandler{svc: svc}
|
||||
}
|
||||
|
||||
// GetTargetName returns target name for a scan
|
||||
// GET /api/worker/scans/:id/target-name
|
||||
func (h *WorkerHandler) GetTargetName(c *gin.Context) {
|
||||
scanID, err := strconv.Atoi(c.Param("id"))
|
||||
if err != nil {
|
||||
dto.BadRequest(c, "Invalid scan ID")
|
||||
return
|
||||
}
|
||||
|
||||
target, err := h.svc.GetTargetName(scanID)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWorkerScanNotFound) {
|
||||
dto.NotFound(c, "Scan not found")
|
||||
return
|
||||
}
|
||||
dto.InternalError(c, "Failed to get target name")
|
||||
return
|
||||
}
|
||||
|
||||
dto.Success(c, dto.WorkerTargetNameResponse{
|
||||
Name: target.Name,
|
||||
Type: target.Type,
|
||||
})
|
||||
}
|
||||
|
||||
// GetProviderConfig returns provider config for a tool
|
||||
// GET /api/worker/scans/:id/provider-config?tool=subfinder
|
||||
func (h *WorkerHandler) GetProviderConfig(c *gin.Context) {
|
||||
scanID, err := strconv.Atoi(c.Param("id"))
|
||||
if err != nil {
|
||||
dto.BadRequest(c, "Invalid scan ID")
|
||||
return
|
||||
}
|
||||
|
||||
toolName := c.Query("tool")
|
||||
config, err := h.svc.GetProviderConfig(scanID, toolName)
|
||||
if err != nil {
|
||||
if errors.Is(err, service.ErrWorkerScanNotFound) {
|
||||
dto.NotFound(c, "Scan not found")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, service.ErrWorkerToolRequired) {
|
||||
dto.BadRequest(c, "Tool parameter required")
|
||||
return
|
||||
}
|
||||
dto.InternalError(c, "Failed to get provider config")
|
||||
return
|
||||
}
|
||||
|
||||
dto.Success(c, config)
|
||||
}
|
||||
30
server/internal/middleware/worker_auth.go
Normal file
30
server/internal/middleware/worker_auth.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// WorkerAuthMiddleware creates a simple token authentication middleware for workers
|
||||
// Workers use a static token via X-Worker-Token header (not JWT)
|
||||
func WorkerAuthMiddleware(workerToken string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
token := c.GetHeader("X-Worker-Token")
|
||||
if token == "" {
|
||||
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{
|
||||
"error": "X-Worker-Token header required",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if token != workerToken {
|
||||
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{
|
||||
"error": "Invalid worker token",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
@@ -66,13 +66,14 @@ func (s *Scan) BeforeCreate(tx *gorm.DB) error {
|
||||
}
|
||||
|
||||
// ScanStatus constants
|
||||
// Status flow: pending → scheduled → running → completed/failed/cancelled
|
||||
const (
|
||||
ScanStatusInitiated = "initiated"
|
||||
ScanStatusRunning = "running"
|
||||
ScanStatusCompleted = "completed"
|
||||
ScanStatusFailed = "failed"
|
||||
ScanStatusStopped = "stopped"
|
||||
ScanStatusPending = "pending"
|
||||
ScanStatusPending = "pending" // Waiting for scheduling (user initiated, waiting for Agent assignment)
|
||||
ScanStatusScheduled = "scheduled" // Assigned to Agent, waiting for execution
|
||||
ScanStatusRunning = "running" // Currently executing
|
||||
ScanStatusCompleted = "completed" // Successfully completed
|
||||
ScanStatusFailed = "failed" // Execution failed
|
||||
ScanStatusCancelled = "cancelled" // Cancelled (by user or system)
|
||||
)
|
||||
|
||||
// ScanMode constants
|
||||
|
||||
@@ -1,34 +1,76 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"gorm.io/datatypes"
|
||||
)
|
||||
|
||||
// SubfinderProviderSettings represents subfinder provider settings (singleton)
|
||||
// SubfinderProviderSettings stores API keys for subfinder data sources (singleton, id=1)
|
||||
type SubfinderProviderSettings struct {
|
||||
ID int `gorm:"primaryKey" json:"id"`
|
||||
Providers datatypes.JSON `gorm:"column:providers;type:jsonb" json:"providers"`
|
||||
CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime" json:"updatedAt"`
|
||||
ID int `gorm:"primaryKey" json:"id"`
|
||||
Providers ProviderConfigs `gorm:"column:providers;type:jsonb" json:"providers"`
|
||||
CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime" json:"updatedAt"`
|
||||
}
|
||||
|
||||
// TableName returns the table name for SubfinderProviderSettings
|
||||
func (SubfinderProviderSettings) TableName() string {
|
||||
return "subfinder_provider_settings"
|
||||
}
|
||||
|
||||
// DefaultProviders returns the default provider configuration
|
||||
func DefaultProviders() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"fofa": map[string]interface{}{"enabled": false, "email": "", "api_key": ""},
|
||||
"hunter": map[string]interface{}{"enabled": false, "api_key": ""},
|
||||
"shodan": map[string]interface{}{"enabled": false, "api_key": ""},
|
||||
"censys": map[string]interface{}{"enabled": false, "api_id": "", "api_secret": ""},
|
||||
"zoomeye": map[string]interface{}{"enabled": false, "api_key": ""},
|
||||
"securitytrails": map[string]interface{}{"enabled": false, "api_key": ""},
|
||||
"threatbook": map[string]interface{}{"enabled": false, "api_key": ""},
|
||||
"quake": map[string]interface{}{"enabled": false, "api_key": ""},
|
||||
}
|
||||
// ProviderConfigs maps provider name to its configuration
|
||||
type ProviderConfigs map[string]ProviderConfig
|
||||
|
||||
// ProviderConfig holds credentials for a single provider
|
||||
type ProviderConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Email string `json:"email,omitempty"`
|
||||
APIKey string `json:"api_key,omitempty"`
|
||||
APIId string `json:"api_id,omitempty"`
|
||||
APISecret string `json:"api_secret,omitempty"`
|
||||
}
|
||||
|
||||
// Scan implements sql.Scanner for GORM
|
||||
func (p *ProviderConfigs) Scan(value any) error {
|
||||
if value == nil {
|
||||
*p = make(ProviderConfigs)
|
||||
return nil
|
||||
}
|
||||
bytes, ok := value.([]byte)
|
||||
if !ok {
|
||||
return errors.New("ProviderConfigs: expected []byte from database")
|
||||
}
|
||||
return json.Unmarshal(bytes, p)
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer for GORM
|
||||
func (p ProviderConfigs) Value() (driver.Value, error) {
|
||||
return json.Marshal(p)
|
||||
}
|
||||
|
||||
// ProviderFormatType defines how provider credentials are formatted
|
||||
type ProviderFormatType string
|
||||
|
||||
const (
|
||||
FormatTypeSingle ProviderFormatType = "single"
|
||||
FormatTypeComposite ProviderFormatType = "composite"
|
||||
)
|
||||
|
||||
// ProviderFormat defines the credential format for a provider
|
||||
type ProviderFormat struct {
|
||||
Type ProviderFormatType
|
||||
Format string // field name for single, template for composite (e.g., "{email}:{api_key}")
|
||||
}
|
||||
|
||||
// ProviderFormats defines credential formats for generating subfinder config YAML
|
||||
var ProviderFormats = map[string]ProviderFormat{
|
||||
"fofa": {Type: FormatTypeComposite, Format: "{email}:{api_key}"},
|
||||
"censys": {Type: FormatTypeComposite, Format: "{api_id}:{api_secret}"},
|
||||
"hunter": {Type: FormatTypeSingle, Format: "api_key"},
|
||||
"shodan": {Type: FormatTypeSingle, Format: "api_key"},
|
||||
"zoomeye": {Type: FormatTypeSingle, Format: "api_key"},
|
||||
"securitytrails": {Type: FormatTypeSingle, Format: "api_key"},
|
||||
"threatbook": {Type: FormatTypeSingle, Format: "api_key"},
|
||||
"quake": {Type: FormatTypeSingle, Format: "api_key"},
|
||||
}
|
||||
|
||||
@@ -1,23 +1,21 @@
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
import "time"
|
||||
|
||||
// Wordlist represents a wordlist file
|
||||
// Wordlist represents a dictionary file for scanning
|
||||
type Wordlist struct {
|
||||
ID int `gorm:"primaryKey;autoIncrement" json:"id"`
|
||||
Name string `gorm:"column:name;size:200;uniqueIndex:unique_wordlist_name" json:"name"`
|
||||
ID int `gorm:"primaryKey" json:"id"`
|
||||
Name string `gorm:"column:name;size:200;uniqueIndex" json:"name"`
|
||||
Description string `gorm:"column:description;size:200" json:"description"`
|
||||
FilePath string `gorm:"column:file_path;size:500" json:"filePath"`
|
||||
FileSize int64 `gorm:"column:file_size;default:0" json:"fileSize"`
|
||||
LineCount int `gorm:"column:line_count;default:0" json:"lineCount"`
|
||||
FileHash string `gorm:"column:file_hash;size:64" json:"fileHash"`
|
||||
CreatedAt time.Time `gorm:"column:created_at;autoCreateTime;index:idx_wordlist_created_at" json:"createdAt"`
|
||||
CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"createdAt"`
|
||||
UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime" json:"updatedAt"`
|
||||
}
|
||||
|
||||
// TableName returns the table name for Wordlist
|
||||
// TableName returns the table name for GORM
|
||||
func (Wordlist) TableName() string {
|
||||
return "wordlist"
|
||||
}
|
||||
|
||||
@@ -56,13 +56,26 @@ func (r *DirectoryRepository) BulkCreate(directories []model.Directory) (int, er
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Use ON CONFLICT DO NOTHING to ignore duplicates (url + target_id unique)
|
||||
result := r.db.Clauses(clause.OnConflict{DoNothing: true}).Create(&directories)
|
||||
if result.Error != nil {
|
||||
return 0, result.Error
|
||||
var totalAffected int
|
||||
|
||||
// Process in batches to avoid SQL statement size limits
|
||||
batchSize := 500
|
||||
for i := 0; i < len(directories); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(directories) {
|
||||
end = len(directories)
|
||||
}
|
||||
batch := directories[i:end]
|
||||
|
||||
// Use ON CONFLICT DO NOTHING to ignore duplicates (url + target_id unique)
|
||||
result := r.db.Clauses(clause.OnConflict{DoNothing: true}).Create(&batch)
|
||||
if result.Error != nil {
|
||||
return totalAffected, result.Error
|
||||
}
|
||||
totalAffected += int(result.RowsAffected)
|
||||
}
|
||||
|
||||
return int(result.RowsAffected), nil
|
||||
return totalAffected, nil
|
||||
}
|
||||
|
||||
// BulkDelete deletes multiple directories by IDs
|
||||
|
||||
@@ -69,13 +69,26 @@ func (r *EndpointRepository) BulkCreate(endpoints []model.Endpoint) (int, error)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Use ON CONFLICT DO NOTHING to ignore duplicates (url + target_id unique)
|
||||
result := r.db.Clauses(clause.OnConflict{DoNothing: true}).Create(&endpoints)
|
||||
if result.Error != nil {
|
||||
return 0, result.Error
|
||||
var totalAffected int
|
||||
|
||||
// Process in batches to avoid SQL statement size limits
|
||||
batchSize := 500
|
||||
for i := 0; i < len(endpoints); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(endpoints) {
|
||||
end = len(endpoints)
|
||||
}
|
||||
batch := endpoints[i:end]
|
||||
|
||||
// Use ON CONFLICT DO NOTHING to ignore duplicates (url + target_id unique)
|
||||
result := r.db.Clauses(clause.OnConflict{DoNothing: true}).Create(&batch)
|
||||
if result.Error != nil {
|
||||
return totalAffected, result.Error
|
||||
}
|
||||
totalAffected += int(result.RowsAffected)
|
||||
}
|
||||
|
||||
return int(result.RowsAffected), nil
|
||||
return totalAffected, nil
|
||||
}
|
||||
|
||||
// Delete deletes an endpoint by ID
|
||||
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/xingrin/server/internal/model"
|
||||
"github.com/xingrin/server/internal/pkg/scope"
|
||||
"github.com/orbit/server/internal/model"
|
||||
"github.com/orbit/server/internal/pkg/scope"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
@@ -94,11 +94,6 @@ func (r *ScanRepository) FindAll(page, pageSize int, targetID int, status, searc
|
||||
return scans, total, err
|
||||
}
|
||||
|
||||
// Update updates a scan
|
||||
func (r *ScanRepository) Update(scan *model.Scan) error {
|
||||
return r.db.Save(scan).Error
|
||||
}
|
||||
|
||||
// SoftDelete soft deletes a scan
|
||||
func (r *ScanRepository) SoftDelete(id int) error {
|
||||
now := time.Now()
|
||||
@@ -142,22 +137,13 @@ func (r *ScanRepository) UpdateStatus(id int, status string, errorMessage ...str
|
||||
if len(errorMessage) > 0 {
|
||||
updates["error_message"] = errorMessage[0]
|
||||
}
|
||||
if status == model.ScanStatusCompleted || status == model.ScanStatusFailed || status == model.ScanStatusStopped {
|
||||
if status == model.ScanStatusCompleted || status == model.ScanStatusFailed || status == model.ScanStatusCancelled {
|
||||
now := time.Now()
|
||||
updates["stopped_at"] = &now
|
||||
}
|
||||
return r.db.Model(&model.Scan{}).Where("id = ?", id).Updates(updates).Error
|
||||
}
|
||||
|
||||
// UpdateProgress updates scan progress
|
||||
func (r *ScanRepository) UpdateProgress(id int, progress int, currentStage string) error {
|
||||
return r.db.Model(&model.Scan{}).Where("id = ?", id).
|
||||
Updates(map[string]interface{}{
|
||||
"progress": progress,
|
||||
"current_stage": currentStage,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// GetStatistics returns scan statistics
|
||||
func (r *ScanRepository) GetStatistics() (*ScanStatistics, error) {
|
||||
stats := &ScanStatistics{}
|
||||
@@ -223,30 +209,6 @@ type ScanStatistics struct {
|
||||
TotalAssets int64
|
||||
}
|
||||
|
||||
// FindByTargetIDs finds scans by target IDs
|
||||
func (r *ScanRepository) FindByTargetIDs(targetIDs []int) ([]model.Scan, error) {
|
||||
if len(targetIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var scans []model.Scan
|
||||
err := r.db.Where("target_id IN ? AND deleted_at IS NULL", targetIDs).
|
||||
Preload("Target").
|
||||
Order("created_at DESC").
|
||||
Find(&scans).Error
|
||||
return scans, err
|
||||
}
|
||||
|
||||
// HasActiveScan checks if target has an active scan
|
||||
func (r *ScanRepository) HasActiveScan(targetID int) (bool, error) {
|
||||
var count int64
|
||||
err := r.db.Model(&model.Scan{}).
|
||||
Where("target_id = ? AND deleted_at IS NULL AND status IN ?", targetID,
|
||||
[]string{model.ScanStatusInitiated, model.ScanStatusRunning, model.ScanStatusPending}).
|
||||
Count(&count).Error
|
||||
return count > 0, err
|
||||
}
|
||||
|
||||
// GetTargetByScanID returns the target associated with a scan
|
||||
func (r *ScanRepository) GetTargetByScanID(scanID int) (*model.Target, error) {
|
||||
var scan model.Scan
|
||||
|
||||
@@ -55,13 +55,26 @@ func (r *SubdomainRepository) BulkCreate(subdomains []model.Subdomain) (int, err
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Use ON CONFLICT DO NOTHING to ignore duplicates (name + target_id unique)
|
||||
result := r.db.Clauses(clause.OnConflict{DoNothing: true}).Create(&subdomains)
|
||||
if result.Error != nil {
|
||||
return 0, result.Error
|
||||
var totalAffected int
|
||||
|
||||
// Process in batches to avoid SQL statement size limits
|
||||
batchSize := 500
|
||||
for i := 0; i < len(subdomains); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(subdomains) {
|
||||
end = len(subdomains)
|
||||
}
|
||||
batch := subdomains[i:end]
|
||||
|
||||
// Use ON CONFLICT DO NOTHING to ignore duplicates (name + target_id unique)
|
||||
result := r.db.Clauses(clause.OnConflict{DoNothing: true}).Create(&batch)
|
||||
if result.Error != nil {
|
||||
return totalAffected, result.Error
|
||||
}
|
||||
totalAffected += int(result.RowsAffected)
|
||||
}
|
||||
|
||||
return int(result.RowsAffected), nil
|
||||
return totalAffected, nil
|
||||
}
|
||||
|
||||
// BulkDelete deletes multiple subdomains by IDs
|
||||
|
||||
@@ -32,7 +32,8 @@ func (r *SubdomainSnapshotRepository) BulkCreate(snapshots []model.SubdomainSnap
|
||||
|
||||
var totalAffected int64
|
||||
|
||||
batchSize := 100
|
||||
// Process in batches to avoid SQL statement size limits
|
||||
batchSize := 500
|
||||
for i := 0; i < len(snapshots); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(snapshots) {
|
||||
|
||||
29
server/internal/repository/subfinder_provider_settings.go
Normal file
29
server/internal/repository/subfinder_provider_settings.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"github.com/orbit/server/internal/model"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type SubfinderProviderSettingsRepository struct {
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
func NewSubfinderProviderSettingsRepository(db *gorm.DB) *SubfinderProviderSettingsRepository {
|
||||
return &SubfinderProviderSettingsRepository{db: db}
|
||||
}
|
||||
|
||||
// GetInstance returns the singleton settings (id=1)
|
||||
func (r *SubfinderProviderSettingsRepository) GetInstance() (*model.SubfinderProviderSettings, error) {
|
||||
var settings model.SubfinderProviderSettings
|
||||
if err := r.db.First(&settings, 1).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &settings, nil
|
||||
}
|
||||
|
||||
// Update updates the settings
|
||||
func (r *SubfinderProviderSettingsRepository) Update(settings *model.SubfinderProviderSettings) error {
|
||||
settings.ID = 1 // Force singleton
|
||||
return r.db.Save(settings).Error
|
||||
}
|
||||
@@ -70,13 +70,26 @@ func (r *WebsiteRepository) BulkCreate(websites []model.Website) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Use ON CONFLICT DO NOTHING to ignore duplicates (url + target_id unique)
|
||||
result := r.db.Clauses(clause.OnConflict{DoNothing: true}).Create(&websites)
|
||||
if result.Error != nil {
|
||||
return 0, result.Error
|
||||
var totalAffected int
|
||||
|
||||
// Process in batches to avoid SQL statement size limits
|
||||
batchSize := 500
|
||||
for i := 0; i < len(websites); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(websites) {
|
||||
end = len(websites)
|
||||
}
|
||||
batch := websites[i:end]
|
||||
|
||||
// Use ON CONFLICT DO NOTHING to ignore duplicates (url + target_id unique)
|
||||
result := r.db.Clauses(clause.OnConflict{DoNothing: true}).Create(&batch)
|
||||
if result.Error != nil {
|
||||
return totalAffected, result.Error
|
||||
}
|
||||
totalAffected += int(result.RowsAffected)
|
||||
}
|
||||
|
||||
return int(result.RowsAffected), nil
|
||||
return totalAffected, nil
|
||||
}
|
||||
|
||||
// Delete deletes a website by ID
|
||||
|
||||
@@ -38,8 +38,8 @@ func (r *WebsiteSnapshotRepository) BulkCreate(snapshots []model.WebsiteSnapshot
|
||||
|
||||
var totalAffected int64
|
||||
|
||||
// Process in batches to avoid parameter limits
|
||||
batchSize := 100
|
||||
// Process in batches to avoid SQL statement size limits
|
||||
batchSize := 500
|
||||
for i := 0; i < len(snapshots); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(snapshots) {
|
||||
|
||||
@@ -2,7 +2,6 @@ package repository
|
||||
|
||||
import (
|
||||
"github.com/orbit/server/internal/model"
|
||||
"github.com/orbit/server/internal/pkg/scope"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
@@ -16,46 +15,64 @@ func NewWordlistRepository(db *gorm.DB) *WordlistRepository {
|
||||
return &WordlistRepository{db: db}
|
||||
}
|
||||
|
||||
// Create creates a new wordlist
|
||||
func (r *WordlistRepository) Create(wordlist *model.Wordlist) error {
|
||||
return r.db.Create(wordlist).Error
|
||||
}
|
||||
|
||||
// FindByID finds a wordlist by ID
|
||||
func (r *WordlistRepository) FindByID(id int) (*model.Wordlist, error) {
|
||||
var wordlist model.Wordlist
|
||||
err := r.db.First(&wordlist, id).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// ExistsByName checks if a wordlist with the given name exists
|
||||
func (r *WordlistRepository) ExistsByName(name string) (bool, error) {
|
||||
var count int64
|
||||
if err := r.db.Model(&model.Wordlist{}).Where("name = ?", name).Count(&count).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
return &wordlist, nil
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
// FindByName finds a wordlist by name
|
||||
func (r *WordlistRepository) FindByName(name string) (*model.Wordlist, error) {
|
||||
var wordlist model.Wordlist
|
||||
err := r.db.Where("name = ?", name).First(&wordlist).Error
|
||||
if err != nil {
|
||||
if err := r.db.Where("name = ?", name).First(&wordlist).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &wordlist, nil
|
||||
}
|
||||
|
||||
// FindAll finds all wordlists with pagination
|
||||
// FindByID finds a wordlist by ID
|
||||
func (r *WordlistRepository) FindByID(id int) (*model.Wordlist, error) {
|
||||
var wordlist model.Wordlist
|
||||
if err := r.db.First(&wordlist, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &wordlist, nil
|
||||
}
|
||||
|
||||
// FindAll returns paginated wordlists
|
||||
func (r *WordlistRepository) FindAll(page, pageSize int) ([]model.Wordlist, int64, error) {
|
||||
var wordlists []model.Wordlist
|
||||
var total int64
|
||||
|
||||
// Count total
|
||||
if err := r.db.Model(&model.Wordlist{}).Count(&total).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
err := r.db.Scopes(
|
||||
scope.WithPagination(page, pageSize),
|
||||
scope.OrderByCreatedAtDesc(),
|
||||
).Find(&wordlists).Error
|
||||
// Get paginated results
|
||||
offset := (page - 1) * pageSize
|
||||
if err := r.db.Order("created_at DESC").Offset(offset).Limit(pageSize).Find(&wordlists).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
return wordlists, total, err
|
||||
return wordlists, total, nil
|
||||
}
|
||||
|
||||
// List returns all wordlists (no pagination)
|
||||
func (r *WordlistRepository) List() ([]model.Wordlist, error) {
|
||||
var wordlists []model.Wordlist
|
||||
if err := r.db.Order("created_at DESC").Find(&wordlists).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return wordlists, nil
|
||||
}
|
||||
|
||||
// Create creates a new wordlist
|
||||
func (r *WordlistRepository) Create(wordlist *model.Wordlist) error {
|
||||
return r.db.Create(wordlist).Error
|
||||
}
|
||||
|
||||
// Update updates a wordlist
|
||||
@@ -63,18 +80,7 @@ func (r *WordlistRepository) Update(wordlist *model.Wordlist) error {
|
||||
return r.db.Save(wordlist).Error
|
||||
}
|
||||
|
||||
// Delete deletes a wordlist
|
||||
// Delete deletes a wordlist by ID
|
||||
func (r *WordlistRepository) Delete(id int) error {
|
||||
return r.db.Delete(&model.Wordlist{}, id).Error
|
||||
}
|
||||
|
||||
// ExistsByName checks if wordlist name exists
|
||||
func (r *WordlistRepository) ExistsByName(name string, excludeID ...int) (bool, error) {
|
||||
var count int64
|
||||
query := r.db.Model(&model.Wordlist{}).Where("name = ?", name)
|
||||
if len(excludeID) > 0 {
|
||||
query = query.Where("id != ?", excludeID[0])
|
||||
}
|
||||
err := query.Count(&count).Error
|
||||
return count > 0, err
|
||||
}
|
||||
|
||||
73
server/internal/service/agent.go
Normal file
73
server/internal/service/agent.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/orbit/server/internal/model"
|
||||
"github.com/orbit/server/internal/repository"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrAgentScanNotFound = errors.New("scan not found")
|
||||
ErrAgentInvalidTransition = errors.New("invalid status transition")
|
||||
)
|
||||
|
||||
// validTransitions defines allowed status transitions
|
||||
// key: current status, value: allowed next statuses
|
||||
var validTransitions = map[string][]string{
|
||||
model.ScanStatusPending: {model.ScanStatusScheduled, model.ScanStatusCancelled},
|
||||
model.ScanStatusScheduled: {model.ScanStatusRunning, model.ScanStatusCancelled},
|
||||
model.ScanStatusRunning: {model.ScanStatusCompleted, model.ScanStatusFailed, model.ScanStatusCancelled},
|
||||
// Terminal states: no transitions allowed
|
||||
model.ScanStatusCompleted: {},
|
||||
model.ScanStatusFailed: {},
|
||||
model.ScanStatusCancelled: {},
|
||||
}
|
||||
|
||||
// AgentService handles agent-related operations
|
||||
type AgentService struct {
|
||||
scanRepo *repository.ScanRepository
|
||||
}
|
||||
|
||||
// NewAgentService creates a new agent service
|
||||
func NewAgentService(scanRepo *repository.ScanRepository) *AgentService {
|
||||
return &AgentService{
|
||||
scanRepo: scanRepo,
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateStatus updates scan status (called by Agent based on Worker exit code)
|
||||
func (s *AgentService) UpdateStatus(scanID int, status string, errorMessage string) error {
|
||||
scan, err := s.scanRepo.FindByID(scanID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return ErrAgentScanNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate status transition
|
||||
if !isValidTransition(scan.Status, status) {
|
||||
return ErrAgentInvalidTransition
|
||||
}
|
||||
|
||||
if errorMessage != "" {
|
||||
return s.scanRepo.UpdateStatus(scanID, status, errorMessage)
|
||||
}
|
||||
return s.scanRepo.UpdateStatus(scanID, status)
|
||||
}
|
||||
|
||||
// isValidTransition checks if the status transition is allowed
|
||||
func isValidTransition(current, next string) bool {
|
||||
allowed, exists := validTransitions[current]
|
||||
if !exists {
|
||||
return false
|
||||
}
|
||||
for _, s := range allowed {
|
||||
if s == next {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -11,10 +11,9 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrScanNotFound = errors.New("scan not found")
|
||||
ErrScanCannotStop = errors.New("scan cannot be stopped in current status")
|
||||
ErrNoTargetsForScan = errors.New("no targets provided for scan")
|
||||
ErrTargetHasActiveScan = errors.New("target already has an active scan")
|
||||
ErrScanNotFound = errors.New("scan not found")
|
||||
ErrScanCannotStop = errors.New("scan cannot be stopped in current status")
|
||||
ErrNoTargetsForScan = errors.New("no targets provided for scan")
|
||||
)
|
||||
|
||||
// ScanService handles scan business logic
|
||||
@@ -109,12 +108,12 @@ func (s *ScanService) Stop(id int) (int, error) {
|
||||
}
|
||||
|
||||
// Check if scan can be stopped
|
||||
if scan.Status != model.ScanStatusRunning && scan.Status != model.ScanStatusInitiated {
|
||||
if scan.Status != model.ScanStatusRunning && scan.Status != model.ScanStatusPending && scan.Status != model.ScanStatusScheduled {
|
||||
return 0, ErrScanCannotStop
|
||||
}
|
||||
|
||||
// Update status to stopped
|
||||
if err := s.repo.UpdateStatus(id, model.ScanStatusStopped); err != nil {
|
||||
// Update status to cancelled
|
||||
if err := s.repo.UpdateStatus(id, model.ScanStatusCancelled); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
|
||||
@@ -17,10 +17,19 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrWordlistNotFound = errors.New("wordlist not found")
|
||||
ErrWordlistExists = errors.New("wordlist name already exists")
|
||||
ErrEmptyName = errors.New("wordlist name cannot be empty")
|
||||
ErrFileNotFound = errors.New("wordlist file not found")
|
||||
ErrWordlistNotFound = errors.New("wordlist not found")
|
||||
ErrWordlistExists = errors.New("wordlist name already exists")
|
||||
ErrEmptyName = errors.New("wordlist name cannot be empty")
|
||||
ErrNameTooLong = errors.New("wordlist name too long (max 200 characters)")
|
||||
ErrInvalidName = errors.New("wordlist name contains invalid characters")
|
||||
ErrFileNotFound = errors.New("wordlist file not found")
|
||||
ErrInvalidFileType = errors.New("file appears to be binary, only text files are allowed")
|
||||
)
|
||||
|
||||
const (
|
||||
maxNameLength = 200
|
||||
maxDescriptionLength = 200
|
||||
binaryCheckSize = 8192 // Check first 8KB for binary content
|
||||
)
|
||||
|
||||
// WordlistService handles wordlist business logic
|
||||
@@ -43,6 +52,20 @@ func (s *WordlistService) Create(name, description, filename string, fileContent
|
||||
if name == "" {
|
||||
return nil, ErrEmptyName
|
||||
}
|
||||
if len(name) > maxNameLength {
|
||||
return nil, ErrNameTooLong
|
||||
}
|
||||
// Reject names with control characters (newlines, tabs, etc.)
|
||||
if containsControlChars(name) {
|
||||
return nil, ErrInvalidName
|
||||
}
|
||||
|
||||
// Truncate description if too long, also sanitize control chars
|
||||
description = strings.TrimSpace(description)
|
||||
description = removeControlChars(description)
|
||||
if len(description) > maxDescriptionLength {
|
||||
description = description[:maxDescriptionLength]
|
||||
}
|
||||
|
||||
exists, err := s.repo.ExistsByName(name)
|
||||
if err != nil {
|
||||
@@ -66,17 +89,23 @@ func (s *WordlistService) Create(name, description, filename string, fileContent
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
hasher := sha256.New()
|
||||
writer := io.MultiWriter(file, hasher)
|
||||
|
||||
written, err := io.Copy(writer, fileContent)
|
||||
if err != nil {
|
||||
os.Remove(fullPath) // Cleanup on error
|
||||
_ = os.Remove(fullPath) // Cleanup on error
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check if file is binary (contains null bytes in first 8KB)
|
||||
if isBinaryFile(fullPath) {
|
||||
_ = os.Remove(fullPath) // Cleanup
|
||||
return nil, ErrInvalidFileType
|
||||
}
|
||||
|
||||
fileHash := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
// Count lines
|
||||
@@ -95,7 +124,7 @@ func (s *WordlistService) Create(name, description, filename string, fileContent
|
||||
}
|
||||
|
||||
if err := s.repo.Create(wordlist); err != nil {
|
||||
os.Remove(fullPath) // Cleanup on error
|
||||
_ = os.Remove(fullPath) // Cleanup on error
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -107,6 +136,21 @@ func (s *WordlistService) List(query *dto.PaginationQuery) ([]model.Wordlist, in
|
||||
return s.repo.FindAll(query.GetPage(), query.GetPageSize())
|
||||
}
|
||||
|
||||
// ListAll returns all wordlists without pagination
|
||||
func (s *WordlistService) ListAll() ([]model.Wordlist, error) {
|
||||
wordlists, err := s.repo.List()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check and update file stats for each wordlist
|
||||
for i := range wordlists {
|
||||
s.checkAndUpdateFileStats(&wordlists[i])
|
||||
}
|
||||
|
||||
return wordlists, nil
|
||||
}
|
||||
|
||||
// GetByID returns a wordlist by ID
|
||||
func (s *WordlistService) GetByID(id int) (*model.Wordlist, error) {
|
||||
wordlist, err := s.repo.FindByID(id)
|
||||
@@ -116,6 +160,10 @@ func (s *WordlistService) GetByID(id int) (*model.Wordlist, error) {
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check if file was modified externally
|
||||
s.checkAndUpdateFileStats(wordlist)
|
||||
|
||||
return wordlist, nil
|
||||
}
|
||||
|
||||
@@ -133,6 +181,10 @@ func (s *WordlistService) GetByName(name string) (*model.Wordlist, error) {
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check if file was modified externally
|
||||
s.checkAndUpdateFileStats(wordlist)
|
||||
|
||||
return wordlist, nil
|
||||
}
|
||||
|
||||
@@ -148,7 +200,7 @@ func (s *WordlistService) Delete(id int) error {
|
||||
|
||||
// Delete file (best effort)
|
||||
if wordlist.FilePath != "" {
|
||||
os.Remove(wordlist.FilePath)
|
||||
_ = os.Remove(wordlist.FilePath)
|
||||
}
|
||||
|
||||
return s.repo.Delete(id)
|
||||
@@ -262,7 +314,7 @@ func countLines(filepath string) (int, error) {
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
count := 0
|
||||
@@ -277,3 +329,90 @@ func fileExists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// containsControlChars checks if string contains control characters (newlines, tabs, etc.)
|
||||
func containsControlChars(s string) bool {
|
||||
for _, r := range s {
|
||||
if r < 32 && r != ' ' { // ASCII control characters except space
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// removeControlChars removes control characters from string
|
||||
func removeControlChars(s string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r < 32 && r != ' ' {
|
||||
return -1 // Remove the character
|
||||
}
|
||||
return r
|
||||
}, s)
|
||||
}
|
||||
|
||||
// isBinaryFile checks if file contains binary content (null bytes in first 8KB)
|
||||
func isBinaryFile(path string) bool {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
buf := make([]byte, binaryCheckSize)
|
||||
n, err := file.Read(buf)
|
||||
if err != nil && err != io.EOF {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for null bytes (common indicator of binary files)
|
||||
for i := 0; i < n; i++ {
|
||||
if buf[i] == 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// checkAndUpdateFileStats checks if file was modified externally and updates stats if needed
|
||||
// Uses mtime + size for quick detection, only recalculates hash when change detected
|
||||
func (s *WordlistService) checkAndUpdateFileStats(wordlist *model.Wordlist) {
|
||||
if wordlist.FilePath == "" {
|
||||
return
|
||||
}
|
||||
|
||||
fileInfo, err := os.Stat(wordlist.FilePath)
|
||||
if err != nil {
|
||||
return // File doesn't exist or can't be accessed
|
||||
}
|
||||
|
||||
// Quick check: compare size and mtime
|
||||
fileModTime := fileInfo.ModTime()
|
||||
if fileInfo.Size() == wordlist.FileSize && !fileModTime.After(wordlist.UpdatedAt) {
|
||||
return // No change detected
|
||||
}
|
||||
|
||||
// File was modified, recalculate stats
|
||||
file, err := os.Open(wordlist.FilePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
// Calculate new hash
|
||||
hasher := sha256.New()
|
||||
if _, err := io.Copy(hasher, file); err != nil {
|
||||
return
|
||||
}
|
||||
newHash := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
// Count lines
|
||||
lineCount, _ := countLines(wordlist.FilePath)
|
||||
|
||||
// Update record
|
||||
wordlist.FileSize = fileInfo.Size()
|
||||
wordlist.FileHash = newHash
|
||||
wordlist.LineCount = lineCount
|
||||
|
||||
// Save to database (best effort, don't fail the request)
|
||||
_ = s.repo.Update(wordlist)
|
||||
}
|
||||
|
||||
161
server/internal/service/worker.go
Normal file
161
server/internal/service/worker.go
Normal file
@@ -0,0 +1,161 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/orbit/server/internal/dto"
|
||||
"github.com/orbit/server/internal/model"
|
||||
"github.com/orbit/server/internal/repository"
|
||||
"gopkg.in/yaml.v3"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrWorkerScanNotFound = errors.New("scan not found")
|
||||
ErrWorkerToolRequired = errors.New("tool parameter required for provider_config")
|
||||
)
|
||||
|
||||
// WorkerService handles scan data for workers
|
||||
type WorkerService struct {
|
||||
scanRepo *repository.ScanRepository
|
||||
subfinderProviderSettingsRepo *repository.SubfinderProviderSettingsRepository
|
||||
}
|
||||
|
||||
// NewWorkerService creates a new worker service
|
||||
func NewWorkerService(
|
||||
scanRepo *repository.ScanRepository,
|
||||
subfinderProviderSettingsRepo *repository.SubfinderProviderSettingsRepository,
|
||||
) *WorkerService {
|
||||
return &WorkerService{
|
||||
scanRepo: scanRepo,
|
||||
subfinderProviderSettingsRepo: subfinderProviderSettingsRepo,
|
||||
}
|
||||
}
|
||||
|
||||
// TargetInfo contains target name and type
|
||||
type TargetInfo struct {
|
||||
Name string
|
||||
Type string
|
||||
}
|
||||
|
||||
// GetTargetName returns target name and type for a scan
|
||||
func (s *WorkerService) GetTargetName(scanID int) (*TargetInfo, error) {
|
||||
target, err := s.scanRepo.GetTargetByScanID(scanID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, ErrWorkerScanNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &TargetInfo{
|
||||
Name: target.Name,
|
||||
Type: target.Type,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetProviderConfig returns provider config (API keys) for a tool
|
||||
func (s *WorkerService) GetProviderConfig(scanID int, toolName string) (*dto.WorkerProviderConfigResponse, error) {
|
||||
if toolName == "" {
|
||||
return nil, ErrWorkerToolRequired
|
||||
}
|
||||
|
||||
// Check scan exists
|
||||
_, err := s.scanRepo.FindByID(scanID)
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, ErrWorkerScanNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config, err := s.generateProviderConfig(toolName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &dto.WorkerProviderConfigResponse{
|
||||
Content: config,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// generateProviderConfig generates provider config YAML for a tool
|
||||
func (s *WorkerService) generateProviderConfig(toolName string) (string, error) {
|
||||
switch toolName {
|
||||
case "subfinder":
|
||||
return s.generateSubfinderConfig()
|
||||
default:
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
|
||||
// generateSubfinderConfig generates subfinder provider-config.yaml content
|
||||
func (s *WorkerService) generateSubfinderConfig() (string, error) {
|
||||
settings, err := s.subfinderProviderSettingsRepo.GetInstance()
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return "", nil // No settings configured
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
|
||||
config := make(map[string][]string)
|
||||
hasEnabled := false
|
||||
|
||||
for providerName, formatInfo := range model.ProviderFormats {
|
||||
providerConfig, exists := settings.Providers[providerName]
|
||||
if !exists || !providerConfig.Enabled {
|
||||
config[providerName] = []string{}
|
||||
continue
|
||||
}
|
||||
|
||||
value := s.buildProviderValue(providerConfig, formatInfo)
|
||||
if value != "" {
|
||||
config[providerName] = []string{value}
|
||||
hasEnabled = true
|
||||
} else {
|
||||
config[providerName] = []string{}
|
||||
}
|
||||
}
|
||||
|
||||
if !hasEnabled {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
yamlBytes, err := yaml.Marshal(config)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal provider config: %w", err)
|
||||
}
|
||||
|
||||
return string(yamlBytes), nil
|
||||
}
|
||||
|
||||
// buildProviderValue builds the config value string for a provider
|
||||
func (s *WorkerService) buildProviderValue(config model.ProviderConfig, formatInfo model.ProviderFormat) string {
|
||||
if formatInfo.Type == model.FormatTypeComposite {
|
||||
// Handle composite formats like "email:api_key"
|
||||
result := formatInfo.Format
|
||||
result = strings.ReplaceAll(result, "{email}", config.Email)
|
||||
result = strings.ReplaceAll(result, "{api_key}", config.APIKey)
|
||||
result = strings.ReplaceAll(result, "{api_id}", config.APIId)
|
||||
result = strings.ReplaceAll(result, "{api_secret}", config.APISecret)
|
||||
|
||||
// Check if all placeholders were replaced (no empty values)
|
||||
if strings.Contains(result, "{}") || result == formatInfo.Format {
|
||||
return ""
|
||||
}
|
||||
// Check for empty segments (e.g., ":key" or "email:")
|
||||
if slices.Contains(strings.Split(result, ":"), "") {
|
||||
return ""
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Single field format
|
||||
return config.APIKey
|
||||
}
|
||||
|
||||
|
||||
BIN
server/server
BIN
server/server
Binary file not shown.
30
worker/.env.example
Normal file
30
worker/.env.example
Normal file
@@ -0,0 +1,30 @@
|
||||
# Worker Configuration Example
|
||||
# Copy this file to .env and modify as needed
|
||||
|
||||
# ===========================================
|
||||
# Server Connection (Required)
|
||||
# ===========================================
|
||||
SERVER_URL=http://localhost:8888
|
||||
SERVER_TOKEN=your_server_token_here
|
||||
|
||||
# ===========================================
|
||||
# Paths
|
||||
# ===========================================
|
||||
# Working directory for scan results
|
||||
WORKSPACE_DIR=/opt/orbit/results
|
||||
RESULTS_BASE_PATH=/opt/orbit/results
|
||||
|
||||
# DNS resolvers file path
|
||||
RESOLVERS_PATH=/opt/orbit/wordlists/resolvers.txt
|
||||
|
||||
# Wordlist base directory (downloaded from Server)
|
||||
WORDLIST_BASE_PATH=/opt/orbit/wordlists
|
||||
|
||||
# ===========================================
|
||||
# Logging
|
||||
# ===========================================
|
||||
# Log level: debug, info, warn, error
|
||||
LOG_LEVEL=info
|
||||
|
||||
# Environment: development or production
|
||||
# ENV=development
|
||||
115
worker/Dockerfile
Normal file
115
worker/Dockerfile
Normal file
@@ -0,0 +1,115 @@
|
||||
# Worker Dockerfile - Go 版本
|
||||
# 多阶段构建:编译扫描工具 + 构建 Go Worker
|
||||
|
||||
# ============================================
|
||||
# 阶段 1: 编译 Go 扫描工具
|
||||
# ============================================
|
||||
FROM golang:1.24 AS tools-builder
|
||||
|
||||
ENV GOPROXY=https://goproxy.cn,direct
|
||||
ENV CGO_ENABLED=1
|
||||
|
||||
# 安装编译依赖(构建阶段,不固定版本以获取最新安全补丁)
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libpcap-dev \
|
||||
git \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 编译 massdns(puredns 依赖)
|
||||
RUN git clone https://github.com/blechschmidt/massdns.git /tmp/massdns && \
|
||||
cd /tmp/massdns && \
|
||||
make && \
|
||||
cp bin/massdns /usr/local/bin/massdns
|
||||
|
||||
# 安装 ProjectDiscovery 等 Go 扫描工具
|
||||
RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest && \
|
||||
go install -v github.com/projectdiscovery/naabu/v2/cmd/naabu@latest && \
|
||||
go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest && \
|
||||
go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest && \
|
||||
go install -v github.com/projectdiscovery/katana/cmd/katana@latest && \
|
||||
go install -v github.com/tomnomnom/assetfinder@latest && \
|
||||
go install -v github.com/ffuf/ffuf/v2@latest && \
|
||||
go install -v github.com/d3mondev/puredns/v2@latest && \
|
||||
go install -v github.com/yyhuni/xingfinger@latest && \
|
||||
go install -v github.com/hahwul/dalfox/v2@latest
|
||||
|
||||
# ============================================
|
||||
# 阶段 2: 构建 Go Worker
|
||||
# ============================================
|
||||
FROM golang:1.24 AS worker-builder
|
||||
|
||||
ENV GOPROXY=https://goproxy.cn,direct
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 先复制依赖文件,利用缓存
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# 复制源码并构建
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 go build -o /worker ./cmd/worker
|
||||
|
||||
# ============================================
|
||||
# 阶段 3: 运行时镜像
|
||||
# ============================================
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 安装运行时依赖 + Python 环境
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
libpcap-dev \
|
||||
nmap \
|
||||
masscan \
|
||||
curl \
|
||||
git \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-venv \
|
||||
pipx \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 建立 python 软链接
|
||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||
|
||||
# 安装 Python 扫描工具
|
||||
ENV PATH="/root/.local/bin:$PATH"
|
||||
RUN pipx install uro && \
|
||||
pipx install waymore && \
|
||||
pipx install dnsgen
|
||||
|
||||
# 安装 Sublist3r
|
||||
RUN git clone https://github.com/aboul3la/Sublist3r.git /opt/orbit/tools/Sublist3r && \
|
||||
pip3 install --no-cache-dir -r /opt/orbit/tools/Sublist3r/requirements.txt --break-system-packages
|
||||
|
||||
# 创建工具目录
|
||||
RUN mkdir -p /opt/orbit/bin /opt/orbit/wordlists
|
||||
|
||||
# 复制 resolvers.txt(DNS 服务器列表,内容固定)
|
||||
COPY resources/resolvers.txt /opt/orbit/wordlists/
|
||||
|
||||
# 从 tools-builder 复制 Go 扫描工具
|
||||
COPY --from=tools-builder /go/bin/* /opt/orbit/bin/
|
||||
COPY --from=tools-builder /usr/local/bin/massdns /opt/orbit/bin/
|
||||
|
||||
# 从 worker-builder 复制 Go Worker
|
||||
COPY --from=worker-builder /worker /usr/local/bin/worker
|
||||
|
||||
# 创建非 root 用户
|
||||
RUN useradd -m -s /bin/bash -u 1000 worker && \
|
||||
chown -R worker:worker /opt/orbit
|
||||
|
||||
# 设置 PATH(Go 工具优先)
|
||||
ENV PATH=/opt/orbit/bin:/root/.local/bin:$PATH
|
||||
|
||||
# 切换到非 root 用户
|
||||
USER worker
|
||||
|
||||
# 默认命令
|
||||
CMD ["worker"]
|
||||
27
worker/Makefile
Normal file
27
worker/Makefile
Normal file
@@ -0,0 +1,27 @@
|
||||
.PHONY: build run test clean tidy
|
||||
|
||||
# Build the worker binary
|
||||
build:
|
||||
go build -o bin/worker ./cmd/worker
|
||||
|
||||
# Run the worker
|
||||
run:
|
||||
go run ./cmd/worker
|
||||
|
||||
# Run tests
|
||||
test:
|
||||
go test -v ./...
|
||||
|
||||
# Run tests with coverage
|
||||
test-coverage:
|
||||
go test -v -coverprofile=coverage.out ./...
|
||||
go tool cover -html=coverage.out -o coverage.html
|
||||
|
||||
# Clean build artifacts
|
||||
clean:
|
||||
rm -rf bin/
|
||||
rm -f coverage.out coverage.html
|
||||
|
||||
# Tidy dependencies
|
||||
tidy:
|
||||
go mod tidy
|
||||
BIN
worker/bin/worker
Executable file
BIN
worker/bin/worker
Executable file
Binary file not shown.
81
worker/cmd/worker/main.go
Normal file
81
worker/cmd/worker/main.go
Normal file
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/orbit/worker/internal/config"
|
||||
"github.com/orbit/worker/internal/pkg"
|
||||
"github.com/orbit/worker/internal/server"
|
||||
"github.com/orbit/worker/internal/workflow"
|
||||
"go.uber.org/zap"
|
||||
|
||||
// Import workflows to trigger init() registration
|
||||
_ "github.com/orbit/worker/internal/workflow/subdomain_discovery"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Load configuration from environment variables
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load config: %v", err)
|
||||
}
|
||||
|
||||
// Initialize logger
|
||||
if err := pkg.InitLogger(cfg.LogLevel); err != nil {
|
||||
log.Fatalf("Failed to initialize logger: %v", err)
|
||||
}
|
||||
defer pkg.Sync()
|
||||
|
||||
pkg.Logger.Info("Worker starting",
|
||||
zap.Int("scanId", cfg.ScanID),
|
||||
zap.Int("targetId", cfg.TargetID),
|
||||
zap.String("targetName", cfg.TargetName),
|
||||
zap.String("targetType", cfg.TargetType),
|
||||
zap.String("workflow", cfg.WorkflowName))
|
||||
|
||||
// Create server client (implements ServerClient, ResultSaver)
|
||||
serverClient := server.NewClient(cfg.ServerURL, cfg.ServerToken)
|
||||
|
||||
// Create workflow params
|
||||
params := &workflow.Params{
|
||||
ScanID: cfg.ScanID,
|
||||
TargetID: cfg.TargetID,
|
||||
TargetName: cfg.TargetName,
|
||||
TargetType: cfg.TargetType,
|
||||
WorkDir: cfg.WorkspaceDir,
|
||||
ScanConfig: cfg.Config,
|
||||
ServerClient: serverClient,
|
||||
}
|
||||
|
||||
// Get and execute the workflow
|
||||
w := workflow.Get(cfg.WorkflowName, cfg.WorkspaceDir)
|
||||
if w == nil {
|
||||
pkg.Logger.Error("Unknown workflow name", zap.String("workflow", cfg.WorkflowName))
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Execute workflow
|
||||
// Status is managed by Agent based on container exit code:
|
||||
// - exit 0 = completed
|
||||
// - exit 1 = failed
|
||||
output, execErr := w.Execute(params)
|
||||
if execErr != nil {
|
||||
pkg.Logger.Error("Workflow execution failed",
|
||||
zap.Int("scanId", cfg.ScanID),
|
||||
zap.Error(execErr))
|
||||
os.Exit(1) // Agent will detect exit code and set status to "failed"
|
||||
}
|
||||
|
||||
// Save results
|
||||
ctx := context.Background()
|
||||
if err := w.SaveResults(ctx, serverClient, params, output); err != nil {
|
||||
pkg.Logger.Error("Failed to save results", zap.Error(err))
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
pkg.Logger.Info("Worker completed successfully",
|
||||
zap.Int("scanId", cfg.ScanID))
|
||||
// exit 0 - Agent will detect and set status to "completed"
|
||||
}
|
||||
@@ -1,14 +1,28 @@
|
||||
module github.com/orbit/worker
|
||||
|
||||
go 1.23.0
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.5
|
||||
|
||||
require (
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/shirou/gopsutil/v3 v3.24.5
|
||||
go.uber.org/zap v1.27.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.12 // indirect
|
||||
github.com/tklauser/numcpus v0.6.1 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
go.uber.org/multierr v1.10.0 // indirect
|
||||
golang.org/x/net v0.49.0 // indirect
|
||||
golang.org/x/sys v0.40.0 // indirect
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
)
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
@@ -8,16 +15,45 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
|
||||
github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
|
||||
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
|
||||
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
|
||||
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
|
||||
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
|
||||
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
|
||||
github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
|
||||
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
|
||||
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
|
||||
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
|
||||
53
worker/internal/activity/command_builder.go
Normal file
53
worker/internal/activity/command_builder.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package activity
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CommandBuilder builds commands from templates
|
||||
type CommandBuilder struct{}
|
||||
|
||||
// NewCommandBuilder creates a new command builder
|
||||
func NewCommandBuilder() *CommandBuilder {
|
||||
return &CommandBuilder{}
|
||||
}
|
||||
|
||||
// Build constructs a command from a template with the given parameters
|
||||
func (b *CommandBuilder) Build(tmpl CommandTemplate, params map[string]string, config map[string]any) (string, error) {
|
||||
if tmpl.Base == "" {
|
||||
return "", fmt.Errorf("template base command is empty")
|
||||
}
|
||||
|
||||
// Start with base command
|
||||
cmd := tmpl.Base
|
||||
|
||||
// Replace required placeholders
|
||||
for key, value := range params {
|
||||
placeholder := "{" + key + "}"
|
||||
cmd = strings.ReplaceAll(cmd, placeholder, value)
|
||||
}
|
||||
|
||||
// Append optional parameters if present in config
|
||||
for configKey, flagTemplate := range tmpl.Optional {
|
||||
if value, ok := getConfigValue(config, configKey); ok {
|
||||
flag := strings.ReplaceAll(flagTemplate, "{"+configKey+"}", fmt.Sprintf("%v", value))
|
||||
cmd = cmd + " " + flag
|
||||
}
|
||||
}
|
||||
|
||||
// Check for unreplaced placeholders (indicates missing required params)
|
||||
if strings.Contains(cmd, "{") && strings.Contains(cmd, "}") {
|
||||
return "", fmt.Errorf("command contains unreplaced placeholders: %s", cmd)
|
||||
}
|
||||
|
||||
return cmd, nil
|
||||
}
|
||||
|
||||
func getConfigValue(config map[string]any, key string) (any, bool) {
|
||||
if config == nil {
|
||||
return nil, false
|
||||
}
|
||||
value, ok := config[key]
|
||||
return value, ok
|
||||
}
|
||||
7
worker/internal/activity/command_template.go
Normal file
7
worker/internal/activity/command_template.go
Normal file
@@ -0,0 +1,7 @@
|
||||
package activity
|
||||
|
||||
// CommandTemplate defines a command template for an activity
|
||||
type CommandTemplate struct {
|
||||
Base string `yaml:"base"` // Base command with required placeholders
|
||||
Optional map[string]string `yaml:"optional"` // Optional parameters and their flags
|
||||
}
|
||||
392
worker/internal/activity/runner.go
Normal file
392
worker/internal/activity/runner.go
Normal file
@@ -0,0 +1,392 @@
|
||||
package activity
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/orbit/worker/internal/pkg"
|
||||
"github.com/shirou/gopsutil/v3/cpu"
|
||||
"github.com/shirou/gopsutil/v3/mem"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// ansiRegex matches ANSI escape sequences (colors, cursor movement, etc.)
|
||||
var ansiRegex = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`)
|
||||
|
||||
// controlCharReplacer removes control characters in a single pass
|
||||
var controlCharReplacer = strings.NewReplacer(
|
||||
"\x00", "", // NUL
|
||||
"\r", "", // CR
|
||||
"\b", "", // Backspace
|
||||
"\f", "", // Form feed
|
||||
"\v", "", // Vertical tab
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultDirPerm = 0755
|
||||
ExitCodeTimeout = -1
|
||||
ExitCodeError = -1
|
||||
|
||||
// System load thresholds
|
||||
DefaultCPUThreshold = 90.0 // CPU usage percentage
|
||||
DefaultMemThreshold = 80.0 // Memory usage percentage
|
||||
LoadCheckInterval = 180 * time.Second // 3 minutes between checks
|
||||
CommandStartupDelay = 5 * time.Second // Initial delay before first check
|
||||
|
||||
// Scanner buffer sizes
|
||||
ScannerInitBufSize = 64 * 1024 // 64KB initial buffer
|
||||
ScannerMaxBufSize = 1024 * 1024 // 1MB max buffer for long lines
|
||||
)
|
||||
|
||||
// Result represents the result of an activity execution
|
||||
type Result struct {
|
||||
Name string
|
||||
OutputFile string
|
||||
LogFile string
|
||||
ExitCode int
|
||||
Duration time.Duration
|
||||
Error error
|
||||
}
|
||||
|
||||
// Command represents a command to execute
|
||||
type Command struct {
|
||||
Name string
|
||||
Command string
|
||||
OutputFile string
|
||||
LogFile string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Runner executes activities (external tools)
|
||||
type Runner struct {
|
||||
workDir string
|
||||
}
|
||||
|
||||
// NewRunner creates a new activity runner
|
||||
func NewRunner(workDir string) *Runner {
|
||||
return &Runner{workDir: workDir}
|
||||
}
|
||||
|
||||
// killProcessGroup terminates the entire process group
|
||||
// When using shell=true (sh -c), the actual tool runs as a child of the shell.
|
||||
// If we only kill the shell process, the child becomes an orphan and keeps running.
|
||||
// By killing the process group, we ensure all child processes are terminated.
|
||||
func killProcessGroup(cmd *exec.Cmd) {
|
||||
if cmd == nil || cmd.Process == nil {
|
||||
return
|
||||
}
|
||||
|
||||
pid := cmd.Process.Pid
|
||||
|
||||
// Try to kill the process group first
|
||||
// The negative PID signals the entire process group
|
||||
if err := syscall.Kill(-pid, syscall.SIGKILL); err != nil {
|
||||
pkg.Logger.Debug("Failed to kill process group, trying single process",
|
||||
zap.Int("pid", pid),
|
||||
zap.Error(err))
|
||||
// Fallback: kill single process
|
||||
_ = cmd.Process.Kill()
|
||||
} else {
|
||||
pkg.Logger.Debug("Killed process group", zap.Int("pgid", pid))
|
||||
}
|
||||
}
|
||||
|
||||
// waitForSystemLoad blocks until system CPU and memory usage are below thresholds.
|
||||
// This prevents OOM when starting multiple concurrent commands.
|
||||
func waitForSystemLoad(ctx context.Context) {
|
||||
// Initial delay to let previous commands consume resources
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(CommandStartupDelay):
|
||||
}
|
||||
|
||||
for {
|
||||
cpu, mem, err := getSystemLoad()
|
||||
if err != nil {
|
||||
pkg.Logger.Debug("Failed to get system load, skipping check", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
if cpu < DefaultCPUThreshold && mem < DefaultMemThreshold {
|
||||
return
|
||||
}
|
||||
|
||||
pkg.Logger.Info("System load high, waiting before starting command",
|
||||
zap.Float64("cpu", cpu),
|
||||
zap.Float64("cpuThreshold", DefaultCPUThreshold),
|
||||
zap.Float64("mem", mem),
|
||||
zap.Float64("memThreshold", DefaultMemThreshold))
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(LoadCheckInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// getSystemLoad returns current CPU and memory usage percentages using gopsutil.
|
||||
// Correctly handles container cgroup limits.
|
||||
func getSystemLoad() (cpuPercent, memPercent float64, err error) {
|
||||
// Get CPU usage (0 interval means since last call, false means aggregate all CPUs)
|
||||
cpuPercents, err := cpu.Percent(0, false)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to get CPU usage: %w", err)
|
||||
}
|
||||
if len(cpuPercents) == 0 {
|
||||
return 0, 0, fmt.Errorf("no CPU data available")
|
||||
}
|
||||
|
||||
// Get memory usage
|
||||
memInfo, err := mem.VirtualMemory()
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to get memory usage: %w", err)
|
||||
}
|
||||
|
||||
return cpuPercents[0], memInfo.UsedPercent, nil
|
||||
}
|
||||
|
||||
// Run executes a single activity with streaming output
|
||||
func (r *Runner) Run(ctx context.Context, cmd Command) *Result {
|
||||
start := time.Now()
|
||||
result := &Result{
|
||||
Name: cmd.Name,
|
||||
OutputFile: cmd.OutputFile,
|
||||
LogFile: cmd.LogFile,
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
result.Error = fmt.Errorf("context cancelled before execution: %w", ctx.Err())
|
||||
result.ExitCode = ExitCodeError
|
||||
return result
|
||||
}
|
||||
|
||||
// Wait for system load to be acceptable before starting
|
||||
waitForSystemLoad(ctx)
|
||||
if ctx.Err() != nil {
|
||||
result.Error = fmt.Errorf("context cancelled while waiting for system load: %w", ctx.Err())
|
||||
result.ExitCode = ExitCodeError
|
||||
return result
|
||||
}
|
||||
|
||||
execCtx, cancel := context.WithTimeout(ctx, cmd.Timeout)
|
||||
defer cancel()
|
||||
|
||||
execCmd := exec.CommandContext(execCtx, "sh", "-c", cmd.Command)
|
||||
execCmd.Dir = r.workDir
|
||||
// Create new process group so we can kill all child processes
|
||||
execCmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
|
||||
// Setup pipes for streaming
|
||||
stdout, err := execCmd.StdoutPipe()
|
||||
if err != nil {
|
||||
result.Error = fmt.Errorf("failed to create stdout pipe: %w", err)
|
||||
result.ExitCode = ExitCodeError
|
||||
return result
|
||||
}
|
||||
|
||||
stderr, err := execCmd.StderrPipe()
|
||||
if err != nil {
|
||||
result.Error = fmt.Errorf("failed to create stderr pipe: %w", err)
|
||||
result.ExitCode = ExitCodeError
|
||||
return result
|
||||
}
|
||||
|
||||
// Prepare log file
|
||||
logFile := r.prepareLogFile(cmd)
|
||||
if logFile != nil {
|
||||
defer func() { _ = logFile.Close() }()
|
||||
r.writeLogHeader(logFile, cmd)
|
||||
}
|
||||
|
||||
// Start command
|
||||
if err := execCmd.Start(); err != nil {
|
||||
result.Error = fmt.Errorf("failed to start command: %w", err)
|
||||
result.ExitCode = ExitCodeError
|
||||
return result
|
||||
}
|
||||
|
||||
// Ensure process cleanup on any exit path
|
||||
defer killProcessGroup(execCmd)
|
||||
|
||||
// Stream output
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
|
||||
go r.streamOutput(&wg, stdout, logFile, cmd.Name, "stdout")
|
||||
go r.streamOutput(&wg, stderr, logFile, cmd.Name, "stderr")
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Wait for command to finish
|
||||
err = execCmd.Wait()
|
||||
result.Duration = time.Since(start)
|
||||
|
||||
// Write duration to log
|
||||
if logFile != nil {
|
||||
r.writeLogFooter(logFile, result)
|
||||
}
|
||||
|
||||
// Handle result
|
||||
if execCtx.Err() == context.DeadlineExceeded {
|
||||
result.Error = fmt.Errorf("activity execution timeout after %v", cmd.Timeout)
|
||||
result.ExitCode = ExitCodeTimeout
|
||||
pkg.Logger.Error("Activity timeout",
|
||||
zap.String("activity", cmd.Name),
|
||||
zap.Duration("timeout", cmd.Timeout))
|
||||
return result
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
result.ExitCode = exitErr.ExitCode()
|
||||
} else {
|
||||
result.ExitCode = ExitCodeError
|
||||
}
|
||||
result.Error = fmt.Errorf("activity execution failed: %w", err)
|
||||
pkg.Logger.Error("Activity failed",
|
||||
zap.String("activity", cmd.Name),
|
||||
zap.Int("exitCode", result.ExitCode),
|
||||
zap.Error(err))
|
||||
return result
|
||||
}
|
||||
|
||||
result.ExitCode = 0
|
||||
pkg.Logger.Info("Activity completed",
|
||||
zap.String("activity", cmd.Name),
|
||||
zap.Duration("duration", result.Duration))
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// RunParallel executes multiple activities in parallel
|
||||
func (r *Runner) RunParallel(ctx context.Context, commands []Command) []*Result {
|
||||
if len(commands) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
results := make([]*Result, len(commands))
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i, cmd := range commands {
|
||||
if ctx.Err() != nil {
|
||||
results[i] = &Result{
|
||||
Name: cmd.Name,
|
||||
ExitCode: ExitCodeError,
|
||||
Error: fmt.Errorf("context cancelled: %w", ctx.Err()),
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func(idx int, c Command) {
|
||||
defer wg.Done()
|
||||
results[idx] = r.Run(ctx, c)
|
||||
}(i, cmd)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
return results
|
||||
}
|
||||
|
||||
func (r *Runner) prepareLogFile(cmd Command) *os.File {
|
||||
if cmd.LogFile == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
dir := filepath.Dir(cmd.LogFile)
|
||||
if err := os.MkdirAll(dir, DefaultDirPerm); err != nil {
|
||||
pkg.Logger.Warn("Failed to create log directory",
|
||||
zap.String("activity", cmd.Name),
|
||||
zap.Error(err))
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := os.Create(cmd.LogFile)
|
||||
if err != nil {
|
||||
pkg.Logger.Warn("Failed to create log file",
|
||||
zap.String("activity", cmd.Name),
|
||||
zap.Error(err))
|
||||
return nil
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func (r *Runner) streamOutput(wg *sync.WaitGroup, reader io.Reader, logFile *os.File, activityName, streamName string) {
|
||||
defer wg.Done()
|
||||
|
||||
scanner := bufio.NewScanner(reader)
|
||||
scanner.Buffer(make([]byte, ScannerInitBufSize), ScannerMaxBufSize)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := cleanLine(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Write to log file
|
||||
if logFile != nil {
|
||||
_, _ = fmt.Fprintln(logFile, line)
|
||||
}
|
||||
|
||||
// Log debug output (optional, can be removed for less verbose logs)
|
||||
pkg.Logger.Debug("Activity output",
|
||||
zap.String("activity", activityName),
|
||||
zap.String("stream", streamName),
|
||||
zap.String("line", line))
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
pkg.Logger.Warn("Error reading output stream",
|
||||
zap.String("activity", activityName),
|
||||
zap.String("stream", streamName),
|
||||
zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// cleanLine removes ANSI escape sequences and control characters from output
|
||||
func cleanLine(line string) string {
|
||||
line = ansiRegex.ReplaceAllString(line, "")
|
||||
line = controlCharReplacer.Replace(line)
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
|
||||
const logSeparator = "============================================================"
|
||||
|
||||
func (r *Runner) writeLogHeader(f *os.File, cmd Command) {
|
||||
_, _ = fmt.Fprintf(f, "$ %s\n", cmd.Command)
|
||||
_, _ = fmt.Fprintln(f, logSeparator)
|
||||
_, _ = fmt.Fprintf(f, "# Tool: %s\n", cmd.Name)
|
||||
_, _ = fmt.Fprintf(f, "# Started: %s\n", time.Now().Format("2006-01-02 15:04:05"))
|
||||
_, _ = fmt.Fprintf(f, "# Timeout: %v\n", cmd.Timeout)
|
||||
_, _ = fmt.Fprintln(f, "# Status: Running...")
|
||||
_, _ = fmt.Fprintln(f, logSeparator)
|
||||
_, _ = fmt.Fprintln(f)
|
||||
}
|
||||
|
||||
func (r *Runner) writeLogFooter(f *os.File, result *Result) {
|
||||
status := "✓ Success"
|
||||
if result.ExitCode != 0 {
|
||||
status = "✗ Failed"
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintln(f)
|
||||
_, _ = fmt.Fprintln(f, logSeparator)
|
||||
_, _ = fmt.Fprintf(f, "# Finished: %s\n", time.Now().Format("2006-01-02 15:04:05"))
|
||||
_, _ = fmt.Fprintf(f, "# Duration: %.2fs\n", result.Duration.Seconds())
|
||||
_, _ = fmt.Fprintf(f, "# Exit Code: %d\n", result.ExitCode)
|
||||
_, _ = fmt.Fprintf(f, "# Status: %s\n", status)
|
||||
_, _ = fmt.Fprintln(f, logSeparator)
|
||||
}
|
||||
23
worker/internal/activity/templates.schema.json
Normal file
23
worker/internal/activity/templates.schema.json
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"description": "Command template schema for workflow tools",
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "object",
|
||||
"description": "Tool template",
|
||||
"required": ["base"],
|
||||
"properties": {
|
||||
"base": {
|
||||
"type": "string",
|
||||
"description": "Base command with placeholders like {domain}, {output-file}"
|
||||
},
|
||||
"optional": {
|
||||
"type": "object",
|
||||
"description": "Optional parameters and their flags",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
43
worker/internal/pkg/logger.go
Normal file
43
worker/internal/pkg/logger.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package pkg
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"go.uber.org/zap/zapcore"
|
||||
)
|
||||
|
||||
var Logger *zap.Logger
|
||||
|
||||
func InitLogger(level string) error {
|
||||
var zapLevel zapcore.Level
|
||||
if err := zapLevel.UnmarshalText([]byte(level)); err != nil {
|
||||
zapLevel = zapcore.InfoLevel
|
||||
}
|
||||
|
||||
// Check if running in development mode
|
||||
isDev := os.Getenv("ENV") == "development"
|
||||
|
||||
var config zap.Config
|
||||
if isDev {
|
||||
config = zap.NewDevelopmentConfig()
|
||||
config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
|
||||
} else {
|
||||
config = zap.NewProductionConfig()
|
||||
}
|
||||
config.Level = zap.NewAtomicLevelAt(zapLevel)
|
||||
|
||||
var err error
|
||||
Logger, err = config.Build()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func Sync() {
|
||||
if Logger != nil {
|
||||
_ = Logger.Sync()
|
||||
}
|
||||
}
|
||||
11
worker/internal/pkg/validator/errors.go
Normal file
11
worker/internal/pkg/validator/errors.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package validator
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
// ErrEmptyDomain is returned when domain is empty
|
||||
ErrEmptyDomain = errors.New("domain cannot be empty")
|
||||
|
||||
// ErrInvalidDomain is returned when domain format is invalid
|
||||
ErrInvalidDomain = errors.New("invalid domain format")
|
||||
)
|
||||
138
worker/internal/server/batch_sender.go
Normal file
138
worker/internal/server/batch_sender.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/orbit/worker/internal/pkg"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// BatchSender handles batched sending of scan results to Server.
|
||||
// It accumulates items and sends them in batches to reduce HTTP overhead.
|
||||
type BatchSender struct {
|
||||
client *Client
|
||||
scanID int
|
||||
targetID int
|
||||
dataType string // "subdomain", "website", "endpoint", "port"
|
||||
batchSize int
|
||||
|
||||
mu sync.Mutex
|
||||
batch []any
|
||||
sent int // total items sent
|
||||
batches int // total batches sent
|
||||
}
|
||||
|
||||
// NewBatchSender creates a new batch sender
|
||||
func NewBatchSender(client *Client, scanID, targetID int, dataType string, batchSize int) *BatchSender {
|
||||
if batchSize <= 0 {
|
||||
batchSize = 1000 // default batch size
|
||||
}
|
||||
return &BatchSender{
|
||||
client: client,
|
||||
scanID: scanID,
|
||||
targetID: targetID,
|
||||
dataType: dataType,
|
||||
batchSize: batchSize,
|
||||
batch: make([]any, 0, batchSize),
|
||||
}
|
||||
}
|
||||
|
||||
// Add adds an item to the batch. Automatically sends when batch is full.
|
||||
func (s *BatchSender) Add(item any) error {
|
||||
s.mu.Lock()
|
||||
s.batch = append(s.batch, item)
|
||||
shouldSend := len(s.batch) >= s.batchSize
|
||||
s.mu.Unlock()
|
||||
|
||||
if shouldSend {
|
||||
return s.sendBatch()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush sends any remaining items in the batch
|
||||
func (s *BatchSender) Flush() error {
|
||||
s.mu.Lock()
|
||||
if len(s.batch) == 0 {
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
return s.sendBatch()
|
||||
}
|
||||
|
||||
// Stats returns the total items and batches sent
|
||||
func (s *BatchSender) Stats() (items, batches int) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.sent, s.batches
|
||||
}
|
||||
|
||||
// sendBatch sends the current batch to the server
|
||||
func (s *BatchSender) sendBatch() error {
|
||||
s.mu.Lock()
|
||||
if len(s.batch) == 0 {
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy batch and clear
|
||||
toSend := make([]any, len(s.batch))
|
||||
copy(toSend, s.batch)
|
||||
s.batch = s.batch[:0] // reset slice but keep capacity
|
||||
s.mu.Unlock()
|
||||
|
||||
// Build URL and body based on data type (RESTful style)
|
||||
var url string
|
||||
var body map[string]any
|
||||
|
||||
switch s.dataType {
|
||||
case "subdomain":
|
||||
url = fmt.Sprintf("%s/api/worker/scans/%d/subdomains/bulk-upsert", s.client.baseURL, s.scanID)
|
||||
body = map[string]any{
|
||||
"targetId": s.targetID,
|
||||
"subdomains": toSend,
|
||||
}
|
||||
case "website":
|
||||
url = fmt.Sprintf("%s/api/worker/scans/%d/websites/bulk-upsert", s.client.baseURL, s.scanID)
|
||||
body = map[string]any{
|
||||
"targetId": s.targetID,
|
||||
"websites": toSend,
|
||||
}
|
||||
case "endpoint":
|
||||
url = fmt.Sprintf("%s/api/worker/scans/%d/endpoints/bulk-upsert", s.client.baseURL, s.scanID)
|
||||
body = map[string]any{
|
||||
"targetId": s.targetID,
|
||||
"endpoints": toSend,
|
||||
}
|
||||
default:
|
||||
// Generic fallback
|
||||
url = fmt.Sprintf("%s/api/worker/scans/%d/%ss/bulk-upsert", s.client.baseURL, s.scanID, s.dataType)
|
||||
body = map[string]any{
|
||||
"targetId": s.targetID,
|
||||
"items": toSend,
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.client.postWithRetry(url, body); err != nil {
|
||||
pkg.Logger.Error("Failed to send batch",
|
||||
zap.String("type", s.dataType),
|
||||
zap.Int("count", len(toSend)),
|
||||
zap.Error(err))
|
||||
return fmt.Errorf("failed to send %s batch: %w", s.dataType, err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.sent += len(toSend)
|
||||
s.batches++
|
||||
s.mu.Unlock()
|
||||
|
||||
pkg.Logger.Debug("Batch sent",
|
||||
zap.String("type", s.dataType),
|
||||
zap.Int("count", len(toSend)),
|
||||
zap.Int("totalSent", s.sent))
|
||||
|
||||
return nil
|
||||
}
|
||||
171
worker/internal/server/client.go
Normal file
171
worker/internal/server/client.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/orbit/worker/internal/pkg"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// Client handles all HTTP communication with Server
|
||||
// Implements Provider, ResultSaver, and StatusUpdater interfaces
|
||||
type Client struct {
|
||||
baseURL string
|
||||
token string
|
||||
httpClient *http.Client
|
||||
maxRetries int
|
||||
}
|
||||
|
||||
// NewClient creates a new server client
|
||||
func NewClient(baseURL, token string) *Client {
|
||||
return &Client{
|
||||
baseURL: baseURL,
|
||||
token: token,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 5 * time.Minute,
|
||||
},
|
||||
maxRetries: 3,
|
||||
}
|
||||
}
|
||||
|
||||
// --- HTTP helpers ---
|
||||
|
||||
func (c *Client) get(ctx context.Context, url string) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
req.Header.Set("X-Worker-Token", c.token)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
return c.httpClient.Do(req)
|
||||
}
|
||||
|
||||
func (c *Client) postWithRetry(ctx context.Context, url string, body any) error {
|
||||
return c.doWithRetry(ctx, "POST", url, body)
|
||||
}
|
||||
|
||||
func (c *Client) doWithRetry(ctx context.Context, method, url string, body any) error {
|
||||
var lastErr error
|
||||
for i := 0; i < c.maxRetries; i++ {
|
||||
// Check context before retry
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if i > 0 {
|
||||
// Use select to allow cancellation during sleep
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(1<<i) * time.Second):
|
||||
}
|
||||
}
|
||||
|
||||
if err := c.doRequest(ctx, method, url, body); err == nil {
|
||||
return nil
|
||||
} else {
|
||||
lastErr = err
|
||||
pkg.Logger.Warn("API call failed, retrying",
|
||||
zap.String("url", url),
|
||||
zap.Int("attempt", i+1),
|
||||
zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
pkg.Logger.Error("All retries failed", zap.String("url", url), zap.Error(lastErr))
|
||||
return lastErr
|
||||
}
|
||||
|
||||
func (c *Client) doRequest(ctx context.Context, method, url string, body any) error {
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal request body: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-Worker-Token", c.token)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("API error: status=%d, body=%s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func fetchJSON[T any](ctx context.Context, c *Client, url string) (T, error) {
|
||||
var result T
|
||||
|
||||
pkg.Logger.Debug("Fetching JSON", zap.String("url", url))
|
||||
|
||||
resp, err := c.get(ctx, url)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return result, fmt.Errorf("server error: status=%d, body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return result, fmt.Errorf("failed to decode JSON: %w", err)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// PostBatch sends a batch of items to the server
|
||||
func (c *Client) PostBatch(ctx context.Context, scanID, targetID int, dataType string, items []any) error {
|
||||
var url string
|
||||
var body map[string]any
|
||||
|
||||
switch dataType {
|
||||
case "subdomain":
|
||||
url = fmt.Sprintf("%s/api/worker/scans/%d/subdomains/bulk-upsert", c.baseURL, scanID)
|
||||
body = map[string]any{
|
||||
"targetId": targetID,
|
||||
"subdomains": items,
|
||||
}
|
||||
case "website":
|
||||
url = fmt.Sprintf("%s/api/worker/scans/%d/websites/bulk-upsert", c.baseURL, scanID)
|
||||
body = map[string]any{
|
||||
"targetId": targetID,
|
||||
"websites": items,
|
||||
}
|
||||
case "endpoint":
|
||||
url = fmt.Sprintf("%s/api/worker/scans/%d/endpoints/bulk-upsert", c.baseURL, scanID)
|
||||
body = map[string]any{
|
||||
"targetId": targetID,
|
||||
"endpoints": items,
|
||||
}
|
||||
default:
|
||||
url = fmt.Sprintf("%s/api/worker/scans/%d/%ss/bulk-upsert", c.baseURL, scanID, dataType)
|
||||
body = map[string]any{
|
||||
"targetId": targetID,
|
||||
"items": items,
|
||||
}
|
||||
}
|
||||
|
||||
return c.postWithRetry(ctx, url, body)
|
||||
}
|
||||
12
worker/internal/server/provider.go
Normal file
12
worker/internal/server/provider.go
Normal file
@@ -0,0 +1,12 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// GetProviderConfig fetches tool-specific configuration from Server
|
||||
func (c *Client) GetProviderConfig(ctx context.Context, scanID int, toolName string) (*ProviderConfig, error) {
|
||||
url := fmt.Sprintf("%s/api/worker/scans/%d/provider-config?tool=%s", c.baseURL, scanID, toolName)
|
||||
return fetchJSON[*ProviderConfig](ctx, c, url)
|
||||
}
|
||||
21
worker/internal/server/types.go
Normal file
21
worker/internal/server/types.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package server
|
||||
|
||||
import "context"
|
||||
|
||||
// ServerClient defines the interface for Worker to communicate with Server.
|
||||
// This interface allows for easier testing and decoupling.
|
||||
type ServerClient interface {
|
||||
// GetProviderConfig fetches tool-specific configuration (e.g., API keys for subfinder)
|
||||
GetProviderConfig(ctx context.Context, scanID int, toolName string) (*ProviderConfig, error)
|
||||
|
||||
// EnsureWordlistLocal ensures a wordlist file exists locally, downloading if needed
|
||||
EnsureWordlistLocal(ctx context.Context, wordlistName, basePath string) (string, error)
|
||||
|
||||
// PostBatch sends a batch of data to the server (used by BatchSender)
|
||||
PostBatch(ctx context.Context, scanID, targetID int, dataType string, items []any) error
|
||||
}
|
||||
|
||||
// ProviderConfig contains tool configuration file content
|
||||
type ProviderConfig struct {
|
||||
Content string `json:"content"`
|
||||
}
|
||||
57
worker/internal/workflow/subdomain_discovery/templates.yaml
Normal file
57
worker/internal/workflow/subdomain_discovery/templates.yaml
Normal file
@@ -0,0 +1,57 @@
|
||||
# yaml-language-server: $schema=../../activity/templates.schema.json
|
||||
# Command templates for subdomain discovery workflow
|
||||
#
|
||||
# Standard Placeholders (kebab-case for multi-word):
|
||||
# Input:
|
||||
# {domain} - Target domain name
|
||||
# {input-file} - Input file path
|
||||
# {wordlist} - Dictionary file path
|
||||
# {resolvers} - DNS resolvers file path
|
||||
# Output:
|
||||
# {output-file} - Output file path
|
||||
# Config:
|
||||
# {provider-config} - API provider config file path
|
||||
# Parameters:
|
||||
# {threads} - Thread/concurrency count
|
||||
# {timeout} - Timeout in seconds
|
||||
# {rate-limit} - Rate limit (requests per second)
|
||||
# {wildcard-tests} - Wildcard detection test count
|
||||
# {wildcard-batch} - Wildcard batch size
|
||||
|
||||
subfinder:
|
||||
base: "subfinder -d {domain} -all -o '{output-file}' -v"
|
||||
optional:
|
||||
threads: "-t {threads}"
|
||||
provider-config: "-pc '{provider-config}'"
|
||||
timeout: "-timeout {timeout}"
|
||||
|
||||
sublist3r:
|
||||
base: "python3 '/usr/local/share/Sublist3r/sublist3r.py' -d {domain} -o '{output-file}'"
|
||||
optional:
|
||||
threads: "-t {threads}"
|
||||
|
||||
assetfinder:
|
||||
base: "assetfinder --subs-only {domain} > '{output-file}'"
|
||||
optional: {}
|
||||
|
||||
subdomain-bruteforce:
|
||||
base: "puredns bruteforce '{wordlist}' {domain} -r '{resolvers}' --write '{output-file}' --quiet"
|
||||
optional:
|
||||
threads: "-t {threads}"
|
||||
rate-limit: "--rate-limit {rate-limit}"
|
||||
wildcard-tests: "--wildcard-tests {wildcard-tests}"
|
||||
wildcard-batch: "--wildcard-batch {wildcard-batch}"
|
||||
|
||||
subdomain-resolve:
|
||||
base: "puredns resolve '{input-file}' -r '{resolvers}' --write '{output-file}' --wildcard-tests 50 --wildcard-batch 1000000 --quiet"
|
||||
optional:
|
||||
threads: "-t {threads}"
|
||||
rate-limit: "--rate-limit {rate-limit}"
|
||||
wildcard-tests: "--wildcard-tests {wildcard-tests}"
|
||||
wildcard-batch: "--wildcard-batch {wildcard-batch}"
|
||||
|
||||
subdomain-permutation-resolve:
|
||||
base: "cat '{input-file}' | dnsgen - | puredns resolve -r '{resolvers}' --write '{output-file}' --wildcard-tests 50 --wildcard-batch 1000000 --quiet"
|
||||
optional:
|
||||
threads: "-t {threads}"
|
||||
rate-limit: "--rate-limit {rate-limit}"
|
||||
39
worker/internal/workflow/types.go
Normal file
39
worker/internal/workflow/types.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package workflow
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/orbit/worker/internal/server"
|
||||
)
|
||||
|
||||
// Params contains parameters for a workflow execution
|
||||
type Params struct {
|
||||
ScanID int
|
||||
TargetID int
|
||||
TargetName string
|
||||
TargetType string
|
||||
WorkDir string
|
||||
ScanConfig map[string]any
|
||||
ServerClient server.ServerClient
|
||||
}
|
||||
|
||||
// Output contains the result of a workflow execution
|
||||
type Output struct {
|
||||
Data any // Workflow-specific data (file paths for streaming, or parsed data)
|
||||
Metrics *Metrics // Execution statistics (optional)
|
||||
}
|
||||
|
||||
// Metrics contains execution statistics
|
||||
type Metrics struct {
|
||||
ProcessedCount int
|
||||
FailedCount int
|
||||
FailedTools []string
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// Workflow defines the interface for scan workflows
|
||||
type Workflow interface {
|
||||
Name() string
|
||||
Execute(params *Params) (*Output, error)
|
||||
SaveResults(client *server.Client, params *Params, output *Output) error
|
||||
}
|
||||
19142
worker/resources/resolvers.txt
Normal file
19142
worker/resources/resolvers.txt
Normal file
File diff suppressed because it is too large
Load Diff
BIN
worker/worker
Executable file
BIN
worker/worker
Executable file
Binary file not shown.
Reference in New Issue
Block a user