Compare commits

..

32 Commits

Author SHA1 Message Date
yyhuni
adb53c9f85 feat(asset,scan): add configurable statement timeout and improve CSV export
- Add statement_timeout_ms parameter to search_service count() and stream_search() methods for long-running exports
- Replace server-side cursors with OFFSET/LIMIT batching for better Django compatibility
- Introduce create_csv_export_response() utility function to standardize CSV export handling
- Add engine-preset-selector and scan-config-editor components for enhanced scan configuration UI
- Update YAML editor component with improved styling and functionality
- Add i18n translations for new scan configuration features in English and Chinese
- Refactor CSV export endpoints to use new utility function instead of manual StreamingHttpResponse
- Remove unused uuid import from search_service.py
- Update nginx configuration for improved performance
- Enhance search service with configurable timeout support for large dataset exports
2026-01-04 08:58:31 +08:00
github-actions[bot]
8dd3f0536e chore: bump version to v1.3.11-dev 2026-01-03 11:54:31 +00:00
yyhuni
8a8062a12d refactor(scan): rename merged_configuration to yaml_configuration
- Rename `merged_configuration` field to `yaml_configuration` in Scan and ScheduledScan models for clarity
- Update all references across scan repositories, services, views, and serializers
- Update database migration to reflect field name change with improved help text
- Update frontend components to use new field naming convention
- Add YAML editor component for improved configuration editing in UI
- Update engine configuration retrieval in initiate_scan_flow to use new field name
- Remove unused asset tasks __init__.py module
- Simplify README feedback section for better clarity
- Update frontend type definitions and internationalization messages for consistency
2026-01-03 19:50:20 +08:00
yyhuni
55908a2da5 fix(asset,scan): improve decorator usage and dialog layout
- Fix transaction.non_atomic_requests decorator usage in AssetSearchExportView by wrapping with method_decorator for proper class-based view compatibility
- Update scan progress dialog to use flexible width (sm:max-w-fit sm:min-w-[450px]) instead of fixed width for better responsiveness
- Refactor engine names display from single Badge to grid layout with multiple badges for improved readability when multiple engines are present
- Add proper spacing and alignment adjustments (gap-4, items-start) to accommodate multi-line engine badge display
- Add text-xs and whitespace-nowrap to engine badges for consistent styling in grid layout
2026-01-03 18:46:44 +08:00
github-actions[bot]
22a7d4f091 chore: bump version to v1.3.10-dev 2026-01-03 10:45:32 +00:00
yyhuni
f287f18134 更新锁定镜像 2026-01-03 18:33:25 +08:00
yyhuni
de27230b7a 更新构建ci 2026-01-03 18:28:57 +08:00
github-actions[bot]
15a6295189 chore: bump version to v1.3.8-dev 2026-01-03 10:24:17 +00:00
yyhuni
674acdac66 refactor(asset): move database extension initialization to migrations
- Remove pg_trgm and pg_ivm extension setup from AssetConfig.ready() method
- Move extension creation to migration 0002 using RunSQL operations
- Add pg_trgm extension creation for text search index support
- Add pg_ivm extension creation for IMMV incremental maintenance
- Generate unique cursor names in search_service to prevent concurrent request conflicts
- Add @transaction.non_atomic_requests decorator to export view for server-side cursor compatibility
- Simplify app initialization by delegating extension setup to database migrations
- Improve thread safety and concurrency handling for streaming exports
2026-01-03 18:20:27 +08:00
github-actions[bot]
c59152bedf chore: bump version to v1.3.7-dev 2026-01-03 09:56:39 +00:00
yyhuni
b4037202dc feat: use registry cache for faster builds 2026-01-03 17:35:54 +08:00
yyhuni
4b4f9862bf ci(docker): add postgres image build configuration and update image tags
- Add xingrin-postgres image build job to docker-build workflow for multi-platform support (linux/amd64,linux/arm64)
- Update docker-compose.dev.yml to use IMAGE_TAG variable with dev as default fallback
- Update docker-compose.yml to use IMAGE_TAG variable with required validation
- Replace hardcoded postgres image tag (15) with dynamic IMAGE_TAG for better version management
- Enable flexible image tagging across development and production environments
2026-01-03 17:26:34 +08:00
github-actions[bot]
1c42e4978f chore: bump version to v1.3.5-dev 2026-01-03 08:44:06 +00:00
github-actions[bot]
57bab63997 chore: bump version to v1.3.3-dev 2026-01-03 05:55:07 +00:00
github-actions[bot]
b1f0f18ac0 chore: bump version to v1.3.4-dev 2026-01-03 05:54:50 +00:00
yyhuni
ccee5471b8 docs(readme): add notification push service documentation
- Add notification push service feature to visualization interface section
- Document support for real-time WeChat Work, Telegram, and Discord message push
- Enhance feature list clarity for notification capabilities
2026-01-03 13:34:36 +08:00
yyhuni
0ccd362535 优化下载逻辑 2026-01-03 13:32:58 +08:00
yyhuni
7f2af7f7e2 feat(search): add result export functionality and pagination limit support
- Add optional limit parameter to AssetSearchService.search() method for controlling result set size
- Implement AssetSearchExportView for exporting search results as CSV files with UTF-8 BOM encoding
- Add CSV export endpoint at GET /api/assets/search/export/ with configurable MAX_EXPORT_ROWS limit (10000)
- Support both website and endpoint asset types with type-specific column mappings in CSV export
- Format array fields (tech, matched_gf_patterns) and dates appropriately in exported CSV
- Update URL routing to include new search export endpoint
- Update views __init__.py to export AssetSearchExportView
- Add CSV generation with streaming response for efficient memory usage on large exports
- Update frontend search service to support export functionality
- Add internationalization strings for export feature in en.json and zh.json
- Update smart-filter-input and search-results-table components to support export UI
- Update installation and Docker startup scripts for deployment compatibility
2026-01-03 13:22:21 +08:00
yyhuni
4bd0f9e8c1 feat(search): implement dual-view IMMV architecture for website and endpoint assets
- Add incremental materialized view (IMMV) support for both Website and Endpoint asset types using pg_ivm extension
- Create asset_search_view IMMV with optimized indexes for host, title, url, headers, body, tech, status_code, and created_at fields
- Create endpoint_search_view IMMV with identical field structure and indexing strategy for endpoint-specific searches
- Extend search_service.py to support asset type routing with VIEW_MAPPING and VALID_ASSET_TYPES configuration
- Add comprehensive field mapping and array field definitions for both asset types
- Implement dual-query execution path in search views to handle website and endpoint searches independently
- Update frontend search components to support asset type filtering and result display
- Add search results table component with improved data presentation and filtering capabilities
- Update installation scripts and Docker configuration for pg_ivm extension deployment
- Add internationalization strings for new search UI elements in English and Chinese
- Consolidate index creation and cleanup logic in migrations for maintainability
- Enable automatic incremental updates on data changes without manual view refresh
2026-01-03 12:41:20 +08:00
yyhuni
68cc996e3b refactor(asset): standardize snapshot and asset model field naming and types
- Rename `status` to `status_code` in WebsiteSnapshotDTO for consistency
- Rename `web_server` to `webserver` in WebsiteSnapshotDTO for consistency
- Make `target_id` required field in EndpointSnapshotDTO and WebsiteSnapshotDTO
- Remove optional validation check for `target_id` in EndpointSnapshotDTO
- Convert CharField to TextField for url, location, title, webserver, and content_type fields in Endpoint and EndpointSnapshot models to support longer values
- Update migration 0001_initial.py to reflect field type changes from CharField to TextField
- Update all related services and repositories to use standardized field names
- Update serializers to map renamed fields correctly
- Ensure consistent field naming across DTOs, models, and database schema
2026-01-03 09:08:25 +08:00
github-actions[bot]
f1e79d638e chore: bump version to v1.3.2-dev 2026-01-03 00:33:26 +00:00
yyhuni
d484133e4c chore(docker): optimize server dockerfile with docker-ce-cli installation
- Replace full docker.io package with lightweight docker-ce-cli to reduce image size
- Add ca-certificates and gnupg dependencies for secure package management
- Improve Docker installation process for local Worker task distribution
- Reduce unnecessary dependencies in server container build
2026-01-03 08:09:03 +08:00
yyhuni
fc977ae029 chore(docker,frontend): optimize docker installation and add auth bypass config
- Replace docker.io installation script with apt-get package manager for better reliability
- Add NEXT_PUBLIC_SKIP_AUTH environment variable to Vercel config for development
- Improve Docker build layer caching by using native package manager instead of curl script
- Simplify frontend deployment configuration for local development workflows
2026-01-03 08:08:40 +08:00
yyhuni
f328474404 feat(frontend): add comprehensive mock data infrastructure for services
- Add mock data modules for auth, engines, notifications, scheduled-scans, and workers
- Implement mock authentication data with user profiles and login/logout responses
- Create mock scan engine configurations with multiple predefined scanning profiles
- Add mock notification system with various severity levels and categories
- Implement mock scheduled scan data with cron expressions and run history
- Add mock worker node data with status and performance metrics
- Update service layer to integrate with new mock data infrastructure
- Provide helper functions for filtering and paginating mock data
- Enable frontend development and testing without backend API dependency
2026-01-03 07:59:20 +08:00
yyhuni
68e726a066 chore(docker): update base image to python 3.10-slim-bookworm
- Update Python base image from 3.10-slim to 3.10-slim-bookworm
- Ensures compatibility with latest Debian stable release
- Improves security with updated system packages and dependencies
2026-01-02 23:19:09 +08:00
yyhuni
77a6f45909 fix:搜索的楼栋统计问题 2026-01-02 23:12:55 +08:00
yyhuni
49d1f1f1bb 采用ivm增量更新方案进行搜索 2026-01-02 22:46:40 +08:00
yyhuni
db8ecb1644 feat(search): add mock data infrastructure and vulnerability detail integration
- Add comprehensive mock data configuration for all major entities (dashboard, endpoints, organizations, scans, subdomains, targets, vulnerabilities, websites)
- Implement mock service layer with centralized config for development and testing
- Add vulnerability detail dialog integration to search results with lazy loading
- Enhance search result card with vulnerability viewing capability
- Update search materialized view migration to include vulnerability name field
- Implement default host fuzzy search fallback for bare text queries without operators
- Add vulnerability data formatting in search view for consistent API response structure
- Configure Vercel deployment settings and update Next.js configuration
- Update all service layers to support mock data injection for development environment
- Extend search types with improved vulnerability data structure
- Add internationalization strings for vulnerability loading errors
- Enable rapid frontend development and testing without backend API dependency
2026-01-02 19:06:09 +08:00
yyhuni
18cc016268 feat(search): implement advanced query parser with expression syntax support
- Add SearchQueryParser class to parse complex search expressions with operators (=, ==, !=)
- Support logical operators && (AND) and || (OR) for combining multiple conditions
- Implement field mapping for frontend to database field translation
- Add support for array field searching (tech stack) with unnest and ANY operators
- Support fuzzy matching (=), exact matching (==), and negation (!=) operators
- Add proper SQL injection prevention through parameterized queries
- Refactor search service to use expression-based filtering instead of simple filters
- Update search views to integrate new query parser
- Enhance frontend search hook and service to support new expression syntax
- Update search types to reflect new query structure
- Improve search page UI to display expression syntax examples and help text
- Enable complex multi-condition searches like: host="api" && tech="nginx" || status=="200"
2026-01-02 17:46:31 +08:00
yyhuni
23bc463283 feat(search): improve technology stack filtering with fuzzy matching
- Replace exact array matching with fuzzy search using ILIKE operator
- Update tech filter to search within array elements using unnest() and EXISTS
- Support partial technology name matching (e.g., "node" matches "nodejs")
- Apply consistent fuzzy matching logic across both search methods
- Enhance user experience by allowing flexible technology stack queries
2026-01-02 17:01:24 +08:00
yyhuni
7b903b91b2 feat(search): implement comprehensive search infrastructure with materialized views and pagination
- Add asset search service with materialized view support for optimized queries
- Implement search refresh service for maintaining up-to-date search indexes
- Create database migrations for AssetStatistics, StatisticsHistory, Directory, and DirectorySnapshot models
- Add PostgreSQL GIN indexes with trigram operators for full-text search capabilities
- Implement search pagination component with configurable page size and navigation
- Add search result card component with enhanced asset display formatting
- Create search API views with filtering and sorting capabilities
- Add use-search hook for client-side search state management
- Implement search service client for API communication
- Update search types with pagination metadata and result structures
- Add English and Chinese translations for search UI components
- Enhance scheduler to support search index refresh tasks
- Refactor asset views into modular search_views and asset_views
- Update URL routing to support new search endpoints
- Improve scan flow handlers for better search index integration
2026-01-02 16:57:54 +08:00
yyhuni
b3136d51b9 搜索页面前端UI设计完成 2026-01-02 10:07:26 +08:00
111 changed files with 8843 additions and 1209 deletions

View File

@@ -19,7 +19,8 @@ permissions:
contents: write
jobs:
build:
# AMD64 构建(原生 x64 runner
build-amd64:
runs-on: ubuntu-latest
strategy:
matrix:
@@ -27,39 +28,30 @@ jobs:
- image: xingrin-server
dockerfile: docker/server/Dockerfile
context: .
platforms: linux/amd64,linux/arm64
- image: xingrin-frontend
dockerfile: docker/frontend/Dockerfile
context: .
platforms: linux/amd64 # ARM64 构建时 Next.js 在 QEMU 下会崩溃
- image: xingrin-worker
dockerfile: docker/worker/Dockerfile
context: .
platforms: linux/amd64,linux/arm64
- image: xingrin-nginx
dockerfile: docker/nginx/Dockerfile
context: .
platforms: linux/amd64,linux/arm64
- image: xingrin-agent
dockerfile: docker/agent/Dockerfile
context: .
platforms: linux/amd64,linux/arm64
- image: xingrin-postgres
dockerfile: docker/postgres/Dockerfile
context: docker/postgres
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Free disk space (for large builds like worker)
- name: Free disk space
run: |
echo "=== Before cleanup ==="
df -h
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
sudo docker image prune -af
echo "=== After cleanup ==="
df -h
- name: Generate SSL certificates for nginx build
if: matrix.image == 'xingrin-nginx'
@@ -69,10 +61,6 @@ jobs:
-keyout docker/nginx/ssl/privkey.pem \
-out docker/nginx/ssl/fullchain.pem \
-subj "/CN=localhost"
echo "SSL certificates generated for CI build"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
@@ -83,7 +71,120 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Get version from git tag
- name: Get version
id: version
run: |
if [[ $GITHUB_REF == refs/tags/* ]]; then
echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
else
echo "VERSION=dev-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
fi
- name: Build and push AMD64
uses: docker/build-push-action@v5
with:
context: ${{ matrix.context }}
file: ${{ matrix.dockerfile }}
platforms: linux/amd64
push: true
tags: ${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}-amd64
build-args: IMAGE_TAG=${{ steps.version.outputs.VERSION }}
cache-from: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-amd64
cache-to: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-amd64,mode=max
provenance: false
sbom: false
# ARM64 构建(原生 ARM64 runner
build-arm64:
runs-on: ubuntu-22.04-arm
strategy:
matrix:
include:
- image: xingrin-server
dockerfile: docker/server/Dockerfile
context: .
- image: xingrin-frontend
dockerfile: docker/frontend/Dockerfile
context: .
- image: xingrin-worker
dockerfile: docker/worker/Dockerfile
context: .
- image: xingrin-nginx
dockerfile: docker/nginx/Dockerfile
context: .
- image: xingrin-agent
dockerfile: docker/agent/Dockerfile
context: .
- image: xingrin-postgres
dockerfile: docker/postgres/Dockerfile
context: docker/postgres
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Generate SSL certificates for nginx build
if: matrix.image == 'xingrin-nginx'
run: |
mkdir -p docker/nginx/ssl
openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
-keyout docker/nginx/ssl/privkey.pem \
-out docker/nginx/ssl/fullchain.pem \
-subj "/CN=localhost"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Get version
id: version
run: |
if [[ $GITHUB_REF == refs/tags/* ]]; then
echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
else
echo "VERSION=dev-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
fi
- name: Build and push ARM64
uses: docker/build-push-action@v5
with:
context: ${{ matrix.context }}
file: ${{ matrix.dockerfile }}
platforms: linux/arm64
push: true
tags: ${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}-arm64
build-args: IMAGE_TAG=${{ steps.version.outputs.VERSION }}
cache-from: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-arm64
cache-to: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-arm64,mode=max
provenance: false
sbom: false
# 合并多架构 manifest
merge-manifests:
runs-on: ubuntu-latest
needs: [build-amd64, build-arm64]
strategy:
matrix:
image:
- xingrin-server
- xingrin-frontend
- xingrin-worker
- xingrin-nginx
- xingrin-agent
- xingrin-postgres
steps:
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Get version
id: version
run: |
if [[ $GITHUB_REF == refs/tags/* ]]; then
@@ -94,28 +195,27 @@ jobs:
echo "IS_RELEASE=false" >> $GITHUB_OUTPUT
fi
- name: Build and push
uses: docker/build-push-action@v5
with:
context: ${{ matrix.context }}
file: ${{ matrix.dockerfile }}
platforms: ${{ matrix.platforms }}
push: true
tags: |
${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}
${{ steps.version.outputs.IS_RELEASE == 'true' && format('{0}/{1}:latest', env.IMAGE_PREFIX, matrix.image) || '' }}
build-args: |
IMAGE_TAG=${{ steps.version.outputs.VERSION }}
cache-from: type=gha,scope=${{ matrix.image }}
cache-to: type=gha,mode=max,scope=${{ matrix.image }}
provenance: false
sbom: false
- name: Create and push multi-arch manifest
run: |
VERSION=${{ steps.version.outputs.VERSION }}
IMAGE=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}
docker manifest create ${IMAGE}:${VERSION} \
${IMAGE}:${VERSION}-amd64 \
${IMAGE}:${VERSION}-arm64
docker manifest push ${IMAGE}:${VERSION}
if [[ "${{ steps.version.outputs.IS_RELEASE }}" == "true" ]]; then
docker manifest create ${IMAGE}:latest \
${IMAGE}:${VERSION}-amd64 \
${IMAGE}:${VERSION}-arm64
docker manifest push ${IMAGE}:latest
fi
# 所有镜像构建成功后,更新 VERSION 文件
# 根据 tag 所在的分支更新对应分支的 VERSION 文件
# 更新 VERSION 文件
update-version:
runs-on: ubuntu-latest
needs: build
needs: merge-manifests
if: startsWith(github.ref, 'refs/tags/v')
steps:
- name: Checkout repository

View File

@@ -13,14 +13,14 @@
<p align="center">
<a href="#-功能特性">功能特性</a> •
<a href="#-全局资产搜索">资产搜索</a> •
<a href="#-快速开始">快速开始</a> •
<a href="#-文档">文档</a> •
<a href="#-技术栈">技术栈</a> •
<a href="#-反馈与贡献">反馈与贡献</a>
</p>
<p align="center">
<sub>🔍 关键词: ASM | 攻击面管理 | 漏洞扫描 | 资产发现 | Bug Bounty | 渗透测试 | Nuclei | 子域名枚举 | EASM</sub>
<sub>🔍 关键词: ASM | 攻击面管理 | 漏洞扫描 | 资产发现 | 资产搜索 | Bug Bounty | 渗透测试 | Nuclei | 子域名枚举 | EASM</sub>
</p>
---
@@ -162,9 +162,34 @@ flowchart TB
W3 -.心跳上报.-> REDIS
```
### 🔎 全局资产搜索
- **多类型搜索** - 支持 Website 和 Endpoint 两种资产类型
- **表达式语法** - 支持 `=`(模糊)、`==`(精确)、`!=`(不等于)操作符
- **逻辑组合** - 支持 `&&` (AND) 和 `||` (OR) 逻辑组合
- **多字段查询** - 支持 host、url、title、tech、status、body、header 字段
- **CSV 导出** - 流式导出全部搜索结果,无数量限制
#### 搜索语法示例
```bash
# 基础搜索
host="api" # host 包含 "api"
status=="200" # 状态码精确等于 200
tech="nginx" # 技术栈包含 nginx
# 组合搜索
host="api" && status=="200" # host 包含 api 且状态码为 200
tech="vue" || tech="react" # 技术栈包含 vue 或 react
# 复杂查询
host="admin" && tech="php" && status=="200"
url="/api/v1" && status!="404"
```
### 📊 可视化界面
- **数据统计** - 资产/漏洞统计仪表盘
- **实时通知** - WebSocket 消息推送
- **通知推送** - 实时企业微信tgdiscard消息推送服务
---
@@ -172,7 +197,7 @@ flowchart TB
### 环境要求
- **操作系统**: Ubuntu 20.04+ / Debian 11+ (推荐)
- **操作系统**: Ubuntu 20.04+ / Debian 11+
- **硬件**: 2核 4G 内存起步20GB+ 磁盘空间
### 一键安装
@@ -197,6 +222,7 @@ sudo ./install.sh --mirror
### 访问服务
- **Web 界面**: `https://ip:8083`
- **默认账号**: admin / admin首次登录后请修改密码
### 常用命令
@@ -216,13 +242,9 @@ sudo ./uninstall.sh
## 🤝 反馈与贡献
- 🐛 **如果发现 Bug** 可以点击右边链接进行提交 [Issue](https://github.com/yyhuni/xingrin/issues)
- 💡 **有新想法比如UI设计功能设计等** 欢迎点击右边链接进行提交建议 [Issue](https://github.com/yyhuni/xingrin/issues)
- 💡 **发现 Bug有新想法比如UI设计功能设计等** 欢迎点击右边链接进行提交建议 [Issue](https://github.com/yyhuni/xingrin/issues) 或者公众号私信
## 📧 联系
- 目前版本就我个人使用,可能会有很多边界问题
- 如有问题,建议,其他,优先提交[Issue](https://github.com/yyhuni/xingrin/issues),也可以直接给我的公众号发消息,我都会回复的
- 微信公众号: **塔罗安全学苑**
<img src="docs/wechat-qrcode.png" alt="微信公众号" width="200">

View File

@@ -1 +1 @@
v1.2.14-dev
v1.3.11-dev

View File

@@ -1,45 +1,6 @@
import logging
from django.apps import AppConfig
logger = logging.getLogger(__name__)
class AssetConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'apps.asset'
def ready(self):
# 导入所有模型以确保Django发现并注册
from . import models
# 启用 pg_trgm 扩展(用于文本模糊搜索索引)
# 用于已有数据库升级场景
self._ensure_pg_trgm_extension()
def _ensure_pg_trgm_extension(self):
"""
确保 pg_trgm 扩展已启用。
该扩展用于 response_body 和 response_headers 字段的 GIN 索引,
支持高效的文本模糊搜索。
"""
from django.db import connection
# 检查是否为 PostgreSQL 数据库
if connection.vendor != 'postgresql':
logger.debug("跳过 pg_trgm 扩展:当前数据库不是 PostgreSQL")
return
try:
with connection.cursor() as cursor:
cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
logger.debug("pg_trgm 扩展已启用")
except Exception as e:
# 记录错误但不阻止应用启动
# 常见原因:权限不足(需要超级用户权限)
logger.warning(
"无法创建 pg_trgm 扩展: %s"
"这可能导致 response_body 和 response_headers 字段的 GIN 索引无法正常工作。"
"请手动执行: CREATE EXTENSION IF NOT EXISTS pg_trgm;",
str(e)
)

View File

@@ -13,6 +13,7 @@ class EndpointSnapshotDTO:
快照只属于 scan。
"""
scan_id: int
target_id: int # 必填,用于同步到资产表
url: str
host: str = '' # 主机名域名或IP地址
title: str = ''
@@ -25,7 +26,6 @@ class EndpointSnapshotDTO:
response_body: str = ''
vhost: Optional[bool] = None
matched_gf_patterns: List[str] = None
target_id: Optional[int] = None # 冗余字段,用于同步到资产表
response_headers: str = ''
def __post_init__(self):
@@ -43,9 +43,6 @@ class EndpointSnapshotDTO:
"""
from apps.asset.dtos.asset import EndpointDTO
if self.target_id is None:
raise ValueError("target_id 不能为 None无法同步到资产表")
return EndpointDTO(
target_id=self.target_id,
url=self.url,

View File

@@ -13,14 +13,14 @@ class WebsiteSnapshotDTO:
快照只属于 scantarget 信息通过 scan.target 获取。
"""
scan_id: int
target_id: int # 仅用于传递数据,不保存到数据库
target_id: int # 必填,用于同步到资产表
url: str
host: str
title: str = ''
status: Optional[int] = None
status_code: Optional[int] = None # 统一命名status -> status_code
content_length: Optional[int] = None
location: str = ''
web_server: str = ''
webserver: str = '' # 统一命名web_server -> webserver
content_type: str = ''
tech: List[str] = None
response_body: str = ''
@@ -45,10 +45,10 @@ class WebsiteSnapshotDTO:
url=self.url,
host=self.host,
title=self.title,
status_code=self.status,
status_code=self.status_code,
content_length=self.content_length,
location=self.location,
webserver=self.web_server,
webserver=self.webserver,
content_type=self.content_type,
tech=self.tech if self.tech else [],
response_body=self.response_body,

View File

@@ -0,0 +1,345 @@
# Generated by Django 5.2.7 on 2026-01-02 04:45
import django.contrib.postgres.fields
import django.contrib.postgres.indexes
import django.core.validators
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
('scan', '0001_initial'),
('targets', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='AssetStatistics',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('total_targets', models.IntegerField(default=0, help_text='目标总数')),
('total_subdomains', models.IntegerField(default=0, help_text='子域名总数')),
('total_ips', models.IntegerField(default=0, help_text='IP地址总数')),
('total_endpoints', models.IntegerField(default=0, help_text='端点总数')),
('total_websites', models.IntegerField(default=0, help_text='网站总数')),
('total_vulns', models.IntegerField(default=0, help_text='漏洞总数')),
('total_assets', models.IntegerField(default=0, help_text='总资产数(子域名+IP+端点+网站)')),
('prev_targets', models.IntegerField(default=0, help_text='上次目标总数')),
('prev_subdomains', models.IntegerField(default=0, help_text='上次子域名总数')),
('prev_ips', models.IntegerField(default=0, help_text='上次IP地址总数')),
('prev_endpoints', models.IntegerField(default=0, help_text='上次端点总数')),
('prev_websites', models.IntegerField(default=0, help_text='上次网站总数')),
('prev_vulns', models.IntegerField(default=0, help_text='上次漏洞总数')),
('prev_assets', models.IntegerField(default=0, help_text='上次总资产数')),
('updated_at', models.DateTimeField(auto_now=True, help_text='最后更新时间')),
],
options={
'verbose_name': '资产统计',
'verbose_name_plural': '资产统计',
'db_table': 'asset_statistics',
},
),
migrations.CreateModel(
name='StatisticsHistory',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('date', models.DateField(help_text='统计日期', unique=True)),
('total_targets', models.IntegerField(default=0, help_text='目标总数')),
('total_subdomains', models.IntegerField(default=0, help_text='子域名总数')),
('total_ips', models.IntegerField(default=0, help_text='IP地址总数')),
('total_endpoints', models.IntegerField(default=0, help_text='端点总数')),
('total_websites', models.IntegerField(default=0, help_text='网站总数')),
('total_vulns', models.IntegerField(default=0, help_text='漏洞总数')),
('total_assets', models.IntegerField(default=0, help_text='总资产数')),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('updated_at', models.DateTimeField(auto_now=True, help_text='更新时间')),
],
options={
'verbose_name': '统计历史',
'verbose_name_plural': '统计历史',
'db_table': 'statistics_history',
'ordering': ['-date'],
'indexes': [models.Index(fields=['date'], name='statistics__date_1d29cd_idx')],
},
),
migrations.CreateModel(
name='Directory',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('url', models.CharField(help_text='完整请求 URL', max_length=2000)),
('status', models.IntegerField(blank=True, help_text='HTTP 响应状态码', null=True)),
('content_length', models.BigIntegerField(blank=True, help_text='响应体字节大小Content-Length 或实际长度)', null=True)),
('words', models.IntegerField(blank=True, help_text='响应体中单词数量(按空格分割)', null=True)),
('lines', models.IntegerField(blank=True, help_text='响应体行数(按换行符分割)', null=True)),
('content_type', models.CharField(blank=True, default='', help_text='响应头 Content-Type 值', max_length=200)),
('duration', models.BigIntegerField(blank=True, help_text='请求耗时(单位:纳秒)', null=True)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('target', models.ForeignKey(help_text='所属的扫描目标', on_delete=django.db.models.deletion.CASCADE, related_name='directories', to='targets.target')),
],
options={
'verbose_name': '目录',
'verbose_name_plural': '目录',
'db_table': 'directory',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['-created_at'], name='directory_created_2cef03_idx'), models.Index(fields=['target'], name='directory_target__e310c8_idx'), models.Index(fields=['url'], name='directory_url_ba40cd_idx'), models.Index(fields=['status'], name='directory_status_40bbe6_idx'), django.contrib.postgres.indexes.GinIndex(fields=['url'], name='directory_url_trgm_idx', opclasses=['gin_trgm_ops'])],
'constraints': [models.UniqueConstraint(fields=('target', 'url'), name='unique_directory_url_target')],
},
),
migrations.CreateModel(
name='DirectorySnapshot',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('url', models.CharField(help_text='目录URL', max_length=2000)),
('status', models.IntegerField(blank=True, help_text='HTTP状态码', null=True)),
('content_length', models.BigIntegerField(blank=True, help_text='内容长度', null=True)),
('words', models.IntegerField(blank=True, help_text='响应体中单词数量(按空格分割)', null=True)),
('lines', models.IntegerField(blank=True, help_text='响应体行数(按换行符分割)', null=True)),
('content_type', models.CharField(blank=True, default='', help_text='响应头 Content-Type 值', max_length=200)),
('duration', models.BigIntegerField(blank=True, help_text='请求耗时(单位:纳秒)', null=True)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('scan', models.ForeignKey(help_text='所属的扫描任务', on_delete=django.db.models.deletion.CASCADE, related_name='directory_snapshots', to='scan.scan')),
],
options={
'verbose_name': '目录快照',
'verbose_name_plural': '目录快照',
'db_table': 'directory_snapshot',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['scan'], name='directory_s_scan_id_c45900_idx'), models.Index(fields=['url'], name='directory_s_url_b4b72b_idx'), models.Index(fields=['status'], name='directory_s_status_e9f57e_idx'), models.Index(fields=['content_type'], name='directory_s_content_45e864_idx'), models.Index(fields=['-created_at'], name='directory_s_created_eb9d27_idx'), django.contrib.postgres.indexes.GinIndex(fields=['url'], name='dir_snap_url_trgm', opclasses=['gin_trgm_ops'])],
'constraints': [models.UniqueConstraint(fields=('scan', 'url'), name='unique_directory_per_scan_snapshot')],
},
),
migrations.CreateModel(
name='Endpoint',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('url', models.TextField(help_text='最终访问的完整URL')),
('host', models.CharField(blank=True, default='', help_text='主机名域名或IP地址', max_length=253)),
('location', models.TextField(blank=True, default='', help_text='重定向地址HTTP 3xx 响应头 Location')),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('title', models.TextField(blank=True, default='', help_text='网页标题HTML <title> 标签内容)')),
('webserver', models.TextField(blank=True, default='', help_text='服务器类型HTTP 响应头 Server 值)')),
('response_body', models.TextField(blank=True, default='', help_text='HTTP响应体')),
('content_type', models.TextField(blank=True, default='', help_text='响应类型HTTP Content-Type 响应头)')),
('tech', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=100), blank=True, default=list, help_text='技术栈(服务器/框架/语言等)', size=None)),
('status_code', models.IntegerField(blank=True, help_text='HTTP状态码', null=True)),
('content_length', models.IntegerField(blank=True, help_text='响应体大小(单位字节)', null=True)),
('vhost', models.BooleanField(blank=True, help_text='是否支持虚拟主机', null=True)),
('matched_gf_patterns', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=100), blank=True, default=list, help_text='匹配的GF模式列表用于识别敏感端点如api, debug, config等', size=None)),
('response_headers', models.TextField(blank=True, default='', help_text='原始HTTP响应头')),
('target', models.ForeignKey(help_text='所属的扫描目标(主关联字段,表示所属关系,不能为空)', on_delete=django.db.models.deletion.CASCADE, related_name='endpoints', to='targets.target')),
],
options={
'verbose_name': '端点',
'verbose_name_plural': '端点',
'db_table': 'endpoint',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['-created_at'], name='endpoint_created_44fe9c_idx'), models.Index(fields=['target'], name='endpoint_target__7f9065_idx'), models.Index(fields=['url'], name='endpoint_url_30f66e_idx'), models.Index(fields=['host'], name='endpoint_host_5b4cc8_idx'), models.Index(fields=['status_code'], name='endpoint_status__5d4fdd_idx'), models.Index(fields=['title'], name='endpoint_title_29e26c_idx'), django.contrib.postgres.indexes.GinIndex(fields=['tech'], name='endpoint_tech_2bfa7c_gin'), django.contrib.postgres.indexes.GinIndex(fields=['response_headers'], name='endpoint_resp_headers_trgm_idx', opclasses=['gin_trgm_ops']), django.contrib.postgres.indexes.GinIndex(fields=['url'], name='endpoint_url_trgm_idx', opclasses=['gin_trgm_ops']), django.contrib.postgres.indexes.GinIndex(fields=['title'], name='endpoint_title_trgm_idx', opclasses=['gin_trgm_ops'])],
'constraints': [models.UniqueConstraint(fields=('url', 'target'), name='unique_endpoint_url_target')],
},
),
migrations.CreateModel(
name='EndpointSnapshot',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('url', models.TextField(help_text='端点URL')),
('host', models.CharField(blank=True, default='', help_text='主机名域名或IP地址', max_length=253)),
('title', models.TextField(blank=True, default='', help_text='页面标题')),
('status_code', models.IntegerField(blank=True, help_text='HTTP状态码', null=True)),
('content_length', models.IntegerField(blank=True, help_text='内容长度', null=True)),
('location', models.TextField(blank=True, default='', help_text='重定向位置')),
('webserver', models.TextField(blank=True, default='', help_text='Web服务器')),
('content_type', models.TextField(blank=True, default='', help_text='内容类型')),
('tech', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=100), blank=True, default=list, help_text='技术栈', size=None)),
('response_body', models.TextField(blank=True, default='', help_text='HTTP响应体')),
('vhost', models.BooleanField(blank=True, help_text='虚拟主机标志', null=True)),
('matched_gf_patterns', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=100), blank=True, default=list, help_text='匹配的GF模式列表', size=None)),
('response_headers', models.TextField(blank=True, default='', help_text='原始HTTP响应头')),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('scan', models.ForeignKey(help_text='所属的扫描任务', on_delete=django.db.models.deletion.CASCADE, related_name='endpoint_snapshots', to='scan.scan')),
],
options={
'verbose_name': '端点快照',
'verbose_name_plural': '端点快照',
'db_table': 'endpoint_snapshot',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['scan'], name='endpoint_sn_scan_id_6ac9a7_idx'), models.Index(fields=['url'], name='endpoint_sn_url_205160_idx'), models.Index(fields=['host'], name='endpoint_sn_host_577bfd_idx'), models.Index(fields=['title'], name='endpoint_sn_title_516a05_idx'), models.Index(fields=['status_code'], name='endpoint_sn_status__83efb0_idx'), models.Index(fields=['webserver'], name='endpoint_sn_webserv_66be83_idx'), models.Index(fields=['-created_at'], name='endpoint_sn_created_21fb5b_idx'), django.contrib.postgres.indexes.GinIndex(fields=['tech'], name='endpoint_sn_tech_0d0752_gin'), django.contrib.postgres.indexes.GinIndex(fields=['response_headers'], name='ep_snap_resp_hdr_trgm', opclasses=['gin_trgm_ops']), django.contrib.postgres.indexes.GinIndex(fields=['url'], name='ep_snap_url_trgm', opclasses=['gin_trgm_ops']), django.contrib.postgres.indexes.GinIndex(fields=['title'], name='ep_snap_title_trgm', opclasses=['gin_trgm_ops'])],
'constraints': [models.UniqueConstraint(fields=('scan', 'url'), name='unique_endpoint_per_scan_snapshot')],
},
),
migrations.CreateModel(
name='HostPortMapping',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('host', models.CharField(help_text='主机名域名或IP', max_length=1000)),
('ip', models.GenericIPAddressField(help_text='IP地址')),
('port', models.IntegerField(help_text='端口号1-65535', validators=[django.core.validators.MinValueValidator(1, message='端口号必须大于等于1'), django.core.validators.MaxValueValidator(65535, message='端口号必须小于等于65535')])),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('target', models.ForeignKey(help_text='所属的扫描目标', on_delete=django.db.models.deletion.CASCADE, related_name='host_port_mappings', to='targets.target')),
],
options={
'verbose_name': '主机端口映射',
'verbose_name_plural': '主机端口映射',
'db_table': 'host_port_mapping',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['target'], name='host_port_m_target__943e9b_idx'), models.Index(fields=['host'], name='host_port_m_host_f78363_idx'), models.Index(fields=['ip'], name='host_port_m_ip_2e6f02_idx'), models.Index(fields=['port'], name='host_port_m_port_9fb9ff_idx'), models.Index(fields=['host', 'ip'], name='host_port_m_host_3ce245_idx'), models.Index(fields=['-created_at'], name='host_port_m_created_11cd22_idx')],
'constraints': [models.UniqueConstraint(fields=('target', 'host', 'ip', 'port'), name='unique_target_host_ip_port')],
},
),
migrations.CreateModel(
name='HostPortMappingSnapshot',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('host', models.CharField(help_text='主机名域名或IP', max_length=1000)),
('ip', models.GenericIPAddressField(help_text='IP地址')),
('port', models.IntegerField(help_text='端口号1-65535', validators=[django.core.validators.MinValueValidator(1, message='端口号必须大于等于1'), django.core.validators.MaxValueValidator(65535, message='端口号必须小于等于65535')])),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('scan', models.ForeignKey(help_text='所属的扫描任务(主关联)', on_delete=django.db.models.deletion.CASCADE, related_name='host_port_mapping_snapshots', to='scan.scan')),
],
options={
'verbose_name': '主机端口映射快照',
'verbose_name_plural': '主机端口映射快照',
'db_table': 'host_port_mapping_snapshot',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['scan'], name='host_port_m_scan_id_50ba0b_idx'), models.Index(fields=['host'], name='host_port_m_host_e99054_idx'), models.Index(fields=['ip'], name='host_port_m_ip_54818c_idx'), models.Index(fields=['port'], name='host_port_m_port_ed7b48_idx'), models.Index(fields=['host', 'ip'], name='host_port_m_host_8a463a_idx'), models.Index(fields=['scan', 'host'], name='host_port_m_scan_id_426fdb_idx'), models.Index(fields=['-created_at'], name='host_port_m_created_fb28b8_idx')],
'constraints': [models.UniqueConstraint(fields=('scan', 'host', 'ip', 'port'), name='unique_scan_host_ip_port_snapshot')],
},
),
migrations.CreateModel(
name='Subdomain',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('name', models.CharField(help_text='子域名名称', max_length=1000)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('target', models.ForeignKey(help_text='所属的扫描目标(主关联字段,表示所属关系,不能为空)', on_delete=django.db.models.deletion.CASCADE, related_name='subdomains', to='targets.target')),
],
options={
'verbose_name': '子域名',
'verbose_name_plural': '子域名',
'db_table': 'subdomain',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['-created_at'], name='subdomain_created_e187a8_idx'), models.Index(fields=['name', 'target'], name='subdomain_name_60e1d0_idx'), models.Index(fields=['target'], name='subdomain_target__e409f0_idx'), models.Index(fields=['name'], name='subdomain_name_d40ba7_idx'), django.contrib.postgres.indexes.GinIndex(fields=['name'], name='subdomain_name_trgm_idx', opclasses=['gin_trgm_ops'])],
'constraints': [models.UniqueConstraint(fields=('name', 'target'), name='unique_subdomain_name_target')],
},
),
migrations.CreateModel(
name='SubdomainSnapshot',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('name', models.CharField(help_text='子域名名称', max_length=1000)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('scan', models.ForeignKey(help_text='所属的扫描任务', on_delete=django.db.models.deletion.CASCADE, related_name='subdomain_snapshots', to='scan.scan')),
],
options={
'verbose_name': '子域名快照',
'verbose_name_plural': '子域名快照',
'db_table': 'subdomain_snapshot',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['scan'], name='subdomain_s_scan_id_68c253_idx'), models.Index(fields=['name'], name='subdomain_s_name_2da42b_idx'), models.Index(fields=['-created_at'], name='subdomain_s_created_d2b48e_idx'), django.contrib.postgres.indexes.GinIndex(fields=['name'], name='subdomain_snap_name_trgm', opclasses=['gin_trgm_ops'])],
'constraints': [models.UniqueConstraint(fields=('scan', 'name'), name='unique_subdomain_per_scan_snapshot')],
},
),
migrations.CreateModel(
name='Vulnerability',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('url', models.CharField(help_text='漏洞所在的URL', max_length=2000)),
('vuln_type', models.CharField(help_text='漏洞类型(如 xss, sqli', max_length=100)),
('severity', models.CharField(choices=[('unknown', '未知'), ('info', '信息'), ('low', ''), ('medium', ''), ('high', ''), ('critical', '危急')], default='unknown', help_text='严重性(未知/信息/低/中/高/危急)', max_length=20)),
('source', models.CharField(blank=True, default='', help_text='来源工具(如 dalfox, nuclei, crlfuzz', max_length=50)),
('cvss_score', models.DecimalField(blank=True, decimal_places=1, help_text='CVSS 评分0.0-10.0', max_digits=3, null=True)),
('description', models.TextField(blank=True, default='', help_text='漏洞描述')),
('raw_output', models.JSONField(blank=True, default=dict, help_text='工具原始输出')),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('target', models.ForeignKey(help_text='所属的扫描目标', on_delete=django.db.models.deletion.CASCADE, related_name='vulnerabilities', to='targets.target')),
],
options={
'verbose_name': '漏洞',
'verbose_name_plural': '漏洞',
'db_table': 'vulnerability',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['target'], name='vulnerabili_target__755a02_idx'), models.Index(fields=['vuln_type'], name='vulnerabili_vuln_ty_3010cd_idx'), models.Index(fields=['severity'], name='vulnerabili_severit_1a798b_idx'), models.Index(fields=['source'], name='vulnerabili_source_7c7552_idx'), models.Index(fields=['url'], name='vulnerabili_url_4dcc4d_idx'), models.Index(fields=['-created_at'], name='vulnerabili_created_e25ff7_idx')],
},
),
migrations.CreateModel(
name='VulnerabilitySnapshot',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('url', models.CharField(help_text='漏洞所在的URL', max_length=2000)),
('vuln_type', models.CharField(help_text='漏洞类型(如 xss, sqli', max_length=100)),
('severity', models.CharField(choices=[('unknown', '未知'), ('info', '信息'), ('low', ''), ('medium', ''), ('high', ''), ('critical', '危急')], default='unknown', help_text='严重性(未知/信息/低/中/高/危急)', max_length=20)),
('source', models.CharField(blank=True, default='', help_text='来源工具(如 dalfox, nuclei, crlfuzz', max_length=50)),
('cvss_score', models.DecimalField(blank=True, decimal_places=1, help_text='CVSS 评分0.0-10.0', max_digits=3, null=True)),
('description', models.TextField(blank=True, default='', help_text='漏洞描述')),
('raw_output', models.JSONField(blank=True, default=dict, help_text='工具原始输出')),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('scan', models.ForeignKey(help_text='所属的扫描任务', on_delete=django.db.models.deletion.CASCADE, related_name='vulnerability_snapshots', to='scan.scan')),
],
options={
'verbose_name': '漏洞快照',
'verbose_name_plural': '漏洞快照',
'db_table': 'vulnerability_snapshot',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['scan'], name='vulnerabili_scan_id_7b81c9_idx'), models.Index(fields=['url'], name='vulnerabili_url_11a707_idx'), models.Index(fields=['vuln_type'], name='vulnerabili_vuln_ty_6b90ee_idx'), models.Index(fields=['severity'], name='vulnerabili_severit_4eae0d_idx'), models.Index(fields=['source'], name='vulnerabili_source_968b1f_idx'), models.Index(fields=['-created_at'], name='vulnerabili_created_53a12e_idx')],
},
),
migrations.CreateModel(
name='WebSite',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('url', models.TextField(help_text='最终访问的完整URL')),
('host', models.CharField(blank=True, default='', help_text='主机名域名或IP地址', max_length=253)),
('location', models.TextField(blank=True, default='', help_text='重定向地址HTTP 3xx 响应头 Location')),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('title', models.TextField(blank=True, default='', help_text='网页标题HTML <title> 标签内容)')),
('webserver', models.TextField(blank=True, default='', help_text='服务器类型HTTP 响应头 Server 值)')),
('response_body', models.TextField(blank=True, default='', help_text='HTTP响应体')),
('content_type', models.TextField(blank=True, default='', help_text='响应类型HTTP Content-Type 响应头)')),
('tech', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=100), blank=True, default=list, help_text='技术栈(服务器/框架/语言等)', size=None)),
('status_code', models.IntegerField(blank=True, help_text='HTTP状态码', null=True)),
('content_length', models.IntegerField(blank=True, help_text='响应体大小(单位字节)', null=True)),
('vhost', models.BooleanField(blank=True, help_text='是否支持虚拟主机', null=True)),
('response_headers', models.TextField(blank=True, default='', help_text='原始HTTP响应头')),
('target', models.ForeignKey(help_text='所属的扫描目标(主关联字段,表示所属关系,不能为空)', on_delete=django.db.models.deletion.CASCADE, related_name='websites', to='targets.target')),
],
options={
'verbose_name': '站点',
'verbose_name_plural': '站点',
'db_table': 'website',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['-created_at'], name='website_created_c9cfd2_idx'), models.Index(fields=['url'], name='website_url_b18883_idx'), models.Index(fields=['host'], name='website_host_996b50_idx'), models.Index(fields=['target'], name='website_target__2a353b_idx'), models.Index(fields=['title'], name='website_title_c2775b_idx'), models.Index(fields=['status_code'], name='website_status__51663d_idx'), django.contrib.postgres.indexes.GinIndex(fields=['tech'], name='website_tech_e3f0cb_gin'), django.contrib.postgres.indexes.GinIndex(fields=['response_headers'], name='website_resp_headers_trgm_idx', opclasses=['gin_trgm_ops']), django.contrib.postgres.indexes.GinIndex(fields=['url'], name='website_url_trgm_idx', opclasses=['gin_trgm_ops']), django.contrib.postgres.indexes.GinIndex(fields=['title'], name='website_title_trgm_idx', opclasses=['gin_trgm_ops'])],
'constraints': [models.UniqueConstraint(fields=('url', 'target'), name='unique_website_url_target')],
},
),
migrations.CreateModel(
name='WebsiteSnapshot',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('url', models.TextField(help_text='站点URL')),
('host', models.CharField(blank=True, default='', help_text='主机名域名或IP地址', max_length=253)),
('title', models.TextField(blank=True, default='', help_text='页面标题')),
('status_code', models.IntegerField(blank=True, help_text='HTTP状态码', null=True)),
('content_length', models.BigIntegerField(blank=True, help_text='内容长度', null=True)),
('location', models.TextField(blank=True, default='', help_text='重定向位置')),
('webserver', models.TextField(blank=True, default='', help_text='Web服务器')),
('content_type', models.TextField(blank=True, default='', help_text='内容类型')),
('tech', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=100), blank=True, default=list, help_text='技术栈', size=None)),
('response_body', models.TextField(blank=True, default='', help_text='HTTP响应体')),
('vhost', models.BooleanField(blank=True, help_text='虚拟主机标志', null=True)),
('response_headers', models.TextField(blank=True, default='', help_text='原始HTTP响应头')),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('scan', models.ForeignKey(help_text='所属的扫描任务', on_delete=django.db.models.deletion.CASCADE, related_name='website_snapshots', to='scan.scan')),
],
options={
'verbose_name': '网站快照',
'verbose_name_plural': '网站快照',
'db_table': 'website_snapshot',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['scan'], name='website_sna_scan_id_26b6dc_idx'), models.Index(fields=['url'], name='website_sna_url_801a70_idx'), models.Index(fields=['host'], name='website_sna_host_348fe1_idx'), models.Index(fields=['title'], name='website_sna_title_b1a5ee_idx'), models.Index(fields=['-created_at'], name='website_sna_created_2c149a_idx'), django.contrib.postgres.indexes.GinIndex(fields=['tech'], name='website_sna_tech_3d6d2f_gin'), django.contrib.postgres.indexes.GinIndex(fields=['response_headers'], name='ws_snap_resp_hdr_trgm', opclasses=['gin_trgm_ops']), django.contrib.postgres.indexes.GinIndex(fields=['url'], name='ws_snap_url_trgm', opclasses=['gin_trgm_ops']), django.contrib.postgres.indexes.GinIndex(fields=['title'], name='ws_snap_title_trgm', opclasses=['gin_trgm_ops'])],
'constraints': [models.UniqueConstraint(fields=('scan', 'url'), name='unique_website_per_scan_snapshot')],
},
),
]

View File

@@ -0,0 +1,193 @@
"""
创建资产搜索 IMMV增量维护物化视图
使用 pg_ivm 扩展创建 IMMV数据变更时自动增量更新无需手动刷新。
包含:
1. asset_search_view - Website 搜索视图
2. endpoint_search_view - Endpoint 搜索视图
"""
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('asset', '0001_initial'),
]
operations = [
# 1. 确保 pg_trgm 扩展已启用(用于文本模糊搜索索引)
migrations.RunSQL(
sql="CREATE EXTENSION IF NOT EXISTS pg_trgm;",
reverse_sql="-- pg_trgm extension kept for other uses"
),
# 2. 确保 pg_ivm 扩展已启用(用于 IMMV 增量维护)
migrations.RunSQL(
sql="CREATE EXTENSION IF NOT EXISTS pg_ivm;",
reverse_sql="-- pg_ivm extension kept for other uses"
),
# ==================== Website IMMV ====================
# 2. 创建 asset_search_view IMMV
migrations.RunSQL(
sql="""
SELECT pgivm.create_immv('asset_search_view', $$
SELECT
w.id,
w.url,
w.host,
w.title,
w.tech,
w.status_code,
w.response_headers,
w.response_body,
w.content_type,
w.content_length,
w.webserver,
w.location,
w.vhost,
w.created_at,
w.target_id
FROM website w
$$);
""",
reverse_sql="SELECT pgivm.drop_immv('asset_search_view');"
),
# 3. 创建 asset_search_view 索引
migrations.RunSQL(
sql="""
-- 唯一索引
CREATE UNIQUE INDEX IF NOT EXISTS asset_search_view_id_idx
ON asset_search_view (id);
-- host 模糊搜索索引
CREATE INDEX IF NOT EXISTS asset_search_view_host_trgm_idx
ON asset_search_view USING gin (host gin_trgm_ops);
-- title 模糊搜索索引
CREATE INDEX IF NOT EXISTS asset_search_view_title_trgm_idx
ON asset_search_view USING gin (title gin_trgm_ops);
-- url 模糊搜索索引
CREATE INDEX IF NOT EXISTS asset_search_view_url_trgm_idx
ON asset_search_view USING gin (url gin_trgm_ops);
-- response_headers 模糊搜索索引
CREATE INDEX IF NOT EXISTS asset_search_view_headers_trgm_idx
ON asset_search_view USING gin (response_headers gin_trgm_ops);
-- response_body 模糊搜索索引
CREATE INDEX IF NOT EXISTS asset_search_view_body_trgm_idx
ON asset_search_view USING gin (response_body gin_trgm_ops);
-- tech 数组索引
CREATE INDEX IF NOT EXISTS asset_search_view_tech_idx
ON asset_search_view USING gin (tech);
-- status_code 索引
CREATE INDEX IF NOT EXISTS asset_search_view_status_idx
ON asset_search_view (status_code);
-- created_at 排序索引
CREATE INDEX IF NOT EXISTS asset_search_view_created_idx
ON asset_search_view (created_at DESC);
""",
reverse_sql="""
DROP INDEX IF EXISTS asset_search_view_id_idx;
DROP INDEX IF EXISTS asset_search_view_host_trgm_idx;
DROP INDEX IF EXISTS asset_search_view_title_trgm_idx;
DROP INDEX IF EXISTS asset_search_view_url_trgm_idx;
DROP INDEX IF EXISTS asset_search_view_headers_trgm_idx;
DROP INDEX IF EXISTS asset_search_view_body_trgm_idx;
DROP INDEX IF EXISTS asset_search_view_tech_idx;
DROP INDEX IF EXISTS asset_search_view_status_idx;
DROP INDEX IF EXISTS asset_search_view_created_idx;
"""
),
# ==================== Endpoint IMMV ====================
# 4. 创建 endpoint_search_view IMMV
migrations.RunSQL(
sql="""
SELECT pgivm.create_immv('endpoint_search_view', $$
SELECT
e.id,
e.url,
e.host,
e.title,
e.tech,
e.status_code,
e.response_headers,
e.response_body,
e.content_type,
e.content_length,
e.webserver,
e.location,
e.vhost,
e.matched_gf_patterns,
e.created_at,
e.target_id
FROM endpoint e
$$);
""",
reverse_sql="SELECT pgivm.drop_immv('endpoint_search_view');"
),
# 5. 创建 endpoint_search_view 索引
migrations.RunSQL(
sql="""
-- 唯一索引
CREATE UNIQUE INDEX IF NOT EXISTS endpoint_search_view_id_idx
ON endpoint_search_view (id);
-- host 模糊搜索索引
CREATE INDEX IF NOT EXISTS endpoint_search_view_host_trgm_idx
ON endpoint_search_view USING gin (host gin_trgm_ops);
-- title 模糊搜索索引
CREATE INDEX IF NOT EXISTS endpoint_search_view_title_trgm_idx
ON endpoint_search_view USING gin (title gin_trgm_ops);
-- url 模糊搜索索引
CREATE INDEX IF NOT EXISTS endpoint_search_view_url_trgm_idx
ON endpoint_search_view USING gin (url gin_trgm_ops);
-- response_headers 模糊搜索索引
CREATE INDEX IF NOT EXISTS endpoint_search_view_headers_trgm_idx
ON endpoint_search_view USING gin (response_headers gin_trgm_ops);
-- response_body 模糊搜索索引
CREATE INDEX IF NOT EXISTS endpoint_search_view_body_trgm_idx
ON endpoint_search_view USING gin (response_body gin_trgm_ops);
-- tech 数组索引
CREATE INDEX IF NOT EXISTS endpoint_search_view_tech_idx
ON endpoint_search_view USING gin (tech);
-- status_code 索引
CREATE INDEX IF NOT EXISTS endpoint_search_view_status_idx
ON endpoint_search_view (status_code);
-- created_at 排序索引
CREATE INDEX IF NOT EXISTS endpoint_search_view_created_idx
ON endpoint_search_view (created_at DESC);
""",
reverse_sql="""
DROP INDEX IF EXISTS endpoint_search_view_id_idx;
DROP INDEX IF EXISTS endpoint_search_view_host_trgm_idx;
DROP INDEX IF EXISTS endpoint_search_view_title_trgm_idx;
DROP INDEX IF EXISTS endpoint_search_view_url_trgm_idx;
DROP INDEX IF EXISTS endpoint_search_view_headers_trgm_idx;
DROP INDEX IF EXISTS endpoint_search_view_body_trgm_idx;
DROP INDEX IF EXISTS endpoint_search_view_tech_idx;
DROP INDEX IF EXISTS endpoint_search_view_status_idx;
DROP INDEX IF EXISTS endpoint_search_view_created_idx;
"""
),
]

View File

@@ -65,28 +65,25 @@ class Endpoint(models.Model):
help_text='所属的扫描目标(主关联字段,表示所属关系,不能为空)'
)
url = models.CharField(max_length=2000, help_text='最终访问的完整URL')
url = models.TextField(help_text='最终访问的完整URL')
host = models.CharField(
max_length=253,
blank=True,
default='',
help_text='主机名域名或IP地址'
)
location = models.CharField(
max_length=1000,
location = models.TextField(
blank=True,
default='',
help_text='重定向地址HTTP 3xx 响应头 Location'
)
created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
title = models.CharField(
max_length=1000,
title = models.TextField(
blank=True,
default='',
help_text='网页标题HTML <title> 标签内容)'
)
webserver = models.CharField(
max_length=200,
webserver = models.TextField(
blank=True,
default='',
help_text='服务器类型HTTP 响应头 Server 值)'
@@ -96,8 +93,7 @@ class Endpoint(models.Model):
default='',
help_text='HTTP响应体'
)
content_type = models.CharField(
max_length=200,
content_type = models.TextField(
blank=True,
default='',
help_text='响应类型HTTP Content-Type 响应头)'
@@ -188,28 +184,25 @@ class WebSite(models.Model):
help_text='所属的扫描目标(主关联字段,表示所属关系,不能为空)'
)
url = models.CharField(max_length=2000, help_text='最终访问的完整URL')
url = models.TextField(help_text='最终访问的完整URL')
host = models.CharField(
max_length=253,
blank=True,
default='',
help_text='主机名域名或IP地址'
)
location = models.CharField(
max_length=1000,
location = models.TextField(
blank=True,
default='',
help_text='重定向地址HTTP 3xx 响应头 Location'
)
created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
title = models.CharField(
max_length=1000,
title = models.TextField(
blank=True,
default='',
help_text='网页标题HTML <title> 标签内容)'
)
webserver = models.CharField(
max_length=200,
webserver = models.TextField(
blank=True,
default='',
help_text='服务器类型HTTP 响应头 Server 值)'
@@ -219,8 +212,7 @@ class WebSite(models.Model):
default='',
help_text='HTTP响应体'
)
content_type = models.CharField(
max_length=200,
content_type = models.TextField(
blank=True,
default='',
help_text='响应类型HTTP Content-Type 响应头)'

View File

@@ -61,14 +61,14 @@ class WebsiteSnapshot(models.Model):
)
# 扫描结果数据
url = models.CharField(max_length=2000, help_text='站点URL')
url = models.TextField(help_text='站点URL')
host = models.CharField(max_length=253, blank=True, default='', help_text='主机名域名或IP地址')
title = models.CharField(max_length=500, blank=True, default='', help_text='页面标题')
status = models.IntegerField(null=True, blank=True, help_text='HTTP状态码')
title = models.TextField(blank=True, default='', help_text='页面标题')
status_code = models.IntegerField(null=True, blank=True, help_text='HTTP状态码')
content_length = models.BigIntegerField(null=True, blank=True, help_text='内容长度')
location = models.CharField(max_length=1000, blank=True, default='', help_text='重定向位置')
web_server = models.CharField(max_length=200, blank=True, default='', help_text='Web服务器')
content_type = models.CharField(max_length=200, blank=True, default='', help_text='内容类型')
location = models.TextField(blank=True, default='', help_text='重定向位置')
webserver = models.TextField(blank=True, default='', help_text='Web服务器')
content_type = models.TextField(blank=True, default='', help_text='内容类型')
tech = ArrayField(
models.CharField(max_length=100),
blank=True,
@@ -267,19 +267,19 @@ class EndpointSnapshot(models.Model):
)
# 扫描结果数据
url = models.CharField(max_length=2000, help_text='端点URL')
url = models.TextField(help_text='端点URL')
host = models.CharField(
max_length=253,
blank=True,
default='',
help_text='主机名域名或IP地址'
)
title = models.CharField(max_length=1000, blank=True, default='', help_text='页面标题')
title = models.TextField(blank=True, default='', help_text='页面标题')
status_code = models.IntegerField(null=True, blank=True, help_text='HTTP状态码')
content_length = models.IntegerField(null=True, blank=True, help_text='内容长度')
location = models.CharField(max_length=1000, blank=True, default='', help_text='重定向位置')
webserver = models.CharField(max_length=200, blank=True, default='', help_text='Web服务器')
content_type = models.CharField(max_length=200, blank=True, default='', help_text='内容类型')
location = models.TextField(blank=True, default='', help_text='重定向位置')
webserver = models.TextField(blank=True, default='', help_text='Web服务器')
content_type = models.TextField(blank=True, default='', help_text='内容类型')
tech = ArrayField(
models.CharField(max_length=100),
blank=True,

View File

@@ -46,10 +46,10 @@ class DjangoWebsiteSnapshotRepository:
url=item.url,
host=item.host,
title=item.title,
status=item.status,
status_code=item.status_code,
content_length=item.content_length,
location=item.location,
web_server=item.web_server,
webserver=item.webserver,
content_type=item.content_type,
tech=item.tech if item.tech else [],
response_body=item.response_body,
@@ -99,27 +99,12 @@ class DjangoWebsiteSnapshotRepository:
WebsiteSnapshot.objects
.filter(scan_id=scan_id)
.values(
'url', 'host', 'location', 'title', 'status',
'content_length', 'content_type', 'web_server', 'tech',
'url', 'host', 'location', 'title', 'status_code',
'content_length', 'content_type', 'webserver', 'tech',
'response_body', 'response_headers', 'vhost', 'created_at'
)
.order_by('url')
)
for row in qs.iterator(chunk_size=batch_size):
# 重命名字段以匹配 CSV 表头
yield {
'url': row['url'],
'host': row['host'],
'location': row['location'],
'title': row['title'],
'status_code': row['status'],
'content_length': row['content_length'],
'content_type': row['content_type'],
'webserver': row['web_server'],
'tech': row['tech'],
'response_body': row['response_body'],
'response_headers': row['response_headers'],
'vhost': row['vhost'],
'created_at': row['created_at'],
}
yield row

View File

@@ -217,8 +217,6 @@ class WebsiteSnapshotSerializer(serializers.ModelSerializer):
"""网站快照序列化器(用于扫描历史)"""
subdomain_name = serializers.CharField(source='subdomain.name', read_only=True)
webserver = serializers.CharField(source='web_server', read_only=True) # 映射字段名
status_code = serializers.IntegerField(source='status', read_only=True) # 映射字段名
responseHeaders = serializers.CharField(source='response_headers', read_only=True) # 原始HTTP响应头
class Meta:
@@ -228,9 +226,9 @@ class WebsiteSnapshotSerializer(serializers.ModelSerializer):
'url',
'location',
'title',
'webserver', # 使用映射后的字段名
'webserver',
'content_type',
'status_code', # 使用映射后的字段名
'status_code',
'content_length',
'response_body',
'tech',

View File

@@ -27,7 +27,7 @@ class EndpointService:
'url': 'url',
'host': 'host',
'title': 'title',
'status': 'status_code',
'status_code': 'status_code',
'tech': 'tech',
}

View File

@@ -19,7 +19,7 @@ class WebSiteService:
'url': 'url',
'host': 'host',
'title': 'title',
'status': 'status_code',
'status_code': 'status_code',
'tech': 'tech',
}

View File

@@ -0,0 +1,460 @@
"""
资产搜索服务
提供资产搜索的核心业务逻辑:
- 从物化视图查询数据
- 支持表达式语法解析
- 支持 =(模糊)、==(精确)、!=(不等于)操作符
- 支持 && (AND) 和 || (OR) 逻辑组合
- 支持 Website 和 Endpoint 两种资产类型
"""
import logging
import re
from typing import Optional, List, Dict, Any, Tuple, Literal, Iterator
from django.db import connection
logger = logging.getLogger(__name__)
# 支持的字段映射(前端字段名 -> 数据库字段名)
FIELD_MAPPING = {
'host': 'host',
'url': 'url',
'title': 'title',
'tech': 'tech',
'status': 'status_code',
'body': 'response_body',
'header': 'response_headers',
}
# 数组类型字段
ARRAY_FIELDS = {'tech'}
# 资产类型到视图名的映射
VIEW_MAPPING = {
'website': 'asset_search_view',
'endpoint': 'endpoint_search_view',
}
# 有效的资产类型
VALID_ASSET_TYPES = {'website', 'endpoint'}
# Website 查询字段
WEBSITE_SELECT_FIELDS = """
id,
url,
host,
title,
tech,
status_code,
response_headers,
response_body,
content_type,
content_length,
webserver,
location,
vhost,
created_at,
target_id
"""
# Endpoint 查询字段(包含 matched_gf_patterns
ENDPOINT_SELECT_FIELDS = """
id,
url,
host,
title,
tech,
status_code,
response_headers,
response_body,
content_type,
content_length,
webserver,
location,
vhost,
matched_gf_patterns,
created_at,
target_id
"""
class SearchQueryParser:
"""
搜索查询解析器
支持语法:
- field="value" 模糊匹配ILIKE %value%
- field=="value" 精确匹配
- field!="value" 不等于
- && AND 连接
- || OR 连接
- () 分组(暂不支持嵌套)
示例:
- host="api" && tech="nginx"
- tech="vue" || tech="react"
- status=="200" && host!="test"
"""
# 匹配单个条件: field="value" 或 field=="value" 或 field!="value"
CONDITION_PATTERN = re.compile(r'(\w+)\s*(==|!=|=)\s*"([^"]*)"')
@classmethod
def parse(cls, query: str) -> Tuple[str, List[Any]]:
"""
解析查询字符串,返回 SQL WHERE 子句和参数
Args:
query: 搜索查询字符串
Returns:
(where_clause, params) 元组
"""
if not query or not query.strip():
return "1=1", []
query = query.strip()
# 检查是否包含操作符语法,如果不包含则作为 host 模糊搜索
if not cls.CONDITION_PATTERN.search(query):
# 裸文本,默认作为 host 模糊搜索
return "host ILIKE %s", [f"%{query}%"]
# 按 || 分割为 OR 组
or_groups = cls._split_by_or(query)
if len(or_groups) == 1:
# 没有 OR直接解析 AND 条件
return cls._parse_and_group(or_groups[0])
# 多个 OR 组
or_clauses = []
all_params = []
for group in or_groups:
clause, params = cls._parse_and_group(group)
if clause and clause != "1=1":
or_clauses.append(f"({clause})")
all_params.extend(params)
if not or_clauses:
return "1=1", []
return " OR ".join(or_clauses), all_params
@classmethod
def _split_by_or(cls, query: str) -> List[str]:
"""按 || 分割查询,但忽略引号内的 ||"""
parts = []
current = ""
in_quotes = False
i = 0
while i < len(query):
char = query[i]
if char == '"':
in_quotes = not in_quotes
current += char
elif not in_quotes and i + 1 < len(query) and query[i:i+2] == '||':
if current.strip():
parts.append(current.strip())
current = ""
i += 1 # 跳过第二个 |
else:
current += char
i += 1
if current.strip():
parts.append(current.strip())
return parts if parts else [query]
@classmethod
def _parse_and_group(cls, group: str) -> Tuple[str, List[Any]]:
"""解析 AND 组(用 && 连接的条件)"""
# 移除外层括号
group = group.strip()
if group.startswith('(') and group.endswith(')'):
group = group[1:-1].strip()
# 按 && 分割
parts = cls._split_by_and(group)
and_clauses = []
all_params = []
for part in parts:
clause, params = cls._parse_condition(part.strip())
if clause:
and_clauses.append(clause)
all_params.extend(params)
if not and_clauses:
return "1=1", []
return " AND ".join(and_clauses), all_params
@classmethod
def _split_by_and(cls, query: str) -> List[str]:
"""按 && 分割查询,但忽略引号内的 &&"""
parts = []
current = ""
in_quotes = False
i = 0
while i < len(query):
char = query[i]
if char == '"':
in_quotes = not in_quotes
current += char
elif not in_quotes and i + 1 < len(query) and query[i:i+2] == '&&':
if current.strip():
parts.append(current.strip())
current = ""
i += 1 # 跳过第二个 &
else:
current += char
i += 1
if current.strip():
parts.append(current.strip())
return parts if parts else [query]
@classmethod
def _parse_condition(cls, condition: str) -> Tuple[Optional[str], List[Any]]:
"""
解析单个条件
Returns:
(sql_clause, params) 或 (None, []) 如果解析失败
"""
# 移除括号
condition = condition.strip()
if condition.startswith('(') and condition.endswith(')'):
condition = condition[1:-1].strip()
match = cls.CONDITION_PATTERN.match(condition)
if not match:
logger.warning(f"无法解析条件: {condition}")
return None, []
field, operator, value = match.groups()
field = field.lower()
# 验证字段
if field not in FIELD_MAPPING:
logger.warning(f"未知字段: {field}")
return None, []
db_field = FIELD_MAPPING[field]
is_array = field in ARRAY_FIELDS
# 根据操作符生成 SQL
if operator == '=':
# 模糊匹配
return cls._build_like_condition(db_field, value, is_array)
elif operator == '==':
# 精确匹配
return cls._build_exact_condition(db_field, value, is_array)
elif operator == '!=':
# 不等于
return cls._build_not_equal_condition(db_field, value, is_array)
return None, []
@classmethod
def _build_like_condition(cls, field: str, value: str, is_array: bool) -> Tuple[str, List[Any]]:
"""构建模糊匹配条件"""
if is_array:
# 数组字段:检查数组中是否有元素包含该值
return f"EXISTS (SELECT 1 FROM unnest({field}) AS t WHERE t ILIKE %s)", [f"%{value}%"]
elif field == 'status_code':
# 状态码是整数,模糊匹配转为精确匹配
try:
return f"{field} = %s", [int(value)]
except ValueError:
return f"{field}::text ILIKE %s", [f"%{value}%"]
else:
return f"{field} ILIKE %s", [f"%{value}%"]
@classmethod
def _build_exact_condition(cls, field: str, value: str, is_array: bool) -> Tuple[str, List[Any]]:
"""构建精确匹配条件"""
if is_array:
# 数组字段:检查数组中是否包含该精确值
return f"%s = ANY({field})", [value]
elif field == 'status_code':
# 状态码是整数
try:
return f"{field} = %s", [int(value)]
except ValueError:
return f"{field}::text = %s", [value]
else:
return f"{field} = %s", [value]
@classmethod
def _build_not_equal_condition(cls, field: str, value: str, is_array: bool) -> Tuple[str, List[Any]]:
"""构建不等于条件"""
if is_array:
# 数组字段:检查数组中不包含该值
return f"NOT (%s = ANY({field}))", [value]
elif field == 'status_code':
try:
return f"({field} IS NULL OR {field} != %s)", [int(value)]
except ValueError:
return f"({field} IS NULL OR {field}::text != %s)", [value]
else:
return f"({field} IS NULL OR {field} != %s)", [value]
AssetType = Literal['website', 'endpoint']
class AssetSearchService:
"""资产搜索服务"""
def search(
self,
query: str,
asset_type: AssetType = 'website',
limit: Optional[int] = None
) -> List[Dict[str, Any]]:
"""
搜索资产
Args:
query: 搜索查询字符串
asset_type: 资产类型 ('website''endpoint')
limit: 最大返回数量(可选)
Returns:
List[Dict]: 搜索结果列表
"""
where_clause, params = SearchQueryParser.parse(query)
# 根据资产类型选择视图和字段
view_name = VIEW_MAPPING.get(asset_type, 'asset_search_view')
select_fields = ENDPOINT_SELECT_FIELDS if asset_type == 'endpoint' else WEBSITE_SELECT_FIELDS
sql = f"""
SELECT {select_fields}
FROM {view_name}
WHERE {where_clause}
ORDER BY created_at DESC
"""
# 添加 LIMIT
if limit is not None and limit > 0:
sql += f" LIMIT {int(limit)}"
try:
with connection.cursor() as cursor:
cursor.execute(sql, params)
columns = [col[0] for col in cursor.description]
results = []
for row in cursor.fetchall():
result = dict(zip(columns, row))
results.append(result)
return results
except Exception as e:
logger.error(f"搜索查询失败: {e}, SQL: {sql}, params: {params}")
raise
def count(self, query: str, asset_type: AssetType = 'website', statement_timeout_ms: int = 300000) -> int:
"""
统计搜索结果数量
Args:
query: 搜索查询字符串
asset_type: 资产类型 ('website''endpoint')
statement_timeout_ms: SQL 语句超时时间(毫秒),默认 5 分钟
Returns:
int: 结果总数
"""
where_clause, params = SearchQueryParser.parse(query)
# 根据资产类型选择视图
view_name = VIEW_MAPPING.get(asset_type, 'asset_search_view')
sql = f"SELECT COUNT(*) FROM {view_name} WHERE {where_clause}"
try:
with connection.cursor() as cursor:
# 为导出设置更长的超时时间(仅影响当前会话)
cursor.execute(f"SET LOCAL statement_timeout = {statement_timeout_ms}")
cursor.execute(sql, params)
return cursor.fetchone()[0]
except Exception as e:
logger.error(f"统计查询失败: {e}")
raise
def search_iter(
self,
query: str,
asset_type: AssetType = 'website',
batch_size: int = 1000,
statement_timeout_ms: int = 300000
) -> Iterator[Dict[str, Any]]:
"""
流式搜索资产(使用分批查询,内存友好)
Args:
query: 搜索查询字符串
asset_type: 资产类型 ('website''endpoint')
batch_size: 每批获取的数量
statement_timeout_ms: SQL 语句超时时间(毫秒),默认 5 分钟
Yields:
Dict: 单条搜索结果
"""
where_clause, params = SearchQueryParser.parse(query)
# 根据资产类型选择视图和字段
view_name = VIEW_MAPPING.get(asset_type, 'asset_search_view')
select_fields = ENDPOINT_SELECT_FIELDS if asset_type == 'endpoint' else WEBSITE_SELECT_FIELDS
# 使用 OFFSET/LIMIT 分批查询Django 不支持命名游标)
offset = 0
try:
while True:
sql = f"""
SELECT {select_fields}
FROM {view_name}
WHERE {where_clause}
ORDER BY created_at DESC
LIMIT {batch_size} OFFSET {offset}
"""
with connection.cursor() as cursor:
# 为导出设置更长的超时时间(仅影响当前会话)
cursor.execute(f"SET LOCAL statement_timeout = {statement_timeout_ms}")
cursor.execute(sql, params)
columns = [col[0] for col in cursor.description]
rows = cursor.fetchall()
if not rows:
break
for row in rows:
yield dict(zip(columns, row))
# 如果返回的行数少于 batch_size说明已经是最后一批
if len(rows) < batch_size:
break
offset += batch_size
except Exception as e:
logger.error(f"流式搜索查询失败: {e}, SQL: {sql}, params: {params}")
raise

View File

@@ -72,7 +72,7 @@ class EndpointSnapshotsService:
'url': 'url',
'host': 'host',
'title': 'title',
'status': 'status_code',
'status_code': 'status_code',
'webserver': 'webserver',
'tech': 'tech',
}

View File

@@ -73,8 +73,8 @@ class WebsiteSnapshotsService:
'url': 'url',
'host': 'host',
'title': 'title',
'status': 'status',
'webserver': 'web_server',
'status_code': 'status_code',
'webserver': 'webserver',
'tech': 'tech',
}

View File

@@ -10,6 +10,8 @@ from .views import (
DirectoryViewSet,
VulnerabilityViewSet,
AssetStatisticsViewSet,
AssetSearchView,
AssetSearchExportView,
)
# 创建 DRF 路由器
@@ -25,4 +27,6 @@ router.register(r'statistics', AssetStatisticsViewSet, basename='asset-statistic
urlpatterns = [
path('assets/', include(router.urls)),
path('assets/search/', AssetSearchView.as_view(), name='asset-search'),
path('assets/search/export/', AssetSearchExportView.as_view(), name='asset-search-export'),
]

View File

@@ -0,0 +1,40 @@
"""
Asset 应用视图模块
重新导出所有视图类以保持向后兼容
"""
from .asset_views import (
AssetStatisticsViewSet,
SubdomainViewSet,
WebSiteViewSet,
DirectoryViewSet,
EndpointViewSet,
HostPortMappingViewSet,
VulnerabilityViewSet,
SubdomainSnapshotViewSet,
WebsiteSnapshotViewSet,
DirectorySnapshotViewSet,
EndpointSnapshotViewSet,
HostPortMappingSnapshotViewSet,
VulnerabilitySnapshotViewSet,
)
from .search_views import AssetSearchView, AssetSearchExportView
__all__ = [
'AssetStatisticsViewSet',
'SubdomainViewSet',
'WebSiteViewSet',
'DirectoryViewSet',
'EndpointViewSet',
'HostPortMappingViewSet',
'VulnerabilityViewSet',
'SubdomainSnapshotViewSet',
'WebsiteSnapshotViewSet',
'DirectorySnapshotViewSet',
'EndpointSnapshotViewSet',
'HostPortMappingSnapshotViewSet',
'VulnerabilitySnapshotViewSet',
'AssetSearchView',
'AssetSearchExportView',
]

View File

@@ -8,19 +8,18 @@ from rest_framework.request import Request
from rest_framework.exceptions import NotFound, ValidationError as DRFValidationError
from django.core.exceptions import ValidationError, ObjectDoesNotExist
from django.db import DatabaseError, IntegrityError, OperationalError
from django.http import StreamingHttpResponse
from .serializers import (
from ..serializers import (
SubdomainListSerializer, WebSiteSerializer, DirectorySerializer,
VulnerabilitySerializer, EndpointListSerializer, IPAddressAggregatedSerializer,
SubdomainSnapshotSerializer, WebsiteSnapshotSerializer, DirectorySnapshotSerializer,
EndpointSnapshotSerializer, VulnerabilitySnapshotSerializer
)
from .services import (
from ..services import (
SubdomainService, WebSiteService, DirectoryService,
VulnerabilityService, AssetStatisticsService, EndpointService, HostPortMappingService
)
from .services.snapshot import (
from ..services.snapshot import (
SubdomainSnapshotsService, WebsiteSnapshotsService, DirectorySnapshotsService,
EndpointSnapshotsService, HostPortMappingSnapshotsService, VulnerabilitySnapshotsService
)
@@ -243,7 +242,7 @@ class SubdomainViewSet(viewsets.ModelViewSet):
CSV name, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime
from apps.common.utils import create_csv_export_response, format_datetime
target_pk = self.kwargs.get('target_pk')
if not target_pk:
@@ -254,12 +253,12 @@ class SubdomainViewSet(viewsets.ModelViewSet):
headers = ['name', 'created_at']
formatters = {'created_at': format_datetime}
response = StreamingHttpResponse(
generate_csv_rows(data_iterator, headers, formatters),
content_type='text/csv; charset=utf-8'
return create_csv_export_response(
data_iterator=data_iterator,
headers=headers,
filename=f"target-{target_pk}-subdomains.csv",
field_formatters=formatters
)
response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-subdomains.csv"'
return response
class WebSiteViewSet(viewsets.ModelViewSet):
@@ -369,7 +368,7 @@ class WebSiteViewSet(viewsets.ModelViewSet):
CSV url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
target_pk = self.kwargs.get('target_pk')
if not target_pk:
@@ -387,12 +386,12 @@ class WebSiteViewSet(viewsets.ModelViewSet):
'tech': lambda x: format_list_field(x, separator=','),
}
response = StreamingHttpResponse(
generate_csv_rows(data_iterator, headers, formatters),
content_type='text/csv; charset=utf-8'
return create_csv_export_response(
data_iterator=data_iterator,
headers=headers,
filename=f"target-{target_pk}-websites.csv",
field_formatters=formatters
)
response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-websites.csv"'
return response
class DirectoryViewSet(viewsets.ModelViewSet):
@@ -499,7 +498,7 @@ class DirectoryViewSet(viewsets.ModelViewSet):
CSV url, status, content_length, words, lines, content_type, duration, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime
from apps.common.utils import create_csv_export_response, format_datetime
target_pk = self.kwargs.get('target_pk')
if not target_pk:
@@ -515,12 +514,12 @@ class DirectoryViewSet(viewsets.ModelViewSet):
'created_at': format_datetime,
}
response = StreamingHttpResponse(
generate_csv_rows(data_iterator, headers, formatters),
content_type='text/csv; charset=utf-8'
return create_csv_export_response(
data_iterator=data_iterator,
headers=headers,
filename=f"target-{target_pk}-directories.csv",
field_formatters=formatters
)
response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-directories.csv"'
return response
class EndpointViewSet(viewsets.ModelViewSet):
@@ -630,7 +629,7 @@ class EndpointViewSet(viewsets.ModelViewSet):
CSV url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, matched_gf_patterns, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
target_pk = self.kwargs.get('target_pk')
if not target_pk:
@@ -649,12 +648,12 @@ class EndpointViewSet(viewsets.ModelViewSet):
'matched_gf_patterns': lambda x: format_list_field(x, separator=','),
}
response = StreamingHttpResponse(
generate_csv_rows(data_iterator, headers, formatters),
content_type='text/csv; charset=utf-8'
return create_csv_export_response(
data_iterator=data_iterator,
headers=headers,
filename=f"target-{target_pk}-endpoints.csv",
field_formatters=formatters
)
response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-endpoints.csv"'
return response
class HostPortMappingViewSet(viewsets.ModelViewSet):
@@ -707,7 +706,7 @@ class HostPortMappingViewSet(viewsets.ModelViewSet):
CSV ip, host, port, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime
from apps.common.utils import create_csv_export_response, format_datetime
target_pk = self.kwargs.get('target_pk')
if not target_pk:
@@ -722,14 +721,12 @@ class HostPortMappingViewSet(viewsets.ModelViewSet):
'created_at': format_datetime
}
# 生成流式响应
response = StreamingHttpResponse(
generate_csv_rows(data_iterator, headers, formatters),
content_type='text/csv; charset=utf-8'
return create_csv_export_response(
data_iterator=data_iterator,
headers=headers,
filename=f"target-{target_pk}-ip-addresses.csv",
field_formatters=formatters
)
response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-ip-addresses.csv"'
return response
class VulnerabilityViewSet(viewsets.ModelViewSet):
@@ -801,7 +798,7 @@ class SubdomainSnapshotViewSet(viewsets.ModelViewSet):
CSV name, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime
from apps.common.utils import create_csv_export_response, format_datetime
scan_pk = self.kwargs.get('scan_pk')
if not scan_pk:
@@ -812,12 +809,12 @@ class SubdomainSnapshotViewSet(viewsets.ModelViewSet):
headers = ['name', 'created_at']
formatters = {'created_at': format_datetime}
response = StreamingHttpResponse(
generate_csv_rows(data_iterator, headers, formatters),
content_type='text/csv; charset=utf-8'
return create_csv_export_response(
data_iterator=data_iterator,
headers=headers,
filename=f"scan-{scan_pk}-subdomains.csv",
field_formatters=formatters
)
response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-subdomains.csv"'
return response
class WebsiteSnapshotViewSet(viewsets.ModelViewSet):
@@ -855,7 +852,7 @@ class WebsiteSnapshotViewSet(viewsets.ModelViewSet):
CSV url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
scan_pk = self.kwargs.get('scan_pk')
if not scan_pk:
@@ -873,12 +870,12 @@ class WebsiteSnapshotViewSet(viewsets.ModelViewSet):
'tech': lambda x: format_list_field(x, separator=','),
}
response = StreamingHttpResponse(
generate_csv_rows(data_iterator, headers, formatters),
content_type='text/csv; charset=utf-8'
return create_csv_export_response(
data_iterator=data_iterator,
headers=headers,
filename=f"scan-{scan_pk}-websites.csv",
field_formatters=formatters
)
response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-websites.csv"'
return response
class DirectorySnapshotViewSet(viewsets.ModelViewSet):
@@ -913,7 +910,7 @@ class DirectorySnapshotViewSet(viewsets.ModelViewSet):
CSV url, status, content_length, words, lines, content_type, duration, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime
from apps.common.utils import create_csv_export_response, format_datetime
scan_pk = self.kwargs.get('scan_pk')
if not scan_pk:
@@ -929,12 +926,12 @@ class DirectorySnapshotViewSet(viewsets.ModelViewSet):
'created_at': format_datetime,
}
response = StreamingHttpResponse(
generate_csv_rows(data_iterator, headers, formatters),
content_type='text/csv; charset=utf-8'
return create_csv_export_response(
data_iterator=data_iterator,
headers=headers,
filename=f"scan-{scan_pk}-directories.csv",
field_formatters=formatters
)
response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-directories.csv"'
return response
class EndpointSnapshotViewSet(viewsets.ModelViewSet):
@@ -972,7 +969,7 @@ class EndpointSnapshotViewSet(viewsets.ModelViewSet):
CSV url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, matched_gf_patterns, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
scan_pk = self.kwargs.get('scan_pk')
if not scan_pk:
@@ -991,12 +988,12 @@ class EndpointSnapshotViewSet(viewsets.ModelViewSet):
'matched_gf_patterns': lambda x: format_list_field(x, separator=','),
}
response = StreamingHttpResponse(
generate_csv_rows(data_iterator, headers, formatters),
content_type='text/csv; charset=utf-8'
return create_csv_export_response(
data_iterator=data_iterator,
headers=headers,
filename=f"scan-{scan_pk}-endpoints.csv",
field_formatters=formatters
)
response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-endpoints.csv"'
return response
class HostPortMappingSnapshotViewSet(viewsets.ModelViewSet):
@@ -1031,7 +1028,7 @@ class HostPortMappingSnapshotViewSet(viewsets.ModelViewSet):
CSV ip, host, port, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime
from apps.common.utils import create_csv_export_response, format_datetime
scan_pk = self.kwargs.get('scan_pk')
if not scan_pk:
@@ -1046,14 +1043,12 @@ class HostPortMappingSnapshotViewSet(viewsets.ModelViewSet):
'created_at': format_datetime
}
# 生成流式响应
response = StreamingHttpResponse(
generate_csv_rows(data_iterator, headers, formatters),
content_type='text/csv; charset=utf-8'
return create_csv_export_response(
data_iterator=data_iterator,
headers=headers,
filename=f"scan-{scan_pk}-ip-addresses.csv",
field_formatters=formatters
)
response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-ip-addresses.csv"'
return response
class VulnerabilitySnapshotViewSet(viewsets.ModelViewSet):

View File

@@ -0,0 +1,361 @@
"""
资产搜索 API 视图
提供资产搜索的 REST API 接口:
- GET /api/assets/search/ - 搜索资产
- GET /api/assets/search/export/ - 导出搜索结果为 CSV
搜索语法:
- field="value" 模糊匹配ILIKE %value%
- field=="value" 精确匹配
- field!="value" 不等于
- && AND 连接
- || OR 连接
支持的字段:
- host: 主机名
- url: URL
- title: 标题
- tech: 技术栈
- status: 状态码
- body: 响应体
- header: 响应头
支持的资产类型:
- website: 站点(默认)
- endpoint: 端点
"""
import logging
import json
from datetime import datetime
from urllib.parse import urlparse, urlunparse
from rest_framework import status
from rest_framework.views import APIView
from rest_framework.request import Request
from django.db import connection
from apps.common.response_helpers import success_response, error_response
from apps.common.error_codes import ErrorCodes
from apps.asset.services.search_service import AssetSearchService, VALID_ASSET_TYPES
logger = logging.getLogger(__name__)
class AssetSearchView(APIView):
"""
资产搜索 API
GET /api/assets/search/
Query Parameters:
q: 搜索查询表达式
asset_type: 资产类型 ('website''endpoint',默认 'website')
page: 页码(从 1 开始,默认 1
pageSize: 每页数量(默认 10最大 100
示例查询:
?q=host="api" && tech="nginx"
?q=tech="vue" || tech="react"&asset_type=endpoint
?q=status=="200" && host!="test"
Response:
{
"results": [...],
"total": 100,
"page": 1,
"pageSize": 10,
"totalPages": 10,
"assetType": "website"
}
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.service = AssetSearchService()
def _parse_headers(self, headers_data) -> dict:
"""解析响应头为字典"""
if not headers_data:
return {}
try:
return json.loads(headers_data)
except (json.JSONDecodeError, TypeError):
result = {}
for line in str(headers_data).split('\n'):
if ':' in line:
key, value = line.split(':', 1)
result[key.strip()] = value.strip()
return result
def _format_result(self, result: dict, vulnerabilities_by_url: dict, asset_type: str) -> dict:
"""格式化单个搜索结果"""
url = result.get('url', '')
vulns = vulnerabilities_by_url.get(url, [])
# 基础字段Website 和 Endpoint 共有)
formatted = {
'id': result.get('id'),
'url': url,
'host': result.get('host', ''),
'title': result.get('title', ''),
'technologies': result.get('tech', []) or [],
'statusCode': result.get('status_code'),
'contentLength': result.get('content_length'),
'contentType': result.get('content_type', ''),
'webserver': result.get('webserver', ''),
'location': result.get('location', ''),
'vhost': result.get('vhost'),
'responseHeaders': self._parse_headers(result.get('response_headers')),
'responseBody': result.get('response_body', ''),
'createdAt': result.get('created_at').isoformat() if result.get('created_at') else None,
'targetId': result.get('target_id'),
}
# Website 特有字段:漏洞关联
if asset_type == 'website':
formatted['vulnerabilities'] = [
{
'id': v.get('id'),
'name': v.get('vuln_type', ''),
'vulnType': v.get('vuln_type', ''),
'severity': v.get('severity', 'info'),
}
for v in vulns
]
# Endpoint 特有字段
if asset_type == 'endpoint':
formatted['matchedGfPatterns'] = result.get('matched_gf_patterns', []) or []
return formatted
def _get_vulnerabilities_by_url_prefix(self, website_urls: list) -> dict:
"""
根据 URL 前缀批量查询漏洞数据
漏洞 URL 是 website URL 的子路径,使用前缀匹配:
- website.url: https://example.com/path?query=1
- vulnerability.url: https://example.com/path/api/users
Args:
website_urls: website URL 列表,格式为 [(url, target_id), ...]
Returns:
dict: {website_url: [vulnerability_list]}
"""
if not website_urls:
return {}
try:
with connection.cursor() as cursor:
# 构建 OR 条件:每个 website URL去掉查询参数作为前缀匹配
conditions = []
params = []
url_mapping = {} # base_url -> original_url
for url, target_id in website_urls:
if not url or target_id is None:
continue
# 使用 urlparse 去掉查询参数和片段,只保留 scheme://netloc/path
parsed = urlparse(url)
base_url = urlunparse((parsed.scheme, parsed.netloc, parsed.path, '', '', ''))
url_mapping[base_url] = url
conditions.append("(v.url LIKE %s AND v.target_id = %s)")
params.extend([base_url + '%', target_id])
if not conditions:
return {}
where_clause = " OR ".join(conditions)
sql = f"""
SELECT v.id, v.vuln_type, v.severity, v.url, v.target_id
FROM vulnerability v
WHERE {where_clause}
ORDER BY
CASE v.severity
WHEN 'critical' THEN 1
WHEN 'high' THEN 2
WHEN 'medium' THEN 3
WHEN 'low' THEN 4
ELSE 5
END
"""
cursor.execute(sql, params)
# 获取所有漏洞
all_vulns = []
for row in cursor.fetchall():
all_vulns.append({
'id': row[0],
'vuln_type': row[1],
'name': row[1],
'severity': row[2],
'url': row[3],
'target_id': row[4],
})
# 按原始 website URL 分组(用于返回结果)
result = {url: [] for url, _ in website_urls}
for vuln in all_vulns:
vuln_url = vuln['url']
# 找到匹配的 website URL最长前缀匹配
for website_url, target_id in website_urls:
parsed = urlparse(website_url)
base_url = urlunparse((parsed.scheme, parsed.netloc, parsed.path, '', '', ''))
if vuln_url.startswith(base_url) and vuln['target_id'] == target_id:
result[website_url].append(vuln)
break
return result
except Exception as e:
logger.error(f"批量查询漏洞失败: {e}")
return {}
def get(self, request: Request):
"""搜索资产"""
# 获取搜索查询
query = request.query_params.get('q', '').strip()
if not query:
return error_response(
code=ErrorCodes.VALIDATION_ERROR,
message='Search query (q) is required',
status_code=status.HTTP_400_BAD_REQUEST
)
# 获取并验证资产类型
asset_type = request.query_params.get('asset_type', 'website').strip().lower()
if asset_type not in VALID_ASSET_TYPES:
return error_response(
code=ErrorCodes.VALIDATION_ERROR,
message=f'Invalid asset_type. Must be one of: {", ".join(VALID_ASSET_TYPES)}',
status_code=status.HTTP_400_BAD_REQUEST
)
# 获取分页参数
try:
page = int(request.query_params.get('page', 1))
page_size = int(request.query_params.get('pageSize', 10))
except (ValueError, TypeError):
page = 1
page_size = 10
# 限制分页参数
page = max(1, page)
page_size = min(max(1, page_size), 100)
# 获取总数和搜索结果
total = self.service.count(query, asset_type)
total_pages = (total + page_size - 1) // page_size if total > 0 else 1
offset = (page - 1) * page_size
all_results = self.service.search(query, asset_type)
results = all_results[offset:offset + page_size]
# 批量查询漏洞数据(仅 Website 类型需要)
vulnerabilities_by_url = {}
if asset_type == 'website':
website_urls = [(r.get('url'), r.get('target_id')) for r in results if r.get('url') and r.get('target_id')]
vulnerabilities_by_url = self._get_vulnerabilities_by_url_prefix(website_urls) if website_urls else {}
# 格式化结果
formatted_results = [self._format_result(r, vulnerabilities_by_url, asset_type) for r in results]
return success_response(data={
'results': formatted_results,
'total': total,
'page': page,
'pageSize': page_size,
'totalPages': total_pages,
'assetType': asset_type,
})
class AssetSearchExportView(APIView):
"""
资产搜索导出 API
GET /api/assets/search/export/
Query Parameters:
q: 搜索查询表达式
asset_type: 资产类型 ('website''endpoint',默认 'website')
Response:
CSV 文件(带 Content-Length支持浏览器显示下载进度
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.service = AssetSearchService()
def _get_headers_and_formatters(self, asset_type: str):
"""获取 CSV 表头和格式化器"""
from apps.common.utils import format_datetime, format_list_field
if asset_type == 'website':
headers = ['url', 'host', 'title', 'status_code', 'content_type', 'content_length',
'webserver', 'location', 'tech', 'vhost', 'created_at']
else:
headers = ['url', 'host', 'title', 'status_code', 'content_type', 'content_length',
'webserver', 'location', 'tech', 'matched_gf_patterns', 'vhost', 'created_at']
formatters = {
'created_at': format_datetime,
'tech': lambda x: format_list_field(x, separator='; '),
'matched_gf_patterns': lambda x: format_list_field(x, separator='; '),
'vhost': lambda x: 'true' if x else ('false' if x is False else ''),
}
return headers, formatters
def get(self, request: Request):
"""导出搜索结果为 CSV带 Content-Length支持下载进度显示"""
from apps.common.utils import create_csv_export_response
# 获取搜索查询
query = request.query_params.get('q', '').strip()
if not query:
return error_response(
code=ErrorCodes.VALIDATION_ERROR,
message='Search query (q) is required',
status_code=status.HTTP_400_BAD_REQUEST
)
# 获取并验证资产类型
asset_type = request.query_params.get('asset_type', 'website').strip().lower()
if asset_type not in VALID_ASSET_TYPES:
return error_response(
code=ErrorCodes.VALIDATION_ERROR,
message=f'Invalid asset_type. Must be one of: {", ".join(VALID_ASSET_TYPES)}',
status_code=status.HTTP_400_BAD_REQUEST
)
# 检查是否有结果(快速检查,避免空导出)
total = self.service.count(query, asset_type)
if total == 0:
return error_response(
code=ErrorCodes.NOT_FOUND,
message='No results to export',
status_code=status.HTTP_404_NOT_FOUND
)
# 获取表头和格式化器
headers, formatters = self._get_headers_and_formatters(asset_type)
# 生成文件名
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
filename = f'search_{asset_type}_{timestamp}.csv'
# 使用通用导出工具
data_iterator = self.service.search_iter(query, asset_type)
return create_csv_export_response(
data_iterator=data_iterator,
headers=headers,
filename=filename,
field_formatters=formatters,
show_progress=True # 显示下载进度
)

View File

@@ -11,6 +11,7 @@ from .csv_utils import (
generate_csv_rows,
format_list_field,
format_datetime,
create_csv_export_response,
UTF8_BOM,
)
@@ -24,5 +25,6 @@ __all__ = [
'generate_csv_rows',
'format_list_field',
'format_datetime',
'create_csv_export_response',
'UTF8_BOM',
]

View File

@@ -4,13 +4,21 @@
- UTF-8 BOMExcel 兼容)
- RFC 4180 规范转义
- 流式生成(内存友好)
- 带 Content-Length 的文件响应(支持浏览器下载进度显示)
"""
import csv
import io
import os
import tempfile
import logging
from datetime import datetime
from typing import Iterator, Dict, Any, List, Callable, Optional
from django.http import FileResponse, StreamingHttpResponse
logger = logging.getLogger(__name__)
# UTF-8 BOM确保 Excel 正确识别编码
UTF8_BOM = '\ufeff'
@@ -114,3 +122,123 @@ def format_datetime(dt: Optional[datetime]) -> str:
dt = timezone.localtime(dt)
return dt.strftime('%Y-%m-%d %H:%M:%S')
def create_csv_export_response(
data_iterator: Iterator[Dict[str, Any]],
headers: List[str],
filename: str,
field_formatters: Optional[Dict[str, Callable]] = None,
show_progress: bool = True
) -> FileResponse | StreamingHttpResponse:
"""
创建 CSV 导出响应
根据 show_progress 参数选择响应类型:
- True: 使用临时文件 + FileResponse带 Content-Length浏览器显示下载进度
- False: 使用 StreamingHttpResponse内存更友好但无下载进度
Args:
data_iterator: 数据迭代器,每个元素是一个字典
headers: CSV 表头列表
filename: 下载文件名(如 "export_2024.csv"
field_formatters: 字段格式化函数字典
show_progress: 是否显示下载进度(默认 True
Returns:
FileResponse 或 StreamingHttpResponse
Example:
>>> data_iter = service.iter_data()
>>> headers = ['url', 'host', 'created_at']
>>> formatters = {'created_at': format_datetime}
>>> response = create_csv_export_response(
... data_iter, headers, 'websites.csv', formatters
... )
>>> return response
"""
if show_progress:
return _create_file_response(data_iterator, headers, filename, field_formatters)
else:
return _create_streaming_response(data_iterator, headers, filename, field_formatters)
def _create_file_response(
data_iterator: Iterator[Dict[str, Any]],
headers: List[str],
filename: str,
field_formatters: Optional[Dict[str, Callable]] = None
) -> FileResponse:
"""
创建带 Content-Length 的文件响应(支持浏览器下载进度)
实现方式:先写入临时文件,再返回 FileResponse
"""
# 创建临时文件
temp_file = tempfile.NamedTemporaryFile(
mode='w',
suffix='.csv',
delete=False,
encoding='utf-8'
)
temp_path = temp_file.name
try:
# 流式写入 CSV 数据到临时文件
for row in generate_csv_rows(data_iterator, headers, field_formatters):
temp_file.write(row)
temp_file.close()
# 获取文件大小
file_size = os.path.getsize(temp_path)
# 创建文件响应
response = FileResponse(
open(temp_path, 'rb'),
content_type='text/csv; charset=utf-8',
as_attachment=True,
filename=filename
)
response['Content-Length'] = file_size
# 设置清理回调:响应完成后删除临时文件
original_close = response.file_to_stream.close
def close_and_cleanup():
original_close()
try:
os.unlink(temp_path)
except OSError:
pass
response.file_to_stream.close = close_and_cleanup
return response
except Exception as e:
# 清理临时文件
try:
temp_file.close()
except:
pass
try:
os.unlink(temp_path)
except OSError:
pass
logger.error(f"创建 CSV 导出响应失败: {e}")
raise
def _create_streaming_response(
data_iterator: Iterator[Dict[str, Any]],
headers: List[str],
filename: str,
field_formatters: Optional[Dict[str, Callable]] = None
) -> StreamingHttpResponse:
"""
创建流式响应(无 Content-Length内存更友好
"""
response = StreamingHttpResponse(
generate_csv_rows(data_iterator, headers, field_formatters),
content_type='text/csv; charset=utf-8'
)
response['Content-Disposition'] = f'attachment; filename="{filename}"'
return response

View File

@@ -0,0 +1,213 @@
# Generated by Django 5.2.7 on 2026-01-02 04:45
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='NucleiTemplateRepo',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(help_text='仓库名称,用于前端展示和配置引用', max_length=200, unique=True)),
('repo_url', models.CharField(help_text='Git 仓库地址', max_length=500)),
('local_path', models.CharField(blank=True, default='', help_text='本地工作目录绝对路径', max_length=500)),
('commit_hash', models.CharField(blank=True, default='', help_text='最后同步的 Git commit hash用于 Worker 版本校验', max_length=40)),
('last_synced_at', models.DateTimeField(blank=True, help_text='最后一次成功同步时间', null=True)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('updated_at', models.DateTimeField(auto_now=True, help_text='更新时间')),
],
options={
'verbose_name': 'Nuclei 模板仓库',
'verbose_name_plural': 'Nuclei 模板仓库',
'db_table': 'nuclei_template_repo',
},
),
migrations.CreateModel(
name='ARLFingerprint',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(help_text='指纹名称', max_length=300, unique=True)),
('rule', models.TextField(help_text='匹配规则表达式')),
('created_at', models.DateTimeField(auto_now_add=True)),
],
options={
'verbose_name': 'ARL 指纹',
'verbose_name_plural': 'ARL 指纹',
'db_table': 'arl_fingerprint',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['name'], name='arl_fingerp_name_c3a305_idx'), models.Index(fields=['-created_at'], name='arl_fingerp_created_ed1060_idx')],
},
),
migrations.CreateModel(
name='EholeFingerprint',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('cms', models.CharField(help_text='产品/CMS名称', max_length=200)),
('method', models.CharField(default='keyword', help_text='匹配方式', max_length=200)),
('location', models.CharField(default='body', help_text='匹配位置', max_length=200)),
('keyword', models.JSONField(default=list, help_text='关键词列表')),
('is_important', models.BooleanField(default=False, help_text='是否重点资产')),
('type', models.CharField(blank=True, default='-', help_text='分类', max_length=100)),
('created_at', models.DateTimeField(auto_now_add=True)),
],
options={
'verbose_name': 'EHole 指纹',
'verbose_name_plural': 'EHole 指纹',
'db_table': 'ehole_fingerprint',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['cms'], name='ehole_finge_cms_72ca2c_idx'), models.Index(fields=['method'], name='ehole_finge_method_17f0db_idx'), models.Index(fields=['location'], name='ehole_finge_locatio_7bb82b_idx'), models.Index(fields=['type'], name='ehole_finge_type_ca2bce_idx'), models.Index(fields=['is_important'], name='ehole_finge_is_impo_d56e64_idx'), models.Index(fields=['-created_at'], name='ehole_finge_created_d862b0_idx')],
'constraints': [models.UniqueConstraint(fields=('cms', 'method', 'location'), name='unique_ehole_fingerprint')],
},
),
migrations.CreateModel(
name='FingerPrintHubFingerprint',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('fp_id', models.CharField(help_text='指纹ID', max_length=200, unique=True)),
('name', models.CharField(help_text='指纹名称', max_length=300)),
('author', models.CharField(blank=True, default='', help_text='作者', max_length=200)),
('tags', models.CharField(blank=True, default='', help_text='标签', max_length=500)),
('severity', models.CharField(blank=True, default='info', help_text='严重程度', max_length=50)),
('metadata', models.JSONField(blank=True, default=dict, help_text='元数据')),
('http', models.JSONField(default=list, help_text='HTTP 匹配规则')),
('source_file', models.CharField(blank=True, default='', help_text='来源文件', max_length=500)),
('created_at', models.DateTimeField(auto_now_add=True)),
],
options={
'verbose_name': 'FingerPrintHub 指纹',
'verbose_name_plural': 'FingerPrintHub 指纹',
'db_table': 'fingerprinthub_fingerprint',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['fp_id'], name='fingerprint_fp_id_df467f_idx'), models.Index(fields=['name'], name='fingerprint_name_95b6fb_idx'), models.Index(fields=['author'], name='fingerprint_author_80f54b_idx'), models.Index(fields=['severity'], name='fingerprint_severit_f70422_idx'), models.Index(fields=['-created_at'], name='fingerprint_created_bec16c_idx')],
},
),
migrations.CreateModel(
name='FingersFingerprint',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(help_text='指纹名称', max_length=300, unique=True)),
('link', models.URLField(blank=True, default='', help_text='相关链接', max_length=500)),
('rule', models.JSONField(default=list, help_text='匹配规则数组')),
('tag', models.JSONField(default=list, help_text='标签数组')),
('focus', models.BooleanField(default=False, help_text='是否重点关注')),
('default_port', models.JSONField(blank=True, default=list, help_text='默认端口数组')),
('created_at', models.DateTimeField(auto_now_add=True)),
],
options={
'verbose_name': 'Fingers 指纹',
'verbose_name_plural': 'Fingers 指纹',
'db_table': 'fingers_fingerprint',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['name'], name='fingers_fin_name_952de0_idx'), models.Index(fields=['link'], name='fingers_fin_link_4c6b7f_idx'), models.Index(fields=['focus'], name='fingers_fin_focus_568c7f_idx'), models.Index(fields=['-created_at'], name='fingers_fin_created_46fc91_idx')],
},
),
migrations.CreateModel(
name='GobyFingerprint',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(help_text='产品名称', max_length=300, unique=True)),
('logic', models.CharField(help_text='逻辑表达式', max_length=500)),
('rule', models.JSONField(default=list, help_text='规则数组')),
('created_at', models.DateTimeField(auto_now_add=True)),
],
options={
'verbose_name': 'Goby 指纹',
'verbose_name_plural': 'Goby 指纹',
'db_table': 'goby_fingerprint',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['name'], name='goby_finger_name_82084c_idx'), models.Index(fields=['logic'], name='goby_finger_logic_a63226_idx'), models.Index(fields=['-created_at'], name='goby_finger_created_50e000_idx')],
},
),
migrations.CreateModel(
name='ScanEngine',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('name', models.CharField(help_text='引擎名称', max_length=200, unique=True)),
('configuration', models.CharField(blank=True, default='', help_text='引擎配置yaml 格式', max_length=10000)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('updated_at', models.DateTimeField(auto_now=True, help_text='更新时间')),
],
options={
'verbose_name': '扫描引擎',
'verbose_name_plural': '扫描引擎',
'db_table': 'scan_engine',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['-created_at'], name='scan_engine_created_da4870_idx')],
},
),
migrations.CreateModel(
name='WappalyzerFingerprint',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(help_text='应用名称', max_length=300, unique=True)),
('cats', models.JSONField(default=list, help_text='分类 ID 数组')),
('cookies', models.JSONField(blank=True, default=dict, help_text='Cookie 检测规则')),
('headers', models.JSONField(blank=True, default=dict, help_text='HTTP Header 检测规则')),
('script_src', models.JSONField(blank=True, default=list, help_text='脚本 URL 正则数组')),
('js', models.JSONField(blank=True, default=list, help_text='JavaScript 变量检测规则')),
('implies', models.JSONField(blank=True, default=list, help_text='依赖关系数组')),
('meta', models.JSONField(blank=True, default=dict, help_text='HTML meta 标签检测规则')),
('html', models.JSONField(blank=True, default=list, help_text='HTML 内容正则数组')),
('description', models.TextField(blank=True, default='', help_text='应用描述')),
('website', models.URLField(blank=True, default='', help_text='官网链接', max_length=500)),
('cpe', models.CharField(blank=True, default='', help_text='CPE 标识符', max_length=300)),
('created_at', models.DateTimeField(auto_now_add=True)),
],
options={
'verbose_name': 'Wappalyzer 指纹',
'verbose_name_plural': 'Wappalyzer 指纹',
'db_table': 'wappalyzer_fingerprint',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['name'], name='wappalyzer__name_63c669_idx'), models.Index(fields=['website'], name='wappalyzer__website_88de1c_idx'), models.Index(fields=['cpe'], name='wappalyzer__cpe_30c761_idx'), models.Index(fields=['-created_at'], name='wappalyzer__created_8e6c21_idx')],
},
),
migrations.CreateModel(
name='Wordlist',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('name', models.CharField(help_text='字典名称,唯一', max_length=200, unique=True)),
('description', models.CharField(blank=True, default='', help_text='字典描述', max_length=200)),
('file_path', models.CharField(help_text='后端保存的字典文件绝对路径', max_length=500)),
('file_size', models.BigIntegerField(default=0, help_text='文件大小(字节)')),
('line_count', models.IntegerField(default=0, help_text='字典行数')),
('file_hash', models.CharField(blank=True, default='', help_text='文件 SHA-256 哈希,用于缓存校验', max_length=64)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('updated_at', models.DateTimeField(auto_now=True, help_text='更新时间')),
],
options={
'verbose_name': '字典文件',
'verbose_name_plural': '字典文件',
'db_table': 'wordlist',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['-created_at'], name='wordlist_created_4afb02_idx')],
},
),
migrations.CreateModel(
name='WorkerNode',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(help_text='节点名称', max_length=100)),
('ip_address', models.GenericIPAddressField(help_text='IP 地址(本地节点为 127.0.0.1')),
('ssh_port', models.IntegerField(default=22, help_text='SSH 端口')),
('username', models.CharField(default='root', help_text='SSH 用户名', max_length=50)),
('password', models.CharField(blank=True, default='', help_text='SSH 密码', max_length=200)),
('is_local', models.BooleanField(default=False, help_text='是否为本地节点Docker 容器内)')),
('status', models.CharField(choices=[('pending', '待部署'), ('deploying', '部署中'), ('online', '在线'), ('offline', '离线'), ('updating', '更新中'), ('outdated', '版本过低')], default='pending', help_text='状态: pending/deploying/online/offline', max_length=20)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
],
options={
'verbose_name': 'Worker 节点',
'db_table': 'worker_node',
'ordering': ['-created_at'],
'constraints': [models.UniqueConstraint(condition=models.Q(('is_local', False)), fields=('ip_address',), name='unique_remote_worker_ip'), models.UniqueConstraint(fields=('name',), name='unique_worker_name')],
},
),
]

View File

@@ -88,6 +88,8 @@ def _register_scheduled_jobs(scheduler: BackgroundScheduler):
replace_existing=True,
)
logger.info(" - 已注册: 扫描结果清理(每天 03:00")
# 注意:搜索物化视图刷新已迁移到 pg_ivm 增量维护,无需定时任务
def _trigger_scheduled_scans():

View File

@@ -115,7 +115,7 @@ def initiate_scan_flow(
# ==================== Task 2: 获取引擎配置 ====================
from apps.scan.models import Scan
scan = Scan.objects.get(id=scan_id)
engine_config = scan.merged_configuration
engine_config = scan.yaml_configuration
# 使用 engine_names 进行显示
display_engine_name = ', '.join(scan.engine_names) if scan.engine_names else engine_name

View File

@@ -204,14 +204,13 @@ def _run_scans_sequentially(
# 流式执行扫描并实时保存结果
result = run_and_stream_save_websites_task(
cmd=command,
tool_name=tool_name, # 新增:工具名称
tool_name=tool_name,
scan_id=scan_id,
target_id=target_id,
cwd=str(site_scan_dir),
shell=True,
batch_size=1000,
timeout=timeout,
log_file=str(log_file) # 新增:日志文件路径
log_file=str(log_file)
)
tool_stats[tool_name] = {

View File

@@ -212,7 +212,6 @@ def _validate_and_stream_save_urls(
target_id=target_id,
cwd=str(url_fetch_dir),
shell=True,
batch_size=500,
timeout=timeout,
log_file=str(log_file)
)

View File

@@ -162,6 +162,8 @@ def on_initiate_scan_flow_completed(flow: Flow, flow_run: FlowRun, state: State)
# 执行状态更新并获取统计数据
stats = _update_completed_status()
# 注意:物化视图刷新已迁移到 pg_ivm 增量维护,无需手动标记刷新
# 发送通知(包含统计摘要)
logger.info("准备发送扫描完成通知 - Scan ID: %s, Target: %s", scan_id, target_name)
try:

View File

@@ -0,0 +1,119 @@
# Generated by Django 5.2.7 on 2026-01-02 04:45
import django.contrib.postgres.fields
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
('engine', '0001_initial'),
('targets', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='NotificationSettings',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('discord_enabled', models.BooleanField(default=False, help_text='是否启用 Discord 通知')),
('discord_webhook_url', models.URLField(blank=True, default='', help_text='Discord Webhook URL')),
('categories', models.JSONField(default=dict, help_text='各分类通知开关,如 {"scan": true, "vulnerability": true, "asset": true, "system": false}')),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
],
options={
'verbose_name': '通知设置',
'verbose_name_plural': '通知设置',
'db_table': 'notification_settings',
},
),
migrations.CreateModel(
name='Notification',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('category', models.CharField(choices=[('scan', '扫描任务'), ('vulnerability', '漏洞发现'), ('asset', '资产发现'), ('system', '系统消息')], db_index=True, default='system', help_text='通知分类', max_length=20)),
('level', models.CharField(choices=[('low', ''), ('medium', ''), ('high', ''), ('critical', '严重')], db_index=True, default='low', help_text='通知级别', max_length=20)),
('title', models.CharField(help_text='通知标题', max_length=200)),
('message', models.CharField(help_text='通知内容', max_length=2000)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('is_read', models.BooleanField(default=False, help_text='是否已读')),
('read_at', models.DateTimeField(blank=True, help_text='阅读时间', null=True)),
],
options={
'verbose_name': '通知',
'verbose_name_plural': '通知',
'db_table': 'notification',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['-created_at'], name='notificatio_created_c430f0_idx'), models.Index(fields=['category', '-created_at'], name='notificatio_categor_df0584_idx'), models.Index(fields=['level', '-created_at'], name='notificatio_level_0e5d12_idx'), models.Index(fields=['is_read', '-created_at'], name='notificatio_is_read_518ce0_idx')],
},
),
migrations.CreateModel(
name='Scan',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('engine_ids', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), default=list, help_text='引擎 ID 列表', size=None)),
('engine_names', models.JSONField(default=list, help_text='引擎名称列表,如 ["引擎A", "引擎B"]')),
('yaml_configuration', models.TextField(default='', help_text='YAML 格式的扫描配置')),
('created_at', models.DateTimeField(auto_now_add=True, help_text='任务创建时间')),
('stopped_at', models.DateTimeField(blank=True, help_text='扫描结束时间', null=True)),
('status', models.CharField(choices=[('cancelled', '已取消'), ('completed', '已完成'), ('failed', '失败'), ('initiated', '初始化'), ('running', '运行中')], db_index=True, default='initiated', help_text='任务状态', max_length=20)),
('results_dir', models.CharField(blank=True, default='', help_text='结果存储目录', max_length=100)),
('container_ids', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=100), blank=True, default=list, help_text='容器 ID 列表Docker Container ID', size=None)),
('error_message', models.CharField(blank=True, default='', help_text='错误信息', max_length=2000)),
('deleted_at', models.DateTimeField(blank=True, db_index=True, help_text='删除时间NULL表示未删除', null=True)),
('progress', models.IntegerField(default=0, help_text='扫描进度 0-100')),
('current_stage', models.CharField(blank=True, default='', help_text='当前扫描阶段', max_length=50)),
('stage_progress', models.JSONField(default=dict, help_text='各阶段进度详情')),
('cached_subdomains_count', models.IntegerField(default=0, help_text='缓存的子域名数量')),
('cached_websites_count', models.IntegerField(default=0, help_text='缓存的网站数量')),
('cached_endpoints_count', models.IntegerField(default=0, help_text='缓存的端点数量')),
('cached_ips_count', models.IntegerField(default=0, help_text='缓存的IP地址数量')),
('cached_directories_count', models.IntegerField(default=0, help_text='缓存的目录数量')),
('cached_vulns_total', models.IntegerField(default=0, help_text='缓存的漏洞总数')),
('cached_vulns_critical', models.IntegerField(default=0, help_text='缓存的严重漏洞数量')),
('cached_vulns_high', models.IntegerField(default=0, help_text='缓存的高危漏洞数量')),
('cached_vulns_medium', models.IntegerField(default=0, help_text='缓存的中危漏洞数量')),
('cached_vulns_low', models.IntegerField(default=0, help_text='缓存的低危漏洞数量')),
('stats_updated_at', models.DateTimeField(blank=True, help_text='统计数据最后更新时间', null=True)),
('target', models.ForeignKey(help_text='扫描目标', on_delete=django.db.models.deletion.CASCADE, related_name='scans', to='targets.target')),
('worker', models.ForeignKey(blank=True, help_text='执行扫描的 Worker 节点', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='scans', to='engine.workernode')),
],
options={
'verbose_name': '扫描任务',
'verbose_name_plural': '扫描任务',
'db_table': 'scan',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['-created_at'], name='scan_created_0bb6c7_idx'), models.Index(fields=['target'], name='scan_target__718b9d_idx'), models.Index(fields=['deleted_at', '-created_at'], name='scan_deleted_eb17e8_idx')],
},
),
migrations.CreateModel(
name='ScheduledScan',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('name', models.CharField(help_text='任务名称', max_length=200)),
('engine_ids', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), default=list, help_text='引擎 ID 列表', size=None)),
('engine_names', models.JSONField(default=list, help_text='引擎名称列表,如 ["引擎A", "引擎B"]')),
('yaml_configuration', models.TextField(default='', help_text='YAML 格式的扫描配置')),
('cron_expression', models.CharField(default='0 2 * * *', help_text='Cron 表达式,格式:分 时 日 月 周', max_length=100)),
('is_enabled', models.BooleanField(db_index=True, default=True, help_text='是否启用')),
('run_count', models.IntegerField(default=0, help_text='已执行次数')),
('last_run_time', models.DateTimeField(blank=True, help_text='上次执行时间', null=True)),
('next_run_time', models.DateTimeField(blank=True, help_text='下次执行时间', null=True)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('updated_at', models.DateTimeField(auto_now=True, help_text='更新时间')),
('organization', models.ForeignKey(blank=True, help_text='扫描组织(设置后执行时动态获取组织下所有目标)', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='scheduled_scans', to='targets.organization')),
('target', models.ForeignKey(blank=True, help_text='扫描单个目标(与 organization 二选一)', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='scheduled_scans', to='targets.target')),
],
options={
'verbose_name': '定时扫描任务',
'verbose_name_plural': '定时扫描任务',
'db_table': 'scheduled_scan',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['-created_at'], name='scheduled_s_created_9b9c2e_idx'), models.Index(fields=['is_enabled', '-created_at'], name='scheduled_s_is_enab_23d660_idx'), models.Index(fields=['name'], name='scheduled_s_name_bf332d_idx')],
},
),
]

View File

@@ -30,9 +30,9 @@ class Scan(models.Model):
default=list,
help_text='引擎名称列表,如 ["引擎A", "引擎B"]'
)
merged_configuration = models.TextField(
yaml_configuration = models.TextField(
default='',
help_text='合并后的 YAML 配置'
help_text='YAML 格式的扫描配置'
)
created_at = models.DateTimeField(auto_now_add=True, help_text='任务创建时间')
@@ -136,9 +136,9 @@ class ScheduledScan(models.Model):
default=list,
help_text='引擎名称列表,如 ["引擎A", "引擎B"]'
)
merged_configuration = models.TextField(
yaml_configuration = models.TextField(
default='',
help_text='合并后的 YAML 配置'
help_text='YAML 格式的扫描配置'
)
# 关联的组织(组织扫描模式:执行时动态获取组织下所有目标)

View File

@@ -104,7 +104,7 @@ class DjangoScanRepository:
target: Target,
engine_ids: List[int],
engine_names: List[str],
merged_configuration: str,
yaml_configuration: str,
results_dir: str,
status: ScanStatus = ScanStatus.INITIATED
) -> Scan:
@@ -115,7 +115,7 @@ class DjangoScanRepository:
target: 扫描目标
engine_ids: 引擎 ID 列表
engine_names: 引擎名称列表
merged_configuration: 合并后的 YAML 配置
yaml_configuration: YAML 格式的扫描配置
results_dir: 结果目录
status: 初始状态
@@ -126,7 +126,7 @@ class DjangoScanRepository:
target=target,
engine_ids=engine_ids,
engine_names=engine_names,
merged_configuration=merged_configuration,
yaml_configuration=yaml_configuration,
results_dir=results_dir,
status=status,
container_ids=[]

View File

@@ -31,7 +31,7 @@ class ScheduledScanDTO:
name: str = ''
engine_ids: List[int] = None # 多引擎支持
engine_names: List[str] = None # 引擎名称列表
merged_configuration: str = '' # 合并后的配置
yaml_configuration: str = '' # YAML 格式的扫描配置
organization_id: Optional[int] = None # 组织扫描模式
target_id: Optional[int] = None # 目标扫描模式
cron_expression: Optional[str] = None
@@ -114,7 +114,7 @@ class DjangoScheduledScanRepository:
name=dto.name,
engine_ids=dto.engine_ids,
engine_names=dto.engine_names,
merged_configuration=dto.merged_configuration,
yaml_configuration=dto.yaml_configuration,
organization_id=dto.organization_id, # 组织扫描模式
target_id=dto.target_id if not dto.organization_id else None, # 目标扫描模式
cron_expression=dto.cron_expression,
@@ -147,8 +147,8 @@ class DjangoScheduledScanRepository:
scheduled_scan.engine_ids = dto.engine_ids
if dto.engine_names is not None:
scheduled_scan.engine_names = dto.engine_names
if dto.merged_configuration is not None:
scheduled_scan.merged_configuration = dto.merged_configuration
if dto.yaml_configuration is not None:
scheduled_scan.yaml_configuration = dto.yaml_configuration
if dto.cron_expression is not None:
scheduled_scan.cron_expression = dto.cron_expression
if dto.is_enabled is not None:

View File

@@ -1,9 +1,71 @@
from rest_framework import serializers
from django.db.models import Count
import yaml
from .models import Scan, ScheduledScan
# ==================== 通用验证 Mixin ====================
class DuplicateKeyLoader(yaml.SafeLoader):
"""自定义 YAML Loader检测重复 key"""
pass
def _check_duplicate_keys(loader, node, deep=False):
"""检测 YAML mapping 中的重复 key"""
mapping = {}
for key_node, value_node in node.value:
key = loader.construct_object(key_node, deep=deep)
if key in mapping:
raise yaml.constructor.ConstructorError(
"while constructing a mapping", node.start_mark,
f"发现重复的配置项 '{key}',后面的配置会覆盖前面的配置,请删除重复项", key_node.start_mark
)
mapping[key] = loader.construct_object(value_node, deep=deep)
return mapping
DuplicateKeyLoader.add_constructor(
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
_check_duplicate_keys
)
class ScanConfigValidationMixin:
"""扫描配置验证 Mixin提供通用的验证方法"""
def validate_configuration(self, value):
"""验证 YAML 配置格式,包括检测重复 key"""
import yaml
if not value or not value.strip():
raise serializers.ValidationError("configuration 不能为空")
try:
# 使用自定义 Loader 检测重复 key
yaml.load(value, Loader=DuplicateKeyLoader)
except yaml.YAMLError as e:
raise serializers.ValidationError(f"无效的 YAML 格式: {str(e)}")
return value
def validate_engine_ids(self, value):
"""验证引擎 ID 列表"""
if not value:
raise serializers.ValidationError("engine_ids 不能为空,请至少选择一个扫描引擎")
return value
def validate_engine_names(self, value):
"""验证引擎名称列表"""
if not value:
raise serializers.ValidationError("engine_names 不能为空")
return value
# ==================== 扫描任务序列化器 ====================
class ScanSerializer(serializers.ModelSerializer):
"""扫描任务序列化器"""
target_name = serializers.SerializerMethodField()
@@ -82,12 +144,12 @@ class ScanHistorySerializer(serializers.ModelSerializer):
return summary
class QuickScanSerializer(serializers.Serializer):
class QuickScanSerializer(ScanConfigValidationMixin, serializers.Serializer):
"""
快速扫描序列化器
功能:
- 接收目标列表和引擎配置
- 接收目标列表和 YAML 配置
- 自动创建/获取目标
- 立即发起扫描
"""
@@ -101,11 +163,24 @@ class QuickScanSerializer(serializers.Serializer):
help_text='目标列表,每个目标包含 name 字段'
)
# 扫描引擎 ID 列表
# YAML 配置(必填)
configuration = serializers.CharField(
required=True,
help_text='YAML 格式的扫描配置(必填)'
)
# 扫描引擎 ID 列表(必填,用于记录和显示)
engine_ids = serializers.ListField(
child=serializers.IntegerField(),
required=True,
help_text='使用的扫描引擎 ID 列表 (必填)'
help_text='使用的扫描引擎 ID 列表必填'
)
# 引擎名称列表(必填,用于记录和显示)
engine_names = serializers.ListField(
child=serializers.CharField(),
required=True,
help_text='引擎名称列表(必填)'
)
def validate_targets(self, value):
@@ -127,12 +202,6 @@ class QuickScanSerializer(serializers.Serializer):
raise serializers.ValidationError(f"{idx + 1} 个目标的 name 不能为空")
return value
def validate_engine_ids(self, value):
"""验证引擎 ID 列表"""
if not value:
raise serializers.ValidationError("engine_ids 不能为空")
return value
# ==================== 定时扫描序列化器 ====================
@@ -171,7 +240,7 @@ class ScheduledScanSerializer(serializers.ModelSerializer):
return 'organization' if obj.organization_id else 'target'
class CreateScheduledScanSerializer(serializers.Serializer):
class CreateScheduledScanSerializer(ScanConfigValidationMixin, serializers.Serializer):
"""创建定时扫描任务序列化器
扫描模式(二选一):
@@ -180,9 +249,25 @@ class CreateScheduledScanSerializer(serializers.Serializer):
"""
name = serializers.CharField(max_length=200, help_text='任务名称')
# YAML 配置(必填)
configuration = serializers.CharField(
required=True,
help_text='YAML 格式的扫描配置(必填)'
)
# 扫描引擎 ID 列表(必填,用于记录和显示)
engine_ids = serializers.ListField(
child=serializers.IntegerField(),
help_text='扫描引擎 ID 列表'
required=True,
help_text='扫描引擎 ID 列表(必填)'
)
# 引擎名称列表(必填,用于记录和显示)
engine_names = serializers.ListField(
child=serializers.CharField(),
required=True,
help_text='引擎名称列表(必填)'
)
# 组织扫描模式
@@ -206,11 +291,61 @@ class CreateScheduledScanSerializer(serializers.Serializer):
)
is_enabled = serializers.BooleanField(default=True, help_text='是否立即启用')
def validate_engine_ids(self, value):
"""验证引擎 ID 列表"""
if not value:
raise serializers.ValidationError("engine_ids 不能为空")
return value
def validate(self, data):
"""验证 organization_id 和 target_id 互斥"""
organization_id = data.get('organization_id')
target_id = data.get('target_id')
if not organization_id and not target_id:
raise serializers.ValidationError('必须提供 organization_id 或 target_id 其中之一')
if organization_id and target_id:
raise serializers.ValidationError('organization_id 和 target_id 只能提供其中之一')
return data
class InitiateScanSerializer(ScanConfigValidationMixin, serializers.Serializer):
"""发起扫描任务序列化器
扫描模式(二选一):
- 组织扫描:提供 organization_id扫描组织下所有目标
- 目标扫描:提供 target_id扫描单个目标
"""
# YAML 配置(必填)
configuration = serializers.CharField(
required=True,
help_text='YAML 格式的扫描配置(必填)'
)
# 扫描引擎 ID 列表(必填)
engine_ids = serializers.ListField(
child=serializers.IntegerField(),
required=True,
help_text='扫描引擎 ID 列表(必填)'
)
# 引擎名称列表(必填)
engine_names = serializers.ListField(
child=serializers.CharField(),
required=True,
help_text='引擎名称列表(必填)'
)
# 组织扫描模式
organization_id = serializers.IntegerField(
required=False,
allow_null=True,
help_text='组织 ID组织扫描模式'
)
# 目标扫描模式
target_id = serializers.IntegerField(
required=False,
allow_null=True,
help_text='目标 ID目标扫描模式'
)
def validate(self, data):
"""验证 organization_id 和 target_id 互斥"""

View File

@@ -282,7 +282,7 @@ class ScanCreationService:
targets: List[Target],
engine_ids: List[int],
engine_names: List[str],
merged_configuration: str,
yaml_configuration: str,
scheduled_scan_name: str | None = None
) -> List[Scan]:
"""
@@ -292,7 +292,7 @@ class ScanCreationService:
targets: 目标列表
engine_ids: 引擎 ID 列表
engine_names: 引擎名称列表
merged_configuration: 合并后的 YAML 配置
yaml_configuration: YAML 格式的扫描配置
scheduled_scan_name: 定时扫描任务名称(可选,用于通知显示)
Returns:
@@ -312,7 +312,7 @@ class ScanCreationService:
target=target,
engine_ids=engine_ids,
engine_names=engine_names,
merged_configuration=merged_configuration,
yaml_configuration=yaml_configuration,
results_dir=scan_workspace_dir,
status=ScanStatus.INITIATED,
container_ids=[],

View File

@@ -117,12 +117,12 @@ class ScanService:
targets: List[Target],
engine_ids: List[int],
engine_names: List[str],
merged_configuration: str,
yaml_configuration: str,
scheduled_scan_name: str | None = None
) -> List[Scan]:
"""批量创建扫描任务(委托给 ScanCreationService"""
return self.creation_service.create_scans(
targets, engine_ids, engine_names, merged_configuration, scheduled_scan_name
targets, engine_ids, engine_names, yaml_configuration, scheduled_scan_name
)
# ==================== 状态管理方法(委托给 ScanStateService ====================

View File

@@ -54,7 +54,7 @@ class ScheduledScanService:
def create(self, dto: ScheduledScanDTO) -> ScheduledScan:
"""
创建定时扫描任务
创建定时扫描任务(使用引擎 ID 合并配置)
流程:
1. 验证参数
@@ -88,7 +88,7 @@ class ScheduledScanService:
# 设置 DTO 的合并配置和引擎名称
dto.engine_names = engine_names
dto.merged_configuration = merged_configuration
dto.yaml_configuration = merged_configuration
# 3. 创建数据库记录
scheduled_scan = self.repo.create(dto)
@@ -107,12 +107,49 @@ class ScheduledScanService:
return scheduled_scan
def _validate_create_dto(self, dto: ScheduledScanDTO) -> None:
"""验证创建 DTO"""
from apps.targets.repositories import DjangoOrganizationRepository
def create_with_configuration(self, dto: ScheduledScanDTO) -> ScheduledScan:
"""
创建定时扫描任务(直接使用前端传递的配置)
if not dto.name:
raise ValidationError('任务名称不能为空')
流程:
1. 验证参数
2. 直接使用 dto.yaml_configuration
3. 创建数据库记录
4. 计算并设置 next_run_time
Args:
dto: 定时扫描 DTO必须包含 yaml_configuration
Returns:
创建的 ScheduledScan 对象
Raises:
ValidationError: 参数验证失败
"""
# 1. 验证参数
self._validate_create_dto_with_configuration(dto)
# 2. 创建数据库记录(直接使用 dto 中的配置)
scheduled_scan = self.repo.create(dto)
# 3. 如果有 cron 表达式且已启用,计算下次执行时间
if scheduled_scan.cron_expression and scheduled_scan.is_enabled:
next_run_time = self._calculate_next_run_time(scheduled_scan)
if next_run_time:
self.repo.update_next_run_time(scheduled_scan.id, next_run_time)
scheduled_scan.next_run_time = next_run_time
logger.info(
"创建定时扫描任务 - ID: %s, 名称: %s, 下次执行: %s",
scheduled_scan.id, scheduled_scan.name, scheduled_scan.next_run_time
)
return scheduled_scan
def _validate_create_dto(self, dto: ScheduledScanDTO) -> None:
"""验证创建 DTO使用引擎 ID"""
# 基础验证
self._validate_base_dto(dto)
if not dto.engine_ids:
raise ValidationError('必须选择扫描引擎')
@@ -121,6 +158,21 @@ class ScheduledScanService:
for engine_id in dto.engine_ids:
if not self.engine_repo.get_by_id(engine_id):
raise ValidationError(f'扫描引擎 ID {engine_id} 不存在')
def _validate_create_dto_with_configuration(self, dto: ScheduledScanDTO) -> None:
"""验证创建 DTO使用前端传递的配置"""
# 基础验证
self._validate_base_dto(dto)
if not dto.yaml_configuration:
raise ValidationError('配置不能为空')
def _validate_base_dto(self, dto: ScheduledScanDTO) -> None:
"""验证 DTO 的基础字段(公共逻辑)"""
from apps.targets.repositories import DjangoOrganizationRepository
if not dto.name:
raise ValidationError('任务名称不能为空')
# 验证扫描模式organization_id 和 target_id 互斥)
if not dto.organization_id and not dto.target_id:
@@ -178,7 +230,7 @@ class ScheduledScanService:
merged_configuration = merge_engine_configs(engines)
dto.engine_names = engine_names
dto.merged_configuration = merged_configuration
dto.yaml_configuration = merged_configuration
# 更新数据库记录
scheduled_scan = self.repo.update(scheduled_scan_id, dto)
@@ -329,7 +381,7 @@ class ScheduledScanService:
立即触发扫描(支持组织扫描和目标扫描两种模式)
复用 ScanService 的逻辑,与 API 调用保持一致。
使用存储的 merged_configuration 而不是重新合并。
使用存储的 yaml_configuration 而不是重新合并。
"""
from apps.scan.services.scan_service import ScanService
@@ -347,7 +399,7 @@ class ScheduledScanService:
targets=targets,
engine_ids=scheduled_scan.engine_ids,
engine_names=scheduled_scan.engine_names,
merged_configuration=scheduled_scan.merged_configuration,
yaml_configuration=scheduled_scan.yaml_configuration,
scheduled_scan_name=scheduled_scan.name
)

View File

@@ -341,9 +341,9 @@ def _process_batch(
url=record['url'],
host=host,
title=record.get('title', '') or '',
status=record.get('status_code'),
status_code=record.get('status_code'),
content_length=record.get('content_length'),
web_server=record.get('server', '') or '',
webserver=record.get('server', '') or '',
tech=record.get('techs', []),
)
snapshot_dtos.append(dto)

View File

@@ -30,7 +30,6 @@ from typing import Generator, Optional, Dict, Any, TYPE_CHECKING
from django.db import IntegrityError, OperationalError, DatabaseError
from dataclasses import dataclass
from urllib.parse import urlparse, urlunparse
from dateutil.parser import parse as parse_datetime
from psycopg2 import InterfaceError
from apps.asset.dtos.snapshot import WebsiteSnapshotDTO
@@ -62,6 +61,18 @@ class ServiceSet:
)
def _sanitize_string(value: str) -> str:
"""
清理字符串中的 NUL 字符和其他不可打印字符
PostgreSQL 不允许字符串字段包含 NUL (0x00) 字符
"""
if not value:
return value
# 移除 NUL 字符
return value.replace('\x00', '')
def normalize_url(url: str) -> str:
"""
标准化 URL移除默认端口号
@@ -117,70 +128,50 @@ def normalize_url(url: str) -> str:
return url
def _extract_hostname(url: str) -> str:
"""
从 URL 提取主机名
Args:
url: URL 字符串
Returns:
str: 提取的主机名(小写)
"""
try:
if url:
parsed = urlparse(url)
if parsed.hostname:
return parsed.hostname
# 降级方案:手动提取
return url.replace('http://', '').replace('https://', '').split('/')[0].split(':')[0]
return ''
except Exception as e:
logger.debug("提取主机名失败: %s", e)
return ''
class HttpxRecord:
"""httpx 扫描记录数据类"""
def __init__(self, data: Dict[str, Any]):
self.url = data.get('url', '')
self.input = data.get('input', '')
self.title = data.get('title', '')
self.status_code = data.get('status_code')
self.content_length = data.get('content_length')
self.content_type = data.get('content_type', '')
self.location = data.get('location', '')
self.webserver = data.get('webserver', '')
self.response_body = data.get('body', '') # 从 body 字段获取完整响应体
self.tech = data.get('tech', [])
self.vhost = data.get('vhost')
self.failed = data.get('failed', False)
self.timestamp = data.get('timestamp')
self.response_headers = data.get('raw_header', '') # 从 raw_header 字段获取原始响应头字符串
self.url = _sanitize_string(data.get('url', ''))
self.input = _sanitize_string(data.get('input', ''))
self.title = _sanitize_string(data.get('title', ''))
self.status_code = data.get('status_code') # int不需要清理
self.content_length = data.get('content_length') # int不需要清理
self.content_type = _sanitize_string(data.get('content_type', ''))
self.location = _sanitize_string(data.get('location', ''))
self.webserver = _sanitize_string(data.get('webserver', ''))
self.response_body = _sanitize_string(data.get('body', ''))
self.tech = [_sanitize_string(t) for t in data.get('tech', []) if isinstance(t, str)] # 列表中的字符串也需要清理
self.vhost = data.get('vhost') # bool不需要清理
self.failed = data.get('failed', False) # bool不需要清理
self.response_headers = _sanitize_string(data.get('raw_header', ''))
# 从 URL 中提取主机名
self.host = self._extract_hostname()
def _extract_hostname(self) -> str:
"""
从 URL 或 input 字段提取主机名
优先级:
1. 使用 urlparse 解析 URL 获取 hostname
2. 从 input 字段提取(处理可能包含协议的情况)
3. 从 URL 字段手动提取(降级方案)
Returns:
str: 提取的主机名(小写)
"""
try:
# 方法 1: 使用 urlparse 解析 URL
if self.url:
parsed = urlparse(self.url)
if parsed.hostname:
return parsed.hostname
# 方法 2: 从 input 字段提取
if self.input:
host = self.input.strip().lower()
# 移除协议前缀
if host.startswith(('http://', 'https://')):
host = host.split('//', 1)[1].split('/')[0]
return host
# 方法 3: 从 URL 手动提取(降级方案)
if self.url:
return self.url.replace('http://', '').replace('https://', '').split('/')[0]
# 兜底:返回空字符串
return ''
except Exception as e:
# 异常处理:尽力从 input 或 URL 提取
logger.debug("提取主机名失败: %s,使用降级方案", e)
if self.input:
return self.input.strip().lower()
if self.url:
return self.url.replace('http://', '').replace('https://', '').split('/')[0]
return ''
# 从 URL 中提取主机名(优先使用 httpx 返回的 host否则自动提取
httpx_host = _sanitize_string(data.get('host', ''))
self.host = httpx_host if httpx_host else _extract_hostname(self.url)
def _save_batch_with_retry(
@@ -228,39 +219,31 @@ def _save_batch_with_retry(
}
except (OperationalError, DatabaseError, InterfaceError) as e:
# 数据库连接/操作错误,可重试
# 数据库级错误(连接中断、表结构不匹配等):按指数退避重试,最终失败时抛出异常让 Flow 失败
if attempt < max_retries - 1:
wait_time = 2 ** attempt # 指数退避: 1s, 2s, 4s
wait_time = 2 ** attempt
logger.warning(
"批次 %d 保存失败(第 %d 次尝试),%d秒后重试: %s",
batch_num, attempt + 1, wait_time, str(e)[:100]
)
time.sleep(wait_time)
else:
logger.error("批次 %d 保存失败(已重试 %d 次): %s", batch_num, max_retries, e)
return {
'success': False,
'created_websites': 0,
'skipped_failed': 0
}
except Exception as e:
# 其他未知错误 - 检查是否为连接问题
error_str = str(e).lower()
if 'connection' in error_str and attempt < max_retries - 1:
logger.warning(
"批次 %d 连接相关错误(尝试 %d/%d: %sRepository 装饰器会自动重连",
batch_num, attempt + 1, max_retries, str(e)
logger.error(
"批次 %d 保存失败(已重试 %d 次),将终止任务: %s",
batch_num,
max_retries,
e,
exc_info=True,
)
time.sleep(2)
else:
logger.error("批次 %d 未知错误: %s", batch_num, e, exc_info=True)
return {
'success': False,
'created_websites': 0,
'skipped_failed': 0
}
# 让上层 Task 感知失败,从而标记整个扫描为失败
raise
except Exception as e:
# 其他未知异常也不再吞掉,直接抛出以便 Flow 标记为失败
logger.error("批次 %d 未知错误: %s", batch_num, e, exc_info=True)
raise
# 理论上不会走到这里,保留兜底返回值以满足类型约束
return {
'success': False,
'created_websites': 0,
@@ -328,43 +311,39 @@ def _save_batch(
skipped_failed += 1
continue
# 解析时间戳
created_at = None
if hasattr(record, 'timestamp') and record.timestamp:
try:
created_at = parse_datetime(record.timestamp)
except (ValueError, TypeError) as e:
logger.warning(f"无法解析时间戳 {record.timestamp}: {e}")
# 使用 input 字段(原始扫描的 URL而不是 url 字段(重定向后的 URL
# 原因:避免多个不同的输入 URL 重定向到同一个 URL 时产生唯一约束冲突
# 例如http://example.com 和 https://example.com 都重定向到 https://example.com
# 如果使用 record.url两条记录会有相同的 url导致数据库冲突
# 如果使用 record.input两条记录保留原始输入不会冲突
normalized_url = normalize_url(record.input)
# 提取 host 字段域名或IP地址
host = record.host if record.host else ''
# 创建 WebsiteSnapshot DTO
snapshot_dto = WebsiteSnapshotDTO(
scan_id=scan_id,
target_id=target_id, # 主关联字段
url=normalized_url, # 保存原始输入 URL归一化后
host=host, # 主机名域名或IP地址
location=record.location, # location 字段保存重定向信息
title=record.title[:1000] if record.title else '',
web_server=record.webserver[:200] if record.webserver else '',
response_body=record.response_body if record.response_body else '',
content_type=record.content_type[:200] if record.content_type else '',
tech=record.tech if isinstance(record.tech, list) else [],
status=record.status_code,
content_length=record.content_length,
vhost=record.vhost,
response_headers=record.response_headers if record.response_headers else '',
)
snapshot_items.append(snapshot_dto)
try:
# 使用 input 字段(原始扫描的 URL而不是 url 字段(重定向后的 URL
# 原因:避免多个不同的输入 URL 重定向到同一个 URL 时产生唯一约束冲突
# 例如http://example.com 和 https://example.com 都重定向到 https://example.com
# 如果使用 record.url两条记录会有相同的 url导致数据库冲突
# 如果使用 record.input两条记录保留原始输入不会冲突
normalized_url = normalize_url(record.input) if record.input else normalize_url(record.url)
# 提取 host 字段域名或IP地址
host = record.host if record.host else ''
# 创建 WebsiteSnapshot DTO
snapshot_dto = WebsiteSnapshotDTO(
scan_id=scan_id,
target_id=target_id, # 主关联字段
url=normalized_url, # 保存原始输入 URL归一化后
host=host, # 主机名域名或IP地址
location=record.location if record.location else '',
title=record.title if record.title else '',
webserver=record.webserver if record.webserver else '',
response_body=record.response_body if record.response_body else '',
content_type=record.content_type if record.content_type else '',
tech=record.tech if isinstance(record.tech, list) else [],
status_code=record.status_code,
content_length=record.content_length,
vhost=record.vhost,
response_headers=record.response_headers if record.response_headers else '',
)
snapshot_items.append(snapshot_dto)
except Exception as e:
logger.error("处理记录失败: %s,错误: %s", record.url, e)
continue
# ========== Step 3: 保存快照并同步到资产表(通过快照 Service==========
if snapshot_items:
@@ -386,28 +365,31 @@ def _parse_and_validate_line(line: str) -> Optional[HttpxRecord]:
Optional[HttpxRecord]: 有效的 httpx 扫描记录,或 None 如果验证失败
验证步骤:
1. 解析 JSON 格式
2. 验证数据类型为字典
3. 创建 HttpxRecord 对象
4. 验证必要字段url
1. 清理 NUL 字符
2. 解析 JSON 格式
3. 验证数据类型为字典
4. 创建 HttpxRecord 对象
5. 验证必要字段url
"""
try:
# 步骤 1: 解析 JSON
# 步骤 1: 清理 NUL 字符后再解析 JSON
line = _sanitize_string(line)
# 步骤 2: 解析 JSON
try:
line_data = json.loads(line, strict=False)
except json.JSONDecodeError:
# logger.info("跳过非 JSON 行: %s", line)
return None
# 步骤 2: 验证数据类型
# 步骤 3: 验证数据类型
if not isinstance(line_data, dict):
logger.info("跳过非字典数据")
return None
# 步骤 3: 创建记录
# 步骤 4: 创建记录
record = HttpxRecord(line_data)
# 步骤 4: 验证必要字段
# 步骤 5: 验证必要字段
if not record.url:
logger.info("URL 为空,跳过 - 数据: %s", str(line_data)[:200])
return None
@@ -416,7 +398,7 @@ def _parse_and_validate_line(line: str) -> Optional[HttpxRecord]:
return record
except Exception:
logger.info("跳过无法解析的行: %s", line[:100])
logger.info("跳过无法解析的行: %s", line[:100] if line else 'empty')
return None
@@ -464,8 +446,8 @@ def _parse_httpx_stream_output(
# yield 一条有效记录
yield record
# 每处理 1000 条记录输出一次进度
if valid_records % 1000 == 0:
# 每处理 5 条记录输出一次进度
if valid_records % 5 == 0:
logger.info("已解析 %d 条有效记录...", valid_records)
except subprocess.TimeoutExpired as e:
@@ -604,8 +586,8 @@ def _process_records_in_batches(
_process_batch(batch, scan_id, target_id, batch_num, total_stats, failed_batches, services)
batch = [] # 清空批次
# 每20个批次输出进度
if batch_num % 20 == 0:
# 每 2 个批次输出进度
if batch_num % 2 == 0:
logger.info("进度: 已处理 %d 批次,%d 条记录", batch_num, total_records)
# 保存最后一批
@@ -676,11 +658,7 @@ def _cleanup_resources(data_generator) -> None:
logger.error("关闭生成器时出错: %s", gen_close_error)
@task(
name='run_and_stream_save_websites',
retries=0,
log_prints=True
)
@task(name='run_and_stream_save_websites', retries=0)
def run_and_stream_save_websites_task(
cmd: str,
tool_name: str,
@@ -688,7 +666,7 @@ def run_and_stream_save_websites_task(
target_id: int,
cwd: Optional[str] = None,
shell: bool = False,
batch_size: int = 1000,
batch_size: int = 10,
timeout: Optional[int] = None,
log_file: Optional[str] = None
) -> dict:

View File

@@ -23,10 +23,11 @@ import subprocess
import time
from pathlib import Path
from prefect import task
from typing import Generator, Optional
from typing import Generator, Optional, Dict, Any
from django.db import IntegrityError, OperationalError, DatabaseError
from psycopg2 import InterfaceError
from dataclasses import dataclass
from urllib.parse import urlparse
from apps.asset.services.snapshot import EndpointSnapshotsService
from apps.scan.utils import execute_stream
@@ -63,7 +64,53 @@ def _sanitize_string(value: str) -> str:
return value.replace('\x00', '')
def _parse_and_validate_line(line: str) -> Optional[dict]:
def _extract_hostname(url: str) -> str:
"""
从 URL 提取主机名
Args:
url: URL 字符串
Returns:
str: 提取的主机名(小写)
"""
try:
if url:
parsed = urlparse(url)
if parsed.hostname:
return parsed.hostname
# 降级方案:手动提取
return url.replace('http://', '').replace('https://', '').split('/')[0].split(':')[0]
return ''
except Exception as e:
logger.debug("提取主机名失败: %s", e)
return ''
class HttpxRecord:
"""httpx 扫描记录数据类"""
def __init__(self, data: Dict[str, Any]):
self.url = _sanitize_string(data.get('url', ''))
self.input = _sanitize_string(data.get('input', ''))
self.title = _sanitize_string(data.get('title', ''))
self.status_code = data.get('status_code') # int不需要清理
self.content_length = data.get('content_length') # int不需要清理
self.content_type = _sanitize_string(data.get('content_type', ''))
self.location = _sanitize_string(data.get('location', ''))
self.webserver = _sanitize_string(data.get('webserver', ''))
self.response_body = _sanitize_string(data.get('body', ''))
self.tech = [_sanitize_string(t) for t in data.get('tech', []) if isinstance(t, str)] # 列表中的字符串也需要清理
self.vhost = data.get('vhost') # bool不需要清理
self.failed = data.get('failed', False) # bool不需要清理
self.response_headers = _sanitize_string(data.get('raw_header', ''))
# 从 URL 中提取主机名(优先使用 httpx 返回的 host否则自动提取
httpx_host = _sanitize_string(data.get('host', ''))
self.host = httpx_host if httpx_host else _extract_hostname(self.url)
def _parse_and_validate_line(line: str) -> Optional[HttpxRecord]:
"""
解析并验证单行 httpx JSON 输出
@@ -71,9 +118,7 @@ def _parse_and_validate_line(line: str) -> Optional[dict]:
line: 单行输出数据
Returns:
Optional[dict]: 有效的 httpx 记录,或 None 如果验证失败
保存所有有效 URL不再过滤状态码安全扫描中 403/404/500 等也有分析价值)
Optional[HttpxRecord]: 有效的 httpx 记录,或 None 如果验证失败
"""
try:
# 清理 NUL 字符后再解析 JSON
@@ -83,7 +128,6 @@ def _parse_and_validate_line(line: str) -> Optional[dict]:
try:
line_data = json.loads(line, strict=False)
except json.JSONDecodeError:
# logger.info("跳过非 JSON 行: %s", line)
return None
# 验证数据类型
@@ -91,29 +135,15 @@ def _parse_and_validate_line(line: str) -> Optional[dict]:
logger.info("跳过非字典数据")
return None
# 获取必要字段
url = line_data.get('url', '').strip()
status_code = line_data.get('status_code')
# 创建记录
record = HttpxRecord(line_data)
if not url:
# 验证必要字段
if not record.url:
logger.info("URL 为空,跳过 - 数据: %s", str(line_data)[:200])
return None
# 保存所有有效 URL不再过滤状态码
return {
'url': _sanitize_string(url),
'host': _sanitize_string(line_data.get('host', '')),
'status_code': status_code,
'title': _sanitize_string(line_data.get('title', '')),
'content_length': line_data.get('content_length', 0),
'content_type': _sanitize_string(line_data.get('content_type', '')),
'webserver': _sanitize_string(line_data.get('webserver', '')),
'location': _sanitize_string(line_data.get('location', '')),
'tech': line_data.get('tech', []),
'response_body': _sanitize_string(line_data.get('body', '')),
'vhost': line_data.get('vhost', False),
'response_headers': _sanitize_string(line_data.get('raw_header', '')),
}
return record
except Exception:
logger.info("跳过无法解析的行: %s", line[:100] if line else 'empty')
@@ -127,7 +157,7 @@ def _parse_httpx_stream_output(
shell: bool = False,
timeout: Optional[int] = None,
log_file: Optional[str] = None
) -> Generator[dict, None, None]:
) -> Generator[HttpxRecord, None, None]:
"""
流式解析 httpx 命令输出
@@ -140,7 +170,7 @@ def _parse_httpx_stream_output(
log_file: 日志文件路径
Yields:
dict: 每次 yield 一条存活的 URL 记录
HttpxRecord: 每次 yield 一条存活的 URL 记录
"""
logger.info("开始流式解析 httpx 输出 - 命令: %s", cmd)
@@ -170,8 +200,8 @@ def _parse_httpx_stream_output(
# yield 一条有效记录(存活的 URL
yield record
# 每处理 500 条记录输出一次进度
if valid_records % 500 == 0:
# 每处理 100 条记录输出一次进度
if valid_records % 100 == 0:
logger.info("已解析 %d 条存活的 URL...", valid_records)
except subprocess.TimeoutExpired as e:
@@ -188,6 +218,78 @@ def _parse_httpx_stream_output(
)
def _validate_task_parameters(cmd: str, target_id: int, scan_id: int, cwd: Optional[str]) -> None:
"""
验证任务参数的有效性
Args:
cmd: 扫描命令
target_id: 目标ID
scan_id: 扫描ID
cwd: 工作目录
Raises:
ValueError: 参数验证失败
"""
if not cmd or not cmd.strip():
raise ValueError("扫描命令不能为空")
if target_id is None:
raise ValueError("target_id 不能为 None必须指定目标ID")
if scan_id is None:
raise ValueError("scan_id 不能为 None必须指定扫描ID")
# 验证工作目录(如果指定)
if cwd and not Path(cwd).exists():
raise ValueError(f"工作目录不存在: {cwd}")
def _build_final_result(stats: dict) -> dict:
"""
构建最终结果并输出日志
Args:
stats: 处理统计信息
Returns:
dict: 最终结果
"""
logger.info(
"✓ URL 验证任务完成 - 处理记录: %d%d 批次),创建端点: %d,跳过(失败): %d",
stats['processed_records'], stats['batch_count'], stats['created_endpoints'],
stats['skipped_failed']
)
# 如果没有创建任何记录,给出明确提示
if stats['created_endpoints'] == 0:
logger.warning(
"⚠️ 没有创建任何端点记录可能原因1) 命令输出格式问题 2) 重复数据被忽略 3) 所有请求都失败"
)
return {
'processed_records': stats['processed_records'],
'created_endpoints': stats['created_endpoints'],
'skipped_failed': stats['skipped_failed']
}
def _cleanup_resources(data_generator) -> None:
"""
清理任务资源
Args:
data_generator: 数据生成器
"""
# 确保生成器被正确关闭
if data_generator is not None:
try:
data_generator.close()
logger.debug("已关闭数据生成器")
except Exception as gen_close_error:
logger.error("关闭生成器时出错: %s", gen_close_error)
def _save_batch_with_retry(
batch: list,
scan_id: int,
@@ -208,14 +310,19 @@ def _save_batch_with_retry(
max_retries: 最大重试次数
Returns:
dict: {'success': bool, 'saved_count': int}
dict: {
'success': bool,
'created_endpoints': int,
'skipped_failed': int
}
"""
for attempt in range(max_retries):
try:
count = _save_batch(batch, scan_id, target_id, batch_num, services)
stats = _save_batch(batch, scan_id, target_id, batch_num, services)
return {
'success': True,
'saved_count': count
'created_endpoints': stats.get('created_endpoints', 0),
'skipped_failed': stats.get('skipped_failed', 0)
}
except IntegrityError as e:
@@ -223,7 +330,8 @@ def _save_batch_with_retry(
logger.error("批次 %d 数据完整性错误,跳过: %s", batch_num, str(e)[:100])
return {
'success': False,
'saved_count': 0
'created_endpoints': 0,
'skipped_failed': 0
}
except (OperationalError, DatabaseError, InterfaceError) as e:
@@ -254,7 +362,8 @@ def _save_batch_with_retry(
# 理论上不会走到这里,保留兜底返回值以满足类型约束
return {
'success': False,
'saved_count': 0
'created_endpoints': 0,
'skipped_failed': 0
}
@@ -264,50 +373,72 @@ def _save_batch(
target_id: int,
batch_num: int,
services: ServiceSet
) -> int:
) -> dict:
"""
保存一个批次的数据到数据库
Args:
batch: 数据批次list of dict
batch: 数据批次list of HttpxRecord
scan_id: 扫描任务 ID
target_id: 目标 ID
batch_num: 批次编号
services: Service 集合
Returns:
int: 创建的记录数
dict: 包含创建和跳过记录的统计信息
"""
# 参数验证
if not isinstance(batch, list):
raise TypeError(f"batch 必须是 list 类型,实际: {type(batch).__name__}")
if not batch:
logger.debug("批次 %d 为空,跳过处理", batch_num)
return 0
return {
'created_endpoints': 0,
'skipped_failed': 0
}
# 统计变量
skipped_failed = 0
# 批量构造 Endpoint 快照 DTO
from apps.asset.dtos.snapshot import EndpointSnapshotDTO
snapshots = []
for record in batch:
# 跳过失败的请求
if record.failed:
skipped_failed += 1
continue
try:
# Endpoint URL 直接使用原始值,不做标准化
# 原因Endpoint URL 来自 waymore/katana包含路径和参数标准化可能改变含义
url = record.input if record.input else record.url
# 提取 host 字段域名或IP地址
host = record.host if record.host else ''
dto = EndpointSnapshotDTO(
scan_id=scan_id,
url=record['url'],
host=record.get('host', ''),
title=record.get('title', ''),
status_code=record.get('status_code'),
content_length=record.get('content_length', 0),
location=record.get('location', ''),
webserver=record.get('webserver', ''),
content_type=record.get('content_type', ''),
tech=record.get('tech', []),
response_body=record.get('response_body', ''),
vhost=record.get('vhost', False),
matched_gf_patterns=[],
target_id=target_id,
response_headers=record.get('response_headers', ''),
url=url,
host=host,
title=record.title if record.title else '',
status_code=record.status_code,
content_length=record.content_length,
location=record.location if record.location else '',
webserver=record.webserver if record.webserver else '',
content_type=record.content_type if record.content_type else '',
tech=record.tech if isinstance(record.tech, list) else [],
response_body=record.response_body if record.response_body else '',
vhost=record.vhost if record.vhost else False,
matched_gf_patterns=[],
response_headers=record.response_headers if record.response_headers else '',
)
snapshots.append(dto)
except Exception as e:
logger.error("处理记录失败: %s,错误: %s", record.get('url', 'Unknown'), e)
logger.error("处理记录失败: %s,错误: %s", record.url, e)
continue
if snapshots:
@@ -316,15 +447,69 @@ def _save_batch(
services.snapshot.save_and_sync(snapshots)
count = len(snapshots)
logger.info(
"批次 %d: 保存了 %d 个存活的 URL%d 个)",
batch_num, count, len(batch)
"批次 %d: 保存了 %d 个存活的 URL%d,跳过失败: %d",
batch_num, count, len(batch), skipped_failed
)
return count
return {
'created_endpoints': count,
'skipped_failed': skipped_failed
}
except Exception as e:
logger.error("批次 %d 批量保存失败: %s", batch_num, e)
raise
return 0
return {
'created_endpoints': 0,
'skipped_failed': skipped_failed
}
def _accumulate_batch_stats(total_stats: dict, batch_result: dict) -> None:
"""
累加批次统计信息
Args:
total_stats: 总统计信息字典
batch_result: 批次结果字典
"""
total_stats['created_endpoints'] += batch_result.get('created_endpoints', 0)
total_stats['skipped_failed'] += batch_result.get('skipped_failed', 0)
def _process_batch(
batch: list,
scan_id: int,
target_id: int,
batch_num: int,
total_stats: dict,
failed_batches: list,
services: ServiceSet
) -> None:
"""
处理单个批次
Args:
batch: 数据批次
scan_id: 扫描ID
target_id: 目标ID
batch_num: 批次编号
total_stats: 总统计信息
failed_batches: 失败批次列表
services: Service 集合(必须,依赖注入)
"""
result = _save_batch_with_retry(
batch, scan_id, target_id, batch_num, services
)
# 累计统计信息(失败时可能有部分数据已保存)
_accumulate_batch_stats(total_stats, result)
if not result['success']:
failed_batches.append(batch_num)
logger.warning(
"批次 %d 保存失败,但已累计统计信息:创建端点=%d",
batch_num, result.get('created_endpoints', 0)
)
def _process_records_in_batches(
@@ -335,7 +520,7 @@ def _process_records_in_batches(
services: ServiceSet
) -> dict:
"""
分批处理记录并保存到数据库
流式处理记录并分批保存
Args:
data_generator: 数据生成器
@@ -345,14 +530,23 @@ def _process_records_in_batches(
services: Service 集合
Returns:
dict: 处理统计结果
dict: 处理统计信息
Raises:
RuntimeError: 存在失败批次时抛出
"""
batch = []
batch_num = 0
total_records = 0
total_saved = 0
batch_num = 0
failed_batches = []
batch = []
# 统计信息
total_stats = {
'created_endpoints': 0,
'skipped_failed': 0
}
# 流式读取生成器并分批保存
for record in data_generator:
batch.append(record)
total_records += 1
@@ -360,46 +554,35 @@ def _process_records_in_batches(
# 达到批次大小,执行保存
if len(batch) >= batch_size:
batch_num += 1
result = _save_batch_with_retry(
batch, scan_id, target_id, batch_num, services
)
if result['success']:
total_saved += result['saved_count']
else:
failed_batches.append(batch_num)
_process_batch(batch, scan_id, target_id, batch_num, total_stats, failed_batches, services)
batch = [] # 清空批次
# 每 10 个批次输出进度
if batch_num % 10 == 0:
logger.info(
"进度: 已处理 %d 批次,%d 条记录,保存 %d",
batch_num, total_records, total_saved
)
logger.info("进度: 已处理 %d 批次,%d 条记录", batch_num, total_records)
# 保存最后一批
if batch:
batch_num += 1
result = _save_batch_with_retry(
batch, scan_id, target_id, batch_num, services
_process_batch(batch, scan_id, target_id, batch_num, total_stats, failed_batches, services)
# 检查失败批次
if failed_batches:
error_msg = (
f"流式保存 URL 验证结果时出现失败批次,处理记录: {total_records}"
f"失败批次: {failed_batches}"
)
if result['success']:
total_saved += result['saved_count']
else:
failed_batches.append(batch_num)
logger.warning(error_msg)
raise RuntimeError(error_msg)
return {
'processed_records': total_records,
'saved_urls': total_saved,
'failed_urls': total_records - total_saved,
'batch_count': batch_num,
'failed_batches': failed_batches
**total_stats
}
@task(name="run_and_stream_save_urls", retries=3, retry_delay_seconds=10)
@task(name="run_and_stream_save_urls", retries=0)
def run_and_stream_save_urls_task(
cmd: str,
tool_name: str,
@@ -407,7 +590,7 @@ def run_and_stream_save_urls_task(
target_id: int,
cwd: Optional[str] = None,
shell: bool = False,
batch_size: int = 500,
batch_size: int = 100,
timeout: Optional[int] = None,
log_file: Optional[str] = None
) -> dict:
@@ -415,17 +598,18 @@ def run_and_stream_save_urls_task(
执行 httpx 验证并流式保存存活的 URL
该任务将:
1. 执行 httpx 命令验证 URL 存活
2. 流式处理输出,实时解析
3. 批量保存存活的 URL 到 Endpoint 表
1. 验证输入参数
2. 初始化资源(缓存、生成器)
3. 流式处理记录并分批保存
4. 构建并返回结果统计
Args:
cmd: httpx 命令
tool_name: 工具名称('httpx'
scan_id: 扫描任务 ID
target_id: 目标 ID
cwd: 工作目录
shell: 是否使用 shell 执行
cwd: 工作目录(可选)
shell: 是否使用 shell 执行(默认 False
batch_size: 批次大小(默认 500
timeout: 超时时间(秒)
log_file: 日志文件路径
@@ -433,11 +617,14 @@ def run_and_stream_save_urls_task(
Returns:
dict: {
'processed_records': int, # 处理的记录总数
'saved_urls': int, # 保存的存活 URL
'failed_urls': int, # 失败/死链
'batch_count': int, # 批次数
'failed_batches': list # 失败的批次号
'created_endpoints': int, # 创建的端点记录
'skipped_failed': int, # 因请求失败跳过的记录
}
Raises:
ValueError: 参数验证失败
RuntimeError: 命令执行或数据库操作失败
subprocess.TimeoutExpired: 命令执行超时
"""
logger.info(
"开始执行流式 URL 验证任务 - target_id=%s, 超时=%s秒, 命令: %s",
@@ -447,33 +634,30 @@ def run_and_stream_save_urls_task(
data_generator = None
try:
# 1. 初始化资源
# 1. 验证参数
_validate_task_parameters(cmd, target_id, scan_id, cwd)
# 2. 初始化资源
data_generator = _parse_httpx_stream_output(
cmd, tool_name, cwd, shell, timeout, log_file
)
services = ServiceSet.create_default()
# 2. 流式处理记录并分批保存
# 3. 流式处理记录并分批保存
stats = _process_records_in_batches(
data_generator, scan_id, target_id, batch_size, services
)
# 3. 输出最终统计
logger.info(
"✓ URL 验证任务完成 - 处理: %d, 存活: %d, 失败: %d",
stats['processed_records'],
stats['saved_urls'],
stats['failed_urls']
)
return stats
# 4. 构建最终结果
return _build_final_result(stats)
except subprocess.TimeoutExpired:
# 超时异常直接向上传播,保留异常类型
logger.warning(
"⚠️ URL 验证任务超时 - target_id=%s, 超时=%s",
target_id, timeout
)
raise
raise # 直接重新抛出,不包装
except Exception as e:
error_msg = f"流式执行 URL 验证任务失败: {e}"
@@ -481,12 +665,5 @@ def run_and_stream_save_urls_task(
raise RuntimeError(error_msg) from e
finally:
# 清理资源
if data_generator is not None:
try:
# 确保生成器被正确关闭
data_generator.close()
except (GeneratorExit, StopIteration):
pass
except Exception as e:
logger.warning("关闭数据生成器时出错: %s", e)
# 5. 清理资源
_cleanup_resources(data_generator)

View File

@@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
from ..models import Scan, ScheduledScan
from ..serializers import (
ScanSerializer, ScanHistorySerializer, QuickScanSerializer,
ScheduledScanSerializer, CreateScheduledScanSerializer,
InitiateScanSerializer, ScheduledScanSerializer, CreateScheduledScanSerializer,
UpdateScheduledScanSerializer, ToggleScheduledScanSerializer
)
from ..services.scan_service import ScanService
@@ -111,7 +111,7 @@ class ScanViewSet(viewsets.ModelViewSet):
快速扫描接口
功能:
1. 接收目标列表和引擎配置
1. 接收目标列表和 YAML 配置
2. 自动解析输入(支持 URL、域名、IP、CIDR
3. 批量创建 Target、Website、Endpoint 资产
4. 立即发起批量扫描
@@ -119,7 +119,9 @@ class ScanViewSet(viewsets.ModelViewSet):
请求参数:
{
"targets": [{"name": "example.com"}, {"name": "https://example.com/api"}],
"engine_ids": [1, 2]
"configuration": "subdomain_discovery:\n enabled: true\n ...",
"engine_ids": [1, 2], // 可选,用于记录
"engine_names": ["引擎A", "引擎B"] // 可选,用于记录
}
支持的输入格式:
@@ -134,7 +136,9 @@ class ScanViewSet(viewsets.ModelViewSet):
serializer.is_valid(raise_exception=True)
targets_data = serializer.validated_data['targets']
engine_ids = serializer.validated_data.get('engine_ids')
configuration = serializer.validated_data['configuration']
engine_ids = serializer.validated_data.get('engine_ids', [])
engine_names = serializer.validated_data.get('engine_names', [])
try:
# 提取输入字符串列表
@@ -154,19 +158,13 @@ class ScanViewSet(viewsets.ModelViewSet):
status_code=status.HTTP_400_BAD_REQUEST
)
# 2. 准备多引擎扫描
# 2. 直接使用前端传递的配置创建扫描
scan_service = ScanService()
_, merged_configuration, engine_names, engine_ids = scan_service.prepare_initiate_scan_multi_engine(
target_id=targets[0].id, # 使用第一个目标来验证引擎
engine_ids=engine_ids
)
# 3. 批量发起扫描
created_scans = scan_service.create_scans(
targets=targets,
engine_ids=engine_ids,
engine_names=engine_names,
merged_configuration=merged_configuration
yaml_configuration=configuration
)
# 检查是否成功创建扫描任务
@@ -195,17 +193,6 @@ class ScanViewSet(viewsets.ModelViewSet):
},
status_code=status.HTTP_201_CREATED
)
except ConfigConflictError as e:
return error_response(
code='CONFIG_CONFLICT',
message=str(e),
details=[
{'key': k, 'engines': [e1, e2]}
for k, e1, e2 in e.conflicts
],
status_code=status.HTTP_400_BAD_REQUEST
)
except ValidationError as e:
return error_response(
@@ -228,48 +215,53 @@ class ScanViewSet(viewsets.ModelViewSet):
请求参数:
- organization_id: 组织ID (int, 可选)
- target_id: 目标ID (int, 可选)
- configuration: YAML 配置字符串 (str, 必填)
- engine_ids: 扫描引擎ID列表 (list[int], 必填)
- engine_names: 引擎名称列表 (list[str], 必填)
注意: organization_id 和 target_id 二选一
返回:
- 扫描任务详情(单个或多个)
"""
# 获取请求数据
organization_id = request.data.get('organization_id')
target_id = request.data.get('target_id')
engine_ids = request.data.get('engine_ids')
# 使用 serializer 验证请求数据
serializer = InitiateScanSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
# 验证 engine_ids
if not engine_ids:
return error_response(
code=ErrorCodes.VALIDATION_ERROR,
message='缺少必填参数: engine_ids',
status_code=status.HTTP_400_BAD_REQUEST
)
if not isinstance(engine_ids, list):
return error_response(
code=ErrorCodes.VALIDATION_ERROR,
message='engine_ids 必须是数组',
status_code=status.HTTP_400_BAD_REQUEST
)
# 获取验证后的数据
organization_id = serializer.validated_data.get('organization_id')
target_id = serializer.validated_data.get('target_id')
configuration = serializer.validated_data['configuration']
engine_ids = serializer.validated_data['engine_ids']
engine_names = serializer.validated_data['engine_names']
try:
# 步骤1准备多引擎扫描所需的数据
# 获取目标列表
scan_service = ScanService()
targets, merged_configuration, engine_names, engine_ids = scan_service.prepare_initiate_scan_multi_engine(
organization_id=organization_id,
target_id=target_id,
engine_ids=engine_ids
)
# 步骤2批量创建扫描记录并分发扫描任务
if organization_id:
from apps.targets.repositories import DjangoOrganizationRepository
org_repo = DjangoOrganizationRepository()
organization = org_repo.get_by_id(organization_id)
if not organization:
raise ObjectDoesNotExist(f'Organization ID {organization_id} 不存在')
targets = org_repo.get_targets(organization_id)
if not targets:
raise ValidationError(f'组织 ID {organization_id} 下没有目标')
else:
from apps.targets.repositories import DjangoTargetRepository
target_repo = DjangoTargetRepository()
target = target_repo.get_by_id(target_id)
if not target:
raise ObjectDoesNotExist(f'Target ID {target_id} 不存在')
targets = [target]
# 直接使用前端传递的配置创建扫描
created_scans = scan_service.create_scans(
targets=targets,
engine_ids=engine_ids,
engine_names=engine_names,
merged_configuration=merged_configuration
yaml_configuration=configuration
)
# 检查是否成功创建扫描任务
@@ -290,17 +282,6 @@ class ScanViewSet(viewsets.ModelViewSet):
},
status_code=status.HTTP_201_CREATED
)
except ConfigConflictError as e:
return error_response(
code='CONFIG_CONFLICT',
message=str(e),
details=[
{'key': k, 'engines': [e1, e2]}
for k, e1, e2 in e.conflicts
],
status_code=status.HTTP_400_BAD_REQUEST
)
except ObjectDoesNotExist as e:
# 资源不存在错误(由 service 层抛出)

View File

@@ -68,30 +68,22 @@ class ScheduledScanViewSet(viewsets.ModelViewSet):
data = serializer.validated_data
dto = ScheduledScanDTO(
name=data['name'],
engine_ids=data['engine_ids'],
engine_ids=data.get('engine_ids', []),
engine_names=data.get('engine_names', []),
yaml_configuration=data['configuration'],
organization_id=data.get('organization_id'),
target_id=data.get('target_id'),
cron_expression=data.get('cron_expression', '0 2 * * *'),
is_enabled=data.get('is_enabled', True),
)
scheduled_scan = self.service.create(dto)
scheduled_scan = self.service.create_with_configuration(dto)
response_serializer = ScheduledScanSerializer(scheduled_scan)
return success_response(
data=response_serializer.data,
status_code=status.HTTP_201_CREATED
)
except ConfigConflictError as e:
return error_response(
code='CONFIG_CONFLICT',
message=str(e),
details=[
{'key': k, 'engines': [e1, e2]}
for k, e1, e2 in e.conflicts
],
status_code=status.HTTP_400_BAD_REQUEST
)
except ValidationError as e:
return error_response(
code=ErrorCodes.VALIDATION_ERROR,

View File

@@ -0,0 +1,52 @@
# Generated by Django 5.2.7 on 2026-01-02 04:45
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Target',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('name', models.CharField(blank=True, default='', help_text='目标标识(域名/IP/CIDR', max_length=300)),
('type', models.CharField(choices=[('domain', '域名'), ('ip', 'IP地址'), ('cidr', 'CIDR范围')], db_index=True, default='domain', help_text='目标类型', max_length=20)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('last_scanned_at', models.DateTimeField(blank=True, help_text='最后扫描时间', null=True)),
('deleted_at', models.DateTimeField(blank=True, db_index=True, help_text='删除时间NULL表示未删除', null=True)),
],
options={
'verbose_name': '扫描目标',
'verbose_name_plural': '扫描目标',
'db_table': 'target',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['type'], name='target_type_36a73c_idx'), models.Index(fields=['-created_at'], name='target_created_67f489_idx'), models.Index(fields=['deleted_at', '-created_at'], name='target_deleted_9fc9da_idx'), models.Index(fields=['deleted_at', 'type'], name='target_deleted_306a89_idx'), models.Index(fields=['name'], name='target_name_f1c641_idx')],
'constraints': [models.UniqueConstraint(condition=models.Q(('deleted_at__isnull', True)), fields=('name',), name='unique_target_name_active')],
},
),
migrations.CreateModel(
name='Organization',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('name', models.CharField(blank=True, default='', help_text='组织名称', max_length=300)),
('description', models.CharField(blank=True, default='', help_text='组织描述', max_length=1000)),
('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
('deleted_at', models.DateTimeField(blank=True, db_index=True, help_text='删除时间NULL表示未删除', null=True)),
('targets', models.ManyToManyField(blank=True, help_text='所属目标列表', related_name='organizations', to='targets.target')),
],
options={
'verbose_name': '组织',
'verbose_name_plural': '组织',
'db_table': 'organization',
'ordering': ['-created_at'],
'indexes': [models.Index(fields=['-created_at'], name='organizatio_created_012eac_idx'), models.Index(fields=['deleted_at', '-created_at'], name='organizatio_deleted_2c604f_idx'), models.Index(fields=['name'], name='organizatio_name_bcc2ee_idx')],
'constraints': [models.UniqueConstraint(condition=models.Q(('deleted_at__isnull', True)), fields=('name',), name='unique_organization_name_active')],
},
),
]

View File

@@ -219,6 +219,8 @@ REST_FRAMEWORK = {
# 允许所有来源(前后端分离项目,安全性由认证系统保障)
CORS_ALLOW_ALL_ORIGINS = os.getenv('CORS_ALLOW_ALL_ORIGINS', 'True').lower() == 'true'
CORS_ALLOW_CREDENTIALS = True
# 暴露额外的响应头给前端Content-Disposition 用于文件下载获取文件名)
CORS_EXPOSE_HEADERS = ['Content-Disposition']
# ==================== CSRF 配置 ====================
CSRF_TRUSTED_ORIGINS = os.getenv('CSRF_TRUSTED_ORIGINS', 'http://localhost:3000,http://127.0.0.1:3000').split(',')

View File

@@ -260,6 +260,12 @@ class TestDataGenerator:
def clear_data(self):
"""清除所有测试数据"""
cur = self.conn.cursor()
# 先删除 IMMV避免 pg_ivm 的 anyarray bug
print(" 删除 IMMV...")
cur.execute("DROP TABLE IF EXISTS asset_search_view CASCADE")
self.conn.commit()
tables = [
# 指纹表
'ehole_fingerprint', 'goby_fingerprint', 'wappalyzer_fingerprint',
@@ -276,6 +282,26 @@ class TestDataGenerator:
for table in tables:
cur.execute(f"DELETE FROM {table}")
self.conn.commit()
# 重建 IMMV
print(" 重建 IMMV...")
cur.execute("""
SELECT pgivm.create_immv('asset_search_view', $$
SELECT
w.id,
w.url,
w.host,
w.title,
w.tech,
w.status_code,
w.response_headers,
w.response_body,
w.created_at,
w.target_id
FROM website w
$$)
""")
self.conn.commit()
print(" ✓ 数据清除完成\n")
def create_workers(self) -> list:
@@ -610,7 +636,7 @@ class TestDataGenerator:
cur.execute("""
INSERT INTO scan (
target_id, engine_ids, engine_names, merged_configuration, status, worker_id, progress, current_stage,
target_id, engine_ids, engine_names, yaml_configuration, status, worker_id, progress, current_stage,
results_dir, error_message, container_ids, stage_progress,
cached_subdomains_count, cached_websites_count, cached_endpoints_count,
cached_ips_count, cached_directories_count, cached_vulns_total,
@@ -723,7 +749,7 @@ class TestDataGenerator:
cur.execute("""
INSERT INTO scheduled_scan (
name, engine_ids, engine_names, merged_configuration, organization_id, target_id, cron_expression, is_enabled,
name, engine_ids, engine_names, yaml_configuration, organization_id, target_id, cron_expression, is_enabled,
run_count, last_run_time, next_run_time, created_at, updated_at
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW() - INTERVAL '%s days', NOW())
ON CONFLICT DO NOTHING
@@ -1248,77 +1274,79 @@ class TestDataGenerator:
print(f" ✓ 创建了 {count} 个主机端口映射\n")
def create_vulnerabilities(self, target_ids: list):
"""创建漏洞"""
"""创建漏洞(基于 website URL 前缀)"""
print("🐛 创建漏洞...")
cur = self.conn.cursor()
vuln_types = [
'sql-injection-authentication-bypass-vulnerability-', # 50 chars
'cross-site-scripting-xss-stored-persistent-attack-', # 50 chars
'cross-site-request-forgery-csrf-token-validation--', # 50 chars
'server-side-request-forgery-ssrf-internal-access--', # 50 chars
'xml-external-entity-xxe-injection-vulnerability---', # 50 chars
'remote-code-execution-rce-command-injection-flaw--', # 50 chars
'local-file-inclusion-lfi-path-traversal-exploit---', # 50 chars
'directory-traversal-arbitrary-file-read-access----', # 50 chars
'authentication-bypass-session-management-flaw-----', # 50 chars
'insecure-direct-object-reference-idor-access-ctrl-', # 50 chars
'sensitive-data-exposure-information-disclosure----', # 50 chars
'security-misconfiguration-default-credentials-----', # 50 chars
'broken-access-control-privilege-escalation-vuln---', # 50 chars
'cors-misconfiguration-cross-origin-data-leakage---', # 50 chars
'subdomain-takeover-dns-misconfiguration-exploit---', # 50 chars
'exposed-admin-panel-unauthorized-access-control---', # 50 chars
'default-credentials-weak-authentication-bypass----', # 50 chars
'information-disclosure-sensitive-data-exposure----', # 50 chars
'command-injection-os-command-execution-exploit----', # 50 chars
'ldap-injection-directory-service-manipulation-----', # 50 chars
'xpath-injection-xml-query-manipulation-attack-----', # 50 chars
'nosql-injection-mongodb-query-manipulation--------', # 50 chars
'template-injection-ssti-server-side-execution-----', # 50 chars
'deserialization-vulnerability-object-injection----', # 50 chars
'jwt-vulnerability-token-forgery-authentication----', # 50 chars
'open-redirect-url-redirection-phishing-attack-----', # 50 chars
'http-request-smuggling-cache-poisoning-attack-----', # 50 chars
'host-header-injection-password-reset-poisoning----', # 50 chars
'clickjacking-ui-redressing-frame-injection--------', # 50 chars
'session-fixation-authentication-session-attack----', # 50 chars
'sql-injection-authentication-bypass-vulnerability-',
'cross-site-scripting-xss-stored-persistent-attack-',
'cross-site-request-forgery-csrf-token-validation--',
'server-side-request-forgery-ssrf-internal-access--',
'xml-external-entity-xxe-injection-vulnerability---',
'remote-code-execution-rce-command-injection-flaw--',
'local-file-inclusion-lfi-path-traversal-exploit---',
'directory-traversal-arbitrary-file-read-access----',
'authentication-bypass-session-management-flaw-----',
'insecure-direct-object-reference-idor-access-ctrl-',
'sensitive-data-exposure-information-disclosure----',
'security-misconfiguration-default-credentials-----',
'broken-access-control-privilege-escalation-vuln---',
'cors-misconfiguration-cross-origin-data-leakage---',
'subdomain-takeover-dns-misconfiguration-exploit---',
'exposed-admin-panel-unauthorized-access-control---',
'default-credentials-weak-authentication-bypass----',
'information-disclosure-sensitive-data-exposure----',
'command-injection-os-command-execution-exploit----',
'ldap-injection-directory-service-manipulation-----',
]
sources = [
'nuclei-vulnerability-scanner--', # 30 chars
'dalfox-xss-parameter-analysis-', # 30 chars
'sqlmap-sql-injection-testing--', # 30 chars
'crlfuzz-crlf-injection-finder-', # 30 chars
'httpx-web-probe-fingerprint---', # 30 chars
'manual-penetration-testing----', # 30 chars
'burp-suite-professional-scan--', # 30 chars
'owasp-zap-security-scanner----', # 30 chars
'nmap-network-service-scanner--', # 30 chars
'nikto-web-server-scanner------', # 30 chars
'wpscan-wordpress-vuln-scan----', # 30 chars
'dirsearch-directory-brute-----', # 30 chars
'ffuf-web-fuzzer-content-disc--', # 30 chars
'amass-subdomain-enumeration---', # 30 chars
'subfinder-passive-subdomain---', # 30 chars
'masscan-port-scanner-fast-----', # 30 chars
'nessus-vulnerability-assess---', # 30 chars
'qualys-cloud-security-scan----', # 30 chars
'acunetix-web-vuln-scanner-----', # 30 chars
'semgrep-static-code-analysis--', # 30 chars
'nuclei-vulnerability-scanner--',
'dalfox-xss-parameter-analysis-',
'sqlmap-sql-injection-testing--',
'crlfuzz-crlf-injection-finder-',
'httpx-web-probe-fingerprint---',
'manual-penetration-testing----',
'burp-suite-professional-scan--',
'owasp-zap-security-scanner----',
]
severities = ['unknown', 'info', 'low', 'medium', 'high', 'critical']
# 获取域名目标
cur.execute("SELECT id, name FROM target WHERE type = 'domain' AND deleted_at IS NULL LIMIT 80")
domain_targets = cur.fetchall()
# 漏洞路径后缀(会追加到 website URL 后面)
vuln_paths = [
'/api/users?id=1',
'/api/admin/config',
'/api/v1/auth/login',
'/api/v2/data/export',
'/admin/settings',
'/debug/console',
'/backup/db.sql',
'/.env',
'/.git/config',
'/wp-admin/',
'/phpmyadmin/',
'/api/graphql',
'/swagger.json',
'/actuator/health',
'/metrics',
]
# 获取所有 website 的 URL 和 target_id
cur.execute("SELECT id, url, target_id FROM website LIMIT 500")
websites = cur.fetchall()
if not websites:
print(" ⚠ 没有 website 数据,跳过漏洞生成\n")
return
count = 0
batch_data = []
for target_id, target_name in domain_targets:
num = random.randint(30, 80)
for website_id, website_url, target_id in websites:
# 每个 website 生成 1-5 个漏洞
num_vulns = random.randint(1, 5)
for idx in range(num):
for idx in range(num_vulns):
severity = random.choice(severities)
cvss_ranges = {
'critical': (9.0, 10.0), 'high': (7.0, 8.9), 'medium': (4.0, 6.9),
@@ -1327,22 +1355,22 @@ class TestDataGenerator:
cvss_range = cvss_ranges.get(severity, (0.0, 10.0))
cvss_score = round(random.uniform(*cvss_range), 1)
# 生成固定 245 长度的 URL
url = generate_fixed_length_url(target_name, length=245, path_hint=f'vuln/{idx:04d}')
# 漏洞 URL = website URL + 漏洞路径
# 先移除 website URL 中的查询参数
base_url = website_url.split('?')[0]
vuln_url = base_url + random.choice(vuln_paths)
# 生成固定 300 长度的描述
description = generate_fixed_length_text(length=300, text_type='description')
raw_output = json.dumps({
'template': f'CVE-2024-{random.randint(10000, 99999)}',
'matcher_name': 'default',
'severity': severity,
'host': target_name,
'matched_at': url,
'matched_at': vuln_url,
})
batch_data.append((
target_id, url, random.choice(vuln_types), severity,
target_id, vuln_url, random.choice(vuln_types), severity,
random.choice(sources), cvss_score, description, raw_output
))
count += 1
@@ -1488,7 +1516,7 @@ class TestDataGenerator:
if batch_data:
execute_values(cur, """
INSERT INTO website_snapshot (
scan_id, url, host, title, web_server, tech, status,
scan_id, url, host, title, webserver, tech, status_code,
content_length, content_type, location, response_body,
response_headers, created_at
) VALUES %s

View File

@@ -27,10 +27,50 @@ BLUE='\033[0;34m'
RED='\033[0;31m'
NC='\033[0m'
log_info() { echo -e "${BLUE}[XingRin]${NC} $1"; }
log_success() { echo -e "${GREEN}[XingRin]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[XingRin]${NC} $1"; }
log_error() { echo -e "${RED}[XingRin]${NC} $1"; }
# 渐变色定义
CYAN='\033[0;36m'
MAGENTA='\033[0;35m'
BOLD='\033[1m'
DIM='\033[2m'
log_info() { echo -e "${CYAN}${NC} $1"; }
log_success() { echo -e "${GREEN}${NC} $1"; }
log_warn() { echo -e "${YELLOW}${NC} $1"; }
log_error() { echo -e "${RED}${NC} $1"; }
# 炫酷 Banner
show_banner() {
echo -e ""
echo -e "${CYAN}${BOLD} ██╗ ██╗██╗███╗ ██╗ ██████╗ ██████╗ ██╗███╗ ██╗${NC}"
echo -e "${CYAN} ╚██╗██╔╝██║████╗ ██║██╔════╝ ██╔══██╗██║████╗ ██║${NC}"
echo -e "${BLUE}${BOLD} ╚███╔╝ ██║██╔██╗ ██║██║ ███╗██████╔╝██║██╔██╗ ██║${NC}"
echo -e "${BLUE} ██╔██╗ ██║██║╚██╗██║██║ ██║██╔══██╗██║██║╚██╗██║${NC}"
echo -e "${MAGENTA}${BOLD} ██╔╝ ██╗██║██║ ╚████║╚██████╔╝██║ ██║██║██║ ╚████║${NC}"
echo -e "${MAGENTA} ╚═╝ ╚═╝╚═╝╚═╝ ╚═══╝ ╚═════╝ ╚═╝ ╚═╝╚═╝╚═╝ ╚═══╝${NC}"
echo -e ""
echo -e "${DIM} ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${BOLD} 🚀 分布式安全扫描平台 │ Worker 节点部署${NC}"
echo -e "${DIM} ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e ""
}
# 完成 Banner
show_complete() {
echo -e ""
echo -e "${GREEN}${BOLD} ╔═══════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN}${BOLD} ║ ║${NC}"
echo -e "${GREEN}${BOLD} ║ ██████╗ ██████╗ ███╗ ██╗███████╗██╗ ║${NC}"
echo -e "${GREEN}${BOLD} ║ ██╔══██╗██╔═══██╗████╗ ██║██╔════╝██║ ║${NC}"
echo -e "${GREEN}${BOLD} ║ ██║ ██║██║ ██║██╔██╗ ██║█████╗ ██║ ║${NC}"
echo -e "${GREEN}${BOLD} ║ ██║ ██║██║ ██║██║╚██╗██║██╔══╝ ╚═╝ ║${NC}"
echo -e "${GREEN}${BOLD} ║ ██████╔╝╚██████╔╝██║ ╚████║███████╗██╗ ║${NC}"
echo -e "${GREEN}${BOLD} ║ ╚═════╝ ╚═════╝ ╚═╝ ╚═══╝╚══════╝╚═╝ ║${NC}"
echo -e "${GREEN}${BOLD} ║ ║${NC}"
echo -e "${GREEN}${BOLD} ║ ✨ XingRin Worker 节点部署完成! ║${NC}"
echo -e "${GREEN}${BOLD} ║ ║${NC}"
echo -e "${GREEN}${BOLD} ╚═══════════════════════════════════════════════════╝${NC}"
echo -e ""
}
# 等待 apt 锁释放
wait_for_apt_lock() {
@@ -150,9 +190,7 @@ pull_image() {
# 主流程
main() {
log_info "=========================================="
log_info " XingRin 节点安装"
log_info "=========================================="
show_banner
detect_os
install_docker
@@ -162,9 +200,7 @@ main() {
touch "$DOCKER_MARKER"
log_success "=========================================="
log_success " ✓ 安装完成"
log_success "=========================================="
show_complete
}
main "$@"

View File

@@ -2,9 +2,13 @@ services:
# PostgreSQL可选使用远程数据库时不启动
# 本地模式: docker compose --profile local-db up -d
# 远程模式: docker compose up -d需配置 DB_HOST 为远程地址)
# 使用自定义镜像,预装 pg_ivm 扩展
postgres:
profiles: ["local-db"]
image: postgres:15
build:
context: ./postgres
dockerfile: Dockerfile
image: ${DOCKER_USER:-yyhuni}/xingrin-postgres:${IMAGE_TAG:-dev}
restart: always
environment:
POSTGRES_DB: ${DB_NAME}
@@ -15,6 +19,9 @@ services:
- ./postgres/init-user-db.sh:/docker-entrypoint-initdb.d/init-user-db.sh
ports:
- "${DB_PORT}:5432"
command: >
postgres
-c shared_preload_libraries=pg_ivm
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${DB_USER}"]
interval: 5s

View File

@@ -8,9 +8,13 @@
services:
# PostgreSQL可选使用远程数据库时不启动
# 使用自定义镜像,预装 pg_ivm 扩展
postgres:
profiles: ["local-db"]
image: postgres:15
build:
context: ./postgres
dockerfile: Dockerfile
image: ${DOCKER_USER:-yyhuni}/xingrin-postgres:${IMAGE_TAG:?IMAGE_TAG is required}
restart: always
environment:
POSTGRES_DB: ${DB_NAME}
@@ -21,6 +25,9 @@ services:
- ./postgres/init-user-db.sh:/docker-entrypoint-initdb.d/init-user-db.sh
ports:
- "${DB_PORT}:5432"
command: >
postgres
-c shared_preload_libraries=pg_ivm
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${DB_USER}"]
interval: 5s

View File

@@ -38,6 +38,8 @@ http {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_read_timeout 300s; # 5分钟支持大数据量导出
proxy_send_timeout 300s;
proxy_pass http://backend;
}

View File

@@ -0,0 +1,19 @@
FROM postgres:15
# 安装编译依赖
RUN apt-get update && apt-get install -y \
build-essential \
postgresql-server-dev-15 \
git \
&& rm -rf /var/lib/apt/lists/*
# 编译安装 pg_ivm
RUN git clone https://github.com/sraoss/pg_ivm.git /tmp/pg_ivm \
&& cd /tmp/pg_ivm \
&& make \
&& make install \
&& rm -rf /tmp/pg_ivm
# 配置 shared_preload_libraries
# 注意: 这个配置会在容器启动时被应用
RUN echo "shared_preload_libraries = 'pg_ivm'" >> /usr/share/postgresql/postgresql.conf.sample

129
docker/scripts/install-pg-ivm.sh Executable file
View File

@@ -0,0 +1,129 @@
#!/bin/bash
# pg_ivm 一键安装脚本(用于远程自建 PostgreSQL 服务器)
# 要求: PostgreSQL 13+ 版本
set -e
echo "=========================================="
echo "pg_ivm 一键安装脚本"
echo "要求: PostgreSQL 13+ 版本"
echo "=========================================="
echo ""
# 检查是否以 root 运行
if [ "$EUID" -ne 0 ]; then
echo "错误: 请使用 sudo 运行此脚本"
exit 1
fi
# 检测 PostgreSQL 版本
detect_pg_version() {
if command -v psql &> /dev/null; then
psql --version | grep -oP '\d+' | head -1
elif [ -n "$PG_VERSION" ]; then
echo "$PG_VERSION"
else
echo "15"
fi
}
PG_VERSION=${PG_VERSION:-$(detect_pg_version)}
# 检测 PostgreSQL
if ! command -v psql &> /dev/null; then
echo "错误: 未检测到 PostgreSQL请先安装 PostgreSQL"
exit 1
fi
echo "检测到 PostgreSQL 版本: $PG_VERSION"
# 检查版本要求
if [ "$PG_VERSION" -lt 13 ]; then
echo "错误: pg_ivm 要求 PostgreSQL 13+ 版本,当前版本: $PG_VERSION"
exit 1
fi
# 安装编译依赖
echo ""
echo "[1/4] 安装编译依赖..."
if command -v apt-get &> /dev/null; then
apt-get update -qq
apt-get install -y -qq build-essential postgresql-server-dev-${PG_VERSION} git
elif command -v yum &> /dev/null; then
yum install -y gcc make git postgresql${PG_VERSION}-devel
else
echo "错误: 不支持的包管理器,请手动安装编译依赖"
exit 1
fi
echo "✓ 编译依赖安装完成"
# 编译安装 pg_ivm
echo ""
echo "[2/4] 编译安装 pg_ivm..."
rm -rf /tmp/pg_ivm
git clone --quiet https://github.com/sraoss/pg_ivm.git /tmp/pg_ivm
cd /tmp/pg_ivm
make -s
make install -s
rm -rf /tmp/pg_ivm
echo "✓ pg_ivm 编译安装完成"
# 配置 shared_preload_libraries
echo ""
echo "[3/4] 配置 shared_preload_libraries..."
PG_CONF_DIRS=(
"/etc/postgresql/${PG_VERSION}/main"
"/var/lib/pgsql/${PG_VERSION}/data"
"/var/lib/postgresql/data"
)
PG_CONF_DIR=""
for dir in "${PG_CONF_DIRS[@]}"; do
if [ -d "$dir" ]; then
PG_CONF_DIR="$dir"
break
fi
done
if [ -z "$PG_CONF_DIR" ]; then
echo "警告: 未找到 PostgreSQL 配置目录,请手动配置 shared_preload_libraries"
echo "在 postgresql.conf 中添加: shared_preload_libraries = 'pg_ivm'"
else
if grep -q "shared_preload_libraries.*pg_ivm" "$PG_CONF_DIR/postgresql.conf" 2>/dev/null; then
echo "✓ shared_preload_libraries 已配置"
else
if [ -d "$PG_CONF_DIR/conf.d" ]; then
echo "shared_preload_libraries = 'pg_ivm'" > "$PG_CONF_DIR/conf.d/pg_ivm.conf"
echo "✓ 配置已写入 $PG_CONF_DIR/conf.d/pg_ivm.conf"
else
if grep -q "^shared_preload_libraries" "$PG_CONF_DIR/postgresql.conf"; then
sed -i "s/^shared_preload_libraries = '\(.*\)'/shared_preload_libraries = '\1,pg_ivm'/" "$PG_CONF_DIR/postgresql.conf"
else
echo "shared_preload_libraries = 'pg_ivm'" >> "$PG_CONF_DIR/postgresql.conf"
fi
echo "✓ 配置已写入 $PG_CONF_DIR/postgresql.conf"
fi
fi
fi
# 重启 PostgreSQL
echo ""
echo "[4/4] 重启 PostgreSQL..."
if systemctl is-active --quiet postgresql; then
systemctl restart postgresql
echo "✓ PostgreSQL 已重启"
elif systemctl is-active --quiet postgresql-${PG_VERSION}; then
systemctl restart postgresql-${PG_VERSION}
echo "✓ PostgreSQL 已重启"
else
echo "警告: 无法自动重启 PostgreSQL请手动重启"
fi
echo ""
echo "=========================================="
echo "✓ pg_ivm 安装完成"
echo "=========================================="
echo ""
echo "验证安装:"
echo " psql -U postgres -c \"CREATE EXTENSION IF NOT EXISTS pg_ivm;\""
echo ""

126
docker/scripts/test-pg-ivm.sh Executable file
View File

@@ -0,0 +1,126 @@
#!/bin/bash
# pg_ivm 安装验证测试
# 在 Docker 容器中测试 install-pg-ivm.sh 的安装流程
set -e
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CONTAINER_NAME="pg_ivm_test_$$"
IMAGE_NAME="postgres:15"
echo "=========================================="
echo "pg_ivm 安装验证测试"
echo "=========================================="
# 清理函数
cleanup() {
echo ""
echo "[清理] 删除测试容器..."
docker rm -f "$CONTAINER_NAME" 2>/dev/null || true
}
trap cleanup EXIT
# 1. 启动临时容器
echo ""
echo "[1/5] 启动临时 PostgreSQL 容器..."
docker run -d --name "$CONTAINER_NAME" \
-e POSTGRES_PASSWORD=test \
-e POSTGRES_USER=postgres \
-e POSTGRES_DB=testdb \
-e PG_VERSION=15 \
"$IMAGE_NAME"
echo "等待 PostgreSQL 启动..."
sleep 10
if ! docker ps | grep -q "$CONTAINER_NAME"; then
echo "错误: 容器启动失败"
exit 1
fi
# 2. 复制并执行安装脚本
echo ""
echo "[2/5] 执行 pg_ivm 安装脚本..."
docker cp "$SCRIPT_DIR/install-pg-ivm.sh" "$CONTAINER_NAME:/tmp/install-pg-ivm.sh"
# 在容器内模拟安装(跳过 systemctl 重启,手动重启容器)
docker exec "$CONTAINER_NAME" bash -c "
set -e
export PG_VERSION=15
echo '安装编译依赖...'
apt-get update -qq
apt-get install -y -qq build-essential postgresql-server-dev-15 git
echo '编译安装 pg_ivm...'
rm -rf /tmp/pg_ivm
git clone --quiet https://github.com/sraoss/pg_ivm.git /tmp/pg_ivm
cd /tmp/pg_ivm
make -s
make install -s
rm -rf /tmp/pg_ivm
echo '✓ pg_ivm 编译安装完成'
"
# 3. 配置 shared_preload_libraries 并重启
echo ""
echo "[3/5] 配置 shared_preload_libraries..."
docker exec "$CONTAINER_NAME" bash -c "
echo \"shared_preload_libraries = 'pg_ivm'\" >> /var/lib/postgresql/data/postgresql.conf
"
echo "重启 PostgreSQL..."
docker restart "$CONTAINER_NAME"
sleep 8
# 4. 验证扩展是否可用
echo ""
echo "[4/5] 验证 pg_ivm 扩展..."
docker exec "$CONTAINER_NAME" psql -U postgres -d testdb -c "CREATE EXTENSION IF NOT EXISTS pg_ivm;" > /dev/null 2>&1
EXTENSION_EXISTS=$(docker exec "$CONTAINER_NAME" psql -U postgres -d testdb -t -c "SELECT COUNT(*) FROM pg_extension WHERE extname = 'pg_ivm';")
if [ "$(echo $EXTENSION_EXISTS | tr -d ' ')" != "1" ]; then
echo "错误: pg_ivm 扩展未正确加载"
exit 1
fi
echo "✓ pg_ivm 扩展已加载"
# 5. 测试 IMMV 功能
echo ""
echo "[5/5] 测试 IMMV 增量更新功能..."
docker exec "$CONTAINER_NAME" psql -U postgres -d testdb -c "
CREATE TABLE test_table (id SERIAL PRIMARY KEY, name TEXT, value INTEGER);
SELECT pgivm.create_immv('test_immv', 'SELECT id, name, value FROM test_table');
INSERT INTO test_table (name, value) VALUES ('test1', 100);
INSERT INTO test_table (name, value) VALUES ('test2', 200);
" > /dev/null 2>&1
IMMV_COUNT=$(docker exec "$CONTAINER_NAME" psql -U postgres -d testdb -t -c "SELECT COUNT(*) FROM test_immv;")
if [ "$(echo $IMMV_COUNT | tr -d ' ')" != "2" ]; then
echo "错误: IMMV 增量更新失败,期望 2 行,实际 $(echo $IMMV_COUNT | tr -d ' ')"
exit 1
fi
echo "✓ IMMV 增量更新正常 (2 行数据)"
# 测试更新
docker exec "$CONTAINER_NAME" psql -U postgres -d testdb -c "UPDATE test_table SET value = 150 WHERE name = 'test1';" > /dev/null 2>&1
UPDATED_VALUE=$(docker exec "$CONTAINER_NAME" psql -U postgres -d testdb -t -c "SELECT value FROM test_immv WHERE name = 'test1';")
if [ "$(echo $UPDATED_VALUE | tr -d ' ')" != "150" ]; then
echo "错误: IMMV 更新同步失败"
exit 1
fi
echo "✓ IMMV 更新同步正常"
# 测试删除
docker exec "$CONTAINER_NAME" psql -U postgres -d testdb -c "DELETE FROM test_table WHERE name = 'test2';" > /dev/null 2>&1
IMMV_COUNT_AFTER=$(docker exec "$CONTAINER_NAME" psql -U postgres -d testdb -t -c "SELECT COUNT(*) FROM test_immv;")
if [ "$(echo $IMMV_COUNT_AFTER | tr -d ' ')" != "1" ]; then
echo "错误: IMMV 删除同步失败"
exit 1
fi
echo "✓ IMMV 删除同步正常"
echo ""
echo "=========================================="
echo "✓ 所有测试通过"
echo "=========================================="
echo ""
echo "pg_ivm 安装验证成功,可以继续构建自定义 PostgreSQL 镜像"

View File

@@ -1,4 +1,4 @@
FROM python:3.10-slim
FROM python:3.10-slim-bookworm
WORKDIR /app
@@ -11,7 +11,16 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/*
# 安装 Docker CLI用于本地 Worker 任务分发)
RUN curl -fsSL https://get.docker.com | sh
# 只安装 docker-ce-cli避免安装完整 Docker 引擎
RUN apt-get update && \
apt-get install -y ca-certificates gnupg && \
install -m 0755 -d /etc/apt/keyrings && \
curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
chmod a+r /etc/apt/keyrings/docker.gpg && \
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian bookworm stable" > /etc/apt/sources.list.d/docker.list && \
apt-get update && \
apt-get install -y docker-ce-cli && \
rm -rf /var/lib/apt/lists/*
# 安装 uv超快的 Python 包管理器)
RUN pip install uv

View File

@@ -15,10 +15,12 @@ NC='\033[0m'
# 解析参数
WITH_FRONTEND=true
DEV_MODE=false
QUIET_MODE=false
for arg in "$@"; do
case $arg in
--no-frontend) WITH_FRONTEND=false ;;
--dev) DEV_MODE=true ;;
--quiet) QUIET_MODE=true ;;
esac
done
@@ -155,6 +157,11 @@ echo -e "${GREEN}[OK]${NC} 服务已启动"
# 数据初始化
./scripts/init-data.sh
# 静默模式下不显示结果(由调用方显示)
if [ "$QUIET_MODE" = true ]; then
exit 0
fi
# 获取访问地址
PUBLIC_HOST=$(grep "^PUBLIC_HOST=" .env 2>/dev/null | cut -d= -f2)
if [ -n "$PUBLIC_HOST" ] && [ "$PUBLIC_HOST" != "server" ]; then

View File

@@ -1,6 +1,5 @@
# 第一阶段:使用 Go 官方镜像编译工具
# 锁定 digest 避免上游更新导致缓存失效
FROM golang:1.24@sha256:7e050c14ae9ca5ae56408a288336545b18632f51402ab0ec8e7be0e649a1fc42 AS go-builder
FROM golang:1.24 AS go-builder
ENV GOPROXY=https://goproxy.cn,direct
# Naabu 需要 CGO 和 libpcap
@@ -37,8 +36,7 @@ RUN CGO_ENABLED=0 go install -v github.com/owasp-amass/amass/v5/cmd/amass@main
RUN go install github.com/hahwul/dalfox/v2@latest
# 第二阶段:运行时镜像
# 锁定 digest 避免上游更新导致缓存失效
FROM ubuntu:24.04@sha256:4fdf0125919d24aec972544669dcd7d6a26a8ad7e6561c73d5549bd6db258ac2
FROM ubuntu:24.04
# 避免交互式提示
ENV DEBIAN_FRONTEND=noninteractive

View File

@@ -0,0 +1,5 @@
import { SearchPage } from "@/components/search"
export default function Search() {
return <SearchPage />
}

View File

@@ -44,7 +44,6 @@
--font-sans: 'Noto Sans SC', system-ui, -apple-system, PingFang SC, sans-serif;
--font-mono: 'JetBrains Mono', 'Fira Code', Consolas, monospace;
--font-serif: Georgia, 'Noto Serif SC', serif;
--radius: 0.625rem;
--tracking-tighter: calc(var(--tracking-normal) - 0.05em);
--tracking-tight: calc(var(--tracking-normal) - 0.025em);
--tracking-wide: calc(var(--tracking-normal) + 0.025em);

View File

@@ -16,6 +16,7 @@ import {
IconTerminal2, // Terminal icon
IconBug, // Vulnerability icon
IconMessageReport, // Feedback icon
IconSearch, // Search icon
} from "@tabler/icons-react"
// Import internationalization hook
import { useTranslations } from 'next-intl'
@@ -76,6 +77,11 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
url: "/dashboard/",
icon: IconDashboard,
},
{
title: t('search'),
url: "/search/",
icon: IconSearch,
},
{
title: t('organization'),
url: "/organization/",

View File

@@ -67,6 +67,45 @@ const DEFAULT_FIELDS: FilterField[] = [
PREDEFINED_FIELDS.host,
]
// History storage key
const FILTER_HISTORY_KEY = 'smart_filter_history'
const MAX_HISTORY_PER_FIELD = 10
// Get history values for a field
function getFieldHistory(field: string): string[] {
if (typeof window === 'undefined') return []
try {
const history = JSON.parse(localStorage.getItem(FILTER_HISTORY_KEY) || '{}')
return history[field] || []
} catch {
return []
}
}
// Save a value to field history
function saveFieldHistory(field: string, value: string) {
if (typeof window === 'undefined' || !value.trim()) return
try {
const history = JSON.parse(localStorage.getItem(FILTER_HISTORY_KEY) || '{}')
const fieldHistory = (history[field] || []).filter((v: string) => v !== value)
fieldHistory.unshift(value)
history[field] = fieldHistory.slice(0, MAX_HISTORY_PER_FIELD)
localStorage.setItem(FILTER_HISTORY_KEY, JSON.stringify(history))
} catch {
// ignore
}
}
// Extract field-value pairs from query and save to history
function saveQueryHistory(query: string) {
const regex = /(\w+)(==|!=|=)"([^"]+)"/g
let match
while ((match = regex.exec(query)) !== null) {
const [, field, , value] = match
saveFieldHistory(field, value)
}
}
// Parse filter expression (FOFA style)
interface ParsedFilter {
field: string
@@ -115,10 +154,114 @@ export function SmartFilterInput({
const [open, setOpen] = React.useState(false)
const [inputValue, setInputValue] = React.useState(value ?? "")
const inputRef = React.useRef<HTMLInputElement>(null)
const ghostRef = React.useRef<HTMLSpanElement>(null)
const listRef = React.useRef<HTMLDivElement>(null)
const savedScrollTop = React.useRef<number | null>(null)
const hasInitialized = React.useRef(false)
// Calculate ghost text suggestion
const ghostText = React.useMemo(() => {
if (!inputValue) return ""
// Get the last word/token being typed
const lastSpaceIndex = inputValue.lastIndexOf(' ')
const currentToken = lastSpaceIndex === -1 ? inputValue : inputValue.slice(lastSpaceIndex + 1)
const lowerToken = currentToken.toLowerCase()
// If empty token after space, check if previous expression is complete
if (!currentToken && inputValue.trim()) {
// Check if last expression is complete (ends with ")
if (inputValue.trimEnd().endsWith('"')) {
return '&& '
}
return ""
}
if (!currentToken) return ""
// Priority 1: Field name completion (no = in token)
if (!currentToken.includes('=') && !currentToken.includes('!')) {
// Find matching field first
const matchingField = fields.find(f =>
f.key.toLowerCase().startsWith(lowerToken) &&
f.key.toLowerCase() !== lowerToken
)
if (matchingField) {
return matchingField.key.slice(currentToken.length) + '="'
}
// If exact match of field name, suggest ="
const exactField = fields.find(f => f.key.toLowerCase() === lowerToken)
if (exactField) {
return '="'
}
// Priority 2: Logical operators (only if no field matches)
if ('&&'.startsWith(currentToken) && currentToken.startsWith('&')) {
return '&&'.slice(currentToken.length) + ' '
}
if ('||'.startsWith(currentToken) && currentToken.startsWith('|')) {
return '||'.slice(currentToken.length) + ' '
}
// 'and' / 'or' only if no field name starts with these
if (!matchingField) {
if ('and'.startsWith(lowerToken) && lowerToken.length > 0 && !fields.some(f => f.key.toLowerCase().startsWith(lowerToken))) {
return 'and'.slice(lowerToken.length) + ' '
}
if ('or'.startsWith(lowerToken) && lowerToken.length > 0 && !fields.some(f => f.key.toLowerCase().startsWith(lowerToken))) {
return 'or'.slice(lowerToken.length) + ' '
}
}
return ""
}
// Check if typing ! for != operator
if (currentToken.match(/^(\w+)!$/)) {
return '="'
}
// Check if typing = and might want ==
const singleEqMatch = currentToken.match(/^(\w+)=$/)
if (singleEqMatch) {
// Suggest " for fuzzy match (most common)
return '"'
}
// Check if typed == or != (no opening quote yet)
const doubleOpMatch = currentToken.match(/^(\w+)(==|!=)$/)
if (doubleOpMatch) {
return '"'
}
// Check if typing a value (has = and opening quote)
const eqMatch = currentToken.match(/^(\w+)(==|!=|=)"([^"]*)$/)
if (eqMatch) {
const [, field, , partialValue] = eqMatch
// Get history for this field
const history = getFieldHistory(field)
// Find matching history value
const matchingValue = history.find(v =>
v.toLowerCase().startsWith(partialValue.toLowerCase()) &&
v.toLowerCase() !== partialValue.toLowerCase()
)
if (matchingValue) {
return matchingValue.slice(partialValue.length) + '"'
}
// If value has content but no closing quote, suggest closing quote
if (partialValue.length > 0) {
return '"'
}
}
// Check if a complete expression just finished (ends with ")
if (currentToken.match(/^\w+(==|!=|=)"[^"]+"$/)) {
return ' && '
}
return ""
}, [inputValue, fields])
// Synchronize external value changes
React.useEffect(() => {
if (value !== undefined) {
@@ -189,12 +332,27 @@ export function SmartFilterInput({
// Handle search
const handleSearch = () => {
// Save query values to history
saveQueryHistory(inputValue)
onSearch?.(parsedFilters, inputValue)
setOpen(false)
}
// Accept ghost text suggestion
const acceptGhostText = () => {
if (ghostText) {
setInputValue(inputValue + ghostText)
return true
}
return false
}
// Handle keyboard events
const handleKeyDown = (e: React.KeyboardEvent) => {
if (e.key === "Tab" && ghostText) {
e.preventDefault()
acceptGhostText()
}
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault()
handleSearch()
@@ -202,6 +360,14 @@ export function SmartFilterInput({
if (e.key === "Escape") {
setOpen(false)
}
// Right arrow at end of input accepts ghost text
if (e.key === "ArrowRight" && ghostText) {
const input = inputRef.current
if (input && input.selectionStart === input.value.length) {
e.preventDefault()
acceptGhostText()
}
}
}
// Append example to input box (not overwrite), then close popover
@@ -215,36 +381,46 @@ export function SmartFilterInput({
return (
<div className={className}>
<Popover open={open} onOpenChange={setOpen} modal={false}>
<PopoverAnchor asChild>
<div className="flex items-center gap-2">
<Input
ref={inputRef}
type="text"
value={inputValue}
onChange={(e) => {
setInputValue(e.target.value)
if (!open) setOpen(true)
}}
onFocus={() => setOpen(true)}
onBlur={(e) => {
// If focus moves to inside Popover or input itself, don't close
const relatedTarget = e.relatedTarget as HTMLElement | null
if (relatedTarget?.closest('[data-radix-popper-content-wrapper]')) {
return
}
// Delay close to let CommandItem's onSelect execute first
setTimeout(() => setOpen(false), 150)
}}
onKeyDown={handleKeyDown}
placeholder={placeholder || defaultPlaceholder}
className="h-8 w-full"
/>
<Button variant="outline" size="sm" onClick={handleSearch}>
<IconSearch className="h-4 w-4" />
</Button>
</div>
</PopoverAnchor>
<div className="flex items-center gap-2">
<Popover open={open} onOpenChange={setOpen} modal={false}>
<PopoverAnchor asChild>
<div className="relative flex-1">
<Input
ref={inputRef}
type="text"
value={inputValue}
onChange={(e) => {
setInputValue(e.target.value)
if (!open) setOpen(true)
}}
onFocus={() => setOpen(true)}
onBlur={(e) => {
// If focus moves to inside Popover or input itself, don't close
const relatedTarget = e.relatedTarget as HTMLElement | null
if (relatedTarget?.closest('[data-radix-popper-content-wrapper]')) {
return
}
// Delay close to let CommandItem's onSelect execute first
setTimeout(() => setOpen(false), 150)
}}
onKeyDown={handleKeyDown}
placeholder={placeholder || defaultPlaceholder}
className="h-8 w-full font-mono text-sm"
/>
{/* Ghost text overlay */}
{ghostText && (
<div
className="absolute inset-0 flex items-center pointer-events-none overflow-hidden px-3"
aria-hidden="true"
>
<span className="font-mono text-sm">
<span className="invisible">{inputValue}</span>
<span ref={ghostRef} className="text-muted-foreground/40">{ghostText}</span>
</span>
</div>
)}
</div>
</PopoverAnchor>
<PopoverContent
className="w-[var(--radix-popover-trigger-width)] p-0"
align="start"
@@ -343,6 +519,10 @@ export function SmartFilterInput({
</Command>
</PopoverContent>
</Popover>
<Button variant="outline" size="sm" onClick={handleSearch}>
<IconSearch className="h-4 w-4" />
</Button>
</div>
</div>
)
}

View File

@@ -0,0 +1,318 @@
"use client"
import React, { useMemo, useCallback } from "react"
import { Play, Server, Settings, Zap } from "lucide-react"
import { useTranslations } from "next-intl"
import { Badge } from "@/components/ui/badge"
import { Checkbox } from "@/components/ui/checkbox"
import { cn } from "@/lib/utils"
import { CAPABILITY_CONFIG, parseEngineCapabilities, mergeEngineConfigurations } from "@/lib/engine-config"
import type { ScanEngine } from "@/types/engine.types"
export interface EnginePreset {
id: string
label: string
description: string
icon: React.ComponentType<{ className?: string }>
engineIds: number[]
}
interface EnginePresetSelectorProps {
engines: ScanEngine[]
selectedEngineIds: number[]
selectedPresetId: string | null
onPresetChange: (presetId: string | null) => void
onEngineIdsChange: (engineIds: number[]) => void
onConfigurationChange: (config: string) => void
disabled?: boolean
className?: string
}
export function EnginePresetSelector({
engines,
selectedEngineIds,
selectedPresetId,
onPresetChange,
onEngineIdsChange,
onConfigurationChange,
disabled = false,
className,
}: EnginePresetSelectorProps) {
const t = useTranslations("scan.initiate")
const tStages = useTranslations("scan.progress.stages")
// Preset definitions with precise engine filtering
const enginePresets = useMemo(() => {
if (!engines?.length) return []
// Categorize engines by their capabilities
const fullScanEngines: number[] = []
const reconEngines: number[] = []
const vulnEngines: number[] = []
engines.forEach(e => {
const caps = parseEngineCapabilities(e.configuration || "")
const hasRecon = caps.includes("subdomain_discovery") || caps.includes("port_scan") || caps.includes("site_scan") || caps.includes("directory_scan") || caps.includes("url_fetch")
const hasVuln = caps.includes("vuln_scan")
if (hasRecon && hasVuln) {
// Full capability engine - only for full scan
fullScanEngines.push(e.id)
} else if (hasRecon && !hasVuln) {
// Recon only engine
reconEngines.push(e.id)
} else if (hasVuln && !hasRecon) {
// Vuln only engine
vulnEngines.push(e.id)
}
})
return [
{
id: "full",
label: t("presets.fullScan"),
description: t("presets.fullScanDesc"),
icon: Zap,
engineIds: fullScanEngines,
},
{
id: "recon",
label: t("presets.recon"),
description: t("presets.reconDesc"),
icon: Server,
engineIds: reconEngines,
},
{
id: "vuln",
label: t("presets.vulnScan"),
description: t("presets.vulnScanDesc"),
icon: Play,
engineIds: vulnEngines,
},
{
id: "custom",
label: t("presets.custom"),
description: t("presets.customDesc"),
icon: Settings,
engineIds: [],
},
]
}, [engines, t])
const selectedEngines = useMemo(() => {
if (!selectedEngineIds.length || !engines) return []
return engines.filter((e) => selectedEngineIds.includes(e.id))
}, [selectedEngineIds, engines])
const selectedCapabilities = useMemo(() => {
if (!selectedEngines.length) return []
const allCaps = new Set<string>()
selectedEngines.forEach((engine) => {
parseEngineCapabilities(engine.configuration || "").forEach((cap) => allCaps.add(cap))
})
return Array.from(allCaps)
}, [selectedEngines])
// Get currently selected preset details
const selectedPreset = useMemo(() => {
return enginePresets.find(p => p.id === selectedPresetId)
}, [enginePresets, selectedPresetId])
// Get engines for the selected preset
const presetEngines = useMemo(() => {
if (!selectedPreset || selectedPreset.id === "custom") return []
return engines?.filter(e => selectedPreset.engineIds.includes(e.id)) || []
}, [selectedPreset, engines])
// Update configuration when engines change
const updateConfigurationFromEngines = useCallback((engineIds: number[]) => {
if (!engines) return
const selectedEngs = engines.filter(e => engineIds.includes(e.id))
const mergedConfig = mergeEngineConfigurations(selectedEngs.map(e => e.configuration || ""))
onConfigurationChange(mergedConfig)
}, [engines, onConfigurationChange])
const handlePresetSelect = useCallback((preset: EnginePreset) => {
onPresetChange(preset.id)
if (preset.id !== "custom") {
onEngineIdsChange(preset.engineIds)
updateConfigurationFromEngines(preset.engineIds)
} else {
// Custom mode - keep current selection or clear
if (selectedEngineIds.length === 0) {
onConfigurationChange("")
}
}
}, [onPresetChange, onEngineIdsChange, updateConfigurationFromEngines, selectedEngineIds.length, onConfigurationChange])
const handleEngineToggle = useCallback((engineId: number, checked: boolean) => {
let newEngineIds: number[]
if (checked) {
newEngineIds = [...selectedEngineIds, engineId]
} else {
newEngineIds = selectedEngineIds.filter((id) => id !== engineId)
}
onEngineIdsChange(newEngineIds)
updateConfigurationFromEngines(newEngineIds)
}, [selectedEngineIds, onEngineIdsChange, updateConfigurationFromEngines])
return (
<div className={cn("flex flex-col h-full", className)}>
<div className="flex-1 overflow-y-auto p-6">
{/* Compact preset cards */}
<div className="grid grid-cols-4 gap-3 mb-4">
{enginePresets.map((preset) => {
const isActive = selectedPresetId === preset.id
const PresetIcon = preset.icon
const matchedEngines = preset.id === "custom"
? []
: engines?.filter(e => preset.engineIds.includes(e.id)) || []
return (
<button
key={preset.id}
type="button"
onClick={() => handlePresetSelect(preset)}
disabled={disabled}
className={cn(
"flex flex-col items-center p-3 rounded-lg border-2 text-center transition-all",
isActive
? "border-primary bg-primary/5"
: "border-border hover:border-primary/50 hover:bg-muted/30",
disabled && "opacity-50 cursor-not-allowed"
)}
>
<div className={cn(
"flex h-10 w-10 items-center justify-center rounded-lg mb-2",
isActive ? "bg-primary text-primary-foreground" : "bg-muted"
)}>
<PresetIcon className="h-5 w-5" />
</div>
<span className="text-sm font-medium">{preset.label}</span>
{preset.id !== "custom" && (
<span className="text-xs text-muted-foreground mt-1">
{matchedEngines.length} {t("presets.enginesCount")}
</span>
)}
</button>
)
})}
</div>
{/* Selected preset details */}
{selectedPresetId && selectedPresetId !== "custom" && (
<div className="border rounded-lg p-4 bg-muted/10">
<div className="flex items-start justify-between mb-3">
<div>
<h3 className="font-medium">{selectedPreset?.label}</h3>
<p className="text-sm text-muted-foreground mt-1">{selectedPreset?.description}</p>
</div>
</div>
{/* Capabilities */}
<div className="mb-4">
<h4 className="text-xs font-medium text-muted-foreground mb-2">{t("presets.capabilities")}</h4>
<div className="flex flex-wrap gap-1.5">
{selectedCapabilities.map((capKey) => {
const config = CAPABILITY_CONFIG[capKey]
return (
<Badge key={capKey} variant="outline" className={cn("text-xs", config?.color)}>
{tStages(capKey)}
</Badge>
)
})}
</div>
</div>
{/* Engines list */}
<div>
<h4 className="text-xs font-medium text-muted-foreground mb-2">{t("presets.usedEngines")}</h4>
<div className="flex flex-wrap gap-2">
{presetEngines.map((engine) => (
<span key={engine.id} className="text-sm px-3 py-1.5 bg-background rounded-md border">
{engine.name}
</span>
))}
</div>
</div>
</div>
)}
{/* Custom mode engine selection */}
{selectedPresetId === "custom" && (
<div className="border rounded-lg p-4 bg-muted/10">
<div className="flex items-start justify-between mb-3">
<div>
<h3 className="font-medium">{selectedPreset?.label}</h3>
<p className="text-sm text-muted-foreground mt-1">{selectedPreset?.description}</p>
</div>
</div>
{/* Capabilities - dynamically calculated from selected engines */}
<div className="mb-4">
<h4 className="text-xs font-medium text-muted-foreground mb-2">{t("presets.capabilities")}</h4>
<div className="flex flex-wrap gap-1.5">
{selectedCapabilities.length > 0 ? (
selectedCapabilities.map((capKey) => {
const config = CAPABILITY_CONFIG[capKey]
return (
<Badge key={capKey} variant="outline" className={cn("text-xs", config?.color)}>
{tStages(capKey)}
</Badge>
)
})
) : (
<span className="text-xs text-muted-foreground">{t("presets.noCapabilities")}</span>
)}
</div>
</div>
{/* Engines list - selectable */}
<div>
<h4 className="text-xs font-medium text-muted-foreground mb-2">{t("presets.usedEngines")}</h4>
<div className="flex flex-wrap gap-2">
{engines?.map((engine) => {
const isSelected = selectedEngineIds.includes(engine.id)
return (
<label
key={engine.id}
htmlFor={`preset-engine-${engine.id}`}
className={cn(
"flex items-center gap-2 px-3 py-1.5 rounded-md cursor-pointer transition-all border",
isSelected
? "bg-primary/10 border-primary/30"
: "hover:bg-muted/50 border-border",
disabled && "opacity-50 cursor-not-allowed"
)}
>
<Checkbox
id={`preset-engine-${engine.id}`}
checked={isSelected}
onCheckedChange={(checked) => {
handleEngineToggle(engine.id, checked as boolean)
}}
disabled={disabled}
className="h-4 w-4"
/>
<span className="text-sm">{engine.name}</span>
</label>
)
})}
</div>
</div>
</div>
)}
{/* Empty state */}
{!selectedPresetId && (
<div className="flex flex-col items-center justify-center py-12 text-muted-foreground">
<Server className="h-12 w-12 mb-4 opacity-50" />
<p className="text-sm">{t("presets.selectHint")}</p>
</div>
)}
</div>
</div>
)
}

View File

@@ -1,7 +1,7 @@
"use client"
import React, { useState, useMemo } from "react"
import { Play, Settings2 } from "lucide-react"
import React, { useState, useMemo, useCallback } from "react"
import { Play, Server, Settings, ChevronLeft, ChevronRight } from "lucide-react"
import { useTranslations } from "next-intl"
import { Button } from "@/components/ui/button"
@@ -9,15 +9,22 @@ import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog"
import { Badge } from "@/components/ui/badge"
import { Checkbox } from "@/components/ui/checkbox"
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog"
import { LoadingSpinner } from "@/components/loading-spinner"
import { cn } from "@/lib/utils"
import { CAPABILITY_CONFIG, getEngineIcon, parseEngineCapabilities } from "@/lib/engine-config"
import { EnginePresetSelector } from "./engine-preset-selector"
import { ScanConfigEditor } from "./scan-config-editor"
import type { Organization } from "@/types/organization.types"
@@ -46,36 +53,78 @@ export function InitiateScanDialog({
}: InitiateScanDialogProps) {
const t = useTranslations("scan.initiate")
const tToast = useTranslations("toast")
const tCommon = useTranslations("common.actions")
const [step, setStep] = useState(1)
const [selectedEngineIds, setSelectedEngineIds] = useState<number[]>([])
const [isSubmitting, setIsSubmitting] = useState(false)
const [selectedPresetId, setSelectedPresetId] = useState<string | null>(null)
// Configuration state management
const [configuration, setConfiguration] = useState("")
const [isConfigEdited, setIsConfigEdited] = useState(false)
const [isYamlValid, setIsYamlValid] = useState(true)
const [showOverwriteConfirm, setShowOverwriteConfirm] = useState(false)
const [pendingConfigChange, setPendingConfigChange] = useState<string | null>(null)
const { data: engines, isLoading, error } = useEngines()
const { data: engines } = useEngines()
const steps = [
{ id: 1, title: t("steps.selectEngine"), icon: Server },
{ id: 2, title: t("steps.editConfig"), icon: Settings },
]
const selectedEngines = useMemo(() => {
if (!selectedEngineIds.length || !engines) return []
return engines.filter((e) => selectedEngineIds.includes(e.id))
}, [selectedEngineIds, engines])
const selectedCapabilities = useMemo(() => {
if (!selectedEngines.length) return []
const allCaps = new Set<string>()
selectedEngines.forEach((engine) => {
parseEngineCapabilities(engine.configuration || "").forEach((cap) => allCaps.add(cap))
})
return Array.from(allCaps)
}, [selectedEngines])
const handleEngineToggle = (engineId: number, checked: boolean) => {
if (checked) {
setSelectedEngineIds((prev) => [...prev, engineId])
// Handle configuration change from preset selector (may need confirmation)
const handlePresetConfigChange = useCallback((value: string) => {
if (isConfigEdited && configuration !== value) {
setPendingConfigChange(value)
setShowOverwriteConfirm(true)
} else {
setSelectedEngineIds((prev) => prev.filter((id) => id !== engineId))
setConfiguration(value)
setIsConfigEdited(false)
}
}, [isConfigEdited, configuration])
// Handle manual config editing
const handleManualConfigChange = useCallback((value: string) => {
setConfiguration(value)
setIsConfigEdited(true)
}, [])
const handleEngineIdsChange = useCallback((engineIds: number[]) => {
setSelectedEngineIds(engineIds)
}, [])
const handleOverwriteConfirm = () => {
if (pendingConfigChange !== null) {
setConfiguration(pendingConfigChange)
setIsConfigEdited(false)
}
setShowOverwriteConfirm(false)
setPendingConfigChange(null)
}
const handleOverwriteCancel = () => {
setShowOverwriteConfirm(false)
setPendingConfigChange(null)
}
const handleYamlValidationChange = (isValid: boolean) => {
setIsYamlValid(isValid)
}
const handleInitiate = async () => {
if (!selectedEngineIds.length) return
if (selectedEngineIds.length === 0) {
toast.error(tToast("noEngineSelected"))
return
}
if (!configuration.trim()) {
toast.error(tToast("emptyConfig"))
return
}
if (!organizationId && !targetId) {
toast.error(tToast("paramError"), { description: tToast("paramErrorDesc") })
return
@@ -85,7 +134,9 @@ export function InitiateScanDialog({
const response = await initiateScan({
organizationId,
targetId,
configuration,
engineIds: selectedEngineIds,
engineNames: selectedEngines.map(e => e.name),
})
// 后端返回 201 说明成功创建扫描任务
@@ -96,19 +147,14 @@ export function InitiateScanDialog({
onSuccess?.()
onOpenChange(false)
setSelectedEngineIds([])
setConfiguration("")
setIsConfigEdited(false)
} catch (err: unknown) {
console.error("Failed to initiate scan:", err)
// 处理配置冲突错误
const error = err as { response?: { data?: { error?: { code?: string; message?: string } } } }
if (error?.response?.data?.error?.code === 'CONFIG_CONFLICT') {
toast.error(tToast("configConflict"), {
description: error.response.data.error.message,
})
} else {
toast.error(tToast("initiateScanFailed"), {
description: err instanceof Error ? err.message : tToast("unknownError"),
})
}
toast.error(tToast("initiateScanFailed"), {
description: error?.response?.data?.error?.message || (err instanceof Error ? err.message : tToast("unknownError")),
})
} finally {
setIsSubmitting(false)
}
@@ -117,158 +163,127 @@ export function InitiateScanDialog({
const handleOpenChange = (newOpen: boolean) => {
if (!isSubmitting) {
onOpenChange(newOpen)
if (!newOpen) setSelectedEngineIds([])
if (!newOpen) {
setStep(1)
setSelectedPresetId(null)
setSelectedEngineIds([])
setConfiguration("")
setIsConfigEdited(false)
}
}
}
const canProceedToStep2 = selectedPresetId !== null && selectedEngineIds.length > 0
const canSubmit = selectedEngineIds.length > 0 && configuration.trim().length > 0 && isYamlValid
return (
<Dialog open={open} onOpenChange={handleOpenChange}>
<DialogContent className="max-w-[90vw] sm:max-w-[900px] p-0 gap-0">
<DialogHeader className="px-6 pt-6 pb-4">
<DialogTitle className="flex items-center gap-2">
<Play className="h-5 w-5" />
{t("title")}
<span className="text-sm font-normal text-muted-foreground">
{targetName ? (
<>
{t("targetDesc")} <span className="font-medium text-foreground">{targetName}</span> {t("selectEngine")}
</>
) : (
<>
{t("orgDesc")} <span className="font-medium text-foreground">{organization?.name}</span> {t("selectEngine")}
</>
)}
</span>
</DialogTitle>
</DialogHeader>
<div className="flex border-t h-[480px]">
{/* Left side engine list */}
<div className="w-[260px] border-r flex flex-col shrink-0">
<div className="px-4 py-3 border-b bg-muted/30 shrink-0">
<h3 className="text-sm font-medium">
{t("selectEngineTitle")}
{selectedEngineIds.length > 0 && (
<span className="text-xs text-muted-foreground font-normal ml-2">
{t("selectedCount", { count: selectedEngineIds.length })}
</span>
)}
</h3>
</div>
<div className="flex-1 overflow-y-auto">
<div className="p-2">
{isLoading ? (
<div className="flex items-center justify-center py-8">
<LoadingSpinner />
<span className="ml-2 text-sm text-muted-foreground">{t("loading")}</span>
</div>
) : error ? (
<div className="py-8 text-center text-sm text-destructive">{t("loadFailed")}</div>
) : !engines?.length ? (
<div className="py-8 text-center text-sm text-muted-foreground">{t("noEngines")}</div>
<div className="flex items-center justify-between">
<div>
<DialogTitle className="flex items-center gap-2">
<Play className="h-5 w-5" />
{t("title")}
</DialogTitle>
<DialogDescription className="mt-1">
{targetName ? (
<>{t("targetDesc")} <span className="font-medium text-foreground">{targetName}</span></>
) : (
<div className="space-y-1">
{engines.map((engine) => {
const capabilities = parseEngineCapabilities(engine.configuration || "")
const EngineIcon = getEngineIcon(capabilities)
const primaryCap = capabilities[0]
const iconConfig = primaryCap ? CAPABILITY_CONFIG[primaryCap] : null
const isSelected = selectedEngineIds.includes(engine.id)
return (
<label
key={engine.id}
htmlFor={`engine-${engine.id}`}
className={cn(
"flex items-center gap-3 p-3 rounded-lg cursor-pointer transition-all",
isSelected
? "bg-primary/10 border border-primary/30"
: "hover:bg-muted/50 border border-transparent"
)}
>
<Checkbox
id={`engine-${engine.id}`}
checked={isSelected}
onCheckedChange={(checked) => handleEngineToggle(engine.id, checked as boolean)}
disabled={isSubmitting}
/>
<div
className={cn(
"flex h-8 w-8 items-center justify-center rounded-md shrink-0",
iconConfig?.color || "bg-muted text-muted-foreground"
)}
>
<EngineIcon className="h-4 w-4" />
</div>
<div className="flex-1 min-w-0">
<div className="font-medium text-sm truncate">{engine.name}</div>
<div className="text-xs text-muted-foreground">
{capabilities.length > 0 ? t("capabilities", { count: capabilities.length }) : t("noConfig")}
</div>
</div>
</label>
)
})}
</div>
<>{t("orgDesc")} <span className="font-medium text-foreground">{organization?.name}</span></>
)}
</div>
</DialogDescription>
</div>
{/* Step indicator */}
<div className="text-sm text-muted-foreground mr-8">
{t("stepIndicator", { current: step, total: steps.length })}
</div>
</div>
</DialogHeader>
{/* Right side engine details */}
<div className="flex-1 flex flex-col min-w-0 overflow-hidden w-0">
{selectedEngines.length > 0 ? (
<>
<div className="px-4 py-3 border-b bg-muted/30 shrink-0 min-w-0">
<div className="flex flex-wrap gap-1.5">
{selectedCapabilities.map((capKey) => {
const config = CAPABILITY_CONFIG[capKey]
return (
<Badge key={capKey} variant="outline" className={cn("text-xs", config?.color)}>
{config?.label || capKey}
</Badge>
)
})}
</div>
</div>
<div className="flex-1 flex flex-col overflow-hidden p-4 min-w-0">
<div className="flex-1 bg-muted/50 rounded-lg border overflow-hidden min-h-0 min-w-0">
<pre className="h-full p-3 text-xs font-mono overflow-auto whitespace-pre-wrap break-all">
{selectedEngines.map((e) => e.configuration || `# ${t("noConfig")}`).join("\n\n")}
</pre>
</div>
</div>
</>
<div className="border-t h-[480px] overflow-hidden">
{/* Step 1: Select preset/engines */}
{step === 1 && engines && (
<EnginePresetSelector
engines={engines}
selectedEngineIds={selectedEngineIds}
selectedPresetId={selectedPresetId}
onPresetChange={setSelectedPresetId}
onEngineIdsChange={handleEngineIdsChange}
onConfigurationChange={handlePresetConfigChange}
disabled={isSubmitting}
/>
)}
{/* Step 2: Edit configuration */}
{step === 2 && (
<ScanConfigEditor
configuration={configuration}
onChange={handleManualConfigChange}
onValidationChange={handleYamlValidationChange}
selectedEngines={selectedEngines}
isConfigEdited={isConfigEdited}
disabled={isSubmitting}
/>
)}
</div>
<div className="px-6 py-4 border-t flex items-center justify-between">
<div className="text-sm text-muted-foreground">
{step === 1 && selectedEngineIds.length > 0 && (
<span className="text-primary">{t("selectedCount", { count: selectedEngineIds.length })}</span>
)}
</div>
<div className="flex gap-2">
{step > 1 && (
<Button variant="outline" onClick={() => setStep(step - 1)} disabled={isSubmitting}>
<ChevronLeft className="h-4 w-4 mr-1" />
{t("back")}
</Button>
)}
{step === 1 ? (
<Button onClick={() => setStep(2)} disabled={!canProceedToStep2}>
{t("next")}
<ChevronRight className="h-4 w-4 ml-1" />
</Button>
) : (
<div className="flex-1 flex items-center justify-center text-muted-foreground">
<div className="text-center">
<Settings2 className="h-10 w-10 mx-auto mb-3 opacity-30" />
<p className="text-sm">{t("selectEngineHint")}</p>
</div>
</div>
<Button onClick={handleInitiate} disabled={!canSubmit || isSubmitting}>
{isSubmitting ? (
<>
<LoadingSpinner />
{t("initiating")}
</>
) : (
<>
<Play className="h-4 w-4" />
{t("startScan")}
</>
)}
</Button>
)}
</div>
</div>
<DialogFooter className="px-6 py-4 border-t">
<Button variant="outline" onClick={() => handleOpenChange(false)} disabled={isSubmitting}>
{tCommon("cancel")}
</Button>
<Button onClick={handleInitiate} disabled={!selectedEngineIds.length || isSubmitting}>
{isSubmitting ? (
<>
<LoadingSpinner />
{t("initiating")}
</>
) : (
<>
<Play className="h-4 w-4" />
{t("startScan")}
</>
)}
</Button>
</DialogFooter>
</DialogContent>
{/* Overwrite confirmation dialog */}
<AlertDialog open={showOverwriteConfirm} onOpenChange={setShowOverwriteConfirm}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>{t("overwriteConfirm.title")}</AlertDialogTitle>
<AlertDialogDescription>
{t("overwriteConfirm.description")}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel onClick={handleOverwriteCancel}>
{t("overwriteConfirm.cancel")}
</AlertDialogCancel>
<AlertDialogAction onClick={handleOverwriteConfirm}>
{t("overwriteConfirm.confirm")}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</Dialog>
)
}

View File

@@ -11,18 +11,27 @@ import {
DialogTitle,
DialogTrigger,
} from "@/components/ui/dialog"
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog"
import { Button } from "@/components/ui/button"
import { Textarea } from "@/components/ui/textarea"
import { Badge } from "@/components/ui/badge"
import { Checkbox } from "@/components/ui/checkbox"
import { LoadingSpinner } from "@/components/loading-spinner"
import { cn } from "@/lib/utils"
import { toast } from "sonner"
import { Zap, Settings2, AlertCircle, ChevronRight, ChevronLeft, Target, Server } from "lucide-react"
import { Zap, AlertCircle, ChevronRight, ChevronLeft, Target, Server, Settings } from "lucide-react"
import { quickScan } from "@/services/scan.service"
import { CAPABILITY_CONFIG, getEngineIcon, parseEngineCapabilities } from "@/lib/engine-config"
import { TargetValidator } from "@/lib/target-validator"
import { useEngines } from "@/hooks/use-engines"
import { EnginePresetSelector } from "./engine-preset-selector"
import { ScanConfigEditor } from "./scan-config-editor"
interface QuickScanDialogProps {
trigger?: React.ReactNode
@@ -36,8 +45,16 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
const [targetInput, setTargetInput] = React.useState("")
const [selectedEngineIds, setSelectedEngineIds] = React.useState<number[]>([])
const [selectedPresetId, setSelectedPresetId] = React.useState<string | null>(null)
const { data: engines, isLoading, error } = useEngines()
// Configuration state management
const [configuration, setConfiguration] = React.useState("")
const [isConfigEdited, setIsConfigEdited] = React.useState(false)
const [isYamlValid, setIsYamlValid] = React.useState(true)
const [showOverwriteConfirm, setShowOverwriteConfirm] = React.useState(false)
const [pendingConfigChange, setPendingConfigChange] = React.useState<string | null>(null)
const { data: engines } = useEngines()
const lineNumbersRef = React.useRef<HTMLDivElement | null>(null)
@@ -61,18 +78,12 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
return engines.filter(e => selectedEngineIds.includes(e.id))
}, [selectedEngineIds, engines])
const selectedCapabilities = React.useMemo(() => {
if (!selectedEngines.length) return []
const allCaps = new Set<string>()
selectedEngines.forEach((engine) => {
parseEngineCapabilities(engine.configuration || "").forEach((cap) => allCaps.add(cap))
})
return Array.from(allCaps)
}, [selectedEngines])
const resetForm = () => {
setTargetInput("")
setSelectedEngineIds([])
setSelectedPresetId(null)
setConfiguration("")
setIsConfigEdited(false)
setStep(1)
}
@@ -81,19 +92,52 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
if (!isOpen) resetForm()
}
const handleEngineToggle = (engineId: number, checked: boolean) => {
if (checked) {
setSelectedEngineIds((prev) => [...prev, engineId])
// Handle configuration change from preset selector (may need confirmation)
const handlePresetConfigChange = React.useCallback((value: string) => {
if (isConfigEdited && configuration !== value) {
setPendingConfigChange(value)
setShowOverwriteConfirm(true)
} else {
setSelectedEngineIds((prev) => prev.filter((id) => id !== engineId))
setConfiguration(value)
setIsConfigEdited(false)
}
}, [isConfigEdited, configuration])
// Handle manual config editing
const handleManualConfigChange = React.useCallback((value: string) => {
setConfiguration(value)
setIsConfigEdited(true)
}, [])
const handleEngineIdsChange = React.useCallback((engineIds: number[]) => {
setSelectedEngineIds(engineIds)
}, [])
const handleOverwriteConfirm = () => {
if (pendingConfigChange !== null) {
setConfiguration(pendingConfigChange)
setIsConfigEdited(false)
}
setShowOverwriteConfirm(false)
setPendingConfigChange(null)
}
const handleOverwriteCancel = () => {
setShowOverwriteConfirm(false)
setPendingConfigChange(null)
}
const handleYamlValidationChange = (isValid: boolean) => {
setIsYamlValid(isValid)
}
const canProceedToStep2 = validInputs.length > 0 && !hasErrors
const canSubmit = selectedEngineIds.length > 0
const canProceedToStep3 = selectedPresetId !== null && selectedEngineIds.length > 0
const canSubmit = selectedEngineIds.length > 0 && configuration.trim().length > 0 && isYamlValid
const handleNext = () => {
if (step === 1 && canProceedToStep2) setStep(2)
else if (step === 2 && canProceedToStep3) setStep(3)
}
const handleBack = () => {
@@ -103,6 +147,7 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
const steps = [
{ id: 1, title: t("step1Title"), icon: Target },
{ id: 2, title: t("step2Title"), icon: Server },
{ id: 3, title: t("step3Title"), icon: Settings },
]
const handleSubmit = async () => {
@@ -118,6 +163,10 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
toast.error(t("toast.selectEngine"))
return
}
if (!configuration.trim()) {
toast.error(t("toast.emptyConfig"))
return
}
const targets = validInputs.map(r => r.originalInput)
@@ -125,7 +174,9 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
try {
const response = await quickScan({
targets: targets.map(name => ({ name })),
configuration,
engineIds: selectedEngineIds,
engineNames: selectedEngines.map(e => e.name),
})
const { targetStats, scans, count } = response
@@ -139,13 +190,7 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
handleClose(false)
} catch (error: unknown) {
const err = error as { response?: { data?: { error?: { code?: string; message?: string }; detail?: string } } }
if (err?.response?.data?.error?.code === 'CONFIG_CONFLICT') {
toast.error(t("toast.configConflict"), {
description: err.response.data.error.message,
})
} else {
toast.error(err?.response?.data?.detail || err?.response?.data?.error?.message || t("toast.createFailed"))
}
toast.error(err?.response?.data?.detail || err?.response?.data?.error?.message || t("toast.createFailed"))
} finally {
setIsSubmitting(false)
}
@@ -179,36 +224,8 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
</DialogDescription>
</div>
{/* Step indicator */}
<div className="flex items-center gap-2 mr-8">
{steps.map((s, index) => (
<React.Fragment key={s.id}>
<button
type="button"
onClick={() => {
if (s.id < step) setStep(s.id)
else if (s.id === 2 && canProceedToStep2) setStep(2)
}}
className={cn(
"flex items-center gap-1.5 px-3 py-1.5 rounded-full text-sm transition-colors",
step === s.id
? "bg-primary text-primary-foreground"
: step > s.id
? "bg-primary/20 text-primary cursor-pointer hover:bg-primary/30"
: "bg-muted text-muted-foreground"
)}
disabled={s.id > step && !(s.id === 2 && canProceedToStep2)}
>
<s.icon className="h-4 w-4" />
<span className="hidden sm:inline">{s.title}</span>
</button>
{index < steps.length - 1 && (
<div className={cn(
"w-8 h-[2px]",
step > s.id ? "bg-primary/50" : "bg-muted"
)} />
)}
</React.Fragment>
))}
<div className="text-sm text-muted-foreground mr-8">
{t("stepIndicator", { current: step, total: steps.length })}
</div>
</div>
</DialogHeader>
@@ -259,118 +276,30 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
</div>
)}
{/* Step 2: Select engines */}
{step === 2 && (
<div className="flex h-full">
<div className="w-[320px] border-r flex flex-col shrink-0">
<div className="px-4 py-3 border-b bg-muted/30 shrink-0">
<h3 className="text-sm font-medium">{t("selectEngine")}</h3>
{selectedEngineIds.length > 0 && (
<p className="text-xs text-muted-foreground mt-1">
{t("selectedCount", { count: selectedEngineIds.length })}
</p>
)}
</div>
<div className="flex-1 overflow-y-auto">
<div className="p-2">
{isLoading ? (
<div className="flex items-center justify-center py-8">
<LoadingSpinner />
<span className="ml-2 text-sm text-muted-foreground">{t("loading")}</span>
</div>
) : error ? (
<div className="py-8 text-center text-sm text-destructive">{t("loadFailed")}</div>
) : !engines?.length ? (
<div className="py-8 text-center text-sm text-muted-foreground">{t("noEngines")}</div>
) : (
<div className="space-y-1">
{engines.map((engine) => {
const capabilities = parseEngineCapabilities(engine.configuration || "")
const EngineIcon = getEngineIcon(capabilities)
const primaryCap = capabilities[0]
const iconConfig = primaryCap ? CAPABILITY_CONFIG[primaryCap] : null
const isSelected = selectedEngineIds.includes(engine.id)
return (
<label
key={engine.id}
htmlFor={`quick-engine-${engine.id}`}
className={cn(
"flex items-center gap-3 p-3 rounded-lg cursor-pointer transition-all",
isSelected
? "bg-primary/10 border border-primary/30"
: "hover:bg-muted/50 border border-transparent"
)}
>
<Checkbox
id={`quick-engine-${engine.id}`}
checked={isSelected}
onCheckedChange={(checked) => handleEngineToggle(engine.id, checked as boolean)}
disabled={isSubmitting}
/>
<div
className={cn(
"flex h-8 w-8 items-center justify-center rounded-md shrink-0",
iconConfig?.color || "bg-muted text-muted-foreground"
)}
>
<EngineIcon className="h-4 w-4" />
</div>
<div className="flex-1 min-w-0">
<div className="font-medium text-sm truncate">{engine.name}</div>
<div className="text-xs text-muted-foreground">
{capabilities.length > 0 ? t("capabilities", { count: capabilities.length }) : t("noConfig")}
</div>
</div>
</label>
)
})}
</div>
)}
</div>
</div>
</div>
<div className="flex-1 flex flex-col min-w-0 overflow-hidden">
{selectedEngines.length > 0 ? (
<>
<div className="px-4 py-3 border-b bg-muted/30 flex items-center gap-2 shrink-0">
<Settings2 className="h-4 w-4 text-muted-foreground" />
<h3 className="text-sm font-medium truncate">
{selectedEngines.map((e) => e.name).join(", ")}
</h3>
</div>
<div className="flex-1 flex flex-col overflow-hidden p-4 gap-3">
{selectedCapabilities.length > 0 && (
<div className="flex flex-wrap gap-1.5 shrink-0">
{selectedCapabilities.map((capKey) => {
const config = CAPABILITY_CONFIG[capKey]
return (
<Badge key={capKey} variant="outline" className={cn("text-xs", config?.color)}>
{config?.label || capKey}
</Badge>
)
})}
</div>
)}
<div className="flex-1 bg-muted/50 rounded-lg border overflow-hidden min-h-0">
<pre className="h-full p-3 text-xs font-mono overflow-auto whitespace-pre-wrap break-all">
{selectedEngines.map((e) => e.configuration || `# ${t("noConfig")}`).join("\n\n")}
</pre>
</div>
</div>
</>
) : (
<div className="flex-1 flex items-center justify-center text-muted-foreground">
<div className="text-center">
<Settings2 className="h-10 w-10 mx-auto mb-3 opacity-30" />
<p className="text-sm">{t("selectEngineHint")}</p>
</div>
</div>
)}
</div>
</div>
{/* Step 2: Select preset/engines */}
{step === 2 && engines && (
<EnginePresetSelector
engines={engines}
selectedEngineIds={selectedEngineIds}
selectedPresetId={selectedPresetId}
onPresetChange={setSelectedPresetId}
onEngineIdsChange={handleEngineIdsChange}
onConfigurationChange={handlePresetConfigChange}
disabled={isSubmitting}
/>
)}
{/* Step 3: Edit configuration */}
{step === 3 && (
<ScanConfigEditor
configuration={configuration}
onChange={handleManualConfigChange}
onValidationChange={handleYamlValidationChange}
selectedEngines={selectedEngines}
isConfigEdited={isConfigEdited}
disabled={isSubmitting}
/>
)}
</div>
<DialogFooter className="px-4 py-4 border-t !flex !items-center !justify-between">
@@ -392,10 +321,10 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
{t("back")}
</Button>
)}
{step === 1 ? (
{step < 3 ? (
<Button
onClick={handleNext}
disabled={!canProceedToStep2}
disabled={step === 1 ? !canProceedToStep2 : !canProceedToStep3}
>
{t("next")}
<ChevronRight className="h-4 w-4 ml-1" />
@@ -418,6 +347,26 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
</div>
</DialogFooter>
</DialogContent>
{/* Overwrite confirmation dialog */}
<AlertDialog open={showOverwriteConfirm} onOpenChange={setShowOverwriteConfirm}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>{t("overwriteConfirm.title")}</AlertDialogTitle>
<AlertDialogDescription>
{t("overwriteConfirm.description")}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel onClick={handleOverwriteCancel}>
{t("overwriteConfirm.cancel")}
</AlertDialogCancel>
<AlertDialogAction onClick={handleOverwriteConfirm}>
{t("overwriteConfirm.confirm")}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</Dialog>
)
}

View File

@@ -0,0 +1,86 @@
"use client"
import React, { useMemo } from "react"
import { useTranslations } from "next-intl"
import { Badge } from "@/components/ui/badge"
import { YamlEditor } from "@/components/ui/yaml-editor"
import { cn } from "@/lib/utils"
import { CAPABILITY_CONFIG, parseEngineCapabilities } from "@/lib/engine-config"
import type { ScanEngine } from "@/types/engine.types"
interface ScanConfigEditorProps {
configuration: string
onChange: (value: string) => void
onValidationChange?: (isValid: boolean) => void
selectedEngines?: ScanEngine[]
selectedCapabilities?: string[]
isConfigEdited?: boolean
disabled?: boolean
showCapabilities?: boolean
className?: string
}
export function ScanConfigEditor({
configuration,
onChange,
onValidationChange,
selectedEngines = [],
selectedCapabilities: propCapabilities,
isConfigEdited = false,
disabled = false,
showCapabilities = true,
className,
}: ScanConfigEditorProps) {
const t = useTranslations("scan.initiate")
const tStages = useTranslations("scan.progress.stages")
// Calculate capabilities from selected engines if not provided
const capabilities = useMemo(() => {
if (propCapabilities) return propCapabilities
if (!selectedEngines.length) return []
const allCaps = new Set<string>()
selectedEngines.forEach((engine) => {
parseEngineCapabilities(engine.configuration || "").forEach((cap) => allCaps.add(cap))
})
return Array.from(allCaps)
}, [selectedEngines, propCapabilities])
return (
<div className={cn("flex flex-col h-full", className)}>
{/* Capabilities header */}
{showCapabilities && (
<div className="px-4 py-2 border-b bg-muted/30 flex items-center gap-2 shrink-0">
{capabilities.length > 0 && (
<div className="flex flex-wrap gap-1">
{capabilities.map((capKey) => {
const config = CAPABILITY_CONFIG[capKey]
return (
<Badge key={capKey} variant="outline" className={cn("text-xs py-0", config?.color)}>
{tStages(capKey)}
</Badge>
)
})}
</div>
)}
{isConfigEdited && (
<Badge variant="outline" className="ml-auto text-xs">
{t("configEdited")}
</Badge>
)}
</div>
)}
{/* YAML Editor */}
<div className="flex-1 overflow-hidden">
<YamlEditor
value={configuration}
onChange={onChange}
disabled={disabled}
onValidationChange={onValidationChange}
/>
</div>
</div>
)
}

View File

@@ -195,7 +195,7 @@ export function ScanProgressDialog({
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="sm:max-w-[500px]">
<DialogContent className="sm:max-w-[500px] sm:min-w-[450px]">
<DialogHeader>
<DialogTitle className="flex items-center gap-2">
<ScanStatusIcon status={data.status} />
@@ -209,9 +209,19 @@ export function ScanProgressDialog({
<span className="text-muted-foreground">{t("target")}</span>
<span className="font-medium">{data.targetName}</span>
</div>
<div className="flex items-center justify-between text-sm">
<span className="text-muted-foreground">{t("engine")}</span>
<Badge variant="secondary">{data.engineNames?.join(", ") || "-"}</Badge>
<div className="flex items-start justify-between text-sm gap-4">
<span className="text-muted-foreground shrink-0">{t("engine")}</span>
<div className="flex flex-wrap gap-1.5 justify-end">
{data.engineNames?.length ? (
data.engineNames.map((name) => (
<Badge key={name} variant="secondary" className="text-xs whitespace-nowrap">
{name}
</Badge>
))
) : (
<span className="text-muted-foreground">-</span>
)}
</div>
</div>
{data.startedAt && (
<div className="flex items-center justify-between text-sm">

View File

@@ -9,6 +9,16 @@ import {
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog"
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog"
import { Button } from "@/components/ui/button"
import { Input } from "@/components/ui/input"
import { Label } from "@/components/ui/label"
@@ -34,6 +44,8 @@ import {
IconClock,
IconInfoCircle,
IconSearch,
IconSettings,
IconCode,
} from "@tabler/icons-react"
import { CronExpressionParser } from "cron-parser"
import cronstrue from "cronstrue/i18n"
@@ -44,9 +56,10 @@ import { useEngines } from "@/hooks/use-engines"
import { useOrganizations } from "@/hooks/use-organizations"
import { useTranslations, useLocale } from "next-intl"
import type { CreateScheduledScanRequest } from "@/types/scheduled-scan.types"
import type { ScanEngine } from "@/types/engine.types"
import type { Target } from "@/types/target.types"
import type { Organization } from "@/types/organization.types"
import { EnginePresetSelector } from "../engine-preset-selector"
import { ScanConfigEditor } from "../scan-config-editor"
interface CreateScheduledScanDialogProps {
@@ -85,14 +98,16 @@ export function CreateScheduledScanDialog({
const FULL_STEPS = [
{ id: 1, title: t("steps.basicInfo"), icon: IconInfoCircle },
{ id: 2, title: t("steps.scanMode"), icon: IconBuilding },
{ id: 3, title: t("steps.selectTarget"), icon: IconTarget },
{ id: 4, title: t("steps.scheduleSettings"), icon: IconClock },
{ id: 2, title: t("steps.selectTarget"), icon: IconTarget },
{ id: 3, title: t("steps.selectEngine"), icon: IconSettings },
{ id: 4, title: t("steps.editConfig"), icon: IconCode },
{ id: 5, title: t("steps.scheduleSettings"), icon: IconClock },
]
const PRESET_STEPS = [
{ id: 1, title: t("steps.basicInfo"), icon: IconInfoCircle },
{ id: 2, title: t("steps.scheduleSettings"), icon: IconClock },
{ id: 1, title: t("steps.selectEngine"), icon: IconSettings },
{ id: 2, title: t("steps.editConfig"), icon: IconCode },
{ id: 3, title: t("steps.scheduleSettings"), icon: IconClock },
]
const [orgSearchInput, setOrgSearchInput] = React.useState("")
@@ -120,10 +135,18 @@ export function CreateScheduledScanDialog({
const [name, setName] = React.useState("")
const [engineIds, setEngineIds] = React.useState<number[]>([])
const [selectedPresetId, setSelectedPresetId] = React.useState<string | null>(null)
const [selectionMode, setSelectionMode] = React.useState<SelectionMode>("organization")
const [selectedOrgId, setSelectedOrgId] = React.useState<number | null>(null)
const [selectedTargetId, setSelectedTargetId] = React.useState<number | null>(null)
const [cronExpression, setCronExpression] = React.useState("0 2 * * *")
// Configuration state management
const [configuration, setConfiguration] = React.useState("")
const [isConfigEdited, setIsConfigEdited] = React.useState(false)
const [isYamlValid, setIsYamlValid] = React.useState(true)
const [showOverwriteConfirm, setShowOverwriteConfirm] = React.useState(false)
const [pendingConfigChange, setPendingConfigChange] = React.useState<string | null>(null)
React.useEffect(() => {
if (open) {
@@ -140,25 +163,65 @@ export function CreateScheduledScanDialog({
}, [open, presetOrganizationId, presetOrganizationName, presetTargetId, presetTargetName, t])
const targets: Target[] = targetsData?.targets || []
const engines: ScanEngine[] = enginesData || []
const engines = enginesData || []
const organizations: Organization[] = organizationsData?.organizations || []
// Get selected engines for display
const selectedEngines = React.useMemo(() => {
if (!engineIds.length || !engines.length) return []
return engines.filter(e => engineIds.includes(e.id))
}, [engineIds, engines])
const resetForm = () => {
setName("")
setEngineIds([])
setSelectedPresetId(null)
setSelectionMode("organization")
setSelectedOrgId(null)
setSelectedTargetId(null)
setCronExpression("0 2 * * *")
setConfiguration("")
setIsConfigEdited(false)
resetStep()
}
const handleEngineToggle = (engineId: number, checked: boolean) => {
if (checked) {
setEngineIds((prev) => [...prev, engineId])
// Handle configuration change from preset selector (may need confirmation)
const handlePresetConfigChange = React.useCallback((value: string) => {
if (isConfigEdited && configuration !== value) {
setPendingConfigChange(value)
setShowOverwriteConfirm(true)
} else {
setEngineIds((prev) => prev.filter((id) => id !== engineId))
setConfiguration(value)
setIsConfigEdited(false)
}
}, [isConfigEdited, configuration])
// Handle manual config editing
const handleManualConfigChange = React.useCallback((value: string) => {
setConfiguration(value)
setIsConfigEdited(true)
}, [])
const handleEngineIdsChange = React.useCallback((newEngineIds: number[]) => {
setEngineIds(newEngineIds)
}, [])
const handleOverwriteConfirm = () => {
if (pendingConfigChange !== null) {
setConfiguration(pendingConfigChange)
setIsConfigEdited(false)
}
setShowOverwriteConfirm(false)
setPendingConfigChange(null)
}
const handleOverwriteCancel = () => {
setShowOverwriteConfirm(false)
setPendingConfigChange(null)
}
const handleYamlValidationChange = (isValid: boolean) => {
setIsYamlValid(isValid)
}
const handleOpenChange = (isOpen: boolean) => {
@@ -177,11 +240,15 @@ export function CreateScheduledScanDialog({
const validateCurrentStep = (): boolean => {
if (hasPreset) {
switch (currentStep) {
case 1:
if (!name.trim()) { toast.error(t("form.taskNameRequired")); return false }
case 1: // Select engine
if (!selectedPresetId) { toast.error(t("form.scanEngineRequired")); return false }
if (engineIds.length === 0) { toast.error(t("form.scanEngineRequired")); return false }
return true
case 2:
case 2: // Edit config
if (!configuration.trim()) { toast.error(t("form.configurationRequired")); return false }
if (!isYamlValid) { toast.error(t("form.yamlInvalid")); return false }
return true
case 3: // Schedule
const parts = cronExpression.trim().split(/\s+/)
if (parts.length !== 5) { toast.error(t("form.cronRequired")); return false }
return true
@@ -190,19 +257,25 @@ export function CreateScheduledScanDialog({
}
switch (currentStep) {
case 1:
case 1: // Basic info
if (!name.trim()) { toast.error(t("form.taskNameRequired")); return false }
if (engineIds.length === 0) { toast.error(t("form.scanEngineRequired")); return false }
return true
case 2: return true
case 3:
case 2: // Select target
if (selectionMode === "organization") {
if (!selectedOrgId) { toast.error(t("toast.selectOrganization")); return false }
} else {
if (!selectedTargetId) { toast.error(t("toast.selectTarget")); return false }
}
return true
case 4:
case 3: // Select engine
if (!selectedPresetId) { toast.error(t("form.scanEngineRequired")); return false }
if (engineIds.length === 0) { toast.error(t("form.scanEngineRequired")); return false }
return true
case 4: // Edit config
if (!configuration.trim()) { toast.error(t("form.configurationRequired")); return false }
if (!isYamlValid) { toast.error(t("form.yamlInvalid")); return false }
return true
case 5: // Schedule
const cronParts = cronExpression.trim().split(/\s+/)
if (cronParts.length !== 5) { toast.error(t("form.cronRequired")); return false }
return true
@@ -216,7 +289,9 @@ export function CreateScheduledScanDialog({
if (!validateCurrentStep()) return
const request: CreateScheduledScanRequest = {
name: name.trim(),
configuration: configuration.trim(),
engineIds: engineIds,
engineNames: selectedEngines.map(e => e.name),
cronExpression: cronExpression.trim(),
}
if (selectionMode === "organization" && selectedOrgId) {
@@ -262,82 +337,30 @@ export function CreateScheduledScanDialog({
return (
<Dialog open={open} onOpenChange={handleOpenChange}>
<DialogContent className="max-w-2xl max-h-[90vh] overflow-hidden flex flex-col">
<DialogHeader>
<DialogTitle>{t("createTitle")}</DialogTitle>
<DialogDescription>{t("createDesc")}</DialogDescription>
<DialogContent className="max-w-[900px] p-0 gap-0">
<DialogHeader className="px-6 pt-6 pb-4">
<div className="flex items-center justify-between">
<div>
<DialogTitle>{t("createTitle")}</DialogTitle>
<DialogDescription className="mt-1">{t("createDesc")}</DialogDescription>
</div>
{/* Step indicator */}
<div className="text-sm text-muted-foreground mr-8">
{t("stepIndicator", { current: currentStep, total: totalSteps })}
</div>
</div>
</DialogHeader>
<div className="flex items-center justify-between px-2 py-4">
{steps.map((step, index) => (
<React.Fragment key={step.id}>
<div className="flex flex-col items-center gap-2">
<div className={cn(
"flex h-10 w-10 items-center justify-center rounded-full border-2 transition-colors",
currentStep > step.id ? "border-primary bg-primary text-primary-foreground"
: currentStep === step.id ? "border-primary text-primary"
: "border-muted text-muted-foreground"
)}>
{currentStep > step.id ? <IconCheck className="h-5 w-5" /> : <step.icon className="h-5 w-5" />}
</div>
<span className={cn("text-xs font-medium", currentStep >= step.id ? "text-foreground" : "text-muted-foreground")}>
{step.title}
</span>
</div>
{index < steps.length - 1 && (
<div className={cn("h-0.5 flex-1 mx-2", currentStep > step.id ? "bg-primary" : "bg-muted")} />
)}
</React.Fragment>
))}
</div>
<Separator />
<div className="flex-1 overflow-y-auto py-4 px-1">
{currentStep === 1 && (
<div className="space-y-6">
<div className="border-t h-[480px] overflow-hidden">
{/* Step 1: Basic Info + Scan Mode */}
{currentStep === 1 && !hasPreset && (
<div className="p-6 space-y-6 overflow-y-auto h-full">
<div className="space-y-2">
<Label htmlFor="name">{t("form.taskName")} *</Label>
<Input id="name" placeholder={t("form.taskNamePlaceholder")} value={name} onChange={(e) => setName(e.target.value)} />
<p className="text-xs text-muted-foreground">{t("form.taskNameDesc")}</p>
</div>
<div className="space-y-2">
<Label>{t("form.scanEngine")} *</Label>
{engineIds.length > 0 && (
<p className="text-xs text-muted-foreground">{t("form.selectedEngines", { count: engineIds.length })}</p>
)}
<div className="border rounded-md p-3 max-h-[200px] overflow-y-auto space-y-2">
{engines.length === 0 ? (
<p className="text-sm text-muted-foreground">{t("form.noEngine")}</p>
) : (
engines.map((engine) => (
<label
key={engine.id}
htmlFor={`engine-${engine.id}`}
className={cn(
"flex items-center gap-3 p-2 rounded-lg cursor-pointer transition-all",
engineIds.includes(engine.id)
? "bg-primary/10 border border-primary/30"
: "hover:bg-muted/50 border border-transparent"
)}
>
<Checkbox
id={`engine-${engine.id}`}
checked={engineIds.includes(engine.id)}
onCheckedChange={(checked) => handleEngineToggle(engine.id, checked as boolean)}
/>
<span className="text-sm">{engine.name}</span>
</label>
))
)}
</div>
<p className="text-xs text-muted-foreground">{t("form.scanEngineDesc")}</p>
</div>
</div>
)}
{currentStep === 2 && !hasPreset && (
<div className="space-y-6">
<Separator />
<div className="space-y-3">
<Label>{t("form.selectScanMode")}</Label>
<div className="grid grid-cols-2 gap-4">
@@ -364,15 +387,16 @@ export function CreateScheduledScanDialog({
{selectionMode === "target" && <IconCheck className="h-5 w-5 text-primary" />}
</div>
</div>
<p className="text-sm text-muted-foreground">
{selectionMode === "organization" ? t("form.organizationScanHint") : t("form.targetScanHint")}
</p>
</div>
<p className="text-sm text-muted-foreground">
{selectionMode === "organization" ? t("form.organizationScanHint") : t("form.targetScanHint")}
</p>
</div>
)}
{currentStep === 3 && !hasPreset && (
<div className="space-y-4">
{/* Step 2: Select Target (Organization or Target) */}
{currentStep === 2 && !hasPreset && (
<div className="p-6 space-y-4 overflow-y-auto h-full">
{selectionMode === "organization" ? (
<>
<Label>{t("form.selectOrganization")}</Label>
@@ -451,8 +475,34 @@ export function CreateScheduledScanDialog({
</div>
)}
{/* Step 3 (full) / Step 1 (preset): Select Engine */}
{((currentStep === 3 && !hasPreset) || (currentStep === 1 && hasPreset)) && engines.length > 0 && (
<EnginePresetSelector
engines={engines}
selectedEngineIds={engineIds}
selectedPresetId={selectedPresetId}
onPresetChange={setSelectedPresetId}
onEngineIdsChange={handleEngineIdsChange}
onConfigurationChange={handlePresetConfigChange}
disabled={isPending}
/>
)}
{/* Step 4 (full) / Step 2 (preset): Edit Configuration */}
{((currentStep === 4 && !hasPreset) || (currentStep === 2 && hasPreset)) && (
<div className="space-y-6">
<ScanConfigEditor
configuration={configuration}
onChange={handleManualConfigChange}
onValidationChange={handleYamlValidationChange}
selectedEngines={selectedEngines}
isConfigEdited={isConfigEdited}
disabled={isPending}
/>
)}
{/* Step 5 (full) / Step 3 (preset): Schedule Settings */}
{((currentStep === 5 && !hasPreset) || (currentStep === 3 && hasPreset)) && (
<div className="p-6 space-y-6 overflow-y-auto h-full">
<div className="space-y-2">
<Label>{t("form.cronExpression")} *</Label>
<Input placeholder={t("form.cronPlaceholder")} value={cronExpression} onChange={(e) => setCronExpression(e.target.value)} className="font-mono" />
@@ -489,9 +539,7 @@ export function CreateScheduledScanDialog({
)}
</div>
<Separator />
<div className="flex justify-between pt-4">
<div className="px-6 py-4 border-t flex justify-between">
<Button variant="outline" onClick={goToPrevStep} disabled={currentStep === 1}>
<IconChevronLeft className="h-4 w-4 mr-1" />{t("buttons.previous")}
</Button>
@@ -504,6 +552,26 @@ export function CreateScheduledScanDialog({
)}
</div>
</DialogContent>
{/* Overwrite confirmation dialog */}
<AlertDialog open={showOverwriteConfirm} onOpenChange={setShowOverwriteConfirm}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>{t("overwriteConfirm.title")}</AlertDialogTitle>
<AlertDialogDescription>
{t("overwriteConfirm.description")}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel onClick={handleOverwriteCancel}>
{t("overwriteConfirm.cancel")}
</AlertDialogCancel>
<AlertDialogAction onClick={handleOverwriteConfirm}>
{t("overwriteConfirm.confirm")}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</Dialog>
)
}

View File

@@ -0,0 +1,4 @@
export { SearchPage } from "./search-page"
export { SearchResultCard } from "./search-result-card"
export { SearchPagination } from "./search-pagination"
export { SearchResultsTable } from "./search-results-table"

View File

@@ -0,0 +1,492 @@
"use client"
import { useState, useCallback, useMemo, useEffect } from "react"
import { useSearchParams } from "next/navigation"
import { motion, AnimatePresence } from "framer-motion"
import { Search, AlertCircle, History, X, Download } from "lucide-react"
import { useTranslations } from "next-intl"
import { toast } from "sonner"
import { SmartFilterInput, type FilterField } from "@/components/common/smart-filter-input"
import { SearchPagination } from "./search-pagination"
import { useAssetSearch } from "@/hooks/use-search"
import { VulnerabilityDetailDialog } from "@/components/vulnerabilities/vulnerability-detail-dialog"
import { VulnerabilityService } from "@/services/vulnerability.service"
import { SearchService } from "@/services/search.service"
import type { SearchParams, SearchState, Vulnerability as SearchVuln, AssetType } from "@/types/search.types"
import type { Vulnerability } from "@/types/vulnerability.types"
import { Alert, AlertDescription } from "@/components/ui/alert"
import { Button } from "@/components/ui/button"
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
import { SearchResultsTable } from "./search-results-table"
import { SearchResultCard } from "./search-result-card"
import { Badge } from "@/components/ui/badge"
import { cn } from "@/lib/utils"
// Website 搜索示例
const WEBSITE_SEARCH_EXAMPLES = [
'host="api"',
'title="Dashboard"',
'tech="nginx"',
'status=="200"',
'host="api" && status=="200"',
'tech="vue" || tech="react"',
'host="admin" && tech="php" && status=="200"',
'status!="404"',
]
// Endpoint 搜索示例
const ENDPOINT_SEARCH_EXAMPLES = [
'host="api"',
'url="/api/v1"',
'title="Dashboard"',
'tech="nginx"',
'status=="200"',
'host="api" && status=="200"',
'url="/admin" && status=="200"',
'tech="vue" || tech="react"',
]
// 快捷搜索标签
const QUICK_SEARCH_TAGS = [
{ label: 'status=="200"', query: 'status=="200"' },
{ label: 'tech="nginx"', query: 'tech="nginx"' },
{ label: 'tech="php"', query: 'tech="php"' },
{ label: 'tech="vue"', query: 'tech="vue"' },
{ label: 'tech="react"', query: 'tech="react"' },
{ label: 'status=="403"', query: 'status=="403"' },
]
// 最近搜索本地存储 key
const RECENT_SEARCHES_KEY = 'xingrin_recent_searches'
const MAX_RECENT_SEARCHES = 5
// 获取最近搜索记录
function getRecentSearches(): string[] {
if (typeof window === 'undefined') return []
try {
const saved = localStorage.getItem(RECENT_SEARCHES_KEY)
return saved ? JSON.parse(saved) : []
} catch {
return []
}
}
// 保存搜索记录
function saveRecentSearch(query: string) {
if (typeof window === 'undefined' || !query.trim()) return
try {
const searches = getRecentSearches().filter(s => s !== query)
searches.unshift(query)
localStorage.setItem(
RECENT_SEARCHES_KEY,
JSON.stringify(searches.slice(0, MAX_RECENT_SEARCHES))
)
} catch {
// ignore
}
}
// 删除搜索记录
function removeRecentSearch(query: string) {
if (typeof window === 'undefined') return
try {
const searches = getRecentSearches().filter(s => s !== query)
localStorage.setItem(RECENT_SEARCHES_KEY, JSON.stringify(searches))
} catch {
// ignore
}
}
export function SearchPage() {
const t = useTranslations('search')
const urlSearchParams = useSearchParams()
const [searchState, setSearchState] = useState<SearchState>("initial")
const [query, setQuery] = useState("")
const [assetType, setAssetType] = useState<AssetType>("website")
const [searchParams, setSearchParams] = useState<SearchParams>({})
const [page, setPage] = useState(1)
const [pageSize, setPageSize] = useState(10)
const [selectedVuln, setSelectedVuln] = useState<Vulnerability | null>(null)
const [vulnDialogOpen, setVulnDialogOpen] = useState(false)
const [, setLoadingVuln] = useState(false)
const [recentSearches, setRecentSearches] = useState<string[]>([])
const [initialQueryProcessed, setInitialQueryProcessed] = useState(false)
// 加载最近搜索记录
useEffect(() => {
setRecentSearches(getRecentSearches())
}, [])
// 处理 URL 参数中的搜索查询
useEffect(() => {
if (initialQueryProcessed) return
const q = urlSearchParams.get('q')
if (q) {
setQuery(q)
setSearchParams({ q, asset_type: assetType })
setSearchState("searching")
saveRecentSearch(q)
setRecentSearches(getRecentSearches())
}
setInitialQueryProcessed(true)
}, [urlSearchParams, assetType, initialQueryProcessed])
// 根据资产类型选择搜索示例
const searchExamples = useMemo(() => {
return assetType === 'endpoint' ? ENDPOINT_SEARCH_EXAMPLES : WEBSITE_SEARCH_EXAMPLES
}, [assetType])
// 搜索过滤字段配置
const SEARCH_FILTER_FIELDS: FilterField[] = [
{ key: "host", label: "Host", description: t('fields.host') },
{ key: "url", label: "URL", description: t('fields.url') },
{ key: "title", label: "Title", description: t('fields.title') },
{ key: "tech", label: "Tech", description: t('fields.tech') },
{ key: "status", label: "Status", description: t('fields.status') },
{ key: "body", label: "Body", description: t('fields.body') },
{ key: "header", label: "Header", description: t('fields.header') },
]
// 使用搜索 Hook
const { data, isLoading, error, isFetching } = useAssetSearch(
{ ...searchParams, page, pageSize },
{ enabled: searchState === "results" || searchState === "searching" }
)
const handleSearch = useCallback((_filters: unknown, rawQuery: string) => {
if (!rawQuery.trim()) return
setQuery(rawQuery)
setSearchParams({ q: rawQuery, asset_type: assetType })
setPage(1)
setSearchState("searching")
// 保存到最近搜索
saveRecentSearch(rawQuery)
setRecentSearches(getRecentSearches())
}, [assetType])
// 处理快捷标签点击
const handleQuickTagClick = useCallback((tagQuery: string) => {
setQuery(tagQuery)
}, [])
// 处理最近搜索点击
const handleRecentSearchClick = useCallback((recentQuery: string) => {
setQuery(recentQuery)
setSearchParams({ q: recentQuery, asset_type: assetType })
setPage(1)
setSearchState("searching")
saveRecentSearch(recentQuery)
setRecentSearches(getRecentSearches())
}, [assetType])
// 删除最近搜索
const handleRemoveRecentSearch = useCallback((e: React.MouseEvent, searchQuery: string) => {
e.stopPropagation()
removeRecentSearch(searchQuery)
setRecentSearches(getRecentSearches())
}, [])
// 导出状态
const [isExporting, setIsExporting] = useState(false)
// 导出 CSV调用后端 API 导出全部结果)
const handleExportCSV = useCallback(async () => {
if (!searchParams.q) return
setIsExporting(true)
try {
await SearchService.exportCSV(searchParams.q, assetType)
toast.success(t('exportSuccess'))
} catch (error) {
console.error('Export failed:', error)
toast.error(t('exportFailed'))
} finally {
setIsExporting(false)
}
}, [searchParams.q, assetType, t])
// 当数据加载完成时更新状态
if (searchState === "searching" && data && !isLoading) {
setSearchState("results")
}
const handleAssetTypeChange = useCallback((value: AssetType) => {
setAssetType(value)
// 清空搜索结果
if (searchState === "results") {
setSearchState("initial")
setSearchParams({})
setQuery("")
}
}, [searchState])
const handlePageChange = useCallback((newPage: number) => {
setPage(newPage)
}, [])
const handlePageSizeChange = useCallback((newPageSize: number) => {
setPageSize(newPageSize)
setPage(1)
}, [])
const handleViewVulnerability = useCallback(async (vuln: SearchVuln) => {
if (!vuln.id) return
setLoadingVuln(true)
try {
const fullVuln = await VulnerabilityService.getVulnerabilityById(vuln.id)
setSelectedVuln(fullVuln)
setVulnDialogOpen(true)
} catch {
toast.error(t('vulnLoadError'))
} finally {
setLoadingVuln(false)
}
}, [t])
// 资产类型选择器组件
const AssetTypeSelector = (
<Select value={assetType} onValueChange={handleAssetTypeChange}>
<SelectTrigger size="sm" className="w-[100px]">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="website">{t('assetTypes.website')}</SelectItem>
<SelectItem value="endpoint">{t('assetTypes.endpoint')}</SelectItem>
</SelectContent>
</Select>
)
return (
<div className="flex-1 w-full flex flex-col">
<AnimatePresence mode="wait">
{searchState === "initial" && (
<motion.div
key="initial"
initial={{ opacity: 0, y: 20 }}
animate={{ opacity: 1, y: 0 }}
exit={{ opacity: 0, y: -50 }}
transition={{ duration: 0.3 }}
className="flex-1 flex flex-col items-center justify-center px-4 relative overflow-hidden"
>
{/* 背景装饰 */}
<div className="absolute inset-0 -z-10 overflow-hidden pointer-events-none">
<div className="absolute left-1/2 top-1/4 -translate-x-1/2 h-[400px] w-[600px] rounded-full bg-primary/5 blur-3xl" />
<div className="absolute right-1/4 top-1/2 h-[200px] w-[300px] rounded-full bg-primary/3 blur-2xl" />
</div>
<div className="flex flex-col items-center gap-6 w-full max-w-3xl -mt-16">
{/* 标题 */}
<div className="flex flex-col items-center gap-2">
<div className="flex items-center justify-center w-16 h-16 rounded-2xl bg-primary/10 mb-2">
<Search className="h-8 w-8 text-primary" />
</div>
<h1 className="text-3xl font-semibold text-foreground">
{t('title')}
</h1>
<p className="text-sm text-muted-foreground">
{t('hint')}
</p>
</div>
{/* 搜索框 */}
<div className="flex items-center gap-3 w-full">
{AssetTypeSelector}
<SmartFilterInput
fields={SEARCH_FILTER_FIELDS}
examples={searchExamples}
placeholder='host="api" && tech="nginx" && status=="200"'
value={query}
onSearch={handleSearch}
className="flex-1"
/>
</div>
{/* 快捷搜索标签 */}
<div className="flex flex-wrap justify-center gap-2">
{QUICK_SEARCH_TAGS.map((tag) => (
<Badge
key={tag.query}
variant="outline"
className="cursor-pointer hover:bg-accent transition-colors px-3 py-1"
onClick={() => handleQuickTagClick(tag.query)}
>
{tag.label}
</Badge>
))}
</div>
{/* 最近搜索 */}
{recentSearches.length > 0 && (
<motion.div
initial={{ opacity: 0 }}
animate={{ opacity: 1 }}
transition={{ delay: 0.3 }}
className="w-full max-w-xl mt-2"
>
<div className="flex items-center gap-2 text-xs text-muted-foreground mb-2">
<History className="h-3.5 w-3.5" />
<span>{t('recentSearches')}</span>
</div>
<div className="flex flex-wrap gap-2">
{recentSearches.map((search) => (
<Badge
key={search}
variant="secondary"
className={cn(
"cursor-pointer hover:bg-secondary/80 transition-colors",
"pl-3 pr-1.5 py-1 gap-1 group"
)}
onClick={() => handleRecentSearchClick(search)}
>
<span className="font-mono text-xs truncate max-w-[200px]">{search}</span>
<button
onClick={(e) => handleRemoveRecentSearch(e, search)}
className="ml-1 p-0.5 rounded hover:bg-muted-foreground/20 opacity-0 group-hover:opacity-100 transition-opacity"
>
<X className="h-3 w-3" />
</button>
</Badge>
))}
</div>
</motion.div>
)}
</div>
</motion.div>
)}
{searchState === "searching" && isLoading && (
<motion.div
key="searching"
initial={{ opacity: 0 }}
animate={{ opacity: 1 }}
exit={{ opacity: 0 }}
transition={{ duration: 0.2 }}
className="h-full flex flex-col items-center justify-center"
>
<div className="flex flex-col items-center gap-4">
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary" />
<span className="text-muted-foreground">{t('searching')}</span>
</div>
</motion.div>
)}
{(searchState === "results" || (searchState === "searching" && !isLoading)) && (
<motion.div
key="results"
initial={{ opacity: 0 }}
animate={{ opacity: 1 }}
transition={{ duration: 0.3 }}
className="h-full flex flex-col"
>
{/* 顶部搜索栏 */}
<motion.div
initial={{ y: -20, opacity: 0 }}
animate={{ y: 0, opacity: 1 }}
transition={{ duration: 0.3, delay: 0.1 }}
className="sticky top-0 z-10 bg-background/95 backdrop-blur supports-[backdrop-filter]:bg-background/60 border-b px-4 py-3"
>
<div className="flex items-center gap-3">
{AssetTypeSelector}
<SmartFilterInput
fields={SEARCH_FILTER_FIELDS}
examples={searchExamples}
placeholder='host="api" && tech="nginx" && status=="200"'
value={query}
onSearch={handleSearch}
className="flex-1"
/>
<span className="text-sm text-muted-foreground whitespace-nowrap">
{isFetching ? t('loading') : t('resultsCount', { count: data?.total ?? 0 })}
</span>
<Button
variant="outline"
size="sm"
onClick={handleExportCSV}
disabled={!data?.results || data.results.length === 0 || isExporting}
>
<Download className="h-4 w-4 mr-1.5" />
{isExporting ? t('exporting') : t('export')}
</Button>
</div>
</motion.div>
{/* 错误提示 */}
{error && (
<div className="p-4 w-full">
<Alert variant="destructive">
<AlertCircle className="h-4 w-4" />
<AlertDescription>
{t('error')}
</AlertDescription>
</Alert>
</div>
)}
{/* 空结果提示 */}
{!error && data?.results.length === 0 && (
<div className="flex-1 flex flex-col items-center justify-center p-4">
<div className="text-center">
<Search className="h-12 w-12 text-muted-foreground mx-auto mb-4" />
<h3 className="text-lg font-medium mb-2">{t('noResults')}</h3>
<p className="text-sm text-muted-foreground">
{t('noResultsHint')}
</p>
</div>
</div>
)}
{/* 搜索结果 */}
{!error && data && data.results.length > 0 && (
<>
<div className="flex-1 overflow-auto p-4">
{assetType === 'website' ? (
// Website 使用卡片样式
<div className="space-y-4 max-w-4xl mx-auto">
{data.results.map((result) => (
<SearchResultCard
key={result.id}
result={result}
onViewVulnerability={handleViewVulnerability}
/>
))}
</div>
) : (
// Endpoint 使用表格样式
<SearchResultsTable
results={data.results}
assetType={assetType}
onViewVulnerability={handleViewVulnerability}
/>
)}
</div>
{/* 分页控制 */}
<div className="border-t px-4 py-3">
<SearchPagination
page={page}
pageSize={pageSize}
total={data.total}
totalPages={data.totalPages}
onPageChange={handlePageChange}
onPageSizeChange={handlePageSizeChange}
/>
</div>
</>
)}
</motion.div>
)}
</AnimatePresence>
{/* 漏洞详情弹窗 - 复用现有组件 */}
<VulnerabilityDetailDialog
vulnerability={selectedVuln}
open={vulnDialogOpen}
onOpenChange={setVulnDialogOpen}
/>
</div>
)
}

View File

@@ -0,0 +1,148 @@
"use client"
import * as React from "react"
import {
IconChevronLeft,
IconChevronRight,
IconChevronsLeft,
IconChevronsRight,
} from "@tabler/icons-react"
import { useTranslations } from 'next-intl'
import { Button } from "@/components/ui/button"
import { Label } from "@/components/ui/label"
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select"
interface SearchPaginationProps {
page: number
pageSize: number
total: number
totalPages: number
onPageChange: (page: number) => void
onPageSizeChange: (pageSize: number) => void
pageSizeOptions?: number[]
}
const DEFAULT_PAGE_SIZE_OPTIONS = [10, 20, 50, 100]
/**
* 搜索结果分页组件
*/
export function SearchPagination({
page,
pageSize,
total,
totalPages,
onPageChange,
onPageSizeChange,
pageSizeOptions = DEFAULT_PAGE_SIZE_OPTIONS,
}: SearchPaginationProps) {
const t = useTranslations('common.pagination')
const handlePageSizeChange = React.useCallback((value: string) => {
onPageSizeChange(Number(value))
}, [onPageSizeChange])
const handleFirstPage = React.useCallback(() => {
onPageChange(1)
}, [onPageChange])
const handlePreviousPage = React.useCallback(() => {
onPageChange(Math.max(1, page - 1))
}, [onPageChange, page])
const handleNextPage = React.useCallback(() => {
onPageChange(Math.min(totalPages, page + 1))
}, [onPageChange, page, totalPages])
const handleLastPage = React.useCallback(() => {
onPageChange(totalPages)
}, [onPageChange, totalPages])
const canPreviousPage = page > 1
const canNextPage = page < totalPages
return (
<div className="flex items-center justify-between">
{/* 总数信息 */}
<div className="flex-1 text-sm text-muted-foreground">
{t('total', { count: total })}
</div>
{/* 分页控制 */}
<div className="flex items-center space-x-6 lg:space-x-8">
{/* 每页条数选择 */}
<div className="flex items-center space-x-2">
<Label htmlFor="rows-per-page" className="text-sm font-medium">
{t('rowsPerPage')}
</Label>
<Select
value={`${pageSize}`}
onValueChange={handlePageSizeChange}
>
<SelectTrigger className="h-8 w-[90px]" id="rows-per-page">
<SelectValue placeholder={pageSize} />
</SelectTrigger>
<SelectContent side="top">
{pageSizeOptions.map((size) => (
<SelectItem key={size} value={`${size}`}>
{size}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
{/* 页码信息 */}
<div className="flex items-center justify-center text-sm font-medium whitespace-nowrap">
{t('page', { current: page, total: totalPages || 1 })}
</div>
{/* 分页按钮 */}
<div className="flex items-center space-x-2">
<Button
variant="outline"
className="hidden h-8 w-8 p-0 lg:flex"
onClick={handleFirstPage}
disabled={!canPreviousPage}
>
<span className="sr-only">{t('first')}</span>
<IconChevronsLeft className="h-4 w-4" />
</Button>
<Button
variant="outline"
className="h-8 w-8 p-0"
onClick={handlePreviousPage}
disabled={!canPreviousPage}
>
<span className="sr-only">{t('previous')}</span>
<IconChevronLeft className="h-4 w-4" />
</Button>
<Button
variant="outline"
className="h-8 w-8 p-0"
onClick={handleNextPage}
disabled={!canNextPage}
>
<span className="sr-only">{t('next')}</span>
<IconChevronRight className="h-4 w-4" />
</Button>
<Button
variant="outline"
className="hidden h-8 w-8 p-0 lg:flex"
onClick={handleLastPage}
disabled={!canNextPage}
>
<span className="sr-only">{t('last')}</span>
<IconChevronsRight className="h-4 w-4" />
</Button>
</div>
</div>
</div>
)
}

View File

@@ -0,0 +1,317 @@
"use client"
import { useState, useRef, useEffect } from "react"
import { ChevronDown, ChevronUp, Eye } from "lucide-react"
import { useTranslations } from "next-intl"
import { Badge } from "@/components/ui/badge"
import { Button } from "@/components/ui/button"
import { Card, CardContent } from "@/components/ui/card"
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
import {
Collapsible,
CollapsibleContent,
CollapsibleTrigger,
} from "@/components/ui/collapsible"
import {
Table,
TableBody,
TableCell,
TableHead,
TableHeader,
TableRow,
} from "@/components/ui/table"
import {
Tooltip,
TooltipContent,
TooltipTrigger,
} from "@/components/ui/tooltip"
import type { SearchResult, Vulnerability, WebsiteSearchResult } from "@/types/search.types"
// 类型守卫:检查是否为 WebsiteSearchResult
function isWebsiteResult(result: SearchResult): result is WebsiteSearchResult {
return 'vulnerabilities' in result
}
interface SearchResultCardProps {
result: SearchResult
onViewVulnerability?: (vuln: Vulnerability) => void
}
// 漏洞严重程度颜色配置
const severityColors: Record<string, string> = {
critical: "bg-[#da3633]/10 text-[#da3633] border border-[#da3633]/20 dark:text-[#f85149]",
high: "bg-[#d29922]/10 text-[#d29922] border border-[#d29922]/20",
medium: "bg-[#d4a72c]/10 text-[#d4a72c] border border-[#d4a72c]/20",
low: "bg-[#238636]/10 text-[#238636] border border-[#238636]/20 dark:text-[#3fb950]",
info: "bg-[#848d97]/10 text-[#848d97] border border-[#848d97]/20",
}
// 状态码 Badge variant
function getStatusVariant(status: number | null): "default" | "secondary" | "destructive" | "outline" {
if (!status) return "outline"
if (status >= 200 && status < 300) return "default"
if (status >= 300 && status < 400) return "secondary"
if (status >= 400) return "destructive"
return "outline"
}
export function SearchResultCard({ result, onViewVulnerability }: SearchResultCardProps) {
const t = useTranslations('search.card')
const [vulnOpen, setVulnOpen] = useState(false)
const [techExpanded, setTechExpanded] = useState(false)
const [isOverflowing, setIsOverflowing] = useState(false)
const containerRef = useRef<HTMLDivElement>(null)
const formatHeaders = (headers: Record<string, string>) => {
return Object.entries(headers)
.map(([key, value]) => `${key}: ${value}`)
.join("\n")
}
// 格式化字节数
const formatBytes = (bytes: number | null) => {
if (bytes === null || bytes === undefined) return null
if (bytes < 1024) return `${bytes} B`
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
}
// 检测内容是否溢出
const maxHeight = 26 * 4
useEffect(() => {
const el = containerRef.current
if (!el || techExpanded) return
const checkOverflow = () => {
setIsOverflowing(el.scrollHeight > maxHeight)
}
checkOverflow()
const resizeObserver = new ResizeObserver(checkOverflow)
resizeObserver.observe(el)
return () => resizeObserver.disconnect()
}, [result.technologies, techExpanded, maxHeight])
const handleViewVulnerability = (vuln: Vulnerability) => {
if (onViewVulnerability) {
onViewVulnerability(vuln)
}
}
return (
<Card className="overflow-hidden py-0 gap-0">
<CardContent className="p-0">
{/* 顶部 URL + Badge 行 */}
<div className="px-4 py-2 bg-muted/30 border-b space-y-2">
<h3 className="font-mono text-sm break-all">
{result.url || result.host}
</h3>
{/* Badge 行 */}
<div className="flex flex-wrap items-center gap-2">
<Badge variant={getStatusVariant(result.statusCode)} className="font-mono text-xs">
{result.statusCode ?? '-'}
</Badge>
{result.webserver && (
<Badge variant="outline" className="font-mono text-xs">
{result.webserver}
</Badge>
)}
{result.contentType && (
<Badge variant="outline" className="font-mono text-xs">
{result.contentType.split(';')[0]}
</Badge>
)}
{formatBytes(result.contentLength) && (
<Badge variant="outline" className="font-mono text-xs">
{formatBytes(result.contentLength)}
</Badge>
)}
</div>
</div>
{/* 中间左右分栏 */}
<div className="flex flex-col md:flex-row">
{/* 左侧信息区 */}
<div className="w-full md:w-[320px] md:shrink-0 px-4 py-3 border-b md:border-b-0 md:border-r flex flex-col">
<div className="space-y-1.5 text-sm">
<div className="flex items-baseline">
<span className="text-muted-foreground w-12 shrink-0">Title</span>
<span className="truncate" title={result.title}>{result.title || '-'}</span>
</div>
<div className="flex items-baseline">
<span className="text-muted-foreground w-12 shrink-0">Host</span>
<span className="font-mono truncate" title={result.host}>{result.host || '-'}</span>
</div>
</div>
{/* Technologies */}
{result.technologies && result.technologies.length > 0 && (
<div className="mt-3 flex flex-col gap-1">
<div
ref={containerRef}
className="flex flex-wrap items-start gap-1 overflow-hidden transition-all duration-200"
style={{ maxHeight: techExpanded ? "none" : `${maxHeight}px` }}
>
{result.technologies.map((tech, index) => (
<Badge
key={`${tech}-${index}`}
variant="secondary"
className="text-xs"
>
{tech}
</Badge>
))}
</div>
{(isOverflowing || techExpanded) && (
<button
onClick={() => setTechExpanded(!techExpanded)}
className="inline-flex items-center gap-0.5 text-xs text-muted-foreground hover:text-foreground transition-colors self-start"
>
{techExpanded ? (
<>
<ChevronUp className="h-3 w-3" />
<span>{t('collapse')}</span>
</>
) : (
<>
<ChevronDown className="h-3 w-3" />
<span>{t('expand')}</span>
</>
)}
</button>
)}
</div>
)}
</div>
{/* 右侧 Tab 区 */}
<div className="w-full md:flex-1 flex flex-col">
<Tabs defaultValue="header" className="w-full h-full flex flex-col gap-0">
<TabsList className="h-[28px] gap-4 rounded-none border-b bg-transparent px-4 pt-1">
<TabsTrigger
value="header"
className="h-full rounded-none border-b-2 border-transparent border-x-0 border-t-0 bg-transparent px-1 text-sm shadow-none focus-visible:ring-0 focus-visible:outline-none data-[state=active]:border-b-primary data-[state=active]:bg-transparent data-[state=active]:shadow-none"
>
Header
</TabsTrigger>
<TabsTrigger
value="body"
className="h-full rounded-none border-b-2 border-transparent border-x-0 border-t-0 bg-transparent px-1 text-sm shadow-none focus-visible:ring-0 focus-visible:outline-none data-[state=active]:border-b-primary data-[state=active]:bg-transparent data-[state=active]:shadow-none"
>
Body
</TabsTrigger>
{result.location && (
<TabsTrigger
value="location"
className="h-full rounded-none border-b-2 border-transparent border-x-0 border-t-0 bg-transparent px-1 text-sm shadow-none focus-visible:ring-0 focus-visible:outline-none data-[state=active]:border-b-primary data-[state=active]:bg-transparent data-[state=active]:shadow-none"
>
Location
</TabsTrigger>
)}
</TabsList>
<TabsContent value="header" className="flex-1 overflow-auto bg-muted/30 px-4 py-2 max-h-[200px]">
<pre className="text-xs font-mono whitespace-pre-wrap">
{result.responseHeaders ? formatHeaders(result.responseHeaders) : '-'}
</pre>
</TabsContent>
<TabsContent value="body" className="flex-1 overflow-auto bg-muted/30 px-4 py-2 max-h-[200px]">
<pre className="text-xs font-mono whitespace-pre-wrap">
{result.responseBody || '-'}
</pre>
</TabsContent>
{result.location && (
<TabsContent value="location" className="flex-1 overflow-auto bg-muted/30 px-4 py-2 max-h-[200px]">
<pre className="text-xs font-mono whitespace-pre-wrap">
{result.location}
</pre>
</TabsContent>
)}
</Tabs>
</div>
</div>
{/* 底部漏洞区 - 仅 Website 类型显示 */}
{isWebsiteResult(result) && result.vulnerabilities && result.vulnerabilities.length > 0 && (
<div className="border-t">
<Collapsible open={vulnOpen} onOpenChange={setVulnOpen}>
<CollapsibleTrigger className="flex items-center gap-1 px-4 py-2 text-sm text-muted-foreground hover:text-foreground transition-colors w-full">
{vulnOpen ? (
<ChevronDown className="size-4" />
) : (
<ChevronUp className="size-4 rotate-90" />
)}
<span>{t('vulnerabilities', { count: result.vulnerabilities.length })}</span>
</CollapsibleTrigger>
<CollapsibleContent>
<div className="px-4 pb-4">
<Table className="table-fixed">
<TableHeader>
<TableRow>
<TableHead className="text-xs w-[50%]">{t('vulnName')}</TableHead>
<TableHead className="text-xs w-[20%]">{t('vulnType')}</TableHead>
<TableHead className="text-xs w-[20%]">{t('severity')}</TableHead>
<TableHead className="text-xs w-[10%]"></TableHead>
</TableRow>
</TableHeader>
<TableBody>
{result.vulnerabilities.map((vuln, index) => (
<TableRow key={`${vuln.name}-${index}`}>
<TableCell className="text-xs font-medium">
<Tooltip>
<TooltipTrigger asChild>
<span className="truncate block max-w-full cursor-default">
{vuln.name}
</span>
</TooltipTrigger>
<TooltipContent side="top" className="max-w-[400px]">
{vuln.name}
</TooltipContent>
</Tooltip>
</TableCell>
<TableCell className="text-xs">
<Tooltip>
<TooltipTrigger asChild>
<span className="truncate block max-w-full cursor-default">
{vuln.vulnType}
</span>
</TooltipTrigger>
<TooltipContent side="top">
{vuln.vulnType}
</TooltipContent>
</Tooltip>
</TableCell>
<TableCell>
<Badge
variant="outline"
className={`text-xs ${severityColors[vuln.severity] || severityColors.info}`}
>
{vuln.severity}
</Badge>
</TableCell>
<TableCell className="text-right">
<Button
variant="ghost"
size="sm"
className="h-7 px-2"
onClick={() => handleViewVulnerability(vuln)}
>
<Eye className="h-3.5 w-3.5" />
</Button>
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</div>
</CollapsibleContent>
</Collapsible>
</div>
)}
</CardContent>
</Card>
)
}

View File

@@ -0,0 +1,285 @@
"use client"
import { useMemo } from "react"
import { useFormatter } from "next-intl"
import type { ColumnDef } from "@tanstack/react-table"
import { Badge } from "@/components/ui/badge"
import { DataTableColumnHeader, UnifiedDataTable } from "@/components/ui/data-table"
import { ExpandableCell, ExpandableTagList } from "@/components/ui/data-table/expandable-cell"
import type { SearchResult, AssetType, Vulnerability, EndpointSearchResult } from "@/types/search.types"
interface SearchResultsTableProps {
results: SearchResult[]
assetType: AssetType
onViewVulnerability?: (vuln: Vulnerability) => void
}
export function SearchResultsTable({ results, assetType }: SearchResultsTableProps) {
const format = useFormatter()
const formatDate = (dateString: string) => {
return format.dateTime(new Date(dateString), {
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
})
}
// 基础列定义Website 和 Endpoint 共用)
const baseColumns: ColumnDef<SearchResult, unknown>[] = useMemo(() => [
{
id: "url",
accessorKey: "url",
meta: { title: "URL" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="URL" />
),
size: 350,
minSize: 200,
maxSize: 600,
cell: ({ row }) => (
<ExpandableCell value={row.getValue("url")} />
),
},
{
id: "host",
accessorKey: "host",
meta: { title: "Host" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="Host" />
),
size: 180,
minSize: 100,
maxSize: 250,
cell: ({ row }) => (
<ExpandableCell value={row.getValue("host")} />
),
},
{
id: "title",
accessorKey: "title",
meta: { title: "Title" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="Title" />
),
size: 150,
minSize: 100,
maxSize: 300,
cell: ({ row }) => (
<ExpandableCell value={row.getValue("title")} />
),
},
{
id: "statusCode",
accessorKey: "statusCode",
meta: { title: "Status" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="Status" />
),
size: 80,
minSize: 60,
maxSize: 100,
cell: ({ row }) => {
const statusCode = row.getValue("statusCode") as number | null
if (!statusCode) return <span className="text-muted-foreground">-</span>
let variant: "default" | "secondary" | "destructive" | "outline" = "outline"
if (statusCode >= 200 && statusCode < 300) {
variant = "outline"
} else if (statusCode >= 300 && statusCode < 400) {
variant = "secondary"
} else if (statusCode >= 400 && statusCode < 500) {
variant = "default"
} else if (statusCode >= 500) {
variant = "destructive"
}
return <Badge variant={variant} className="font-mono">{statusCode}</Badge>
},
},
{
id: "technologies",
accessorKey: "technologies",
meta: { title: "Tech" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="Tech" />
),
size: 180,
minSize: 120,
cell: ({ row }) => {
const tech = row.getValue("technologies") as string[] | null
if (!tech || tech.length === 0) return <span className="text-muted-foreground">-</span>
return <ExpandableTagList items={tech} maxLines={2} variant="outline" />
},
},
{
id: "contentLength",
accessorKey: "contentLength",
meta: { title: "Length" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="Length" />
),
size: 100,
minSize: 80,
maxSize: 150,
cell: ({ row }) => {
const len = row.getValue("contentLength") as number | null
if (len === null || len === undefined) return <span className="text-muted-foreground">-</span>
return <span className="font-mono tabular-nums">{new Intl.NumberFormat().format(len)}</span>
},
},
{
id: "location",
accessorKey: "location",
meta: { title: "Location" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="Location" />
),
size: 150,
minSize: 100,
maxSize: 300,
cell: ({ row }) => (
<ExpandableCell value={row.getValue("location")} />
),
},
{
id: "webserver",
accessorKey: "webserver",
meta: { title: "Server" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="Server" />
),
size: 120,
minSize: 80,
maxSize: 200,
cell: ({ row }) => (
<ExpandableCell value={row.getValue("webserver")} />
),
},
{
id: "contentType",
accessorKey: "contentType",
meta: { title: "Type" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="Type" />
),
size: 120,
minSize: 80,
maxSize: 200,
cell: ({ row }) => (
<ExpandableCell value={row.getValue("contentType")} />
),
},
{
id: "responseBody",
accessorKey: "responseBody",
meta: { title: "Body" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="Body" />
),
size: 300,
minSize: 200,
cell: ({ row }) => (
<ExpandableCell value={row.getValue("responseBody")} maxLines={3} />
),
},
{
id: "responseHeaders",
accessorKey: "responseHeaders",
meta: { title: "Headers" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="Headers" />
),
size: 250,
minSize: 150,
maxSize: 400,
cell: ({ row }) => {
const headers = row.getValue("responseHeaders") as Record<string, string> | null
if (!headers || Object.keys(headers).length === 0) {
return <span className="text-muted-foreground">-</span>
}
const headersStr = Object.entries(headers)
.map(([k, v]) => `${k}: ${v}`)
.join('\n')
return <ExpandableCell value={headersStr} maxLines={3} />
},
},
{
id: "vhost",
accessorKey: "vhost",
meta: { title: "VHost" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="VHost" />
),
size: 80,
minSize: 60,
maxSize: 100,
cell: ({ row }) => {
const vhost = row.getValue("vhost") as boolean | null
if (vhost === null || vhost === undefined) return <span className="text-muted-foreground">-</span>
return <span className="font-mono text-sm">{vhost ? "true" : "false"}</span>
},
},
{
id: "createdAt",
accessorKey: "createdAt",
meta: { title: "Created" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="Created" />
),
size: 150,
minSize: 120,
maxSize: 200,
cell: ({ row }) => {
const createdAt = row.getValue("createdAt") as string | null
if (!createdAt) return <span className="text-muted-foreground">-</span>
return <span className="text-sm">{formatDate(createdAt)}</span>
},
},
], [formatDate])
// Endpoint 特有列
const endpointColumns: ColumnDef<SearchResult, unknown>[] = useMemo(() => [
{
id: "matchedGfPatterns",
accessorKey: "matchedGfPatterns",
meta: { title: "GF Patterns" },
header: ({ column }) => (
<DataTableColumnHeader column={column} title="GF Patterns" />
),
size: 150,
minSize: 100,
maxSize: 250,
cell: ({ row }) => {
const patterns = (row.original as EndpointSearchResult).matchedGfPatterns
if (!patterns || patterns.length === 0) return <span className="text-muted-foreground">-</span>
return <ExpandableTagList items={patterns} maxLines={2} variant="secondary" />
},
},
], [])
// 根据资产类型组合列
const columns = useMemo(() => {
if (assetType === 'endpoint') {
// 在 technologies 后面插入 gfPatterns
const techIndex = baseColumns.findIndex(col => col.id === 'technologies')
const cols = [...baseColumns]
cols.splice(techIndex + 1, 0, ...endpointColumns)
return cols
}
return baseColumns
}, [assetType, baseColumns, endpointColumns])
return (
<UnifiedDataTable
columns={columns}
data={results}
getRowId={(row) => String(row.id)}
hideToolbar
hidePagination
enableRowSelection={false}
/>
)
}

View File

@@ -0,0 +1,194 @@
"use client"
import React, { useState, useCallback, useEffect } from "react"
import Editor from "@monaco-editor/react"
import * as yaml from "js-yaml"
import { AlertCircle } from "lucide-react"
import { useColorTheme } from "@/hooks/use-color-theme"
import { useTranslations } from "next-intl"
import { cn } from "@/lib/utils"
interface YamlEditorProps {
value: string
onChange: (value: string) => void
placeholder?: string
disabled?: boolean
height?: string
className?: string
onValidationChange?: (isValid: boolean, error?: { message: string; line?: number; column?: number }) => void
}
/**
* YAML Editor component with Monaco Editor
* Provides VSCode-level editing experience with syntax highlighting and validation
*/
export function YamlEditor({
value,
onChange,
placeholder,
disabled = false,
height = "100%",
className,
onValidationChange,
}: YamlEditorProps) {
const t = useTranslations("common.yamlEditor")
const { currentTheme } = useColorTheme()
const [shouldMount, setShouldMount] = useState(false)
const [yamlError, setYamlError] = useState<{ message: string; line?: number; column?: number } | null>(null)
// Delay mounting to avoid Monaco hitTest error on rapid container changes
useEffect(() => {
const timer = setTimeout(() => setShouldMount(true), 50)
return () => clearTimeout(timer)
}, [])
// Check for duplicate keys in YAML content
const checkDuplicateKeys = useCallback((content: string): { key: string; line: number } | null => {
const lines = content.split('\n')
const keyStack: { indent: number; keys: Set<string> }[] = [{ indent: -1, keys: new Set() }]
for (let i = 0; i < lines.length; i++) {
const line = lines[i]
// Skip empty lines and comments
if (!line.trim() || line.trim().startsWith('#')) continue
// Match top-level keys (no leading whitespace, ends with colon)
const topLevelMatch = line.match(/^([a-zA-Z_][a-zA-Z0-9_-]*):\s*(?:#.*)?$/)
if (topLevelMatch) {
const key = topLevelMatch[1]
const currentLevel = keyStack[0]
if (currentLevel.keys.has(key)) {
return { key, line: i + 1 }
}
currentLevel.keys.add(key)
}
}
return null
}, [])
// Validate YAML syntax
const validateYaml = useCallback((content: string) => {
if (!content.trim()) {
setYamlError(null)
onValidationChange?.(true)
return true
}
// First check for duplicate keys
const duplicateKey = checkDuplicateKeys(content)
if (duplicateKey) {
const errorInfo = {
message: t("duplicateKey", { key: duplicateKey.key }),
line: duplicateKey.line,
column: 1,
}
setYamlError(errorInfo)
onValidationChange?.(false, errorInfo)
return false
}
try {
yaml.load(content)
setYamlError(null)
onValidationChange?.(true)
return true
} catch (error) {
const yamlException = error as yaml.YAMLException
const errorInfo = {
message: yamlException.message,
line: yamlException.mark?.line ? yamlException.mark.line + 1 : undefined,
column: yamlException.mark?.column ? yamlException.mark.column + 1 : undefined,
}
setYamlError(errorInfo)
onValidationChange?.(false, errorInfo)
return false
}
}, [onValidationChange, checkDuplicateKeys, t])
// Handle editor content change
const handleEditorChange = useCallback((newValue: string | undefined) => {
const content = newValue || ""
onChange(content)
validateYaml(content)
}, [onChange, validateYaml])
// Handle editor mount
const handleEditorDidMount = useCallback(() => {
// Validate initial content
validateYaml(value)
}, [validateYaml, value])
return (
<div className={cn("flex flex-col h-full", className)}>
{/* Monaco Editor */}
<div className={cn("flex-1 overflow-hidden", yamlError ? 'border-destructive' : '')}>
{shouldMount ? (
<Editor
height={height}
defaultLanguage="yaml"
value={value}
onChange={handleEditorChange}
onMount={handleEditorDidMount}
theme={currentTheme.isDark ? "vs-dark" : "light"}
options={{
minimap: { enabled: false },
fontSize: 12,
lineNumbers: "off",
wordWrap: "off",
scrollBeyondLastLine: false,
automaticLayout: true,
tabSize: 2,
insertSpaces: true,
formatOnPaste: true,
formatOnType: true,
folding: true,
foldingStrategy: "indentation",
showFoldingControls: "mouseover",
bracketPairColorization: {
enabled: true,
},
padding: {
top: 8,
bottom: 8,
},
readOnly: disabled,
placeholder: placeholder,
}}
loading={
<div className="flex items-center justify-center h-full bg-muted/30">
<div className="flex flex-col items-center gap-2">
<div className="h-6 w-6 animate-spin rounded-full border-2 border-primary border-t-transparent" />
<p className="text-xs text-muted-foreground">{t("loading")}</p>
</div>
</div>
}
/>
) : (
<div className="flex items-center justify-center h-full bg-muted/30">
<div className="flex flex-col items-center gap-2">
<div className="h-6 w-6 animate-spin rounded-full border-2 border-primary border-t-transparent" />
<p className="text-xs text-muted-foreground">{t("loading")}</p>
</div>
</div>
)}
</div>
{/* Error message display */}
{yamlError && (
<div className="flex items-start gap-2 p-2 bg-destructive/10 border-t border-destructive/20">
<AlertCircle className="h-3.5 w-3.5 text-destructive mt-0.5 flex-shrink-0" />
<div className="flex-1 text-xs">
<p className="font-medium text-destructive">
{yamlError.line && yamlError.column
? t("errorLocation", { line: yamlError.line, column: yamlError.column })
: t("syntaxError")}
</p>
<p className="text-muted-foreground truncate">{yamlError.message}</p>
</div>
</div>
)}
</div>
)
}

View File

@@ -0,0 +1,26 @@
import { useQuery, keepPreviousData } from '@tanstack/react-query'
import { SearchService } from '@/services/search.service'
import type { SearchParams, SearchResponse } from '@/types/search.types'
/**
* 资产搜索 Hook
*
* @param params 搜索参数
* @param options 查询选项
* @returns 搜索结果
*/
export function useAssetSearch(
params: SearchParams,
options?: { enabled?: boolean }
) {
// 检查是否有有效的搜索查询
const hasSearchParams = !!(params.q && params.q.trim())
return useQuery<SearchResponse>({
queryKey: ['asset-search', params],
queryFn: () => SearchService.search(params),
enabled: (options?.enabled ?? true) && hasSearchParams,
placeholderData: keepPreviousData,
staleTime: 30000, // 30 秒内不重新请求
})
}

View File

@@ -80,3 +80,14 @@ export function parseEngineCapabilities(configuration: string): string[] {
return []
}
}
/**
* Merge multiple engine configurations into a single YAML string
* Simply concatenates configurations with separators
*/
export function mergeEngineConfigurations(configurations: string[]): string {
const validConfigs = configurations.filter(c => c && c.trim())
if (validConfigs.length === 0) return ""
if (validConfigs.length === 1) return validConfigs[0]
return validConfigs.join("\n\n# ---\n\n")
}

View File

@@ -175,6 +175,13 @@
"website": "Website",
"description": "Description"
},
"yamlEditor": {
"syntaxError": "Syntax Error",
"syntaxValid": "Syntax Valid",
"errorLocation": "Line {line}, Column {column}",
"loading": "Loading editor...",
"duplicateKey": "Duplicate key '{key}' found. Later values will override earlier ones. Please remove duplicates."
},
"theme": {
"switchToLight": "Switch to light mode",
"switchToDark": "Switch to dark mode",
@@ -296,6 +303,7 @@
"navigation": {
"mainFeatures": "Main Features",
"dashboard": "Dashboard",
"search": "Search",
"organization": "Organization",
"target": "Targets",
"vulnerabilities": "Vulnerabilities",
@@ -314,6 +322,71 @@
"help": "Get Help",
"feedback": "Feedback"
},
"search": {
"title": "Asset Search",
"hint": "Click search box to view available fields and syntax. Plain text defaults to hostname search",
"searching": "Searching...",
"loading": "Loading...",
"resultsCount": "Found {count} results",
"error": "Search failed, please try again later",
"noResults": "No matching assets found",
"noResultsHint": "Try adjusting your search criteria",
"vulnLoadError": "Failed to load vulnerability details",
"recentSearches": "Recent Searches",
"export": "Export",
"exporting": "Exporting...",
"exportSuccess": "Export successful",
"exportFailed": "Export failed",
"stats": {
"vulnerabilities": "Vulnerabilities"
},
"assetTypes": {
"website": "Website",
"endpoint": "Endpoint"
},
"fields": {
"host": "Hostname",
"url": "URL address",
"title": "Page title",
"tech": "Technology stack",
"status": "HTTP status code",
"body": "Response body content",
"header": "Response header content"
},
"table": {
"url": "URL",
"host": "Host",
"title": "Title",
"status": "Status",
"technologies": "Technologies",
"contentLength": "Content Length",
"location": "Location",
"webserver": "Web Server",
"contentType": "Content Type",
"responseBody": "Response Body",
"responseHeaders": "Response Headers",
"vhost": "VHost",
"createdAt": "Created At",
"gfPatterns": "GF Patterns"
},
"card": {
"title": "Title",
"expand": "Expand",
"collapse": "Collapse",
"vulnerabilities": "Vulnerabilities ({count})",
"vulnName": "Vulnerability Name",
"severity": "Severity",
"source": "Source",
"vulnType": "Vuln Type"
},
"vulnDetail": {
"title": "Vulnerability Details",
"name": "Vulnerability Name",
"source": "Source",
"type": "Vulnerability Type",
"url": "Vulnerability URL"
}
},
"dashboard": {
"title": "Dashboard",
"stats": {
@@ -588,7 +661,40 @@
"noConfig": "No config",
"initiating": "Initiating...",
"startScan": "Start Scan",
"selectedCount": "{count} engines selected"
"selectedCount": "{count} engines selected",
"configTitle": "Scan Configuration",
"configEdited": "Edited",
"stepIndicator": "Step {current}/{total}",
"back": "Back",
"next": "Next",
"steps": {
"selectEngine": "Select Engine",
"editConfig": "Edit Config"
},
"presets": {
"title": "Quick Select",
"fullScan": "Full Scan",
"fullScanDesc": "Complete security assessment covering asset discovery to vulnerability detection",
"recon": "Reconnaissance",
"reconDesc": "Discover and identify target assets including subdomains, ports, sites and fingerprints",
"vulnScan": "Vulnerability Scan",
"vulnScanDesc": "Detect security vulnerabilities on known assets",
"custom": "Custom",
"customDesc": "Manually select engine combination",
"customHint": "Click to manually select engines",
"selectHint": "Please select a scan preset",
"selectEngines": "Select Engines",
"enginesCount": "engines",
"capabilities": "Capabilities",
"usedEngines": "Used Engines",
"noCapabilities": "Please select engines"
},
"overwriteConfirm": {
"title": "Overwrite Configuration",
"description": "You have manually edited the configuration. Changing engines will overwrite your changes. Continue?",
"cancel": "Cancel",
"confirm": "Overwrite"
}
},
"cron": {
"everyMinute": "Every minute",
@@ -670,10 +776,13 @@
"createDesc": "Configure scheduled scan task and set execution plan",
"editTitle": "Edit Scheduled Scan",
"editDesc": "Modify scheduled scan task configuration",
"stepIndicator": "Step {current}/{total}",
"steps": {
"basicInfo": "Basic Info",
"scanMode": "Scan Mode",
"selectTarget": "Select Target",
"selectEngine": "Select Engine",
"editConfig": "Edit Config",
"scheduleSettings": "Schedule Settings"
},
"form": {
@@ -683,8 +792,14 @@
"taskNameRequired": "Please enter task name",
"scanEngine": "Scan Engine",
"scanEnginePlaceholder": "Select scan engine",
"scanEngineDesc": "Select the scan engine configuration to use",
"scanEngineDesc": "Select engine to auto-fill configuration, or edit directly",
"scanEngineRequired": "Please select a scan engine",
"configuration": "Scan Configuration",
"configurationPlaceholder": "Enter YAML scan configuration...",
"configurationDesc": "YAML format scan configuration, select engine to auto-fill or edit manually",
"configurationRequired": "Please enter scan configuration",
"yamlInvalid": "Invalid YAML configuration, please check syntax",
"configEdited": "Edited",
"selectScanMode": "Select Scan Mode",
"organizationScan": "Organization Scan",
"organizationScanDesc": "Select organization, dynamically fetch all targets at execution",
@@ -715,6 +830,9 @@
"organizationMode": "Organization Scan",
"organizationModeHint": "In organization scan mode, all targets under this organization will be dynamically fetched at execution",
"noAvailableTarget": "No available targets",
"noEngine": "No engines available",
"noConfig": "No config",
"capabilitiesCount": "{count} capabilities",
"selected": "Selected",
"selectedEngines": "{count} engines selected"
},
@@ -736,7 +854,14 @@
},
"toast": {
"selectOrganization": "Please select an organization",
"selectTarget": "Please select a scan target"
"selectTarget": "Please select a scan target",
"configConflict": "Configuration conflict"
},
"overwriteConfirm": {
"title": "Overwrite Configuration",
"description": "You have manually edited the configuration. Changing engines will overwrite your changes. Do you want to continue?",
"cancel": "Cancel",
"confirm": "Overwrite"
}
},
"engine": {
@@ -1338,6 +1463,8 @@
"initiateScanFailed": "Failed to initiate scan",
"noScansCreated": "No scan tasks were created",
"unknownError": "Unknown error",
"noEngineSelected": "Please select at least one scan engine",
"emptyConfig": "Scan configuration cannot be empty",
"engineNameRequired": "Please enter engine name",
"configRequired": "Configuration content is required",
"yamlSyntaxError": "YAML syntax error",
@@ -1642,7 +1769,8 @@
},
"step1Title": "Enter Targets",
"step2Title": "Select Engines",
"step3Title": "Confirm",
"step3Title": "Edit Config",
"stepIndicator": "Step {current}/{total}",
"step1Hint": "Enter scan targets in the left input box, one per line",
"step": "Step {current}/{total} · {title}",
"targetPlaceholder": "Enter one target per line, supported formats:\n\nDomain: example.com, sub.example.com\nIP Address: 192.168.1.1, 10.0.0.1\nCIDR: 192.168.1.0/24, 10.0.0.0/8\nURL: https://example.com/api/v1",
@@ -1670,10 +1798,19 @@
"andMore": "{count} more...",
"selectedEngines": "Selected Engines",
"confirmSummary": "Will scan {targetCount} targets with {engineCount} engines",
"configTitle": "Scan Configuration",
"configEdited": "Edited",
"overwriteConfirm": {
"title": "Overwrite Configuration",
"description": "You have manually edited the configuration. Changing engines will overwrite your changes. Continue?",
"cancel": "Cancel",
"confirm": "Overwrite"
},
"toast": {
"noValidTarget": "Please enter at least one valid target",
"hasInvalidInputs": "{count} invalid inputs, please fix before continuing",
"selectEngine": "Please select a scan engine",
"emptyConfig": "Scan configuration cannot be empty",
"getEnginesFailed": "Failed to get engine list",
"createFailed": "Failed to create scan task",
"createSuccess": "Created {count} scan tasks",
@@ -1908,6 +2045,16 @@
"formatInvalid": "Invalid format"
}
},
"globalSearch": {
"search": "Search",
"placeholder": "Search assets... (host=\"api\" && tech=\"nginx\")",
"noResults": "No results found",
"searchFor": "Search for",
"recent": "Recent Searches",
"quickSearch": "Quick Search",
"hint": "Supports FOFA-style syntax",
"toSearch": "to search"
},
"errors": {
"unknown": "Operation failed, please try again later",
"validation": "Invalid input data",

View File

@@ -175,6 +175,13 @@
"website": "官网",
"description": "描述"
},
"yamlEditor": {
"syntaxError": "语法错误",
"syntaxValid": "语法正确",
"errorLocation": "第 {line} 行,第 {column} 列",
"loading": "加载编辑器...",
"duplicateKey": "发现重复的配置项 '{key}',后面的配置会覆盖前面的,请删除重复项"
},
"theme": {
"switchToLight": "切换到亮色模式",
"switchToDark": "切换到暗色模式",
@@ -296,6 +303,7 @@
"navigation": {
"mainFeatures": "主要功能",
"dashboard": "仪表盘",
"search": "搜索",
"organization": "组织",
"target": "目标",
"vulnerabilities": "漏洞",
@@ -314,6 +322,71 @@
"help": "获取帮助",
"feedback": "反馈建议"
},
"search": {
"title": "资产搜索",
"hint": "点击搜索框查看可用字段和语法,直接输入文本默认搜索主机名",
"searching": "搜索中...",
"loading": "加载中...",
"resultsCount": "找到 {count} 条结果",
"error": "搜索失败,请稍后重试",
"noResults": "未找到匹配的资产",
"noResultsHint": "请尝试调整搜索条件",
"vulnLoadError": "加载漏洞详情失败",
"recentSearches": "最近搜索",
"export": "导出",
"exporting": "导出中...",
"exportSuccess": "导出成功",
"exportFailed": "导出失败",
"stats": {
"vulnerabilities": "漏洞"
},
"assetTypes": {
"website": "网站",
"endpoint": "URL"
},
"fields": {
"host": "主机名",
"url": "URL 地址",
"title": "页面标题",
"tech": "技术栈",
"status": "HTTP 状态码",
"body": "响应体内容",
"header": "响应头内容"
},
"table": {
"url": "URL",
"host": "主机名",
"title": "标题",
"status": "状态码",
"technologies": "技术栈",
"contentLength": "内容长度",
"location": "跳转地址",
"webserver": "Web 服务器",
"contentType": "内容类型",
"responseBody": "响应体",
"responseHeaders": "响应头",
"vhost": "VHost",
"createdAt": "创建时间",
"gfPatterns": "GF 模式"
},
"card": {
"title": "标题",
"expand": "展开",
"collapse": "收起",
"vulnerabilities": "关联漏洞 ({count})",
"vulnName": "漏洞名称",
"severity": "严重程度",
"source": "来源",
"vulnType": "漏洞类型"
},
"vulnDetail": {
"title": "漏洞详情",
"name": "漏洞名称",
"source": "来源",
"type": "漏洞类型",
"url": "漏洞 URL"
}
},
"dashboard": {
"title": "仪表盘",
"stats": {
@@ -588,7 +661,40 @@
"noConfig": "无配置",
"initiating": "发起中...",
"startScan": "开始扫描",
"selectedCount": "已选择 {count} 个引擎"
"selectedCount": "已选择 {count} 个引擎",
"configTitle": "扫描配置",
"configEdited": "已编辑",
"stepIndicator": "步骤 {current}/{total}",
"back": "上一步",
"next": "下一步",
"steps": {
"selectEngine": "选择引擎",
"editConfig": "编辑配置"
},
"presets": {
"title": "推荐组合",
"fullScan": "全量扫描",
"fullScanDesc": "完整的安全评估,覆盖资产发现到漏洞检测的全部流程",
"recon": "信息收集",
"reconDesc": "发现和识别目标资产,包括子域名、端口、站点和指纹",
"vulnScan": "漏洞扫描",
"vulnScanDesc": "对已知资产进行安全漏洞检测",
"custom": "自定义",
"customDesc": "手动选择引擎组合",
"customHint": "点击选择后手动勾选引擎",
"selectHint": "请选择一个扫描方案",
"selectEngines": "选择引擎",
"enginesCount": "个引擎",
"capabilities": "涉及能力",
"usedEngines": "使用引擎",
"noCapabilities": "请选择引擎"
},
"overwriteConfirm": {
"title": "覆盖配置确认",
"description": "您已手动编辑过配置,切换引擎将覆盖当前配置。是否继续?",
"cancel": "取消",
"confirm": "确认覆盖"
}
},
"cron": {
"everyMinute": "每分钟",
@@ -670,10 +776,13 @@
"createDesc": "配置定时扫描任务,设置执行计划",
"editTitle": "编辑定时扫描",
"editDesc": "修改定时扫描任务配置",
"stepIndicator": "步骤 {current}/{total}",
"steps": {
"basicInfo": "基本信息",
"scanMode": "扫描模式",
"selectTarget": "选择目标",
"selectEngine": "选择引擎",
"editConfig": "编辑配置",
"scheduleSettings": "调度设置"
},
"form": {
@@ -683,8 +792,14 @@
"taskNameRequired": "请输入任务名称",
"scanEngine": "扫描引擎",
"scanEnginePlaceholder": "选择扫描引擎",
"scanEngineDesc": "选择要使用的扫描引擎配置",
"scanEngineDesc": "选择引擎可快速填充配置,也可直接编辑配置",
"scanEngineRequired": "请选择扫描引擎",
"configuration": "扫描配置",
"configurationPlaceholder": "请输入 YAML 格式的扫描配置...",
"configurationDesc": "YAML 格式的扫描配置,可选择引擎自动填充或手动编辑",
"configurationRequired": "请输入扫描配置",
"yamlInvalid": "YAML 配置格式错误,请检查语法",
"configEdited": "已编辑",
"selectScanMode": "选择扫描模式",
"organizationScan": "组织扫描",
"organizationScanDesc": "选择组织,执行时动态获取其下所有目标",
@@ -715,6 +830,9 @@
"organizationMode": "组织扫描",
"organizationModeHint": "组织扫描模式下,执行时将动态获取该组织下所有目标",
"noAvailableTarget": "暂无可用目标",
"noEngine": "暂无可用引擎",
"noConfig": "无配置",
"capabilitiesCount": "{count} 项能力",
"selected": "已选择",
"selectedEngines": "已选择 {count} 个引擎"
},
@@ -736,7 +854,14 @@
},
"toast": {
"selectOrganization": "请选择一个组织",
"selectTarget": "请选择一个扫描目标"
"selectTarget": "请选择一个扫描目标",
"configConflict": "配置冲突"
},
"overwriteConfirm": {
"title": "覆盖配置确认",
"description": "您已手动编辑了配置,切换引擎将覆盖当前配置。确定要继续吗?",
"cancel": "取消",
"confirm": "确定覆盖"
}
},
"engine": {
@@ -1338,6 +1463,8 @@
"initiateScanFailed": "发起扫描失败",
"noScansCreated": "未创建任何扫描任务",
"unknownError": "未知错误",
"noEngineSelected": "请选择至少一个扫描引擎",
"emptyConfig": "扫描配置不能为空",
"engineNameRequired": "请输入引擎名称",
"configRequired": "配置内容不能为空",
"yamlSyntaxError": "YAML 语法错误",
@@ -1642,7 +1769,8 @@
},
"step1Title": "输入目标",
"step2Title": "选择引擎",
"step3Title": "确认",
"step3Title": "编辑配置",
"stepIndicator": "步骤 {current}/{total}",
"step1Hint": "在左侧输入框中输入扫描目标,每行一个",
"step": "步骤 {current}/{total} · {title}",
"targetPlaceholder": "每行输入一个目标,支持以下格式:\n\n域名: example.com, sub.example.com\nIP地址: 192.168.1.1, 10.0.0.1\nCIDR网段: 192.168.1.0/24, 10.0.0.0/8\nURL: https://example.com/api/v1",
@@ -1670,10 +1798,19 @@
"andMore": "还有 {count} 个...",
"selectedEngines": "已选引擎",
"confirmSummary": "将使用 {engineCount} 个引擎扫描 {targetCount} 个目标",
"configTitle": "扫描配置",
"configEdited": "已编辑",
"overwriteConfirm": {
"title": "覆盖配置确认",
"description": "您已手动编辑过配置,切换引擎将覆盖当前配置。是否继续?",
"cancel": "取消",
"confirm": "确认覆盖"
},
"toast": {
"noValidTarget": "请输入至少一个有效目标",
"hasInvalidInputs": "存在 {count} 个无效输入,请修正后继续",
"selectEngine": "请选择扫描引擎",
"emptyConfig": "扫描配置不能为空",
"getEnginesFailed": "获取引擎列表失败",
"createFailed": "创建扫描任务失败",
"createSuccess": "已创建 {count} 个扫描任务",
@@ -1908,6 +2045,16 @@
"formatInvalid": "格式无效"
}
},
"globalSearch": {
"search": "搜索",
"placeholder": "搜索资产... (host=\"api\" && tech=\"nginx\")",
"noResults": "未找到结果",
"searchFor": "搜索",
"recent": "最近搜索",
"quickSearch": "快捷搜索",
"hint": "支持 FOFA 风格语法",
"toSearch": "搜索"
},
"errors": {
"unknown": "操作失败,请稍后重试",
"validation": "输入数据无效",

23
frontend/mock/config.ts Normal file
View File

@@ -0,0 +1,23 @@
/**
* Mock 数据配置
*
* 使用方式:
* 1. 在 .env.local 中设置 NEXT_PUBLIC_USE_MOCK=true 启用 mock 数据
* 2. 或者直接修改下面的 FORCE_MOCK 为 true
*/
// 强制使用 mock 数据(一般保持 false通过环境变量控制
const FORCE_MOCK = false
// 从环境变量读取 mock 配置
export const USE_MOCK = FORCE_MOCK || process.env.NEXT_PUBLIC_USE_MOCK === 'true'
// Mock 数据延迟(模拟网络请求)
export const MOCK_DELAY = 300 // ms
/**
* 模拟网络延迟
*/
export function mockDelay(ms: number = MOCK_DELAY): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms))
}

View File

@@ -0,0 +1,22 @@
import type { User, MeResponse, LoginResponse, LogoutResponse } from '@/types/auth.types'
export const mockUser: User = {
id: 1,
username: 'admin',
isStaff: true,
isSuperuser: true,
}
export const mockMeResponse: MeResponse = {
authenticated: true,
user: mockUser,
}
export const mockLoginResponse: LoginResponse = {
message: 'Login successful',
user: mockUser,
}
export const mockLogoutResponse: LogoutResponse = {
message: 'Logout successful',
}

View File

@@ -0,0 +1,71 @@
import type { AssetStatistics, StatisticsHistoryItem, DashboardStats } from '@/types/dashboard.types'
export const mockDashboardStats: DashboardStats = {
totalTargets: 156,
totalSubdomains: 4823,
totalEndpoints: 12456,
totalVulnerabilities: 89,
}
export const mockAssetStatistics: AssetStatistics = {
totalTargets: 156,
totalSubdomains: 4823,
totalIps: 892,
totalEndpoints: 12456,
totalWebsites: 3421,
totalVulns: 89,
totalAssets: 21638,
runningScans: 3,
updatedAt: new Date().toISOString(),
// 变化值
changeTargets: 12,
changeSubdomains: 234,
changeIps: 45,
changeEndpoints: 567,
changeWebsites: 89,
changeVulns: -5,
changeAssets: 942,
// 漏洞严重程度分布
vulnBySeverity: {
critical: 3,
high: 12,
medium: 28,
low: 34,
info: 12,
},
}
// 生成过去 N 天的历史数据
function generateHistoryData(days: number): StatisticsHistoryItem[] {
const data: StatisticsHistoryItem[] = []
const now = new Date()
for (let i = days - 1; i >= 0; i--) {
const date = new Date(now)
date.setDate(date.getDate() - i)
// 模拟逐渐增长的趋势
const factor = 1 + (days - i) * 0.02
data.push({
date: date.toISOString().split('T')[0],
totalTargets: Math.floor(140 * factor),
totalSubdomains: Math.floor(4200 * factor),
totalIps: Math.floor(780 * factor),
totalEndpoints: Math.floor(10800 * factor),
totalWebsites: Math.floor(2980 * factor),
totalVulns: Math.floor(75 * factor),
totalAssets: Math.floor(18900 * factor),
})
}
return data
}
export const mockStatisticsHistory7Days = generateHistoryData(7)
export const mockStatisticsHistory30Days = generateHistoryData(30)
export function getMockStatisticsHistory(days: number): StatisticsHistoryItem[] {
if (days <= 7) return mockStatisticsHistory7Days
return generateHistoryData(days)
}

View File

@@ -0,0 +1,257 @@
import type { Endpoint, GetEndpointsResponse } from '@/types/endpoint.types'
export const mockEndpoints: Endpoint[] = [
{
id: 1,
url: 'https://acme.com/',
method: 'GET',
statusCode: 200,
title: 'Acme Corporation - Home',
contentLength: 45678,
contentType: 'text/html; charset=utf-8',
responseTime: 0.234,
host: 'acme.com',
webserver: 'nginx/1.24.0',
tech: ['React', 'Next.js', 'Node.js'],
createdAt: '2024-12-28T10:00:00Z',
},
{
id: 2,
url: 'https://acme.com/login',
method: 'GET',
statusCode: 200,
title: 'Login - Acme',
contentLength: 12345,
contentType: 'text/html; charset=utf-8',
responseTime: 0.156,
host: 'acme.com',
webserver: 'nginx/1.24.0',
tech: ['React', 'Next.js'],
createdAt: '2024-12-28T10:01:00Z',
},
{
id: 3,
url: 'https://api.acme.com/v1/users',
method: 'GET',
statusCode: 200,
title: '',
contentLength: 8923,
contentType: 'application/json',
responseTime: 0.089,
host: 'api.acme.com',
webserver: 'nginx/1.24.0',
tech: ['Django', 'Python'],
gfPatterns: ['api', 'json'],
createdAt: '2024-12-28T10:02:00Z',
},
{
id: 4,
url: 'https://api.acme.com/v1/products',
method: 'GET',
statusCode: 200,
title: '',
contentLength: 23456,
contentType: 'application/json',
responseTime: 0.145,
host: 'api.acme.com',
webserver: 'nginx/1.24.0',
tech: ['Django', 'Python'],
gfPatterns: ['api', 'json'],
createdAt: '2024-12-28T10:03:00Z',
},
{
id: 5,
url: 'https://acme.io/docs',
method: 'GET',
statusCode: 200,
title: 'Documentation - Acme.io',
contentLength: 67890,
contentType: 'text/html; charset=utf-8',
responseTime: 0.312,
host: 'acme.io',
webserver: 'cloudflare',
tech: ['Vue.js', 'Vitepress'],
createdAt: '2024-12-27T14:30:00Z',
},
{
id: 6,
url: 'https://acme.io/api/config',
method: 'GET',
statusCode: 200,
title: '',
contentLength: 1234,
contentType: 'application/json',
responseTime: 0.067,
host: 'acme.io',
webserver: 'cloudflare',
tech: ['Node.js', 'Express'],
gfPatterns: ['config', 'json'],
createdAt: '2024-12-27T14:31:00Z',
},
{
id: 7,
url: 'https://techstart.io/',
method: 'GET',
statusCode: 200,
title: 'TechStart - Innovation Hub',
contentLength: 34567,
contentType: 'text/html; charset=utf-8',
responseTime: 0.278,
host: 'techstart.io',
webserver: 'Apache/2.4.54',
tech: ['WordPress', 'PHP'],
createdAt: '2024-12-26T08:45:00Z',
},
{
id: 8,
url: 'https://techstart.io/admin',
method: 'GET',
statusCode: 302,
title: '',
contentLength: 0,
contentType: 'text/html',
responseTime: 0.045,
location: 'https://techstart.io/admin/login',
host: 'techstart.io',
webserver: 'Apache/2.4.54',
tech: ['WordPress', 'PHP'],
createdAt: '2024-12-26T08:46:00Z',
},
{
id: 9,
url: 'https://globalfinance.com/',
method: 'GET',
statusCode: 200,
title: 'Global Finance - Your Financial Partner',
contentLength: 56789,
contentType: 'text/html; charset=utf-8',
responseTime: 0.456,
host: 'globalfinance.com',
webserver: 'Microsoft-IIS/10.0',
tech: ['ASP.NET', 'C#', 'jQuery'],
createdAt: '2024-12-25T16:20:00Z',
},
{
id: 10,
url: 'https://globalfinance.com/.git/config',
method: 'GET',
statusCode: 200,
title: '',
contentLength: 456,
contentType: 'text/plain',
responseTime: 0.034,
host: 'globalfinance.com',
webserver: 'Microsoft-IIS/10.0',
gfPatterns: ['git', 'config'],
createdAt: '2024-12-25T16:21:00Z',
},
{
id: 11,
url: 'https://retailmax.com/',
method: 'GET',
statusCode: 200,
title: 'RetailMax - Shop Everything',
contentLength: 89012,
contentType: 'text/html; charset=utf-8',
responseTime: 0.567,
host: 'retailmax.com',
webserver: 'nginx/1.22.0',
tech: ['React', 'Redux', 'Node.js'],
createdAt: '2024-12-21T10:45:00Z',
},
{
id: 12,
url: 'https://retailmax.com/product?id=1',
method: 'GET',
statusCode: 200,
title: 'Product Detail - RetailMax',
contentLength: 23456,
contentType: 'text/html; charset=utf-8',
responseTime: 0.234,
host: 'retailmax.com',
webserver: 'nginx/1.22.0',
tech: ['React', 'Redux'],
gfPatterns: ['param', 'id'],
createdAt: '2024-12-21T10:46:00Z',
},
{
id: 13,
url: 'https://healthcareplus.com/',
method: 'GET',
statusCode: 200,
title: 'HealthCare Plus - Digital Health',
contentLength: 45678,
contentType: 'text/html; charset=utf-8',
responseTime: 0.345,
host: 'healthcareplus.com',
webserver: 'nginx/1.24.0',
tech: ['Angular', 'TypeScript'],
createdAt: '2024-12-23T11:00:00Z',
},
{
id: 14,
url: 'https://edutech.io/',
method: 'GET',
statusCode: 200,
title: 'EduTech - Learn Anywhere',
contentLength: 67890,
contentType: 'text/html; charset=utf-8',
responseTime: 0.289,
host: 'edutech.io',
webserver: 'cloudflare',
tech: ['Vue.js', 'Nuxt.js'],
createdAt: '2024-12-22T13:30:00Z',
},
{
id: 15,
url: 'https://cloudnine.host/',
method: 'GET',
statusCode: 200,
title: 'CloudNine Hosting',
contentLength: 34567,
contentType: 'text/html; charset=utf-8',
responseTime: 0.178,
host: 'cloudnine.host',
webserver: 'LiteSpeed',
tech: ['PHP', 'Laravel'],
createdAt: '2024-12-19T16:00:00Z',
},
]
export function getMockEndpoints(params?: {
page?: number
pageSize?: number
search?: string
}): GetEndpointsResponse {
const page = params?.page || 1
const pageSize = params?.pageSize || 10
const search = params?.search?.toLowerCase() || ''
let filtered = mockEndpoints
if (search) {
filtered = mockEndpoints.filter(
ep =>
ep.url.toLowerCase().includes(search) ||
ep.title.toLowerCase().includes(search) ||
ep.host?.toLowerCase().includes(search)
)
}
const total = filtered.length
const totalPages = Math.ceil(total / pageSize)
const start = (page - 1) * pageSize
const endpoints = filtered.slice(start, start + pageSize)
return {
endpoints,
total,
page,
pageSize,
totalPages,
}
}
export function getMockEndpointById(id: number): Endpoint | undefined {
return mockEndpoints.find(ep => ep.id === id)
}

View File

@@ -0,0 +1,78 @@
import type { ScanEngine } from '@/types/engine.types'
export const mockEngines: ScanEngine[] = [
{
id: 1,
name: 'Full Scan',
configuration: `# Full reconnaissance scan
stages:
- name: subdomain_discovery
tools:
- subfinder
- amass
- name: port_scan
tools:
- nmap
- name: web_crawling
tools:
- httpx
- katana
- name: vulnerability_scan
tools:
- nuclei
`,
createdAt: '2024-01-15T08:00:00Z',
updatedAt: '2024-12-20T10:30:00Z',
},
{
id: 2,
name: 'Quick Scan',
configuration: `# Quick scan - subdomain and web only
stages:
- name: subdomain_discovery
tools:
- subfinder
- name: web_crawling
tools:
- httpx
`,
createdAt: '2024-02-10T09:00:00Z',
updatedAt: '2024-12-18T14:00:00Z',
},
{
id: 3,
name: 'Vulnerability Only',
configuration: `# Vulnerability scan only
stages:
- name: vulnerability_scan
tools:
- nuclei
options:
severity: critical,high,medium
`,
createdAt: '2024-03-05T11:00:00Z',
updatedAt: '2024-12-15T16:20:00Z',
},
{
id: 4,
name: 'Subdomain Discovery',
configuration: `# Subdomain enumeration only
stages:
- name: subdomain_discovery
tools:
- subfinder
- amass
- findomain
`,
createdAt: '2024-04-12T08:30:00Z',
updatedAt: '2024-12-10T09:00:00Z',
},
]
export function getMockEngines(): ScanEngine[] {
return mockEngines
}
export function getMockEngineById(id: number): ScanEngine | undefined {
return mockEngines.find(e => e.id === id)
}

View File

@@ -0,0 +1,110 @@
import type { BackendNotification, GetNotificationsResponse } from '@/types/notification.types'
export const mockNotifications: BackendNotification[] = [
{
id: 1,
category: 'vulnerability',
title: 'Critical Vulnerability Found',
message: 'SQL Injection detected in retailmax.com/product endpoint',
level: 'critical',
createdAt: '2024-12-29T10:30:00Z',
isRead: false,
},
{
id: 2,
category: 'scan',
title: 'Scan Completed',
message: 'Scan for acme.com completed successfully with 23 vulnerabilities found',
level: 'medium',
createdAt: '2024-12-29T09:00:00Z',
isRead: false,
},
{
id: 3,
category: 'vulnerability',
title: 'High Severity Vulnerability',
message: 'XSS vulnerability found in acme.com/search',
level: 'high',
createdAt: '2024-12-28T16:45:00Z',
isRead: true,
},
{
id: 4,
category: 'scan',
title: 'Scan Failed',
message: 'Scan for globalfinance.com failed: Connection timeout',
level: 'high',
createdAt: '2024-12-28T14:20:00Z',
isRead: true,
},
{
id: 5,
category: 'asset',
title: 'New Subdomains Discovered',
message: '15 new subdomains discovered for techstart.io',
level: 'low',
createdAt: '2024-12-27T11:00:00Z',
isRead: true,
},
{
id: 6,
category: 'system',
title: 'Worker Offline',
message: 'Worker node worker-03 is now offline',
level: 'medium',
createdAt: '2024-12-27T08:30:00Z',
isRead: true,
},
{
id: 7,
category: 'scan',
title: 'Scheduled Scan Started',
message: 'Scheduled scan for Acme Corporation started',
level: 'low',
createdAt: '2024-12-26T06:00:00Z',
isRead: true,
},
{
id: 8,
category: 'system',
title: 'System Update Available',
message: 'A new version of the scanner is available',
level: 'low',
createdAt: '2024-12-25T10:00:00Z',
isRead: true,
},
]
export function getMockNotifications(params?: {
page?: number
pageSize?: number
unread?: boolean
}): GetNotificationsResponse {
const page = params?.page || 1
const pageSize = params?.pageSize || 10
let filtered = mockNotifications
if (params?.unread) {
filtered = filtered.filter(n => !n.isRead)
}
const total = filtered.length
const totalPages = Math.ceil(total / pageSize)
const start = (page - 1) * pageSize
const results = filtered.slice(start, start + pageSize)
return {
results,
total,
page,
pageSize,
totalPages,
}
}
export function getMockUnreadCount(): { count: number } {
return {
count: mockNotifications.filter(n => !n.isRead).length,
}
}

View File

@@ -0,0 +1,145 @@
import type { Organization, OrganizationsResponse } from '@/types/organization.types'
export const mockOrganizations: Organization[] = [
{
id: 1,
name: 'Acme Corporation',
description: '全球领先的科技公司,专注于云计算和人工智能领域',
createdAt: '2024-01-15T08:30:00Z',
updatedAt: '2024-12-28T14:20:00Z',
targetCount: 12,
domainCount: 156,
endpointCount: 2341,
targets: [
{ id: 1, name: 'acme.com' },
{ id: 2, name: 'acme.io' },
],
},
{
id: 2,
name: 'TechStart Inc',
description: '创新型初创企业,主营 SaaS 产品开发',
createdAt: '2024-02-20T10:15:00Z',
updatedAt: '2024-12-27T09:45:00Z',
targetCount: 5,
domainCount: 78,
endpointCount: 892,
targets: [
{ id: 3, name: 'techstart.io' },
],
},
{
id: 3,
name: 'Global Finance Ltd',
description: '国际金融服务公司,提供银行和投资解决方案',
createdAt: '2024-03-10T14:00:00Z',
updatedAt: '2024-12-26T16:30:00Z',
targetCount: 8,
domainCount: 234,
endpointCount: 1567,
targets: [
{ id: 4, name: 'globalfinance.com' },
{ id: 5, name: 'gf-bank.net' },
],
},
{
id: 4,
name: 'HealthCare Plus',
description: '医疗健康科技公司,专注于数字化医疗解决方案',
createdAt: '2024-04-05T09:20:00Z',
updatedAt: '2024-12-25T11:10:00Z',
targetCount: 6,
domainCount: 89,
endpointCount: 723,
targets: [
{ id: 6, name: 'healthcareplus.com' },
],
},
{
id: 5,
name: 'EduTech Solutions',
description: '在线教育平台,提供 K-12 和职业培训课程',
createdAt: '2024-05-12T11:45:00Z',
updatedAt: '2024-12-24T13:55:00Z',
targetCount: 4,
domainCount: 45,
endpointCount: 456,
targets: [
{ id: 7, name: 'edutech.io' },
],
},
{
id: 6,
name: 'RetailMax',
description: '电子商务零售平台,覆盖多品类商品销售',
createdAt: '2024-06-08T16:30:00Z',
updatedAt: '2024-12-23T10:20:00Z',
targetCount: 15,
domainCount: 312,
endpointCount: 4521,
targets: [
{ id: 8, name: 'retailmax.com' },
{ id: 9, name: 'retailmax.cn' },
],
},
{
id: 7,
name: 'CloudNine Hosting',
description: '云托管服务提供商,提供 VPS 和专用服务器',
createdAt: '2024-07-20T08:00:00Z',
updatedAt: '2024-12-22T15:40:00Z',
targetCount: 3,
domainCount: 67,
endpointCount: 389,
targets: [
{ id: 10, name: 'cloudnine.host' },
],
},
{
id: 8,
name: 'MediaStream Corp',
description: '流媒体内容分发平台,提供视频和音频服务',
createdAt: '2024-08-15T12:10:00Z',
updatedAt: '2024-12-21T08:25:00Z',
targetCount: 7,
domainCount: 123,
endpointCount: 1234,
targets: [
{ id: 11, name: 'mediastream.tv' },
],
},
]
export function getMockOrganizations(params?: {
page?: number
pageSize?: number
search?: string
}): OrganizationsResponse<Organization> {
const page = params?.page || 1
const pageSize = params?.pageSize || 10
const search = params?.search?.toLowerCase() || ''
// 过滤搜索
let filtered = mockOrganizations
if (search) {
filtered = mockOrganizations.filter(
org =>
org.name.toLowerCase().includes(search) ||
org.description.toLowerCase().includes(search)
)
}
// 分页
const total = filtered.length
const totalPages = Math.ceil(total / pageSize)
const start = (page - 1) * pageSize
const results = filtered.slice(start, start + pageSize)
return {
results,
total,
page,
pageSize,
totalPages,
}
}

309
frontend/mock/data/scans.ts Normal file
View File

@@ -0,0 +1,309 @@
import type { ScanRecord, GetScansResponse, ScanStatus } from '@/types/scan.types'
import type { ScanStatistics } from '@/services/scan.service'
export const mockScans: ScanRecord[] = [
{
id: 1,
target: 1,
targetName: 'acme.com',
workerName: 'worker-01',
summary: {
subdomains: 156,
websites: 89,
directories: 234,
endpoints: 2341,
ips: 45,
vulnerabilities: {
total: 23,
critical: 1,
high: 4,
medium: 8,
low: 10,
},
},
engineIds: [1, 2, 3],
engineNames: ['Subdomain Discovery', 'Web Crawling', 'Nuclei Scanner'],
createdAt: '2024-12-28T10:00:00Z',
status: 'completed',
progress: 100,
},
{
id: 2,
target: 2,
targetName: 'acme.io',
workerName: 'worker-02',
summary: {
subdomains: 78,
websites: 45,
directories: 123,
endpoints: 892,
ips: 23,
vulnerabilities: {
total: 12,
critical: 0,
high: 2,
medium: 5,
low: 5,
},
},
engineIds: [1, 2],
engineNames: ['Subdomain Discovery', 'Web Crawling'],
createdAt: '2024-12-27T14:30:00Z',
status: 'running',
progress: 65,
currentStage: 'web_crawling',
stageProgress: {
subdomain_discovery: {
status: 'completed',
order: 0,
startedAt: '2024-12-27T14:30:00Z',
duration: 1200,
detail: 'Found 78 subdomains',
},
web_crawling: {
status: 'running',
order: 1,
startedAt: '2024-12-27T14:50:00Z',
},
},
},
{
id: 3,
target: 3,
targetName: 'techstart.io',
workerName: 'worker-01',
summary: {
subdomains: 45,
websites: 28,
directories: 89,
endpoints: 567,
ips: 12,
vulnerabilities: {
total: 8,
critical: 0,
high: 1,
medium: 3,
low: 4,
},
},
engineIds: [1, 2, 3],
engineNames: ['Subdomain Discovery', 'Web Crawling', 'Nuclei Scanner'],
createdAt: '2024-12-26T08:45:00Z',
status: 'completed',
progress: 100,
},
{
id: 4,
target: 4,
targetName: 'globalfinance.com',
workerName: 'worker-03',
summary: {
subdomains: 0,
websites: 0,
directories: 0,
endpoints: 0,
ips: 0,
vulnerabilities: {
total: 0,
critical: 0,
high: 0,
medium: 0,
low: 0,
},
},
engineIds: [1],
engineNames: ['Subdomain Discovery'],
createdAt: '2024-12-25T16:20:00Z',
status: 'failed',
progress: 15,
errorMessage: 'Connection timeout: Unable to reach target',
},
{
id: 5,
target: 6,
targetName: 'healthcareplus.com',
workerName: 'worker-02',
summary: {
subdomains: 34,
websites: 0,
directories: 0,
endpoints: 0,
ips: 8,
vulnerabilities: {
total: 0,
critical: 0,
high: 0,
medium: 0,
low: 0,
},
},
engineIds: [1, 2, 3],
engineNames: ['Subdomain Discovery', 'Web Crawling', 'Nuclei Scanner'],
createdAt: '2024-12-29T09:00:00Z',
status: 'running',
progress: 25,
currentStage: 'subdomain_discovery',
stageProgress: {
subdomain_discovery: {
status: 'running',
order: 0,
startedAt: '2024-12-29T09:00:00Z',
},
web_crawling: {
status: 'pending',
order: 1,
},
nuclei_scan: {
status: 'pending',
order: 2,
},
},
},
{
id: 6,
target: 7,
targetName: 'edutech.io',
workerName: null,
summary: {
subdomains: 0,
websites: 0,
directories: 0,
endpoints: 0,
ips: 0,
vulnerabilities: {
total: 0,
critical: 0,
high: 0,
medium: 0,
low: 0,
},
},
engineIds: [1, 2],
engineNames: ['Subdomain Discovery', 'Web Crawling'],
createdAt: '2024-12-29T10:30:00Z',
status: 'initiated',
progress: 0,
},
{
id: 7,
target: 8,
targetName: 'retailmax.com',
workerName: 'worker-01',
summary: {
subdomains: 89,
websites: 56,
directories: 178,
endpoints: 1234,
ips: 28,
vulnerabilities: {
total: 15,
critical: 0,
high: 3,
medium: 6,
low: 6,
},
},
engineIds: [1, 2, 3],
engineNames: ['Subdomain Discovery', 'Web Crawling', 'Nuclei Scanner'],
createdAt: '2024-12-21T10:45:00Z',
status: 'completed',
progress: 100,
},
{
id: 8,
target: 11,
targetName: 'mediastream.tv',
workerName: 'worker-02',
summary: {
subdomains: 67,
websites: 0,
directories: 0,
endpoints: 0,
ips: 15,
vulnerabilities: {
total: 0,
critical: 0,
high: 0,
medium: 0,
low: 0,
},
},
engineIds: [1, 2, 3],
engineNames: ['Subdomain Discovery', 'Web Crawling', 'Nuclei Scanner'],
createdAt: '2024-12-29T08:00:00Z',
status: 'running',
progress: 45,
currentStage: 'web_crawling',
stageProgress: {
subdomain_discovery: {
status: 'completed',
order: 0,
startedAt: '2024-12-29T08:00:00Z',
duration: 900,
detail: 'Found 67 subdomains',
},
web_crawling: {
status: 'running',
order: 1,
startedAt: '2024-12-29T08:15:00Z',
},
nuclei_scan: {
status: 'pending',
order: 2,
},
},
},
]
export const mockScanStatistics: ScanStatistics = {
total: 156,
running: 3,
completed: 142,
failed: 11,
totalVulns: 89,
totalSubdomains: 4823,
totalEndpoints: 12456,
totalWebsites: 3421,
totalAssets: 21638,
}
export function getMockScans(params?: {
page?: number
pageSize?: number
status?: ScanStatus
search?: string
}): GetScansResponse {
const page = params?.page || 1
const pageSize = params?.pageSize || 10
const status = params?.status
const search = params?.search?.toLowerCase() || ''
let filtered = mockScans
if (status) {
filtered = filtered.filter(scan => scan.status === status)
}
if (search) {
filtered = filtered.filter(scan =>
scan.targetName.toLowerCase().includes(search)
)
}
const total = filtered.length
const totalPages = Math.ceil(total / pageSize)
const start = (page - 1) * pageSize
const results = filtered.slice(start, start + pageSize)
return {
results,
total,
page,
pageSize,
totalPages,
}
}
export function getMockScanById(id: number): ScanRecord | undefined {
return mockScans.find(scan => scan.id === id)
}

View File

@@ -0,0 +1,132 @@
import type { ScheduledScan, GetScheduledScansResponse } from '@/types/scheduled-scan.types'
export const mockScheduledScans: ScheduledScan[] = [
{
id: 1,
name: 'Daily Acme Scan',
engineIds: [1],
engineNames: ['Full Scan'],
organizationId: 1,
organizationName: 'Acme Corporation',
targetId: null,
targetName: null,
scanMode: 'organization',
cronExpression: '0 2 * * *',
isEnabled: true,
nextRunTime: '2024-12-30T02:00:00Z',
lastRunTime: '2024-12-29T02:00:00Z',
runCount: 45,
createdAt: '2024-11-15T08:00:00Z',
updatedAt: '2024-12-29T02:00:00Z',
},
{
id: 2,
name: 'Weekly TechStart Vuln Scan',
engineIds: [3],
engineNames: ['Vulnerability Only'],
organizationId: 2,
organizationName: 'TechStart Inc',
targetId: null,
targetName: null,
scanMode: 'organization',
cronExpression: '0 3 * * 0',
isEnabled: true,
nextRunTime: '2025-01-05T03:00:00Z',
lastRunTime: '2024-12-29T03:00:00Z',
runCount: 12,
createdAt: '2024-10-01T10:00:00Z',
updatedAt: '2024-12-29T03:00:00Z',
},
{
id: 3,
name: 'Hourly API Monitoring',
engineIds: [2],
engineNames: ['Quick Scan'],
organizationId: null,
organizationName: null,
targetId: 12,
targetName: 'api.acme.com',
scanMode: 'target',
cronExpression: '0 * * * *',
isEnabled: true,
nextRunTime: '2024-12-29T12:00:00Z',
lastRunTime: '2024-12-29T11:00:00Z',
runCount: 720,
createdAt: '2024-12-01T00:00:00Z',
updatedAt: '2024-12-29T11:00:00Z',
},
{
id: 4,
name: 'Monthly Full Scan - Finance',
engineIds: [1],
engineNames: ['Full Scan'],
organizationId: 3,
organizationName: 'Global Finance Ltd',
targetId: null,
targetName: null,
scanMode: 'organization',
cronExpression: '0 0 1 * *',
isEnabled: false,
nextRunTime: '2025-01-01T00:00:00Z',
lastRunTime: '2024-12-01T00:00:00Z',
runCount: 6,
createdAt: '2024-06-01T08:00:00Z',
updatedAt: '2024-12-20T15:00:00Z',
},
{
id: 5,
name: 'RetailMax Daily Quick',
engineIds: [2, 3],
engineNames: ['Quick Scan', 'Vulnerability Only'],
organizationId: null,
organizationName: null,
targetId: 8,
targetName: 'retailmax.com',
scanMode: 'target',
cronExpression: '0 4 * * *',
isEnabled: true,
nextRunTime: '2024-12-30T04:00:00Z',
lastRunTime: '2024-12-29T04:00:00Z',
runCount: 30,
createdAt: '2024-11-29T09:00:00Z',
updatedAt: '2024-12-29T04:00:00Z',
},
]
export function getMockScheduledScans(params?: {
page?: number
pageSize?: number
search?: string
}): GetScheduledScansResponse {
const page = params?.page || 1
const pageSize = params?.pageSize || 10
const search = params?.search?.toLowerCase() || ''
let filtered = mockScheduledScans
if (search) {
filtered = filtered.filter(
s =>
s.name.toLowerCase().includes(search) ||
s.organizationName?.toLowerCase().includes(search) ||
s.targetName?.toLowerCase().includes(search)
)
}
const total = filtered.length
const totalPages = Math.ceil(total / pageSize)
const start = (page - 1) * pageSize
const results = filtered.slice(start, start + pageSize)
return {
results,
total,
page,
pageSize,
totalPages,
}
}
export function getMockScheduledScanById(id: number): ScheduledScan | undefined {
return mockScheduledScans.find(s => s.id === id)
}

View File

@@ -0,0 +1,78 @@
import type { Subdomain, GetAllSubdomainsResponse } from '@/types/subdomain.types'
export const mockSubdomains: Subdomain[] = [
{ id: 1, name: 'acme.com', createdAt: '2024-12-28T10:00:00Z' },
{ id: 2, name: 'www.acme.com', createdAt: '2024-12-28T10:01:00Z' },
{ id: 3, name: 'api.acme.com', createdAt: '2024-12-28T10:02:00Z' },
{ id: 4, name: 'admin.acme.com', createdAt: '2024-12-28T10:03:00Z' },
{ id: 5, name: 'mail.acme.com', createdAt: '2024-12-28T10:04:00Z' },
{ id: 6, name: 'blog.acme.com', createdAt: '2024-12-28T10:05:00Z' },
{ id: 7, name: 'shop.acme.com', createdAt: '2024-12-28T10:06:00Z' },
{ id: 8, name: 'cdn.acme.com', createdAt: '2024-12-28T10:07:00Z' },
{ id: 9, name: 'static.acme.com', createdAt: '2024-12-28T10:08:00Z' },
{ id: 10, name: 'dev.acme.com', createdAt: '2024-12-28T10:09:00Z' },
{ id: 11, name: 'staging.acme.com', createdAt: '2024-12-28T10:10:00Z' },
{ id: 12, name: 'test.acme.com', createdAt: '2024-12-28T10:11:00Z' },
{ id: 13, name: 'acme.io', createdAt: '2024-12-27T14:30:00Z' },
{ id: 14, name: 'docs.acme.io', createdAt: '2024-12-27T14:31:00Z' },
{ id: 15, name: 'api.acme.io', createdAt: '2024-12-27T14:32:00Z' },
{ id: 16, name: 'status.acme.io', createdAt: '2024-12-27T14:33:00Z' },
{ id: 17, name: 'techstart.io', createdAt: '2024-12-26T08:45:00Z' },
{ id: 18, name: 'www.techstart.io', createdAt: '2024-12-26T08:46:00Z' },
{ id: 19, name: 'app.techstart.io', createdAt: '2024-12-26T08:47:00Z' },
{ id: 20, name: 'globalfinance.com', createdAt: '2024-12-25T16:20:00Z' },
{ id: 21, name: 'www.globalfinance.com', createdAt: '2024-12-25T16:21:00Z' },
{ id: 22, name: 'secure.globalfinance.com', createdAt: '2024-12-25T16:22:00Z' },
{ id: 23, name: 'portal.globalfinance.com', createdAt: '2024-12-25T16:23:00Z' },
{ id: 24, name: 'healthcareplus.com', createdAt: '2024-12-23T11:00:00Z' },
{ id: 25, name: 'www.healthcareplus.com', createdAt: '2024-12-23T11:01:00Z' },
{ id: 26, name: 'patient.healthcareplus.com', createdAt: '2024-12-23T11:02:00Z' },
{ id: 27, name: 'edutech.io', createdAt: '2024-12-22T13:30:00Z' },
{ id: 28, name: 'learn.edutech.io', createdAt: '2024-12-22T13:31:00Z' },
{ id: 29, name: 'retailmax.com', createdAt: '2024-12-21T10:45:00Z' },
{ id: 30, name: 'www.retailmax.com', createdAt: '2024-12-21T10:46:00Z' },
{ id: 31, name: 'm.retailmax.com', createdAt: '2024-12-21T10:47:00Z' },
{ id: 32, name: 'api.retailmax.com', createdAt: '2024-12-21T10:48:00Z' },
{ id: 33, name: 'cloudnine.host', createdAt: '2024-12-19T16:00:00Z' },
{ id: 34, name: 'panel.cloudnine.host', createdAt: '2024-12-19T16:01:00Z' },
{ id: 35, name: 'mediastream.tv', createdAt: '2024-12-18T09:30:00Z' },
{ id: 36, name: 'www.mediastream.tv', createdAt: '2024-12-18T09:31:00Z' },
{ id: 37, name: 'cdn.mediastream.tv', createdAt: '2024-12-18T09:32:00Z' },
{ id: 38, name: 'stream.mediastream.tv', createdAt: '2024-12-18T09:33:00Z' },
]
export function getMockSubdomains(params?: {
page?: number
pageSize?: number
search?: string
organizationId?: number
}): GetAllSubdomainsResponse {
const page = params?.page || 1
const pageSize = params?.pageSize || 10
const search = params?.search?.toLowerCase() || ''
let filtered = mockSubdomains
if (search) {
filtered = mockSubdomains.filter(sub =>
sub.name.toLowerCase().includes(search)
)
}
const total = filtered.length
const totalPages = Math.ceil(total / pageSize)
const start = (page - 1) * pageSize
const domains = filtered.slice(start, start + pageSize)
return {
domains,
total,
page,
pageSize,
totalPages,
}
}
export function getMockSubdomainById(id: number): Subdomain | undefined {
return mockSubdomains.find(sub => sub.id === id)
}

View File

@@ -0,0 +1,205 @@
import type { Target, TargetsResponse, TargetDetail } from '@/types/target.types'
export const mockTargets: Target[] = [
{
id: 1,
name: 'acme.com',
type: 'domain',
description: 'Acme Corporation 主站',
createdAt: '2024-01-15T08:30:00Z',
lastScannedAt: '2024-12-28T10:00:00Z',
organizations: [{ id: 1, name: 'Acme Corporation' }],
},
{
id: 2,
name: 'acme.io',
type: 'domain',
description: 'Acme Corporation 开发者平台',
createdAt: '2024-01-16T09:00:00Z',
lastScannedAt: '2024-12-27T14:30:00Z',
organizations: [{ id: 1, name: 'Acme Corporation' }],
},
{
id: 3,
name: 'techstart.io',
type: 'domain',
description: 'TechStart 官网',
createdAt: '2024-02-20T10:15:00Z',
lastScannedAt: '2024-12-26T08:45:00Z',
organizations: [{ id: 2, name: 'TechStart Inc' }],
},
{
id: 4,
name: 'globalfinance.com',
type: 'domain',
description: 'Global Finance 主站',
createdAt: '2024-03-10T14:00:00Z',
lastScannedAt: '2024-12-25T16:20:00Z',
organizations: [{ id: 3, name: 'Global Finance Ltd' }],
},
{
id: 5,
name: '192.168.1.0/24',
type: 'cidr',
description: '内网 IP 段',
createdAt: '2024-03-15T11:30:00Z',
lastScannedAt: '2024-12-24T09:15:00Z',
organizations: [{ id: 3, name: 'Global Finance Ltd' }],
},
{
id: 6,
name: 'healthcareplus.com',
type: 'domain',
description: 'HealthCare Plus 官网',
createdAt: '2024-04-05T09:20:00Z',
lastScannedAt: '2024-12-23T11:00:00Z',
organizations: [{ id: 4, name: 'HealthCare Plus' }],
},
{
id: 7,
name: 'edutech.io',
type: 'domain',
description: 'EduTech 在线教育平台',
createdAt: '2024-05-12T11:45:00Z',
lastScannedAt: '2024-12-22T13:30:00Z',
organizations: [{ id: 5, name: 'EduTech Solutions' }],
},
{
id: 8,
name: 'retailmax.com',
type: 'domain',
description: 'RetailMax 电商主站',
createdAt: '2024-06-08T16:30:00Z',
lastScannedAt: '2024-12-21T10:45:00Z',
organizations: [{ id: 6, name: 'RetailMax' }],
},
{
id: 9,
name: '10.0.0.1',
type: 'ip',
description: '核心服务器 IP',
createdAt: '2024-07-01T08:00:00Z',
lastScannedAt: '2024-12-20T14:20:00Z',
organizations: [{ id: 7, name: 'CloudNine Hosting' }],
},
{
id: 10,
name: 'cloudnine.host',
type: 'domain',
description: 'CloudNine 托管服务',
createdAt: '2024-07-20T08:00:00Z',
lastScannedAt: '2024-12-19T16:00:00Z',
organizations: [{ id: 7, name: 'CloudNine Hosting' }],
},
{
id: 11,
name: 'mediastream.tv',
type: 'domain',
description: 'MediaStream 流媒体平台',
createdAt: '2024-08-15T12:10:00Z',
lastScannedAt: '2024-12-18T09:30:00Z',
organizations: [{ id: 8, name: 'MediaStream Corp' }],
},
{
id: 12,
name: 'api.acme.com',
type: 'domain',
description: 'Acme API 服务',
createdAt: '2024-09-01T10:00:00Z',
lastScannedAt: '2024-12-17T11:15:00Z',
organizations: [{ id: 1, name: 'Acme Corporation' }],
},
]
export const mockTargetDetails: Record<number, TargetDetail> = {
1: {
...mockTargets[0],
summary: {
subdomains: 156,
websites: 89,
endpoints: 2341,
ips: 45,
vulnerabilities: {
total: 23,
critical: 1,
high: 4,
medium: 8,
low: 10,
},
},
},
2: {
...mockTargets[1],
summary: {
subdomains: 78,
websites: 45,
endpoints: 892,
ips: 23,
vulnerabilities: {
total: 12,
critical: 0,
high: 2,
medium: 5,
low: 5,
},
},
},
}
export function getMockTargets(params?: {
page?: number
pageSize?: number
search?: string
}): TargetsResponse {
const page = params?.page || 1
const pageSize = params?.pageSize || 10
const search = params?.search?.toLowerCase() || ''
let filtered = mockTargets
if (search) {
filtered = mockTargets.filter(
target =>
target.name.toLowerCase().includes(search) ||
target.description?.toLowerCase().includes(search)
)
}
const total = filtered.length
const totalPages = Math.ceil(total / pageSize)
const start = (page - 1) * pageSize
const results = filtered.slice(start, start + pageSize)
return {
results,
total,
page,
pageSize,
totalPages,
}
}
export function getMockTargetById(id: number): TargetDetail | undefined {
if (mockTargetDetails[id]) {
return mockTargetDetails[id]
}
const target = mockTargets.find(t => t.id === id)
if (target) {
return {
...target,
summary: {
subdomains: Math.floor(Math.random() * 100) + 10,
websites: Math.floor(Math.random() * 50) + 5,
endpoints: Math.floor(Math.random() * 1000) + 100,
ips: Math.floor(Math.random() * 30) + 5,
vulnerabilities: {
total: Math.floor(Math.random() * 20) + 1,
critical: Math.floor(Math.random() * 2),
high: Math.floor(Math.random() * 5),
medium: Math.floor(Math.random() * 8),
low: Math.floor(Math.random() * 10),
},
},
}
}
return undefined
}

View File

@@ -0,0 +1,275 @@
import type { Vulnerability, GetVulnerabilitiesResponse, VulnerabilitySeverity } from '@/types/vulnerability.types'
export const mockVulnerabilities: Vulnerability[] = [
{
id: 1,
target: 1,
url: 'https://acme.com/search?q=test',
vulnType: 'xss-reflected',
severity: 'critical',
source: 'dalfox',
cvssScore: 9.1,
description: 'Reflected XSS in search parameter',
rawOutput: {
type: 'R',
inject_type: 'inHTML-URL',
method: 'GET',
data: 'https://acme.com/search?q=<script>alert(1)</script>',
param: 'q',
payload: '<script>alert(1)</script>',
evidence: '<script>alert(1)</script>',
cwe: 'CWE-79',
},
createdAt: '2024-12-28T10:30:00Z',
},
{
id: 2,
target: 1,
url: 'https://api.acme.com/v1/users',
vulnType: 'CVE-2024-1234',
severity: 'high',
source: 'nuclei',
cvssScore: 8.5,
description: 'SQL Injection in user API endpoint',
rawOutput: {
'template-id': 'CVE-2024-1234',
'matched-at': 'https://api.acme.com/v1/users',
host: 'api.acme.com',
info: {
name: 'SQL Injection',
description: 'SQL injection vulnerability in user endpoint',
severity: 'high',
tags: ['sqli', 'cve'],
reference: ['https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-1234'],
classification: {
'cve-id': 'CVE-2024-1234',
'cwe-id': ['CWE-89'],
},
},
},
createdAt: '2024-12-28T10:45:00Z',
},
{
id: 3,
target: 1,
url: 'https://acme.com/login',
vulnType: 'xss-stored',
severity: 'high',
source: 'dalfox',
cvssScore: 8.2,
description: 'Stored XSS in user profile',
rawOutput: {
type: 'S',
inject_type: 'inHTML-TAG',
method: 'POST',
param: 'bio',
payload: '<img src=x onerror=alert(1)>',
},
createdAt: '2024-12-27T14:20:00Z',
},
{
id: 4,
target: 2,
url: 'https://acme.io/api/config',
vulnType: 'information-disclosure',
severity: 'medium',
source: 'nuclei',
cvssScore: 5.3,
description: 'Exposed configuration file',
rawOutput: {
'template-id': 'exposed-config',
'matched-at': 'https://acme.io/api/config',
host: 'acme.io',
info: {
name: 'Exposed Configuration',
description: 'Configuration file accessible without authentication',
severity: 'medium',
tags: ['exposure', 'config'],
},
},
createdAt: '2024-12-27T15:00:00Z',
},
{
id: 5,
target: 3,
url: 'https://techstart.io/admin',
vulnType: 'open-redirect',
severity: 'medium',
source: 'nuclei',
cvssScore: 4.7,
description: 'Open redirect vulnerability',
rawOutput: {
'template-id': 'open-redirect',
'matched-at': 'https://techstart.io/admin?redirect=evil.com',
host: 'techstart.io',
info: {
name: 'Open Redirect',
description: 'URL redirect without validation',
severity: 'medium',
tags: ['redirect'],
},
},
createdAt: '2024-12-26T09:30:00Z',
},
{
id: 6,
target: 4,
url: 'https://globalfinance.com/.git/config',
vulnType: 'git-config-exposure',
severity: 'high',
source: 'nuclei',
cvssScore: 7.5,
description: 'Git configuration file exposed',
rawOutput: {
'template-id': 'git-config',
'matched-at': 'https://globalfinance.com/.git/config',
host: 'globalfinance.com',
info: {
name: 'Git Config Exposure',
description: 'Git configuration file is publicly accessible',
severity: 'high',
tags: ['git', 'exposure'],
},
},
createdAt: '2024-12-25T11:15:00Z',
},
{
id: 7,
target: 8,
url: 'https://retailmax.com/product?id=1',
vulnType: 'sqli',
severity: 'critical',
source: 'nuclei',
cvssScore: 9.8,
description: 'SQL Injection in product parameter',
rawOutput: {
'template-id': 'generic-sqli',
'matched-at': "https://retailmax.com/product?id=1'",
host: 'retailmax.com',
info: {
name: 'SQL Injection',
description: 'SQL injection in product ID parameter',
severity: 'critical',
tags: ['sqli'],
classification: {
'cwe-id': ['CWE-89'],
},
},
},
createdAt: '2024-12-21T12:00:00Z',
},
{
id: 8,
target: 1,
url: 'https://acme.com/robots.txt',
vulnType: 'robots-txt-exposure',
severity: 'info',
source: 'nuclei',
description: 'Robots.txt file found',
rawOutput: {
'template-id': 'robots-txt',
'matched-at': 'https://acme.com/robots.txt',
host: 'acme.com',
info: {
name: 'Robots.txt',
description: 'Robots.txt file detected',
severity: 'info',
tags: ['misc'],
},
},
createdAt: '2024-12-28T10:00:00Z',
},
{
id: 9,
target: 2,
url: 'https://acme.io/sitemap.xml',
vulnType: 'sitemap-exposure',
severity: 'info',
source: 'nuclei',
description: 'Sitemap.xml file found',
rawOutput: {
'template-id': 'sitemap-xml',
'matched-at': 'https://acme.io/sitemap.xml',
host: 'acme.io',
info: {
name: 'Sitemap.xml',
description: 'Sitemap.xml file detected',
severity: 'info',
tags: ['misc'],
},
},
createdAt: '2024-12-27T14:00:00Z',
},
{
id: 10,
target: 3,
url: 'https://techstart.io/api/v2/debug',
vulnType: 'debug-endpoint',
severity: 'low',
source: 'nuclei',
cvssScore: 3.1,
description: 'Debug endpoint exposed',
rawOutput: {
'template-id': 'debug-endpoint',
'matched-at': 'https://techstart.io/api/v2/debug',
host: 'techstart.io',
info: {
name: 'Debug Endpoint',
description: 'Debug endpoint accessible in production',
severity: 'low',
tags: ['debug', 'exposure'],
},
},
createdAt: '2024-12-26T10:00:00Z',
},
]
export function getMockVulnerabilities(params?: {
page?: number
pageSize?: number
targetId?: number
severity?: VulnerabilitySeverity
search?: string
}): GetVulnerabilitiesResponse {
const page = params?.page || 1
const pageSize = params?.pageSize || 10
const targetId = params?.targetId
const severity = params?.severity
const search = params?.search?.toLowerCase() || ''
let filtered = mockVulnerabilities
if (targetId) {
filtered = filtered.filter(v => v.target === targetId)
}
if (severity) {
filtered = filtered.filter(v => v.severity === severity)
}
if (search) {
filtered = filtered.filter(
v =>
v.url.toLowerCase().includes(search) ||
v.vulnType.toLowerCase().includes(search) ||
v.description?.toLowerCase().includes(search)
)
}
const total = filtered.length
const totalPages = Math.ceil(total / pageSize)
const start = (page - 1) * pageSize
const vulnerabilities = filtered.slice(start, start + pageSize)
return {
vulnerabilities,
total,
page,
pageSize,
totalPages,
}
}
export function getMockVulnerabilityById(id: number): Vulnerability | undefined {
return mockVulnerabilities.find(v => v.id === id)
}

View File

@@ -0,0 +1,252 @@
import type { WebSite, WebSiteListResponse } from '@/types/website.types'
export const mockWebsites: WebSite[] = [
{
id: 1,
target: 1,
url: 'https://acme.com',
host: 'acme.com',
location: '',
title: 'Acme Corporation - Home',
webserver: 'nginx/1.24.0',
contentType: 'text/html; charset=utf-8',
statusCode: 200,
contentLength: 45678,
responseBody: '<!DOCTYPE html>...',
tech: ['React', 'Next.js', 'Node.js', 'Tailwind CSS'],
vhost: false,
subdomain: 'acme.com',
createdAt: '2024-12-28T10:00:00Z',
},
{
id: 2,
target: 1,
url: 'https://www.acme.com',
host: 'www.acme.com',
location: 'https://acme.com',
title: 'Acme Corporation - Home',
webserver: 'nginx/1.24.0',
contentType: 'text/html; charset=utf-8',
statusCode: 301,
contentLength: 0,
responseBody: '',
tech: [],
vhost: false,
subdomain: 'www.acme.com',
createdAt: '2024-12-28T10:01:00Z',
},
{
id: 3,
target: 1,
url: 'https://api.acme.com',
host: 'api.acme.com',
location: '',
title: 'Acme API',
webserver: 'nginx/1.24.0',
contentType: 'application/json',
statusCode: 200,
contentLength: 234,
responseBody: '{"status":"ok","version":"1.0"}',
tech: ['Django', 'Python', 'PostgreSQL'],
vhost: false,
subdomain: 'api.acme.com',
createdAt: '2024-12-28T10:02:00Z',
},
{
id: 4,
target: 1,
url: 'https://admin.acme.com',
host: 'admin.acme.com',
location: '',
title: 'Admin Panel - Acme',
webserver: 'nginx/1.24.0',
contentType: 'text/html; charset=utf-8',
statusCode: 200,
contentLength: 23456,
responseBody: '<!DOCTYPE html>...',
tech: ['React', 'Ant Design'],
vhost: false,
subdomain: 'admin.acme.com',
createdAt: '2024-12-28T10:03:00Z',
},
{
id: 5,
target: 2,
url: 'https://acme.io',
host: 'acme.io',
location: '',
title: 'Acme Developer Platform',
webserver: 'cloudflare',
contentType: 'text/html; charset=utf-8',
statusCode: 200,
contentLength: 56789,
responseBody: '<!DOCTYPE html>...',
tech: ['Vue.js', 'Vitepress', 'CloudFlare'],
vhost: false,
subdomain: 'acme.io',
createdAt: '2024-12-27T14:30:00Z',
},
{
id: 6,
target: 2,
url: 'https://docs.acme.io',
host: 'docs.acme.io',
location: '',
title: 'Documentation - Acme.io',
webserver: 'cloudflare',
contentType: 'text/html; charset=utf-8',
statusCode: 200,
contentLength: 67890,
responseBody: '<!DOCTYPE html>...',
tech: ['Vue.js', 'Vitepress'],
vhost: false,
subdomain: 'docs.acme.io',
createdAt: '2024-12-27T14:31:00Z',
},
{
id: 7,
target: 3,
url: 'https://techstart.io',
host: 'techstart.io',
location: '',
title: 'TechStart - Innovation Hub',
webserver: 'Apache/2.4.54',
contentType: 'text/html; charset=utf-8',
statusCode: 200,
contentLength: 34567,
responseBody: '<!DOCTYPE html>...',
tech: ['WordPress', 'PHP', 'MySQL'],
vhost: false,
subdomain: 'techstart.io',
createdAt: '2024-12-26T08:45:00Z',
},
{
id: 8,
target: 4,
url: 'https://globalfinance.com',
host: 'globalfinance.com',
location: '',
title: 'Global Finance - Your Financial Partner',
webserver: 'Microsoft-IIS/10.0',
contentType: 'text/html; charset=utf-8',
statusCode: 200,
contentLength: 56789,
responseBody: '<!DOCTYPE html>...',
tech: ['ASP.NET', 'C#', 'jQuery', 'SQL Server'],
vhost: false,
subdomain: 'globalfinance.com',
createdAt: '2024-12-25T16:20:00Z',
},
{
id: 9,
target: 6,
url: 'https://healthcareplus.com',
host: 'healthcareplus.com',
location: '',
title: 'HealthCare Plus - Digital Health',
webserver: 'nginx/1.24.0',
contentType: 'text/html; charset=utf-8',
statusCode: 200,
contentLength: 45678,
responseBody: '<!DOCTYPE html>...',
tech: ['Angular', 'TypeScript', 'Node.js'],
vhost: false,
subdomain: 'healthcareplus.com',
createdAt: '2024-12-23T11:00:00Z',
},
{
id: 10,
target: 7,
url: 'https://edutech.io',
host: 'edutech.io',
location: '',
title: 'EduTech - Learn Anywhere',
webserver: 'cloudflare',
contentType: 'text/html; charset=utf-8',
statusCode: 200,
contentLength: 67890,
responseBody: '<!DOCTYPE html>...',
tech: ['Vue.js', 'Nuxt.js', 'PostgreSQL'],
vhost: false,
subdomain: 'edutech.io',
createdAt: '2024-12-22T13:30:00Z',
},
{
id: 11,
target: 8,
url: 'https://retailmax.com',
host: 'retailmax.com',
location: '',
title: 'RetailMax - Shop Everything',
webserver: 'nginx/1.22.0',
contentType: 'text/html; charset=utf-8',
statusCode: 200,
contentLength: 89012,
responseBody: '<!DOCTYPE html>...',
tech: ['React', 'Redux', 'Node.js', 'MongoDB'],
vhost: false,
subdomain: 'retailmax.com',
createdAt: '2024-12-21T10:45:00Z',
},
{
id: 12,
target: 10,
url: 'https://cloudnine.host',
host: 'cloudnine.host',
location: '',
title: 'CloudNine Hosting',
webserver: 'LiteSpeed',
contentType: 'text/html; charset=utf-8',
statusCode: 200,
contentLength: 34567,
responseBody: '<!DOCTYPE html>...',
tech: ['PHP', 'Laravel', 'MySQL'],
vhost: false,
subdomain: 'cloudnine.host',
createdAt: '2024-12-19T16:00:00Z',
},
]
export function getMockWebsites(params?: {
page?: number
pageSize?: number
search?: string
targetId?: number
}): WebSiteListResponse {
const page = params?.page || 1
const pageSize = params?.pageSize || 10
const search = params?.search?.toLowerCase() || ''
const targetId = params?.targetId
let filtered = mockWebsites
if (targetId) {
filtered = filtered.filter(w => w.target === targetId)
}
if (search) {
filtered = filtered.filter(
w =>
w.url.toLowerCase().includes(search) ||
w.title.toLowerCase().includes(search) ||
w.host.toLowerCase().includes(search)
)
}
const total = filtered.length
const totalPages = Math.ceil(total / pageSize)
const start = (page - 1) * pageSize
const results = filtered.slice(start, start + pageSize)
return {
results,
total,
page,
pageSize,
totalPages,
}
}
export function getMockWebsiteById(id: number): WebSite | undefined {
return mockWebsites.find(w => w.id === id)
}

View File

@@ -0,0 +1,78 @@
import type { WorkerNode, WorkersResponse } from '@/types/worker.types'
export const mockWorkers: WorkerNode[] = [
{
id: 1,
name: 'local-worker',
ipAddress: '127.0.0.1',
sshPort: 22,
username: 'root',
status: 'online',
isLocal: true,
createdAt: '2024-01-01T00:00:00Z',
updatedAt: '2024-12-29T10:00:00Z',
info: {
cpuPercent: 23.5,
memoryPercent: 45.2,
},
},
{
id: 2,
name: 'worker-01',
ipAddress: '192.168.1.101',
sshPort: 22,
username: 'scanner',
status: 'online',
isLocal: false,
createdAt: '2024-06-15T08:00:00Z',
updatedAt: '2024-12-29T09:30:00Z',
info: {
cpuPercent: 56.8,
memoryPercent: 72.1,
},
},
{
id: 3,
name: 'worker-02',
ipAddress: '192.168.1.102',
sshPort: 22,
username: 'scanner',
status: 'online',
isLocal: false,
createdAt: '2024-07-20T10:00:00Z',
updatedAt: '2024-12-29T09:45:00Z',
info: {
cpuPercent: 34.2,
memoryPercent: 58.9,
},
},
{
id: 4,
name: 'worker-03',
ipAddress: '192.168.1.103',
sshPort: 22,
username: 'scanner',
status: 'offline',
isLocal: false,
createdAt: '2024-08-10T14:00:00Z',
updatedAt: '2024-12-28T16:00:00Z',
},
]
export function getMockWorkers(page = 1, pageSize = 10): WorkersResponse {
const total = mockWorkers.length
const totalPages = Math.ceil(total / pageSize)
const start = (page - 1) * pageSize
const results = mockWorkers.slice(start, start + pageSize)
return {
results,
total,
page,
pageSize,
}
}
export function getMockWorkerById(id: number): WorkerNode | undefined {
return mockWorkers.find(w => w.id === id)
}

107
frontend/mock/index.ts Normal file
View File

@@ -0,0 +1,107 @@
/**
* Mock 数据统一导出
*
* 使用方式:
* import { USE_MOCK, mockData } from '@/mock'
*
* if (USE_MOCK) {
* return mockData.dashboard.assetStatistics
* }
*/
export { USE_MOCK, MOCK_DELAY, mockDelay } from './config'
// Dashboard
export {
mockDashboardStats,
mockAssetStatistics,
mockStatisticsHistory7Days,
mockStatisticsHistory30Days,
getMockStatisticsHistory,
} from './data/dashboard'
// Organizations
export {
mockOrganizations,
getMockOrganizations,
} from './data/organizations'
// Targets
export {
mockTargets,
mockTargetDetails,
getMockTargets,
getMockTargetById,
} from './data/targets'
// Scans
export {
mockScans,
mockScanStatistics,
getMockScans,
getMockScanById,
} from './data/scans'
// Vulnerabilities
export {
mockVulnerabilities,
getMockVulnerabilities,
getMockVulnerabilityById,
} from './data/vulnerabilities'
// Endpoints
export {
mockEndpoints,
getMockEndpoints,
getMockEndpointById,
} from './data/endpoints'
// Websites
export {
mockWebsites,
getMockWebsites,
getMockWebsiteById,
} from './data/websites'
// Subdomains
export {
mockSubdomains,
getMockSubdomains,
getMockSubdomainById,
} from './data/subdomains'
// Auth
export {
mockUser,
mockMeResponse,
mockLoginResponse,
mockLogoutResponse,
} from './data/auth'
// Engines
export {
mockEngines,
getMockEngines,
getMockEngineById,
} from './data/engines'
// Workers
export {
mockWorkers,
getMockWorkers,
getMockWorkerById,
} from './data/workers'
// Notifications
export {
mockNotifications,
getMockNotifications,
getMockUnreadCount,
} from './data/notifications'
// Scheduled Scans
export {
mockScheduledScans,
getMockScheduledScans,
getMockScheduledScanById,
} from './data/scheduled-scans'

View File

@@ -3,9 +3,12 @@ import createNextIntlPlugin from 'next-intl/plugin';
const withNextIntl = createNextIntlPlugin('./i18n/request.ts');
// Check if running on Vercel
const isVercel = process.env.VERCEL === '1';
const nextConfig: NextConfig = {
// Use standalone mode for Docker deployment
output: 'standalone',
// Use standalone mode for Docker deployment (not needed on Vercel)
...(isVercel ? {} : { output: 'standalone' }),
// Disable Next.js automatic add/remove trailing slash behavior
// Let us manually control URL format
skipTrailingSlashRedirect: true,
@@ -17,6 +20,10 @@ const nextConfig: NextConfig = {
allowedDevOrigins: ['192.168.*.*', '10.*.*.*', '172.16.*.*'],
async rewrites() {
// Skip rewrites on Vercel when using mock data
if (isVercel) {
return [];
}
// Use server service name in Docker environment, localhost for local development
const apiHost = process.env.API_HOST || 'localhost';
return [

View File

@@ -4,6 +4,7 @@
"private": true,
"scripts": {
"dev": "next dev --turbopack",
"dev:mock": "NEXT_PUBLIC_USE_MOCK=true next dev --turbopack",
"dev:noauth": "NEXT_PUBLIC_SKIP_AUTH=true next dev --turbopack",
"build": "next build --turbopack",
"start": "next start",
@@ -53,6 +54,7 @@
"cron-parser": "^5.4.0",
"cronstrue": "^3.9.0",
"date-fns": "^4.1.0",
"framer-motion": "^12.23.26",
"geist": "^1.5.1",
"is-ip": "^5.0.1",
"js-yaml": "^4.1.0",

View File

@@ -137,6 +137,9 @@ importers:
date-fns:
specifier: ^4.1.0
version: 4.1.0
framer-motion:
specifier: ^12.23.26
version: 12.23.26(react-dom@19.1.2(react@19.1.2))(react@19.1.2)
geist:
specifier: ^1.5.1
version: 1.5.1(next@15.5.9(react-dom@19.1.2(react@19.1.2))(react@19.1.2))
@@ -2311,6 +2314,20 @@ packages:
resolution: {integrity: sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==}
engines: {node: '>= 6'}
framer-motion@12.23.26:
resolution: {integrity: sha512-cPcIhgR42xBn1Uj+PzOyheMtZ73H927+uWPDVhUMqxy8UHt6Okavb6xIz9J/phFUHUj0OncR6UvMfJTXoc/LKA==}
peerDependencies:
'@emotion/is-prop-valid': '*'
react: ^18.0.0 || ^19.0.0
react-dom: ^18.0.0 || ^19.0.0
peerDependenciesMeta:
'@emotion/is-prop-valid':
optional: true
react:
optional: true
react-dom:
optional: true
function-bind@1.1.2:
resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
@@ -2767,6 +2784,12 @@ packages:
monaco-editor@0.55.1:
resolution: {integrity: sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A==}
motion-dom@12.23.23:
resolution: {integrity: sha512-n5yolOs0TQQBRUFImrRfs/+6X4p3Q4n1dUEqt/H58Vx7OW6RF+foWEgmTVDhIWJIMXOuNNL0apKH2S16en9eiA==}
motion-utils@12.23.6:
resolution: {integrity: sha512-eAWoPgr4eFEOFfg2WjIsMoqJTW6Z8MTUCgn/GZ3VRpClWBdnbjryiA3ZSNLyxCTmCQx4RmYX6jX1iWHbenUPNQ==}
ms@2.1.3:
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
@@ -5577,6 +5600,15 @@ snapshots:
hasown: 2.0.2
mime-types: 2.1.35
framer-motion@12.23.26(react-dom@19.1.2(react@19.1.2))(react@19.1.2):
dependencies:
motion-dom: 12.23.23
motion-utils: 12.23.6
tslib: 2.8.1
optionalDependencies:
react: 19.1.2
react-dom: 19.1.2(react@19.1.2)
function-bind@1.1.2: {}
function-timeout@0.1.1: {}
@@ -5996,6 +6028,12 @@ snapshots:
dompurify: 3.2.7
marked: 14.0.0
motion-dom@12.23.23:
dependencies:
motion-utils: 12.23.6
motion-utils@12.23.6: {}
ms@2.1.3: {}
msw@2.11.6(@types/node@20.19.19)(typescript@5.9.3):

View File

@@ -10,11 +10,16 @@ import type {
ChangePasswordRequest,
ChangePasswordResponse
} from '@/types/auth.types'
import { USE_MOCK, mockDelay, mockLoginResponse, mockLogoutResponse, mockMeResponse } from '@/mock'
/**
* User login
*/
export async function login(data: LoginRequest): Promise<LoginResponse> {
if (USE_MOCK) {
await mockDelay()
return mockLoginResponse
}
const res = await api.post<LoginResponse>('/auth/login/', data)
return res.data
}
@@ -23,6 +28,10 @@ export async function login(data: LoginRequest): Promise<LoginResponse> {
* User logout
*/
export async function logout(): Promise<LogoutResponse> {
if (USE_MOCK) {
await mockDelay()
return mockLogoutResponse
}
const res = await api.post<LogoutResponse>('/auth/logout/')
return res.data
}
@@ -31,6 +40,10 @@ export async function logout(): Promise<LogoutResponse> {
* Get current user information
*/
export async function getMe(): Promise<MeResponse> {
if (USE_MOCK) {
await mockDelay()
return mockMeResponse
}
const res = await api.get<MeResponse>('/auth/me/')
return res.data
}
@@ -39,6 +52,10 @@ export async function getMe(): Promise<MeResponse> {
* Change password
*/
export async function changePassword(data: ChangePasswordRequest): Promise<ChangePasswordResponse> {
if (USE_MOCK) {
await mockDelay()
return { message: 'Password changed successfully' }
}
const res = await api.post<ChangePasswordResponse>('/auth/change-password/', data)
return res.data
}

View File

@@ -1,7 +1,12 @@
import { api } from '@/lib/api-client'
import type { DashboardStats, AssetStatistics, StatisticsHistoryItem } from '@/types/dashboard.types'
import { USE_MOCK, mockDelay, mockDashboardStats, mockAssetStatistics, getMockStatisticsHistory } from '@/mock'
export async function getDashboardStats(): Promise<DashboardStats> {
if (USE_MOCK) {
await mockDelay()
return mockDashboardStats
}
const res = await api.get<DashboardStats>('/dashboard/stats/')
return res.data
}
@@ -10,6 +15,10 @@ export async function getDashboardStats(): Promise<DashboardStats> {
* Get asset statistics data (pre-aggregated)
*/
export async function getAssetStatistics(): Promise<AssetStatistics> {
if (USE_MOCK) {
await mockDelay()
return mockAssetStatistics
}
const res = await api.get<AssetStatistics>('/assets/statistics/')
return res.data
}
@@ -18,6 +27,10 @@ export async function getAssetStatistics(): Promise<AssetStatistics> {
* Get statistics history data (for line charts)
*/
export async function getStatisticsHistory(days: number = 7): Promise<StatisticsHistoryItem[]> {
if (USE_MOCK) {
await mockDelay()
return getMockStatisticsHistory(days)
}
const res = await api.get<StatisticsHistoryItem[]>('/assets/statistics/history/', {
params: { days }
})

View File

@@ -8,6 +8,7 @@ import type {
BatchDeleteEndpointsRequest,
BatchDeleteEndpointsResponse
} from "@/types/endpoint.types"
import { USE_MOCK, mockDelay, getMockEndpoints, getMockEndpointById } from '@/mock'
// Bulk create endpoints response type
export interface BulkCreateEndpointsResponse {
@@ -38,6 +39,12 @@ export class EndpointService {
* @returns Promise<Endpoint>
*/
static async getEndpointById(id: number): Promise<Endpoint> {
if (USE_MOCK) {
await mockDelay()
const endpoint = getMockEndpointById(id)
if (!endpoint) throw new Error('Endpoint not found')
return endpoint
}
const response = await api.get<Endpoint>(`/endpoints/${id}/`)
return response.data
}
@@ -48,6 +55,10 @@ export class EndpointService {
* @returns Promise<GetEndpointsResponse>
*/
static async getEndpoints(params: GetEndpointsRequest): Promise<GetEndpointsResponse> {
if (USE_MOCK) {
await mockDelay()
return getMockEndpoints(params)
}
// api-client.ts automatically converts camelCase params to snake_case
const response = await api.get<GetEndpointsResponse>('/endpoints/', {
params

View File

@@ -1,5 +1,6 @@
import apiClient from '@/lib/api-client'
import type { ScanEngine } from '@/types/engine.types'
import { USE_MOCK, mockDelay, getMockEngines, getMockEngineById } from '@/mock'
/**
* Engine API service
@@ -9,6 +10,10 @@ import type { ScanEngine } from '@/types/engine.types'
* Get engine list
*/
export async function getEngines(): Promise<ScanEngine[]> {
if (USE_MOCK) {
await mockDelay()
return getMockEngines()
}
// Engines are usually not many, get all
const response = await apiClient.get('/engines/', {
params: { pageSize: 1000 }
@@ -21,6 +26,12 @@ export async function getEngines(): Promise<ScanEngine[]> {
* Get engine details
*/
export async function getEngine(id: number): Promise<ScanEngine> {
if (USE_MOCK) {
await mockDelay()
const engine = getMockEngineById(id)
if (!engine) throw new Error('Engine not found')
return engine
}
const response = await apiClient.get(`/engines/${id}/`)
return response.data
}

View File

@@ -9,6 +9,7 @@ import type {
GetNotificationsRequest,
GetNotificationsResponse,
} from '@/types/notification.types'
import { USE_MOCK, mockDelay, getMockNotifications, getMockUnreadCount } from '@/mock'
export class NotificationService {
/**
@@ -18,6 +19,10 @@ export class NotificationService {
static async getNotifications(
params: GetNotificationsRequest = {}
): Promise<GetNotificationsResponse> {
if (USE_MOCK) {
await mockDelay()
return getMockNotifications(params)
}
const response = await api.get<GetNotificationsResponse>('/notifications/', {
params,
})
@@ -29,6 +34,10 @@ export class NotificationService {
* 后端返回: { updated: number }
*/
static async markAllAsRead(): Promise<{ updated: number }> {
if (USE_MOCK) {
await mockDelay()
return { updated: 2 }
}
const response = await api.post<{ updated: number }>('/notifications/mark-all-as-read/')
return response.data
}
@@ -38,6 +47,10 @@ export class NotificationService {
* 后端返回: { count: number }
*/
static async getUnreadCount(): Promise<{ count: number }> {
if (USE_MOCK) {
await mockDelay()
return getMockUnreadCount()
}
const response = await api.get<{ count: number }>('/notifications/unread-count/')
return response.data
}

View File

@@ -1,5 +1,6 @@
import { api } from "@/lib/api-client"
import type { Organization, OrganizationsResponse } from "@/types/organization.types"
import { USE_MOCK, mockDelay, getMockOrganizations, mockOrganizations } from '@/mock'
export class OrganizationService {
@@ -18,6 +19,10 @@ export class OrganizationService {
pageSize?: number
search?: string
}): Promise<OrganizationsResponse<Organization>> {
if (USE_MOCK) {
await mockDelay()
return getMockOrganizations(params)
}
const response = await api.get<OrganizationsResponse<Organization>>(
'/organizations/',
{ params }
@@ -31,6 +36,12 @@ export class OrganizationService {
* @returns Promise<Organization>
*/
static async getOrganizationById(id: string | number): Promise<Organization> {
if (USE_MOCK) {
await mockDelay()
const org = mockOrganizations.find(o => o.id === Number(id))
if (!org) throw new Error('Organization not found')
return org
}
const response = await api.get<Organization>(`/organizations/${id}/`)
return response.data
}

View File

@@ -8,11 +8,16 @@ import type {
QuickScanResponse,
ScanRecord
} from '@/types/scan.types'
import { USE_MOCK, mockDelay, getMockScans, getMockScanById, mockScanStatistics } from '@/mock'
/**
* Get scan list
*/
export async function getScans(params?: GetScansParams): Promise<GetScansResponse> {
if (USE_MOCK) {
await mockDelay()
return getMockScans(params)
}
const res = await api.get<GetScansResponse>('/scans/', { params })
return res.data
}
@@ -23,6 +28,12 @@ export async function getScans(params?: GetScansParams): Promise<GetScansRespons
* @returns Scan details
*/
export async function getScan(id: number): Promise<ScanRecord> {
if (USE_MOCK) {
await mockDelay()
const scan = getMockScanById(id)
if (!scan) throw new Error('Scan not found')
return scan
}
const res = await api.get<ScanRecord>(`/scans/${id}/`)
return res.data
}
@@ -95,6 +106,10 @@ export interface ScanStatistics {
* @returns Statistics data
*/
export async function getScanStatistics(): Promise<ScanStatistics> {
if (USE_MOCK) {
await mockDelay()
return mockScanStatistics
}
const res = await api.get<ScanStatistics>('/scans/statistics/')
return res.data
}

Some files were not shown because too many files have changed in this diff Show More