Compare commits

...

14 Commits

Author SHA1 Message Date
yyhuni
fb13bb74d8 feat(filter): add array fuzzy search support with PostgreSQL array_to_string
- Add ArrayToString custom PostgreSQL function for converting arrays to delimited strings
- Implement array field annotation in QueryBuilder to support fuzzy matching on JSON array fields
- Enhance _build_single_q to handle three operators for JSON arrays: exact match (==), negation (!=), and fuzzy search (=)
- Update target navigation routes from subdomain to website view for consistency
- Enable fuzzy search on array fields by converting them to text during query building
2026-01-01 22:41:57 +08:00
yyhuni
f076c682b6 feat(scan): add multi-engine support and config merging with enhanced indexing
- Add multi-engine support to Scan model with engine_ids and engine_names fields
- Implement config_merger utility for merging multiple engine configurations
- Add merged_configuration property to Scan model for unified config access
- Update scan creation and scheduling services to handle multiple engines
- Add pg_trgm GIN indexes to asset and snapshot models for fuzzy search on url, title, and name fields
- Update scan views and serializers to support multi-engine selection and display
- Enhance frontend components for multi-engine scan initiation and scheduling
- Update test data generation script for multi-engine scan scenarios
- Add internationalization strings for multi-engine UI elements
- Refactor scan flow to use merged configuration instead of single engine config
- Update Docker compose files with latest configuration
2026-01-01 22:35:05 +08:00
yyhuni
9eda2caceb feat(asset): add response headers and body tracking with pg_trgm indexing
- Rename body_preview to response_body across endpoint and website models for consistency
- Change response_headers from Dict to string type for efficient text indexing
- Add pg_trgm PostgreSQL extension initialization in AssetConfig for GIN index support
- Update all DTOs to reflect response_body and response_headers field changes
- Modify repositories to handle new response_body and response_headers formats
- Update serializers and views to work with string-based response headers
- Add response_headers and response_body columns to frontend endpoint and website tables
- Update command templates and scan tasks to populate response_body and response_headers
- Add database initialization script for pg_trgm extension in PostgreSQL setup
- Update frontend types and translations for new field names
- Enable efficient full-text search on response headers and body content through GIN indexes
2026-01-01 19:34:11 +08:00
yyhuni
b1c9e202dd feat(sidebar): add feedback link to secondary navigation menu
- Import IconMessageReport icon from tabler/icons-react for feedback menu item
- Add feedback navigation item linking to GitHub issues page
- Add "feedback" translation key to English messages (en.json)
- Add "feedback" translation key to Chinese messages (zh.json) as "反馈建议"
- Improves user engagement by providing direct access to issue reporting
2026-01-01 18:31:34 +08:00
yyhuni
918669bc29 style(ui): update expandable cell whitespace handling for better formatting
- Change whitespace class from `whitespace-normal` to `whitespace-pre-wrap` in expandable cell component
- Improves text rendering by preserving whitespace and line breaks in cell content
- Ensures consistent formatting display across different content types (mono, url, muted variants)
2026-01-01 16:41:47 +08:00
yyhuni
fd70b0544d docs(frontend): update Chinese translations to English for consistency
- Change "响应头" to "Response Headers" in endpoint messages
- Change "响应头" to "Response Headers" in website messages
- Maintain consistency across frontend message translations
- Improve clarity for international users by standardizing field labels
2026-01-01 16:23:03 +08:00
github-actions[bot]
0f2df7a5f3 chore: bump version to v1.2.14-dev 2026-01-01 05:13:25 +00:00
yyhuni
857ab737b5 feat(fingerprint): enhance xingfinger task with snapshot tracking and field merging
- Replace `not_found_count` with `created_count` and `snapshot_count` metrics in fingerprint detect flow
- Initialize and aggregate `snapshot_count` across tool statistics
- Refactor `parse_xingfinger_line()` to return structured dict with url, techs, server, title, status_code, and content_length
- Replace `bulk_merge_tech_field()` with `bulk_merge_website_fields()` to support merging multiple WebSite fields
- Implement smart merge strategy: arrays deduplicated, scalar fields only updated when empty/NULL
- Remove dynamic model loading via importlib in favor of direct WebSite model import
- Add WebsiteSnapshotDTO and DjangoWebsiteSnapshotRepository imports for snapshot handling
- Improve xingfinger output parsing to capture server, title, and HTTP metadata alongside technology detection
2026-01-01 12:40:49 +08:00
yyhuni
ee2d99edda feat(asset): add response headers tracking to endpoints and websites
- Add response_headers field to Endpoint and WebSite models as JSONField
- Add response_headers field to EndpointSnapshot and WebsiteSnapshot models
- Update all related DTOs to include response_headers with Dict[str, Any] type
- Add GIN indexes on response_headers fields for optimized JSON queries
- Update endpoint and website repositories to handle response_headers data
- Update serializers to include response_headers in API responses
- Update frontend components to display response headers in detail views
- Add response_headers to fingerprint detection and site scan tasks
- Update command templates and engine config to support header extraction
- Add internationalization strings for response headers in en.json and zh.json
- Update TypeScript types for endpoint and website to include response_headers
- Enhance scan history and target detail pages to show response header information
2026-01-01 12:25:22 +08:00
github-actions[bot]
db6ce16aca chore: bump version to v1.2.13-dev 2026-01-01 02:24:08 +00:00
yyhuni
ab800eca06 feat(frontend): reorder navigation tabs for improved UX
- Move "Websites" tab to first position in scan history and target layouts
- Reposition "IP Addresses" tab before "Ports" for better logical flow
- Maintain consistent tab ordering across both scan history and target pages
- Improve navigation hierarchy by placing primary discovery results first
2026-01-01 09:47:30 +08:00
yyhuni
e8e5572339 perf(asset): add GIN indexes for tech array fields and improve query parser
- Add GinIndex for tech array field in Endpoint model to optimize __contains queries
- Add GinIndex for tech array field in WebSite model to optimize __contains queries
- Import GinIndex from django.contrib.postgres.indexes
- Refactor QueryParser to protect quoted filter values during tokenization
- Implement placeholder-based filter extraction to preserve spaces within quoted values
- Replace filter tokens with placeholders before logical operator normalization
- Restore original filter conditions from placeholders during parsing
- Fix spacing in comments for consistency (add space after "从")
- Improves query performance for technology stack filtering on large datasets
2026-01-01 08:58:03 +08:00
github-actions[bot]
d48d4bbcad chore: bump version to v1.2.12-dev 2025-12-31 16:01:48 +00:00
yyhuni
d1cca4c083 base timeout set 10s 2025-12-31 23:27:02 +08:00
68 changed files with 1790 additions and 791 deletions

View File

@@ -1 +1 @@
v1.2.9-dev
v1.2.14-dev

View File

@@ -1,5 +1,9 @@
import logging
from django.apps import AppConfig
logger = logging.getLogger(__name__)
class AssetConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
@@ -8,3 +12,34 @@ class AssetConfig(AppConfig):
def ready(self):
# 导入所有模型以确保Django发现并注册
from . import models
# 启用 pg_trgm 扩展(用于文本模糊搜索索引)
# 用于已有数据库升级场景
self._ensure_pg_trgm_extension()
def _ensure_pg_trgm_extension(self):
"""
确保 pg_trgm 扩展已启用。
该扩展用于 response_body 和 response_headers 字段的 GIN 索引,
支持高效的文本模糊搜索。
"""
from django.db import connection
# 检查是否为 PostgreSQL 数据库
if connection.vendor != 'postgresql':
logger.debug("跳过 pg_trgm 扩展:当前数据库不是 PostgreSQL")
return
try:
with connection.cursor() as cursor:
cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
logger.debug("pg_trgm 扩展已启用")
except Exception as e:
# 记录错误但不阻止应用启动
# 常见原因:权限不足(需要超级用户权限)
logger.warning(
"无法创建 pg_trgm 扩展: %s"
"这可能导致 response_body 和 response_headers 字段的 GIN 索引无法正常工作。"
"请手动执行: CREATE EXTENSION IF NOT EXISTS pg_trgm;",
str(e)
)

View File

@@ -14,12 +14,13 @@ class EndpointDTO:
status_code: Optional[int] = None
content_length: Optional[int] = None
webserver: Optional[str] = None
body_preview: Optional[str] = None
response_body: Optional[str] = None
content_type: Optional[str] = None
tech: Optional[List[str]] = None
vhost: Optional[bool] = None
location: Optional[str] = None
matched_gf_patterns: Optional[List[str]] = None
response_headers: Optional[str] = None
def __post_init__(self):
if self.tech is None:

View File

@@ -17,9 +17,10 @@ class WebSiteDTO:
webserver: str = ''
content_type: str = ''
tech: List[str] = None
body_preview: str = ''
response_body: str = ''
vhost: Optional[bool] = None
created_at: str = None
response_headers: str = ''
def __post_init__(self):
if self.tech is None:

View File

@@ -22,10 +22,11 @@ class EndpointSnapshotDTO:
webserver: str = ''
content_type: str = ''
tech: List[str] = None
body_preview: str = ''
response_body: str = ''
vhost: Optional[bool] = None
matched_gf_patterns: List[str] = None
target_id: Optional[int] = None # 冗余字段,用于同步到资产表
response_headers: str = ''
def __post_init__(self):
if self.tech is None:
@@ -53,10 +54,11 @@ class EndpointSnapshotDTO:
status_code=self.status_code,
content_length=self.content_length,
webserver=self.webserver,
body_preview=self.body_preview,
response_body=self.response_body,
content_type=self.content_type,
tech=self.tech if self.tech else [],
vhost=self.vhost,
location=self.location,
matched_gf_patterns=self.matched_gf_patterns if self.matched_gf_patterns else []
matched_gf_patterns=self.matched_gf_patterns if self.matched_gf_patterns else [],
response_headers=self.response_headers,
)

View File

@@ -23,8 +23,9 @@ class WebsiteSnapshotDTO:
web_server: str = ''
content_type: str = ''
tech: List[str] = None
body_preview: str = ''
response_body: str = ''
vhost: Optional[bool] = None
response_headers: str = ''
def __post_init__(self):
if self.tech is None:
@@ -50,6 +51,7 @@ class WebsiteSnapshotDTO:
webserver=self.web_server,
content_type=self.content_type,
tech=self.tech if self.tech else [],
body_preview=self.body_preview,
vhost=self.vhost
response_body=self.response_body,
vhost=self.vhost,
response_headers=self.response_headers,
)

View File

@@ -1,6 +1,7 @@
from django.db import models
from django.contrib.postgres.fields import ArrayField
from django.contrib.postgres.indexes import GinIndex
from django.core.validators import MinValueValidator, MaxValueValidator
@@ -34,6 +35,12 @@ class Subdomain(models.Model):
models.Index(fields=['name', 'target']), # 复合索引,优化 get_by_names_and_target_id 批量查询
models.Index(fields=['target']), # 优化从target_id快速查找下面的子域名
models.Index(fields=['name']), # 优化从name快速查找子域名搜索场景
# pg_trgm GIN 索引,支持 LIKE '%keyword%' 模糊搜索
GinIndex(
name='subdomain_name_trgm_idx',
fields=['name'],
opclasses=['gin_trgm_ops']
),
]
constraints = [
# 普通唯一约束name + target 组合唯一
@@ -84,11 +91,10 @@ class Endpoint(models.Model):
default='',
help_text='服务器类型HTTP 响应头 Server 值)'
)
body_preview = models.CharField(
max_length=1000,
response_body = models.TextField(
blank=True,
default='',
help_text='响应正文前N个字符默认100个字符'
help_text='HTTP响应体'
)
content_type = models.CharField(
max_length=200,
@@ -123,6 +129,11 @@ class Endpoint(models.Model):
default=list,
help_text='匹配的GF模式列表用于识别敏感端点如api, debug, config等'
)
response_headers = models.TextField(
blank=True,
default='',
help_text='原始HTTP响应头'
)
class Meta:
db_table = 'endpoint'
@@ -131,11 +142,28 @@ class Endpoint(models.Model):
ordering = ['-created_at']
indexes = [
models.Index(fields=['-created_at']),
models.Index(fields=['target']), # 优化从target_id快速查找下面的端点主关联字段
models.Index(fields=['target']), # 优化从 target_id快速查找下面的端点主关联字段
models.Index(fields=['url']), # URL索引优化查询性能
models.Index(fields=['host']), # host索引优化根据主机名查询
models.Index(fields=['status_code']), # 状态码索引,优化筛选
models.Index(fields=['title']), # title索引优化智能过滤搜索
GinIndex(fields=['tech']), # GIN索引优化 tech 数组字段的 __contains 查询
# pg_trgm GIN 索引,支持 LIKE '%keyword%' 模糊搜索
GinIndex(
name='endpoint_resp_headers_trgm_idx',
fields=['response_headers'],
opclasses=['gin_trgm_ops']
),
GinIndex(
name='endpoint_url_trgm_idx',
fields=['url'],
opclasses=['gin_trgm_ops']
),
GinIndex(
name='endpoint_title_trgm_idx',
fields=['title'],
opclasses=['gin_trgm_ops']
),
]
constraints = [
# 普通唯一约束url + target 组合唯一
@@ -186,11 +214,10 @@ class WebSite(models.Model):
default='',
help_text='服务器类型HTTP 响应头 Server 值)'
)
body_preview = models.CharField(
max_length=1000,
response_body = models.TextField(
blank=True,
default='',
help_text='响应正文前N个字符默认100个字符'
help_text='HTTP响应体'
)
content_type = models.CharField(
max_length=200,
@@ -219,6 +246,11 @@ class WebSite(models.Model):
blank=True,
help_text='是否支持虚拟主机'
)
response_headers = models.TextField(
blank=True,
default='',
help_text='原始HTTP响应头'
)
class Meta:
db_table = 'website'
@@ -229,9 +261,26 @@ class WebSite(models.Model):
models.Index(fields=['-created_at']),
models.Index(fields=['url']), # URL索引优化查询性能
models.Index(fields=['host']), # host索引优化根据主机名查询
models.Index(fields=['target']), # 优化从target_id快速查找下面的站点
models.Index(fields=['target']), # 优化从 target_id快速查找下面的站点
models.Index(fields=['title']), # title索引优化智能过滤搜索
models.Index(fields=['status_code']), # 状态码索引,优化智能过滤搜索
GinIndex(fields=['tech']), # GIN索引优化 tech 数组字段的 __contains 查询
# pg_trgm GIN 索引,支持 LIKE '%keyword%' 模糊搜索
GinIndex(
name='website_resp_headers_trgm_idx',
fields=['response_headers'],
opclasses=['gin_trgm_ops']
),
GinIndex(
name='website_url_trgm_idx',
fields=['url'],
opclasses=['gin_trgm_ops']
),
GinIndex(
name='website_title_trgm_idx',
fields=['title'],
opclasses=['gin_trgm_ops']
),
]
constraints = [
# 普通唯一约束url + target 组合唯一
@@ -308,6 +357,12 @@ class Directory(models.Model):
models.Index(fields=['target']), # 优化从target_id快速查找下面的目录
models.Index(fields=['url']), # URL索引优化搜索和唯一约束
models.Index(fields=['status']), # 状态码索引,优化筛选
# pg_trgm GIN 索引,支持 LIKE '%keyword%' 模糊搜索
GinIndex(
name='directory_url_trgm_idx',
fields=['url'],
opclasses=['gin_trgm_ops']
),
]
constraints = [
# 普通唯一约束target + url 组合唯一

View File

@@ -1,5 +1,6 @@
from django.db import models
from django.contrib.postgres.fields import ArrayField
from django.contrib.postgres.indexes import GinIndex
from django.core.validators import MinValueValidator, MaxValueValidator
@@ -26,6 +27,12 @@ class SubdomainSnapshot(models.Model):
models.Index(fields=['scan']),
models.Index(fields=['name']),
models.Index(fields=['-created_at']),
# pg_trgm GIN 索引,支持 LIKE '%keyword%' 模糊搜索
GinIndex(
name='subdomain_snap_name_trgm',
fields=['name'],
opclasses=['gin_trgm_ops']
),
]
constraints = [
# 唯一约束:同一次扫描中,同一个子域名只能记录一次
@@ -68,8 +75,13 @@ class WebsiteSnapshot(models.Model):
default=list,
help_text='技术栈'
)
body_preview = models.TextField(blank=True, default='', help_text='响应体预览')
response_body = models.TextField(blank=True, default='', help_text='HTTP响应体')
vhost = models.BooleanField(null=True, blank=True, help_text='虚拟主机标志')
response_headers = models.TextField(
blank=True,
default='',
help_text='原始HTTP响应头'
)
created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
class Meta:
@@ -83,6 +95,23 @@ class WebsiteSnapshot(models.Model):
models.Index(fields=['host']), # host索引优化根据主机名查询
models.Index(fields=['title']), # title索引优化标题搜索
models.Index(fields=['-created_at']),
GinIndex(fields=['tech']), # GIN索引优化数组字段查询
# pg_trgm GIN 索引,支持 LIKE '%keyword%' 模糊搜索
GinIndex(
name='ws_snap_resp_hdr_trgm',
fields=['response_headers'],
opclasses=['gin_trgm_ops']
),
GinIndex(
name='ws_snap_url_trgm',
fields=['url'],
opclasses=['gin_trgm_ops']
),
GinIndex(
name='ws_snap_title_trgm',
fields=['title'],
opclasses=['gin_trgm_ops']
),
]
constraints = [
# 唯一约束同一次扫描中同一个URL只能记录一次
@@ -132,6 +161,12 @@ class DirectorySnapshot(models.Model):
models.Index(fields=['status']), # 状态码索引,优化筛选
models.Index(fields=['content_type']), # content_type索引优化内容类型搜索
models.Index(fields=['-created_at']),
# pg_trgm GIN 索引,支持 LIKE '%keyword%' 模糊搜索
GinIndex(
name='dir_snap_url_trgm',
fields=['url'],
opclasses=['gin_trgm_ops']
),
]
constraints = [
# 唯一约束同一次扫描中同一个目录URL只能记录一次
@@ -251,7 +286,7 @@ class EndpointSnapshot(models.Model):
default=list,
help_text='技术栈'
)
body_preview = models.CharField(max_length=1000, blank=True, default='', help_text='响应体预览')
response_body = models.TextField(blank=True, default='', help_text='HTTP响应体')
vhost = models.BooleanField(null=True, blank=True, help_text='虚拟主机标志')
matched_gf_patterns = ArrayField(
models.CharField(max_length=100),
@@ -259,6 +294,11 @@ class EndpointSnapshot(models.Model):
default=list,
help_text='匹配的GF模式列表'
)
response_headers = models.TextField(
blank=True,
default='',
help_text='原始HTTP响应头'
)
created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
class Meta:
@@ -274,6 +314,23 @@ class EndpointSnapshot(models.Model):
models.Index(fields=['status_code']), # 状态码索引,优化筛选
models.Index(fields=['webserver']), # webserver索引优化服务器搜索
models.Index(fields=['-created_at']),
GinIndex(fields=['tech']), # GIN索引优化数组字段查询
# pg_trgm GIN 索引,支持 LIKE '%keyword%' 模糊搜索
GinIndex(
name='ep_snap_resp_hdr_trgm',
fields=['response_headers'],
opclasses=['gin_trgm_ops']
),
GinIndex(
name='ep_snap_url_trgm',
fields=['url'],
opclasses=['gin_trgm_ops']
),
GinIndex(
name='ep_snap_title_trgm',
fields=['title'],
opclasses=['gin_trgm_ops']
),
]
constraints = [
# 唯一约束同一次扫描中同一个URL只能记录一次

View File

@@ -48,12 +48,13 @@ class DjangoEndpointRepository:
status_code=item.status_code,
content_length=item.content_length,
webserver=item.webserver or '',
body_preview=item.body_preview or '',
response_body=item.response_body or '',
content_type=item.content_type or '',
tech=item.tech if item.tech else [],
vhost=item.vhost,
location=item.location or '',
matched_gf_patterns=item.matched_gf_patterns if item.matched_gf_patterns else []
matched_gf_patterns=item.matched_gf_patterns if item.matched_gf_patterns else [],
response_headers=item.response_headers if item.response_headers else ''
)
for item in unique_items
]
@@ -65,8 +66,8 @@ class DjangoEndpointRepository:
unique_fields=['url', 'target'],
update_fields=[
'host', 'title', 'status_code', 'content_length',
'webserver', 'body_preview', 'content_type', 'tech',
'vhost', 'location', 'matched_gf_patterns'
'webserver', 'response_body', 'content_type', 'tech',
'vhost', 'location', 'matched_gf_patterns', 'response_headers'
],
batch_size=1000
)
@@ -138,12 +139,13 @@ class DjangoEndpointRepository:
status_code=item.status_code,
content_length=item.content_length,
webserver=item.webserver or '',
body_preview=item.body_preview or '',
response_body=item.response_body or '',
content_type=item.content_type or '',
tech=item.tech if item.tech else [],
vhost=item.vhost,
location=item.location or '',
matched_gf_patterns=item.matched_gf_patterns if item.matched_gf_patterns else []
matched_gf_patterns=item.matched_gf_patterns if item.matched_gf_patterns else [],
response_headers=item.response_headers if item.response_headers else ''
)
for item in unique_items
]
@@ -183,7 +185,7 @@ class DjangoEndpointRepository:
.values(
'url', 'host', 'location', 'title', 'status_code',
'content_length', 'content_type', 'webserver', 'tech',
'body_preview', 'vhost', 'matched_gf_patterns', 'created_at'
'response_body', 'response_headers', 'vhost', 'matched_gf_patterns', 'created_at'
)
.order_by('url')
)

View File

@@ -49,12 +49,13 @@ class DjangoWebSiteRepository:
location=item.location or '',
title=item.title or '',
webserver=item.webserver or '',
body_preview=item.body_preview or '',
response_body=item.response_body or '',
content_type=item.content_type or '',
tech=item.tech if item.tech else [],
status_code=item.status_code,
content_length=item.content_length,
vhost=item.vhost
vhost=item.vhost,
response_headers=item.response_headers if item.response_headers else ''
)
for item in unique_items
]
@@ -66,8 +67,8 @@ class DjangoWebSiteRepository:
unique_fields=['url', 'target'],
update_fields=[
'host', 'location', 'title', 'webserver',
'body_preview', 'content_type', 'tech',
'status_code', 'content_length', 'vhost'
'response_body', 'content_type', 'tech',
'status_code', 'content_length', 'vhost', 'response_headers'
],
batch_size=1000
)
@@ -132,12 +133,13 @@ class DjangoWebSiteRepository:
location=item.location or '',
title=item.title or '',
webserver=item.webserver or '',
body_preview=item.body_preview or '',
response_body=item.response_body or '',
content_type=item.content_type or '',
tech=item.tech if item.tech else [],
status_code=item.status_code,
content_length=item.content_length,
vhost=item.vhost
vhost=item.vhost,
response_headers=item.response_headers if item.response_headers else ''
)
for item in unique_items
]
@@ -177,7 +179,7 @@ class DjangoWebSiteRepository:
.values(
'url', 'host', 'location', 'title', 'status_code',
'content_length', 'content_type', 'webserver', 'tech',
'body_preview', 'vhost', 'created_at'
'response_body', 'response_headers', 'vhost', 'created_at'
)
.order_by('url')
)

View File

@@ -44,6 +44,7 @@ class DjangoEndpointSnapshotRepository:
snapshots.append(EndpointSnapshot(
scan_id=item.scan_id,
url=item.url,
host=item.host if item.host else '',
title=item.title,
status_code=item.status_code,
content_length=item.content_length,
@@ -51,9 +52,10 @@ class DjangoEndpointSnapshotRepository:
webserver=item.webserver,
content_type=item.content_type,
tech=item.tech if item.tech else [],
body_preview=item.body_preview,
response_body=item.response_body,
vhost=item.vhost,
matched_gf_patterns=item.matched_gf_patterns if item.matched_gf_patterns else []
matched_gf_patterns=item.matched_gf_patterns if item.matched_gf_patterns else [],
response_headers=item.response_headers if item.response_headers else ''
))
# 批量创建(忽略冲突,基于唯一约束去重)
@@ -100,7 +102,7 @@ class DjangoEndpointSnapshotRepository:
.values(
'url', 'host', 'location', 'title', 'status_code',
'content_length', 'content_type', 'webserver', 'tech',
'body_preview', 'vhost', 'matched_gf_patterns', 'created_at'
'response_body', 'response_headers', 'vhost', 'matched_gf_patterns', 'created_at'
)
.order_by('url')
)

View File

@@ -52,8 +52,9 @@ class DjangoWebsiteSnapshotRepository:
web_server=item.web_server,
content_type=item.content_type,
tech=item.tech if item.tech else [],
body_preview=item.body_preview,
vhost=item.vhost
response_body=item.response_body,
vhost=item.vhost,
response_headers=item.response_headers if item.response_headers else ''
))
# 批量创建(忽略冲突,基于唯一约束去重)
@@ -100,7 +101,7 @@ class DjangoWebsiteSnapshotRepository:
.values(
'url', 'host', 'location', 'title', 'status',
'content_length', 'content_type', 'web_server', 'tech',
'body_preview', 'vhost', 'created_at'
'response_body', 'response_headers', 'vhost', 'created_at'
)
.order_by('url')
)
@@ -117,7 +118,8 @@ class DjangoWebsiteSnapshotRepository:
'content_type': row['content_type'],
'webserver': row['web_server'],
'tech': row['tech'],
'body_preview': row['body_preview'],
'response_body': row['response_body'],
'response_headers': row['response_headers'],
'vhost': row['vhost'],
'created_at': row['created_at'],
}

View File

@@ -67,9 +67,10 @@ class SubdomainListSerializer(serializers.ModelSerializer):
class WebSiteSerializer(serializers.ModelSerializer):
"""站点序列化器"""
"""站点序列化器(目标详情页)"""
subdomain = serializers.CharField(source='subdomain.name', allow_blank=True, default='')
responseHeaders = serializers.CharField(source='response_headers', read_only=True) # 原始HTTP响应头
class Meta:
model = WebSite
@@ -83,9 +84,10 @@ class WebSiteSerializer(serializers.ModelSerializer):
'content_type',
'status_code',
'content_length',
'body_preview',
'response_body',
'tech',
'vhost',
'responseHeaders', # HTTP响应头
'subdomain',
'created_at',
]
@@ -140,6 +142,7 @@ class EndpointListSerializer(serializers.ModelSerializer):
source='matched_gf_patterns',
read_only=True,
)
responseHeaders = serializers.CharField(source='response_headers', read_only=True) # 原始HTTP响应头
class Meta:
model = Endpoint
@@ -152,9 +155,10 @@ class EndpointListSerializer(serializers.ModelSerializer):
'content_length',
'content_type',
'webserver',
'body_preview',
'response_body',
'tech',
'vhost',
'responseHeaders', # HTTP响应头
'gfPatterns',
'created_at',
]
@@ -215,6 +219,7 @@ class WebsiteSnapshotSerializer(serializers.ModelSerializer):
subdomain_name = serializers.CharField(source='subdomain.name', read_only=True)
webserver = serializers.CharField(source='web_server', read_only=True) # 映射字段名
status_code = serializers.IntegerField(source='status', read_only=True) # 映射字段名
responseHeaders = serializers.CharField(source='response_headers', read_only=True) # 原始HTTP响应头
class Meta:
model = WebsiteSnapshot
@@ -227,9 +232,10 @@ class WebsiteSnapshotSerializer(serializers.ModelSerializer):
'content_type',
'status_code', # 使用映射后的字段名
'content_length',
'body_preview',
'response_body',
'tech',
'vhost',
'responseHeaders', # HTTP响应头
'subdomain_name',
'created_at',
]
@@ -264,6 +270,7 @@ class EndpointSnapshotSerializer(serializers.ModelSerializer):
source='matched_gf_patterns',
read_only=True,
)
responseHeaders = serializers.CharField(source='response_headers', read_only=True) # 原始HTTP响应头
class Meta:
model = EndpointSnapshot
@@ -277,9 +284,10 @@ class EndpointSnapshotSerializer(serializers.ModelSerializer):
'content_type',
'status_code',
'content_length',
'body_preview',
'response_body',
'tech',
'vhost',
'responseHeaders', # HTTP响应头
'gfPatterns',
'created_at',
]

View File

@@ -367,7 +367,7 @@ class WebSiteViewSet(viewsets.ModelViewSet):
def export(self, request, **kwargs):
"""导出网站为 CSV 格式
CSV 列url, host, location, title, status_code, content_length, content_type, webserver, tech, body_preview, vhost, created_at
CSV 列url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
@@ -380,7 +380,7 @@ class WebSiteViewSet(viewsets.ModelViewSet):
headers = [
'url', 'host', 'location', 'title', 'status_code',
'content_length', 'content_type', 'webserver', 'tech',
'body_preview', 'vhost', 'created_at'
'response_body', 'response_headers', 'vhost', 'created_at'
]
formatters = {
'created_at': format_datetime,
@@ -628,7 +628,7 @@ class EndpointViewSet(viewsets.ModelViewSet):
def export(self, request, **kwargs):
"""导出端点为 CSV 格式
CSV 列url, host, location, title, status_code, content_length, content_type, webserver, tech, body_preview, vhost, matched_gf_patterns, created_at
CSV 列url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, matched_gf_patterns, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
@@ -641,7 +641,7 @@ class EndpointViewSet(viewsets.ModelViewSet):
headers = [
'url', 'host', 'location', 'title', 'status_code',
'content_length', 'content_type', 'webserver', 'tech',
'body_preview', 'vhost', 'matched_gf_patterns', 'created_at'
'response_body', 'response_headers', 'vhost', 'matched_gf_patterns', 'created_at'
]
formatters = {
'created_at': format_datetime,
@@ -853,7 +853,7 @@ class WebsiteSnapshotViewSet(viewsets.ModelViewSet):
def export(self, request, **kwargs):
"""导出网站快照为 CSV 格式
CSV 列url, host, location, title, status_code, content_length, content_type, webserver, tech, body_preview, vhost, created_at
CSV 列url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
@@ -866,7 +866,7 @@ class WebsiteSnapshotViewSet(viewsets.ModelViewSet):
headers = [
'url', 'host', 'location', 'title', 'status_code',
'content_length', 'content_type', 'webserver', 'tech',
'body_preview', 'vhost', 'created_at'
'response_body', 'response_headers', 'vhost', 'created_at'
]
formatters = {
'created_at': format_datetime,
@@ -970,7 +970,7 @@ class EndpointSnapshotViewSet(viewsets.ModelViewSet):
def export(self, request, **kwargs):
"""导出端点快照为 CSV 格式
CSV 列url, host, location, title, status_code, content_length, content_type, webserver, tech, body_preview, vhost, matched_gf_patterns, created_at
CSV 列url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, matched_gf_patterns, created_at
"""
from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
@@ -983,7 +983,7 @@ class EndpointSnapshotViewSet(viewsets.ModelViewSet):
headers = [
'url', 'host', 'location', 'title', 'status_code',
'content_length', 'content_type', 'webserver', 'tech',
'body_preview', 'vhost', 'matched_gf_patterns', 'created_at'
'response_body', 'response_headers', 'vhost', 'matched_gf_patterns', 'created_at'
]
formatters = {
'created_at': format_datetime,

View File

@@ -29,11 +29,19 @@ from dataclasses import dataclass
from typing import List, Dict, Optional, Union
from enum import Enum
from django.db.models import QuerySet, Q
from django.db.models import QuerySet, Q, F, Func, CharField
from django.db.models.functions import Cast
logger = logging.getLogger(__name__)
class ArrayToString(Func):
"""PostgreSQL array_to_string 函数"""
function = 'array_to_string'
template = "%(function)s(%(expressions)s, ',')"
output_field = CharField()
class LogicalOp(Enum):
"""逻辑运算符"""
AND = 'AND'
@@ -86,9 +94,21 @@ class QueryParser:
if not query_string or not query_string.strip():
return []
# 第一步:提取所有过滤条件并用占位符替换,保护引号内的空格
filters_found = []
placeholder_pattern = '__FILTER_{}__'
def replace_filter(match):
idx = len(filters_found)
filters_found.append(match.group(0))
return placeholder_pattern.format(idx)
# 先用正则提取所有 field="value" 形式的条件
protected = cls.FILTER_PATTERN.sub(replace_filter, query_string)
# 标准化逻辑运算符
# 先处理 || 和 or -> __OR__
normalized = cls.OR_PATTERN.sub(' __OR__ ', query_string)
normalized = cls.OR_PATTERN.sub(' __OR__ ', protected)
# 再处理 && 和 and -> __AND__
normalized = cls.AND_PATTERN.sub(' __AND__ ', normalized)
@@ -103,20 +123,26 @@ class QueryParser:
pending_op = LogicalOp.OR
elif token == '__AND__':
pending_op = LogicalOp.AND
else:
# 尝试解析为过滤条件
match = cls.FILTER_PATTERN.match(token)
if match:
field, operator, value = match.groups()
groups.append(FilterGroup(
filter=ParsedFilter(
field=field.lower(),
operator=operator,
value=value
),
logical_op=pending_op if groups else LogicalOp.AND # 第一个条件默认 AND
))
pending_op = LogicalOp.AND # 重置为默认 AND
elif token.startswith('__FILTER_') and token.endswith('__'):
# 还原占位符为原始过滤条件
try:
idx = int(token[9:-2]) # 提取索引
original_filter = filters_found[idx]
match = cls.FILTER_PATTERN.match(original_filter)
if match:
field, operator, value = match.groups()
groups.append(FilterGroup(
filter=ParsedFilter(
field=field.lower(),
operator=operator,
value=value
),
logical_op=pending_op if groups else LogicalOp.AND
))
pending_op = LogicalOp.AND # 重置为默认 AND
except (ValueError, IndexError):
pass
# 其他 token 忽略(无效输入)
return groups
@@ -151,6 +177,21 @@ class QueryBuilder:
json_array_fields = json_array_fields or []
# 收集需要 annotate 的数组模糊搜索字段
array_fuzzy_fields = set()
# 第一遍:检查是否有数组模糊匹配
for group in filter_groups:
f = group.filter
db_field = field_mapping.get(f.field)
if db_field and db_field in json_array_fields and f.operator == '=':
array_fuzzy_fields.add(db_field)
# 对数组模糊搜索字段做 annotate
for field in array_fuzzy_fields:
annotate_name = f'{field}_text'
queryset = queryset.annotate(**{annotate_name: ArrayToString(F(field))})
# 构建 Q 对象
combined_q = None
@@ -187,8 +228,17 @@ class QueryBuilder:
def _build_single_q(cls, field: str, operator: str, value: str, is_json_array: bool = False) -> Optional[Q]:
"""构建单个条件的 Q 对象"""
if is_json_array:
# JSON 数组字段使用 __contains 查询
return Q(**{f'{field}__contains': [value]})
if operator == '==':
# 精确匹配:数组中包含完全等于 value 的元素
return Q(**{f'{field}__contains': [value]})
elif operator == '!=':
# 不包含:数组中不包含完全等于 value 的元素
return ~Q(**{f'{field}__contains': [value]})
else: # '=' 模糊匹配
# 使用 annotate 后的字段进行模糊搜索
# 字段已在 build_query 中通过 ArrayToString 转换为文本
annotate_name = f'{field}_text'
return Q(**{f'{annotate_name}__icontains': value})
if operator == '!=':
return cls._build_not_equal_q(field, value)

View File

@@ -15,9 +15,10 @@
"""
from django.core.management.base import BaseCommand
from io import StringIO
from pathlib import Path
import yaml
from ruamel.yaml import YAML
from apps.engine.models import ScanEngine
@@ -44,10 +45,12 @@ class Command(BaseCommand):
with open(config_path, 'r', encoding='utf-8') as f:
default_config = f.read()
# 解析 YAML 为字典,后续用于生成子引擎配置
# 使用 ruamel.yaml 解析,保留注释
yaml_parser = YAML()
yaml_parser.preserve_quotes = True
try:
config_dict = yaml.safe_load(default_config) or {}
except yaml.YAMLError as e:
config_dict = yaml_parser.load(default_config) or {}
except Exception as e:
self.stdout.write(self.style.ERROR(f'引擎配置 YAML 解析失败: {e}'))
return
@@ -83,16 +86,13 @@ class Command(BaseCommand):
if scan_type != 'subdomain_discovery' and 'tools' not in scan_cfg:
continue
# 构造只包含当前扫描类型配置的 YAML
# 构造只包含当前扫描类型配置的 YAML(保留注释)
single_config = {scan_type: scan_cfg}
try:
single_yaml = yaml.safe_dump(
single_config,
sort_keys=False,
allow_unicode=True,
default_flow_style=None,
)
except yaml.YAMLError as e:
stream = StringIO()
yaml_parser.dump(single_config, stream)
single_yaml = stream.getvalue()
except Exception as e:
self.stdout.write(self.style.ERROR(f'生成子引擎 {scan_type} 配置失败: {e}'))
continue

View File

@@ -97,9 +97,11 @@ SITE_SCAN_COMMANDS = {
'base': (
"'{scan_tools_base}/httpx' -l '{url_file}' "
'-status-code -content-type -content-length '
'-location -title -server -body-preview '
'-location -title -server '
'-tech-detect -cdn -vhost '
'-random-agent -no-color -json'
'-include-response '
'-rstr 2000 '
'-random-agent -no-color -json -silent'
),
'optional': {
'threads': '-threads {threads}',
@@ -169,9 +171,11 @@ URL_FETCH_COMMANDS = {
'base': (
"'{scan_tools_base}/httpx' -l '{url_file}' "
'-status-code -content-type -content-length '
'-location -title -server -body-preview '
'-location -title -server '
'-tech-detect -cdn -vhost '
'-random-agent -no-color -json'
'-include-response '
'-rstr 2000 '
'-random-agent -no-color -json -silent'
),
'optional': {
'threads': '-threads {threads}',

View File

@@ -4,14 +4,12 @@
# 必需参数enabled是否启用
# 可选参数timeout超时秒数默认 auto 自动计算)
# ==================== 子域名发现 ====================
#
# Stage 1: 被动收集(并行) - 必选,至少启用一个工具
# Stage 2: 字典爆破(可选) - 使用字典暴力枚举子域名
# Stage 3: 变异生成 + 验证(可选) - 基于已发现域名生成变异,流式验证存活
# Stage 4: DNS 存活验证(可选) - 验证所有候选域名是否能解析
#
subdomain_discovery:
# ==================== 子域名发现 ====================
# Stage 1: 被动收集(并行) - 必选,至少启用一个工具
# Stage 2: 字典爆破(可选) - 使用字典暴力枚举子域名
# Stage 3: 变异生成 + 验证(可选) - 基于已发现域名生成变异,流式验证存活
# Stage 4: DNS 存活验证(可选) - 验证所有候选域名是否能解析
# === Stage 1: 被动收集工具(并行执行)===
passive_tools:
subfinder:
@@ -55,8 +53,8 @@ subdomain_discovery:
subdomain_resolve:
timeout: auto # 自动根据候选子域数量计算
# ==================== 端口扫描 ====================
port_scan:
# ==================== 端口扫描 ====================
tools:
naabu_active:
enabled: true
@@ -70,8 +68,8 @@ port_scan:
enabled: true
# timeout: auto # 被动扫描通常较快
# ==================== 站点扫描 ====================
site_scan:
# ==================== 站点扫描 ====================
tools:
httpx:
enabled: true
@@ -81,16 +79,16 @@ site_scan:
# request-timeout: 10 # 单个请求超时秒数(默认 10
# retries: 2 # 请求失败重试次数
# ==================== 指纹识别 ====================
# 在 site_scan 后串行执行,识别 WebSite 的技术栈
fingerprint_detect:
# ==================== 指纹识别 ====================
# 在 站点扫描 后串行执行,识别 WebSite 的技术栈
tools:
xingfinger:
enabled: true
fingerprint-libs: [ehole, goby, wappalyzer, fingers, fingerprinthub, arl] # 全部指纹库
fingerprint-libs: [ehole, goby, wappalyzer, fingers, fingerprinthub, arl] # 默认启动全部指纹库
# ==================== 目录扫描 ====================
directory_scan:
# ==================== 目录扫描 ====================
tools:
ffuf:
enabled: true
@@ -103,8 +101,8 @@ directory_scan:
match-codes: 200,201,301,302,401,403 # 匹配的 HTTP 状态码
# rate: 0 # 每秒请求数(默认 0 不限制)
# ==================== URL 获取 ====================
url_fetch:
# ==================== URL 获取 ====================
tools:
waymore:
enabled: true
@@ -142,8 +140,8 @@ url_fetch:
# request-timeout: 10 # 单个请求超时秒数(默认 10
# retries: 2 # 请求失败重试次数
# ==================== 漏洞扫描 ====================
vuln_scan:
# ==================== 漏洞扫描 ====================
tools:
dalfox_xss:
enabled: true

View File

@@ -37,7 +37,7 @@ logger = logging.getLogger(__name__)
def calculate_fingerprint_detect_timeout(
url_count: int,
base_per_url: float = 5.0,
base_per_url: float = 10.0,
min_timeout: int = 300
) -> int:
"""
@@ -49,7 +49,7 @@ def calculate_fingerprint_detect_timeout(
Args:
url_count: URL 数量
base_per_url: 每 URL 基础时间(秒),默认 5
base_per_url: 每 URL 基础时间(秒),默认 10
min_timeout: 最小超时时间(秒),默认 300秒
Returns:
@@ -256,7 +256,8 @@ def fingerprint_detect_flow(
'url_count': int,
'processed_records': int,
'updated_count': int,
'not_found_count': int,
'created_count': int,
'snapshot_count': int,
'executed_tasks': list,
'tool_stats': dict
}
@@ -303,6 +304,7 @@ def fingerprint_detect_flow(
'processed_records': 0,
'updated_count': 0,
'created_count': 0,
'snapshot_count': 0,
'executed_tasks': ['export_urls_for_fingerprint'],
'tool_stats': {
'total': 0,
@@ -340,6 +342,7 @@ def fingerprint_detect_flow(
total_processed = sum(stats['result'].get('processed_records', 0) for stats in tool_stats.values())
total_updated = sum(stats['result'].get('updated_count', 0) for stats in tool_stats.values())
total_created = sum(stats['result'].get('created_count', 0) for stats in tool_stats.values())
total_snapshots = sum(stats['result'].get('snapshot_count', 0) for stats in tool_stats.values())
successful_tools = [name for name in enabled_tools.keys()
if name not in [f['tool'] for f in failed_tools]]
@@ -354,6 +357,7 @@ def fingerprint_detect_flow(
'processed_records': total_processed,
'updated_count': total_updated,
'created_count': total_created,
'snapshot_count': total_snapshots,
'executed_tasks': executed_tasks,
'tool_stats': {
'total': len(enabled_tools),

View File

@@ -114,8 +114,11 @@ def initiate_scan_flow(
# ==================== Task 2: 获取引擎配置 ====================
from apps.scan.models import Scan
scan = Scan.objects.select_related('engine').get(id=scan_id)
engine_config = scan.engine.configuration
scan = Scan.objects.get(id=scan_id)
engine_config = scan.merged_configuration
# 使用 engine_names 进行显示
display_engine_name = ', '.join(scan.engine_names) if scan.engine_names else engine_name
# ==================== Task 3: 解析配置,生成执行计划 ====================
orchestrator = FlowOrchestrator(engine_config)

View File

@@ -20,11 +20,19 @@ class Scan(models.Model):
target = models.ForeignKey('targets.Target', on_delete=models.CASCADE, related_name='scans', help_text='扫描目标')
engine = models.ForeignKey(
'engine.ScanEngine',
on_delete=models.CASCADE,
related_name='scans',
help_text='使用的扫描引擎'
# 多引擎支持字段
engine_ids = ArrayField(
models.IntegerField(),
default=list,
help_text='引擎 ID 列表'
)
engine_names = models.JSONField(
default=list,
help_text='引擎名称列表,如 ["引擎A", "引擎B"]'
)
merged_configuration = models.TextField(
default='',
help_text='合并后的 YAML 配置'
)
created_at = models.DateTimeField(auto_now_add=True, help_text='任务创建时间')
@@ -118,12 +126,19 @@ class ScheduledScan(models.Model):
# 基本信息
name = models.CharField(max_length=200, help_text='任务名称')
# 关联的扫描引擎
engine = models.ForeignKey(
'engine.ScanEngine',
on_delete=models.CASCADE,
related_name='scheduled_scans',
help_text='使用的扫描引擎'
# 多引擎支持字段
engine_ids = ArrayField(
models.IntegerField(),
default=list,
help_text='引擎 ID 列表'
)
engine_names = models.JSONField(
default=list,
help_text='引擎名称列表,如 ["引擎A", "引擎B"]'
)
merged_configuration = models.TextField(
default='',
help_text='合并后的 YAML 配置'
)
# 关联的组织(组织扫描模式:执行时动态获取组织下所有目标)

View File

@@ -16,7 +16,6 @@ from django.utils import timezone
from apps.scan.models import Scan
from apps.targets.models import Target
from apps.engine.models import ScanEngine
from apps.common.definitions import ScanStatus
from apps.common.decorators import auto_ensure_db_connection
@@ -40,7 +39,7 @@ class DjangoScanRepository:
Args:
scan_id: 扫描任务 ID
prefetch_relations: 是否预加载关联对象(engine, target
prefetch_relations: 是否预加载关联对象(target, worker
默认 False只在需要展示关联信息时设为 True
for_update: 是否加锁(用于更新场景)
@@ -56,7 +55,7 @@ class DjangoScanRepository:
# 预加载关联对象(性能优化:默认不加载)
if prefetch_relations:
queryset = queryset.select_related('engine', 'target')
queryset = queryset.select_related('target', 'worker')
return queryset.get(id=scan_id)
except Scan.DoesNotExist: # type: ignore # pylint: disable=no-member
@@ -79,7 +78,7 @@ class DjangoScanRepository:
Note:
- 使用默认的阻塞模式(等待锁释放)
- 不包含关联对象(engine, target),如需关联对象请使用 get_by_id()
- 不包含关联对象(target, worker),如需关联对象请使用 get_by_id()
"""
try:
return Scan.objects.select_for_update().get(id=scan_id) # type: ignore # pylint: disable=no-member
@@ -103,7 +102,9 @@ class DjangoScanRepository:
def create(self,
target: Target,
engine: ScanEngine,
engine_ids: List[int],
engine_names: List[str],
merged_configuration: str,
results_dir: str,
status: ScanStatus = ScanStatus.INITIATED
) -> Scan:
@@ -112,7 +113,9 @@ class DjangoScanRepository:
Args:
target: 扫描目标
engine: 扫描引擎
engine_ids: 引擎 ID 列表
engine_names: 引擎名称列表
merged_configuration: 合并后的 YAML 配置
results_dir: 结果目录
status: 初始状态
@@ -121,7 +124,9 @@ class DjangoScanRepository:
"""
scan = Scan(
target=target,
engine=engine,
engine_ids=engine_ids,
engine_names=engine_names,
merged_configuration=merged_configuration,
results_dir=results_dir,
status=status,
container_ids=[]
@@ -231,14 +236,14 @@ class DjangoScanRepository:
获取所有扫描任务
Args:
prefetch_relations: 是否预加载关联对象(engine, target
prefetch_relations: 是否预加载关联对象(target, worker
Returns:
Scan QuerySet
"""
queryset = Scan.objects.all() # type: ignore # pylint: disable=no-member
if prefetch_relations:
queryset = queryset.select_related('engine', 'target')
queryset = queryset.select_related('target', 'worker')
return queryset.order_by('-created_at')

View File

@@ -29,7 +29,9 @@ class ScheduledScanDTO:
"""
id: Optional[int] = None
name: str = ''
engine_id: int = 0
engine_ids: List[int] = None # 多引擎支持
engine_names: List[str] = None # 引擎名称列表
merged_configuration: str = '' # 合并后的配置
organization_id: Optional[int] = None # 组织扫描模式
target_id: Optional[int] = None # 目标扫描模式
cron_expression: Optional[str] = None
@@ -40,6 +42,11 @@ class ScheduledScanDTO:
created_at: Optional[datetime] = None
updated_at: Optional[datetime] = None
def __post_init__(self):
if self.engine_ids is None:
self.engine_ids = []
if self.engine_names is None:
self.engine_names = []
@auto_ensure_db_connection
@@ -56,7 +63,7 @@ class DjangoScheduledScanRepository:
def get_by_id(self, scheduled_scan_id: int) -> Optional[ScheduledScan]:
"""根据 ID 查询定时扫描任务"""
try:
return ScheduledScan.objects.select_related('engine', 'organization', 'target').get(id=scheduled_scan_id)
return ScheduledScan.objects.select_related('organization', 'target').get(id=scheduled_scan_id)
except ScheduledScan.DoesNotExist:
return None
@@ -67,7 +74,7 @@ class DjangoScheduledScanRepository:
Returns:
QuerySet
"""
return ScheduledScan.objects.select_related('engine', 'organization', 'target').order_by('-created_at')
return ScheduledScan.objects.select_related('organization', 'target').order_by('-created_at')
def get_all(self, page: int = 1, page_size: int = 10) -> Tuple[List[ScheduledScan], int]:
"""
@@ -87,7 +94,7 @@ class DjangoScheduledScanRepository:
def get_enabled(self) -> List[ScheduledScan]:
"""获取所有启用的定时扫描任务"""
return list(
ScheduledScan.objects.select_related('engine', 'target')
ScheduledScan.objects.select_related('target')
.filter(is_enabled=True)
.order_by('-created_at')
)
@@ -105,7 +112,9 @@ class DjangoScheduledScanRepository:
with transaction.atomic():
scheduled_scan = ScheduledScan.objects.create(
name=dto.name,
engine_id=dto.engine_id,
engine_ids=dto.engine_ids,
engine_names=dto.engine_names,
merged_configuration=dto.merged_configuration,
organization_id=dto.organization_id, # 组织扫描模式
target_id=dto.target_id if not dto.organization_id else None, # 目标扫描模式
cron_expression=dto.cron_expression,
@@ -134,8 +143,12 @@ class DjangoScheduledScanRepository:
# 更新基本字段
if dto.name:
scheduled_scan.name = dto.name
if dto.engine_id:
scheduled_scan.engine_id = dto.engine_id
if dto.engine_ids is not None:
scheduled_scan.engine_ids = dto.engine_ids
if dto.engine_names is not None:
scheduled_scan.engine_names = dto.engine_names
if dto.merged_configuration is not None:
scheduled_scan.merged_configuration = dto.merged_configuration
if dto.cron_expression is not None:
scheduled_scan.cron_expression = dto.cron_expression
if dto.is_enabled is not None:

View File

@@ -7,12 +7,11 @@ from .models import Scan, ScheduledScan
class ScanSerializer(serializers.ModelSerializer):
"""扫描任务序列化器"""
target_name = serializers.SerializerMethodField()
engine_name = serializers.SerializerMethodField()
class Meta:
model = Scan
fields = [
'id', 'target', 'target_name', 'engine', 'engine_name',
'id', 'target', 'target_name', 'engine_ids', 'engine_names',
'created_at', 'stopped_at', 'status', 'results_dir',
'container_ids', 'error_message'
]
@@ -24,10 +23,6 @@ class ScanSerializer(serializers.ModelSerializer):
def get_target_name(self, obj):
"""获取目标名称"""
return obj.target.name if obj.target else None
def get_engine_name(self, obj):
"""获取引擎名称"""
return obj.engine.name if obj.engine else None
class ScanHistorySerializer(serializers.ModelSerializer):
@@ -36,11 +31,12 @@ class ScanHistorySerializer(serializers.ModelSerializer):
为前端扫描历史页面提供优化的数据格式,包括:
- 扫描汇总统计(子域名、端点、漏洞数量)
- 进度百分比和当前阶段
- 执行节点信息
"""
# 字段映射
target_name = serializers.CharField(source='target.name', read_only=True)
engine_name = serializers.CharField(source='engine.name', read_only=True)
worker_name = serializers.CharField(source='worker.name', read_only=True, allow_null=True)
# 计算字段
summary = serializers.SerializerMethodField()
@@ -53,9 +49,9 @@ class ScanHistorySerializer(serializers.ModelSerializer):
class Meta:
model = Scan
fields = [
'id', 'target', 'target_name', 'engine', 'engine_name',
'created_at', 'status', 'error_message', 'summary', 'progress',
'current_stage', 'stage_progress'
'id', 'target', 'target_name', 'engine_ids', 'engine_names',
'worker_name', 'created_at', 'status', 'error_message', 'summary',
'progress', 'current_stage', 'stage_progress'
]
def get_summary(self, obj):
@@ -105,10 +101,11 @@ class QuickScanSerializer(serializers.Serializer):
help_text='目标列表,每个目标包含 name 字段'
)
# 扫描引擎 ID
engine_id = serializers.IntegerField(
# 扫描引擎 ID 列表
engine_ids = serializers.ListField(
child=serializers.IntegerField(),
required=True,
help_text='使用的扫描引擎 ID (必填)'
help_text='使用的扫描引擎 ID 列表 (必填)'
)
def validate_targets(self, value):
@@ -130,6 +127,12 @@ class QuickScanSerializer(serializers.Serializer):
raise serializers.ValidationError(f"{idx + 1} 个目标的 name 不能为空")
return value
def validate_engine_ids(self, value):
"""验证引擎 ID 列表"""
if not value:
raise serializers.ValidationError("engine_ids 不能为空")
return value
# ==================== 定时扫描序列化器 ====================
@@ -138,7 +141,6 @@ class ScheduledScanSerializer(serializers.ModelSerializer):
"""定时扫描任务序列化器(用于列表和详情)"""
# 关联字段
engine_name = serializers.CharField(source='engine.name', read_only=True)
organization_id = serializers.IntegerField(source='organization.id', read_only=True, allow_null=True)
organization_name = serializers.CharField(source='organization.name', read_only=True, allow_null=True)
target_id = serializers.IntegerField(source='target.id', read_only=True, allow_null=True)
@@ -149,7 +151,7 @@ class ScheduledScanSerializer(serializers.ModelSerializer):
model = ScheduledScan
fields = [
'id', 'name',
'engine', 'engine_name',
'engine_ids', 'engine_names',
'organization_id', 'organization_name',
'target_id', 'target_name',
'scan_mode',
@@ -178,7 +180,10 @@ class CreateScheduledScanSerializer(serializers.Serializer):
"""
name = serializers.CharField(max_length=200, help_text='任务名称')
engine_id = serializers.IntegerField(help_text='扫描引擎 ID')
engine_ids = serializers.ListField(
child=serializers.IntegerField(),
help_text='扫描引擎 ID 列表'
)
# 组织扫描模式
organization_id = serializers.IntegerField(
@@ -201,6 +206,12 @@ class CreateScheduledScanSerializer(serializers.Serializer):
)
is_enabled = serializers.BooleanField(default=True, help_text='是否立即启用')
def validate_engine_ids(self, value):
"""验证引擎 ID 列表"""
if not value:
raise serializers.ValidationError("engine_ids 不能为空")
return value
def validate(self, data):
"""验证 organization_id 和 target_id 互斥"""
organization_id = data.get('organization_id')
@@ -219,7 +230,11 @@ class UpdateScheduledScanSerializer(serializers.Serializer):
"""更新定时扫描任务序列化器"""
name = serializers.CharField(max_length=200, required=False, help_text='任务名称')
engine_id = serializers.IntegerField(required=False, help_text='扫描引擎 ID')
engine_ids = serializers.ListField(
child=serializers.IntegerField(),
required=False,
help_text='扫描引擎 ID 列表'
)
# 组织扫描模式
organization_id = serializers.IntegerField(
@@ -237,6 +252,12 @@ class UpdateScheduledScanSerializer(serializers.Serializer):
cron_expression = serializers.CharField(max_length=100, required=False, help_text='Cron 表达式')
is_enabled = serializers.BooleanField(required=False, help_text='是否启用')
def validate_engine_ids(self, value):
"""验证引擎 ID 列表"""
if value is not None and not value:
raise serializers.ValidationError("engine_ids 不能为空")
return value
class ToggleScheduledScanSerializer(serializers.Serializer):

View File

@@ -10,7 +10,7 @@
import uuid
import logging
import threading
from typing import List
from typing import List, Tuple
from datetime import datetime
from pathlib import Path
from django.conf import settings
@@ -20,6 +20,7 @@ from django.core.exceptions import ValidationError, ObjectDoesNotExist
from apps.scan.models import Scan
from apps.scan.repositories import DjangoScanRepository
from apps.scan.utils.config_merger import merge_engine_configs, ConfigConflictError
from apps.targets.repositories import DjangoTargetRepository, DjangoOrganizationRepository
from apps.engine.repositories import DjangoEngineRepository
from apps.targets.models import Target
@@ -142,6 +143,106 @@ class ScanCreationService:
return targets, engine
def prepare_initiate_scan_multi_engine(
self,
organization_id: int | None = None,
target_id: int | None = None,
engine_ids: List[int] | None = None
) -> Tuple[List[Target], str, List[str], List[int]]:
"""
准备多引擎扫描任务所需的数据
职责:
1. 参数验证(必填项、互斥参数)
2. 资源查询Engines、Organization、Target
3. 合并引擎配置(检测冲突)
4. 返回准备好的目标列表、合并配置和引擎信息
Args:
organization_id: 组织ID可选
target_id: 目标ID可选
engine_ids: 扫描引擎ID列表必填
Returns:
(目标列表, 合并配置, 引擎名称列表, 引擎ID列表) - 供 create_scans 方法使用
Raises:
ValidationError: 参数验证失败或业务规则不满足
ObjectDoesNotExist: 资源不存在Organization/Target/ScanEngine
ConfigConflictError: 引擎配置存在冲突
Note:
- organization_id 和 target_id 必须二选一
- 如果提供 organization_id返回该组织下所有目标
- 如果提供 target_id返回单个目标列表
"""
# 1. 参数验证
if not engine_ids:
raise ValidationError('缺少必填参数: engine_ids')
if not organization_id and not target_id:
raise ValidationError('必须提供 organization_id 或 target_id 其中之一')
if organization_id and target_id:
raise ValidationError('organization_id 和 target_id 只能提供其中之一')
# 2. 查询所有扫描引擎
engines = []
for engine_id in engine_ids:
engine = self.engine_repo.get_by_id(engine_id)
if not engine:
logger.error("扫描引擎不存在 - Engine ID: %s", engine_id)
raise ObjectDoesNotExist(f'ScanEngine ID {engine_id} 不存在')
engines.append(engine)
# 3. 合并引擎配置(可能抛出 ConfigConflictError
engine_configs = [(e.name, e.configuration or '') for e in engines]
merged_configuration = merge_engine_configs(engine_configs)
engine_names = [e.name for e in engines]
logger.debug(
"引擎配置合并成功 - 引擎: %s",
', '.join(engine_names)
)
# 4. 根据参数获取目标列表
targets = []
if organization_id:
# 根据组织ID获取所有目标
organization = self.organization_repo.get_by_id(organization_id)
if not organization:
logger.error("组织不存在 - Organization ID: %s", organization_id)
raise ObjectDoesNotExist(f'Organization ID {organization_id} 不存在')
targets = self.organization_repo.get_targets(organization_id)
if not targets:
raise ValidationError(f'组织 ID {organization_id} 下没有目标')
logger.debug(
"准备发起扫描 - 组织: %s, 目标数量: %d, 引擎: %s",
organization.name,
len(targets),
', '.join(engine_names)
)
else:
# 根据目标ID获取单个目标
target = self.target_repo.get_by_id(target_id)
if not target:
logger.error("目标不存在 - Target ID: %s", target_id)
raise ObjectDoesNotExist(f'Target ID {target_id} 不存在')
targets = [target]
logger.debug(
"准备发起扫描 - 目标: %s, 引擎: %s",
target.name,
', '.join(engine_names)
)
return targets, merged_configuration, engine_names, engine_ids
def _generate_scan_workspace_dir(self) -> str:
"""
生成 Scan 工作空间目录路径
@@ -179,7 +280,9 @@ class ScanCreationService:
def create_scans(
self,
targets: List[Target],
engine: ScanEngine,
engine_ids: List[int],
engine_names: List[str],
merged_configuration: str,
scheduled_scan_name: str | None = None
) -> List[Scan]:
"""
@@ -187,7 +290,9 @@ class ScanCreationService:
Args:
targets: 目标列表
engine: 扫描引擎对象
engine_ids: 引擎 ID 列表
engine_names: 引擎名称列表
merged_configuration: 合并后的 YAML 配置
scheduled_scan_name: 定时扫描任务名称(可选,用于通知显示)
Returns:
@@ -205,7 +310,9 @@ class ScanCreationService:
scan_workspace_dir = self._generate_scan_workspace_dir()
scan = Scan(
target=target,
engine=engine,
engine_ids=engine_ids,
engine_names=engine_names,
merged_configuration=merged_configuration,
results_dir=scan_workspace_dir,
status=ScanStatus.INITIATED,
container_ids=[],
@@ -236,13 +343,15 @@ class ScanCreationService:
return []
# 第三步:分发任务到 Workers
# 使用第一个引擎名称作为显示名称,或者合并显示
display_engine_name = ', '.join(engine_names) if engine_names else ''
scan_data = [
{
'scan_id': scan.id,
'target_name': scan.target.name,
'target_id': scan.target.id,
'results_dir': scan.results_dir,
'engine_name': scan.engine.name,
'engine_name': display_engine_name,
'scheduled_scan_name': scheduled_scan_name,
}
for scan in created_scans

View File

@@ -96,14 +96,34 @@ class ScanService:
organization_id, target_id, engine_id
)
def prepare_initiate_scan_multi_engine(
self,
organization_id: int | None = None,
target_id: int | None = None,
engine_ids: List[int] | None = None
) -> tuple[List[Target], str, List[str], List[int]]:
"""
为创建多引擎扫描任务做准备
Returns:
(目标列表, 合并配置, 引擎名称列表, 引擎ID列表)
"""
return self.creation_service.prepare_initiate_scan_multi_engine(
organization_id, target_id, engine_ids
)
def create_scans(
self,
targets: List[Target],
engine: ScanEngine,
engine_ids: List[int],
engine_names: List[str],
merged_configuration: str,
scheduled_scan_name: str | None = None
) -> List[Scan]:
"""批量创建扫描任务(委托给 ScanCreationService"""
return self.creation_service.create_scans(targets, engine, scheduled_scan_name)
return self.creation_service.create_scans(
targets, engine_ids, engine_names, merged_configuration, scheduled_scan_name
)
# ==================== 状态管理方法(委托给 ScanStateService ====================

View File

@@ -14,6 +14,7 @@ from django.core.exceptions import ValidationError
from apps.scan.models import ScheduledScan
from apps.scan.repositories import DjangoScheduledScanRepository, ScheduledScanDTO
from apps.scan.utils.config_merger import merge_engine_configs, ConfigConflictError
from apps.engine.repositories import DjangoEngineRepository
from apps.targets.services import TargetService
@@ -57,8 +58,9 @@ class ScheduledScanService:
流程:
1. 验证参数
2. 创建数据库记录
3. 计算并设置 next_run_time
2. 合并引擎配置
3. 创建数据库记录
4. 计算并设置 next_run_time
Args:
dto: 定时扫描 DTO
@@ -68,14 +70,30 @@ class ScheduledScanService:
Raises:
ValidationError: 参数验证失败
ConfigConflictError: 引擎配置冲突
"""
# 1. 验证参数
self._validate_create_dto(dto)
# 2. 创建数据库记录
# 2. 合并引擎配置
engines = []
engine_names = []
for engine_id in dto.engine_ids:
engine = self.engine_repo.get_by_id(engine_id)
if engine:
engines.append((engine.name, engine.configuration or ''))
engine_names.append(engine.name)
merged_configuration = merge_engine_configs(engines)
# 设置 DTO 的合并配置和引擎名称
dto.engine_names = engine_names
dto.merged_configuration = merged_configuration
# 3. 创建数据库记录
scheduled_scan = self.repo.create(dto)
# 3. 如果有 cron 表达式且已启用,计算下次执行时间
# 4. 如果有 cron 表达式且已启用,计算下次执行时间
if scheduled_scan.cron_expression and scheduled_scan.is_enabled:
next_run_time = self._calculate_next_run_time(scheduled_scan)
if next_run_time:
@@ -96,11 +114,13 @@ class ScheduledScanService:
if not dto.name:
raise ValidationError('任务名称不能为空')
if not dto.engine_id:
if not dto.engine_ids:
raise ValidationError('必须选择扫描引擎')
if not self.engine_repo.get_by_id(dto.engine_id):
raise ValidationError(f'扫描引擎 ID {dto.engine_id} 不存在')
# 验证所有引擎是否存在
for engine_id in dto.engine_ids:
if not self.engine_repo.get_by_id(engine_id):
raise ValidationError(f'扫描引擎 ID {engine_id} 不存在')
# 验证扫描模式organization_id 和 target_id 互斥)
if not dto.organization_id and not dto.target_id:
@@ -138,11 +158,28 @@ class ScheduledScanService:
Returns:
更新后的 ScheduledScan 对象
Raises:
ConfigConflictError: 引擎配置冲突
"""
existing = self.repo.get_by_id(scheduled_scan_id)
if not existing:
return None
# 如果引擎变更,重新合并配置
if dto.engine_ids is not None:
engines = []
engine_names = []
for engine_id in dto.engine_ids:
engine = self.engine_repo.get_by_id(engine_id)
if engine:
engines.append((engine.name, engine.configuration or ''))
engine_names.append(engine.name)
merged_configuration = merge_engine_configs(engines)
dto.engine_names = engine_names
dto.merged_configuration = merged_configuration
# 更新数据库记录
scheduled_scan = self.repo.update(scheduled_scan_id, dto)
if not scheduled_scan:
@@ -292,21 +329,25 @@ class ScheduledScanService:
立即触发扫描(支持组织扫描和目标扫描两种模式)
复用 ScanService 的逻辑,与 API 调用保持一致。
使用存储的 merged_configuration 而不是重新合并。
"""
from apps.scan.services.scan_service import ScanService
scan_service = ScanService()
# 1. 准备扫描所需数据(复用 API 的逻辑
targets, engine = scan_service.prepare_initiate_scan(
# 1. 准备扫描所需数据(使用存储的多引擎配置
targets, _, _, _ = scan_service.prepare_initiate_scan_multi_engine(
organization_id=scheduled_scan.organization_id,
target_id=scheduled_scan.target_id,
engine_id=scheduled_scan.engine_id
engine_ids=scheduled_scan.engine_ids
)
# 2. 创建扫描任务,传递定时扫描名称用于通知显示
# 2. 创建扫描任务,使用存储的合并配置
created_scans = scan_service.create_scans(
targets, engine,
targets=targets,
engine_ids=scheduled_scan.engine_ids,
engine_names=scheduled_scan.engine_names,
merged_configuration=scheduled_scan.merged_configuration,
scheduled_scan_name=scheduled_scan.name
)

View File

@@ -4,7 +4,6 @@ xingfinger 执行任务
流式执行 xingfinger 命令并实时更新 tech 字段
"""
import importlib
import json
import logging
import subprocess
@@ -15,93 +14,97 @@ from django.db import connection
from prefect import task
from apps.scan.utils import execute_stream
from apps.asset.dtos.snapshot import WebsiteSnapshotDTO
from apps.asset.repositories.snapshot import DjangoWebsiteSnapshotRepository
logger = logging.getLogger(__name__)
# 数据源映射source → (module_path, model_name, url_field)
SOURCE_MODEL_MAP = {
'website': ('apps.asset.models', 'WebSite', 'url'),
# 以后扩展:
# 'endpoint': ('apps.asset.models', 'Endpoint', 'url'),
# 'directory': ('apps.asset.models', 'Directory', 'url'),
}
def _get_model_class(source: str):
"""根据数据源类型获取 Model 类"""
if source not in SOURCE_MODEL_MAP:
raise ValueError(f"不支持的数据源: {source}")
module_path, model_name, _ = SOURCE_MODEL_MAP[source]
module = importlib.import_module(module_path)
return getattr(module, model_name)
def parse_xingfinger_line(line: str) -> tuple[str, list[str]] | None:
def parse_xingfinger_line(line: str) -> dict | None:
"""
解析 xingfinger 单行 JSON 输出
xingfinger 静默模式输出格式:
{"url": "https://example.com", "cms": "WordPress,PHP,nginx", ...}
xingfinger 输出格式:
{"url": "...", "cms": "...", "server": "BWS/1.1", "status_code": 200, "length": 642831, "title": "..."}
Returns:
tuple: (url, tech_list) 或 None解析失败时
dict: 包含 url, techs, server, title, status_code, content_length 的字典
None: 解析失败或 URL 为空时
"""
try:
item = json.loads(line)
url = item.get('url', '').strip()
cms = item.get('cms', '')
if not url or not cms:
if not url:
return None
# cms 字段按逗号分割,去除空白
techs = [t.strip() for t in cms.split(',') if t.strip()]
cms = item.get('cms', '')
techs = [t.strip() for t in cms.split(',') if t.strip()] if cms else []
return (url, techs) if techs else None
return {
'url': url,
'techs': techs,
'server': item.get('server', ''),
'title': item.get('title', ''),
'status_code': item.get('status_code'),
'content_length': item.get('length'),
}
except json.JSONDecodeError:
return None
def bulk_merge_tech_field(
source: str,
url_techs_map: dict[str, list[str]],
def bulk_merge_website_fields(
records: list[dict],
target_id: int
) -> dict:
"""
批量合并 tech 数组字段PostgreSQL 原生 SQL
批量合并更新 WebSite 字段PostgreSQL 原生 SQL
合并策略:
- tech数组合并去重
- title, webserver, status_code, content_length只在原值为空/NULL 时更新
使用 PostgreSQL 原生 SQL 实现高效的数组合并去重操作。
如果 URL 对应的记录不存在,会自动创建新记录。
Args:
records: 解析后的记录列表,每个包含 {url, techs, server, title, status_code, content_length}
target_id: 目标 ID
Returns:
dict: {'updated_count': int, 'created_count': int}
"""
Model = _get_model_class(source)
table_name = Model._meta.db_table
from apps.asset.models import WebSite
table_name = WebSite._meta.db_table
updated_count = 0
created_count = 0
with connection.cursor() as cursor:
for url, techs in url_techs_map.items():
if not techs:
continue
for record in records:
url = record['url']
techs = record.get('techs', [])
server = record.get('server', '') or ''
title = record.get('title', '') or ''
status_code = record.get('status_code')
content_length = record.get('content_length')
# 先尝试更新(PostgreSQL 数组合并去重
sql = f"""
# 先尝试更新(合并策略
update_sql = f"""
UPDATE {table_name}
SET tech = (
SELECT ARRAY(SELECT DISTINCT unnest(
SET
tech = (SELECT ARRAY(SELECT DISTINCT unnest(
COALESCE(tech, ARRAY[]::varchar[]) || %s::varchar[]
))
)
))),
title = CASE WHEN title = '' OR title IS NULL THEN %s ELSE title END,
webserver = CASE WHEN webserver = '' OR webserver IS NULL THEN %s ELSE webserver END,
status_code = CASE WHEN status_code IS NULL THEN %s ELSE status_code END,
content_length = CASE WHEN content_length IS NULL THEN %s ELSE content_length END
WHERE url = %s AND target_id = %s
"""
cursor.execute(sql, [techs, url, target_id])
cursor.execute(update_sql, [techs, title, server, status_code, content_length, url, target_id])
if cursor.rowcount > 0:
updated_count += cursor.rowcount
@@ -113,22 +116,27 @@ def bulk_merge_tech_field(
host = parsed.hostname or ''
# 插入新记录(带冲突处理)
# 显式传入所有 NOT NULL 字段的默认值
insert_sql = f"""
INSERT INTO {table_name} (target_id, url, host, location, title, webserver, body_preview, content_type, tech, created_at)
VALUES (%s, %s, %s, '', '', '', '', '', %s::varchar[], NOW())
INSERT INTO {table_name} (
target_id, url, host, location, title, webserver,
response_body, content_type, tech, status_code, content_length,
response_headers, created_at
)
VALUES (%s, %s, %s, '', %s, %s, '', '', %s::varchar[], %s, %s, '', NOW())
ON CONFLICT (target_id, url) DO UPDATE SET
tech = (
SELECT ARRAY(SELECT DISTINCT unnest(
COALESCE({table_name}.tech, ARRAY[]::varchar[]) || EXCLUDED.tech
))
)
tech = (SELECT ARRAY(SELECT DISTINCT unnest(
COALESCE({table_name}.tech, ARRAY[]::varchar[]) || EXCLUDED.tech
))),
title = CASE WHEN {table_name}.title = '' OR {table_name}.title IS NULL THEN EXCLUDED.title ELSE {table_name}.title END,
webserver = CASE WHEN {table_name}.webserver = '' OR {table_name}.webserver IS NULL THEN EXCLUDED.webserver ELSE {table_name}.webserver END,
status_code = CASE WHEN {table_name}.status_code IS NULL THEN EXCLUDED.status_code ELSE {table_name}.status_code END,
content_length = CASE WHEN {table_name}.content_length IS NULL THEN EXCLUDED.content_length ELSE {table_name}.content_length END
"""
cursor.execute(insert_sql, [target_id, url, host, techs])
cursor.execute(insert_sql, [target_id, url, host, title, server, techs, status_code, content_length])
created_count += 1
except Exception as e:
logger.warning("创建 %s 记录失败 (url=%s): %s", source, url, e)
logger.warning("创建 WebSite 记录失败 (url=%s): %s", url, e)
return {
'updated_count': updated_count,
@@ -142,12 +150,12 @@ def _parse_xingfinger_stream_output(
cwd: Optional[str] = None,
timeout: Optional[int] = None,
log_file: Optional[str] = None
) -> Generator[tuple[str, list[str]], None, None]:
) -> Generator[dict, None, None]:
"""
流式解析 xingfinger 命令输出
基于 execute_stream 实时处理 xingfinger 命令的 stdout将每行 JSON 输出
转换为 (url, tech_list) 格式
转换为完整字段字典
"""
logger.info("开始流式解析 xingfinger 命令输出 - 命令: %s", cmd)
@@ -194,43 +202,46 @@ def run_xingfinger_and_stream_update_tech_task(
batch_size: int = 100
) -> dict:
"""
流式执行 xingfinger 命令并实时更新 tech 字段
根据 source 参数更新对应表的 tech 字段:
- website → WebSite.tech
- endpoint → Endpoint.tech以后扩展
流式执行 xingfinger 命令,保存快照并合并更新资产表
处理流程:
1. 流式执行 xingfinger 命令
2. 实时解析 JSON 输出
3. 累积到 batch_size 条后批量更新数据库
4. 使用 PostgreSQL 原生 SQL 进行数组合并去重
5. 如果记录不存在,自动创建
2. 实时解析 JSON 输出(完整字段)
3. 累积到 batch_size 条后批量处理:
- 保存快照WebsiteSnapshot
- 合并更新资产表WebSite
合并策略:
- tech数组合并去重
- title, webserver, status_code, content_length只在原值为空时更新
Returns:
dict: {
'processed_records': int,
'updated_count': int,
'created_count': int,
'snapshot_count': int,
'batch_count': int
}
"""
logger.info(
"开始执行 xingfinger 并更新 tech - target_id=%s, source=%s, timeout=%s",
target_id, source, timeout
"开始执行 xingfinger - scan_id=%s, target_id=%s, timeout=%s",
scan_id, target_id, timeout
)
data_generator = None
snapshot_repo = DjangoWebsiteSnapshotRepository()
try:
# 初始化统计
processed_records = 0
updated_count = 0
created_count = 0
snapshot_count = 0
batch_count = 0
# 当前批次的 URL -> techs 映射
url_techs_map = {}
# 当前批次的记录列表
batch_records = []
# 流式处理
data_generator = _parse_xingfinger_stream_output(
@@ -241,47 +252,43 @@ def run_xingfinger_and_stream_update_tech_task(
log_file=log_file
)
for url, techs in data_generator:
for record in data_generator:
processed_records += 1
batch_records.append(record)
# 累积到 url_techs_map
if url in url_techs_map:
# 合并同一 URL 的多次识别结果
url_techs_map[url].extend(techs)
else:
url_techs_map[url] = techs
# 达到批次大小,执行批量更新
if len(url_techs_map) >= batch_size:
# 达到批次大小,执行批量处理
if len(batch_records) >= batch_size:
batch_count += 1
result = bulk_merge_tech_field(source, url_techs_map, target_id)
updated_count += result['updated_count']
created_count += result.get('created_count', 0)
logger.debug(
"批次 %d 完成 - 更新: %d, 创建: %d",
batch_count, result['updated_count'], result.get('created_count', 0)
result = _process_batch(
batch_records, scan_id, target_id, batch_count, snapshot_repo
)
updated_count += result['updated_count']
created_count += result['created_count']
snapshot_count += result['snapshot_count']
# 清空批次
url_techs_map = {}
batch_records = []
# 处理最后一批
if url_techs_map:
if batch_records:
batch_count += 1
result = bulk_merge_tech_field(source, url_techs_map, target_id)
result = _process_batch(
batch_records, scan_id, target_id, batch_count, snapshot_repo
)
updated_count += result['updated_count']
created_count += result.get('created_count', 0)
created_count += result['created_count']
snapshot_count += result['snapshot_count']
logger.info(
"✓ xingfinger 执行完成 - 处理记录: %d, 更新: %d, 创建: %d, 批次: %d",
processed_records, updated_count, created_count, batch_count
"✓ xingfinger 执行完成 - 处理: %d, 更新: %d, 创建: %d, 快照: %d, 批次: %d",
processed_records, updated_count, created_count, snapshot_count, batch_count
)
return {
'processed_records': processed_records,
'updated_count': updated_count,
'created_count': created_count,
'snapshot_count': snapshot_count,
'batch_count': batch_count
}
@@ -299,3 +306,67 @@ def run_xingfinger_and_stream_update_tech_task(
data_generator.close()
except Exception as e:
logger.debug("关闭生成器时出错: %s", e)
def _process_batch(
records: list[dict],
scan_id: int,
target_id: int,
batch_num: int,
snapshot_repo: DjangoWebsiteSnapshotRepository
) -> dict:
"""
处理一个批次的数据:保存快照 + 合并更新资产表
Args:
records: 解析后的记录列表
scan_id: 扫描任务 ID
target_id: 目标 ID
batch_num: 批次编号
snapshot_repo: 快照仓库
Returns:
dict: {'updated_count': int, 'created_count': int, 'snapshot_count': int}
"""
# 1. 构建快照 DTO 列表
snapshot_dtos = []
for record in records:
# 从 URL 提取 host
parsed = urlparse(record['url'])
host = parsed.hostname or ''
dto = WebsiteSnapshotDTO(
scan_id=scan_id,
target_id=target_id,
url=record['url'],
host=host,
title=record.get('title', '') or '',
status=record.get('status_code'),
content_length=record.get('content_length'),
web_server=record.get('server', '') or '',
tech=record.get('techs', []),
)
snapshot_dtos.append(dto)
# 2. 保存快照
snapshot_count = 0
if snapshot_dtos:
try:
snapshot_repo.save_snapshots(snapshot_dtos)
snapshot_count = len(snapshot_dtos)
except Exception as e:
logger.warning("批次 %d 保存快照失败: %s", batch_num, e)
# 3. 合并更新资产表
merge_result = bulk_merge_website_fields(records, target_id)
logger.debug(
"批次 %d 完成 - 更新: %d, 创建: %d, 快照: %d",
batch_num, merge_result['updated_count'], merge_result['created_count'], snapshot_count
)
return {
'updated_count': merge_result['updated_count'],
'created_count': merge_result['created_count'],
'snapshot_count': snapshot_count
}

View File

@@ -129,11 +129,12 @@ class HttpxRecord:
self.content_type = data.get('content_type', '')
self.location = data.get('location', '')
self.webserver = data.get('webserver', '')
self.body_preview = data.get('body_preview', '')
self.response_body = data.get('body', '') # 从 body 字段获取完整响应体
self.tech = data.get('tech', [])
self.vhost = data.get('vhost')
self.failed = data.get('failed', False)
self.timestamp = data.get('timestamp')
self.response_headers = data.get('raw_header', '') # 从 raw_header 字段获取原始响应头字符串
# 从 URL 中提取主机名
self.host = self._extract_hostname()
@@ -354,12 +355,13 @@ def _save_batch(
location=record.location, # location 字段保存重定向信息
title=record.title[:1000] if record.title else '',
web_server=record.webserver[:200] if record.webserver else '',
body_preview=record.body_preview[:1000] if record.body_preview else '',
response_body=record.response_body if record.response_body else '',
content_type=record.content_type[:200] if record.content_type else '',
tech=record.tech if isinstance(record.tech, list) else [],
status=record.status_code,
content_length=record.content_length,
vhost=record.vhost
vhost=record.vhost,
response_headers=record.response_headers if record.response_headers else '',
)
snapshot_items.append(snapshot_dto)

View File

@@ -2,8 +2,8 @@
基于 execute_stream 的流式 URL 验证任务
主要功能:
1. 实时执行 httpx 命令验证 URL 存活
2. 流式处理命令输出,解析存活的 URL
1. 实时执行 httpx 命令验证 URL
2. 流式处理命令输出,解析 URL 信息
3. 批量保存到数据库Endpoint 表)
4. 避免一次性加载所有 URL 到内存
@@ -14,7 +14,7 @@
- 使用 execute_stream 实时处理输出
- 流式处理避免内存溢出
- 批量操作减少数据库交互
- 保存存活的 URLstatus 2xx/3xx
- 保存所有有效 URL包括 4xx/5xx,便于安全分析
"""
import logging
@@ -73,7 +73,7 @@ def _parse_and_validate_line(line: str) -> Optional[dict]:
Returns:
Optional[dict]: 有效的 httpx 记录,或 None 如果验证失败
只返回存活的 URL2xx/3xx 状态码
保存所有有效 URL不再过滤状态码安全扫描中 403/404/500 等也有分析价值
"""
try:
# 清理 NUL 字符后再解析 JSON
@@ -99,24 +99,21 @@ def _parse_and_validate_line(line: str) -> Optional[dict]:
logger.info("URL 为空,跳过 - 数据: %s", str(line_data)[:200])
return None
# 保存存活的 URL2xx 或 3xx
if status_code and (200 <= status_code < 400):
return {
'url': _sanitize_string(url),
'host': _sanitize_string(line_data.get('host', '')),
'status_code': status_code,
'title': _sanitize_string(line_data.get('title', '')),
'content_length': line_data.get('content_length', 0),
'content_type': _sanitize_string(line_data.get('content_type', '')),
'webserver': _sanitize_string(line_data.get('webserver', '')),
'location': _sanitize_string(line_data.get('location', '')),
'tech': line_data.get('tech', []),
'body_preview': _sanitize_string(line_data.get('body_preview', '')),
'vhost': line_data.get('vhost', False),
}
else:
logger.debug("URL 不存活(状态码: %s),跳过: %s", status_code, url)
return None
# 保存所有有效 URL不再过滤状态码
return {
'url': _sanitize_string(url),
'host': _sanitize_string(line_data.get('host', '')),
'status_code': status_code,
'title': _sanitize_string(line_data.get('title', '')),
'content_length': line_data.get('content_length', 0),
'content_type': _sanitize_string(line_data.get('content_type', '')),
'webserver': _sanitize_string(line_data.get('webserver', '')),
'location': _sanitize_string(line_data.get('location', '')),
'tech': line_data.get('tech', []),
'response_body': _sanitize_string(line_data.get('body', '')),
'vhost': line_data.get('vhost', False),
'response_headers': _sanitize_string(line_data.get('raw_header', '')),
}
except Exception:
logger.info("跳过无法解析的行: %s", line[:100] if line else 'empty')
@@ -302,10 +299,11 @@ def _save_batch(
webserver=record.get('webserver', ''),
content_type=record.get('content_type', ''),
tech=record.get('tech', []),
body_preview=record.get('body_preview', ''),
response_body=record.get('response_body', ''),
vhost=record.get('vhost', False),
matched_gf_patterns=[],
target_id=target_id,
response_headers=record.get('response_headers', ''),
)
snapshots.append(dto)
except Exception as e:

View File

@@ -0,0 +1,80 @@
"""
配置合并工具模块
提供多引擎 YAML 配置的冲突检测和合并功能。
"""
from typing import List, Tuple
import yaml
class ConfigConflictError(Exception):
"""配置冲突异常
当两个或多个引擎定义相同的顶层扫描类型键时抛出。
"""
def __init__(self, conflicts: List[Tuple[str, str, str]]):
"""
参数:
conflicts: (键, 引擎1名称, 引擎2名称) 元组列表
"""
self.conflicts = conflicts
msg = "; ".join([f"{k} 同时存在于「{e1}」和「{e2}" for k, e1, e2 in conflicts])
super().__init__(f"扫描类型冲突: {msg}")
def merge_engine_configs(engines: List[Tuple[str, str]]) -> str:
"""
合并多个引擎的 YAML 配置。
参数:
engines: (引擎名称, 配置YAML) 元组列表
返回:
合并后的 YAML 字符串
异常:
ConfigConflictError: 当顶层键冲突时
"""
if not engines:
return ""
if len(engines) == 1:
return engines[0][1]
# 追踪每个顶层键属于哪个引擎
key_to_engine: dict[str, str] = {}
conflicts: List[Tuple[str, str, str]] = []
for engine_name, config_yaml in engines:
if not config_yaml or not config_yaml.strip():
continue
try:
parsed = yaml.safe_load(config_yaml)
except yaml.YAMLError:
# 无效 YAML 跳过
continue
if not isinstance(parsed, dict):
continue
# 检查顶层键冲突
for key in parsed.keys():
if key in key_to_engine:
conflicts.append((key, key_to_engine[key], engine_name))
else:
key_to_engine[key] = engine_name
if conflicts:
raise ConfigConflictError(conflicts)
# 无冲突,用双换行符连接配置
configs = []
for _, config_yaml in engines:
if config_yaml and config_yaml.strip():
configs.append(config_yaml.strip())
return "\n\n".join(configs)

View File

@@ -9,6 +9,7 @@ import logging
from apps.common.response_helpers import success_response, error_response
from apps.common.error_codes import ErrorCodes
from apps.scan.utils.config_merger import ConfigConflictError
logger = logging.getLogger(__name__)
@@ -118,7 +119,7 @@ class ScanViewSet(viewsets.ModelViewSet):
请求参数:
{
"targets": [{"name": "example.com"}, {"name": "https://example.com/api"}],
"engine_id": 1
"engine_ids": [1, 2]
}
支持的输入格式:
@@ -133,7 +134,7 @@ class ScanViewSet(viewsets.ModelViewSet):
serializer.is_valid(raise_exception=True)
targets_data = serializer.validated_data['targets']
engine_id = serializer.validated_data.get('engine_id')
engine_ids = serializer.validated_data.get('engine_ids')
try:
# 提取输入字符串列表
@@ -141,7 +142,7 @@ class ScanViewSet(viewsets.ModelViewSet):
# 1. 使用 QuickScanService 解析输入并创建资产
quick_scan_service = QuickScanService()
result = quick_scan_service.process_quick_scan(inputs, engine_id)
result = quick_scan_service.process_quick_scan(inputs, engine_ids[0] if engine_ids else None)
targets = result['targets']
@@ -153,17 +154,19 @@ class ScanViewSet(viewsets.ModelViewSet):
status_code=status.HTTP_400_BAD_REQUEST
)
# 2. 获取扫描引擎
engine_service = EngineService()
engine = engine_service.get_engine(engine_id)
if not engine:
raise ValidationError(f'扫描引擎 ID {engine_id} 不存在')
# 2. 准备多引擎扫描
scan_service = ScanService()
_, merged_configuration, engine_names, engine_ids = scan_service.prepare_initiate_scan_multi_engine(
target_id=targets[0].id, # 使用第一个目标来验证引擎
engine_ids=engine_ids
)
# 3. 批量发起扫描
scan_service = ScanService()
created_scans = scan_service.create_scans(
targets=targets,
engine=engine
engine_ids=engine_ids,
engine_names=engine_names,
merged_configuration=merged_configuration
)
# 检查是否成功创建扫描任务
@@ -192,6 +195,17 @@ class ScanViewSet(viewsets.ModelViewSet):
},
status_code=status.HTTP_201_CREATED
)
except ConfigConflictError as e:
return error_response(
code='CONFIG_CONFLICT',
message=str(e),
details=[
{'key': k, 'engines': [e1, e2]}
for k, e1, e2 in e.conflicts
],
status_code=status.HTTP_400_BAD_REQUEST
)
except ValidationError as e:
return error_response(
@@ -214,7 +228,7 @@ class ScanViewSet(viewsets.ModelViewSet):
请求参数:
- organization_id: 组织ID (int, 可选)
- target_id: 目标ID (int, 可选)
- engine_id: 扫描引擎ID (int, 必填)
- engine_ids: 扫描引擎ID列表 (list[int], 必填)
注意: organization_id 和 target_id 二选一
@@ -224,21 +238,38 @@ class ScanViewSet(viewsets.ModelViewSet):
# 获取请求数据
organization_id = request.data.get('organization_id')
target_id = request.data.get('target_id')
engine_id = request.data.get('engine_id')
engine_ids = request.data.get('engine_ids')
# 验证 engine_ids
if not engine_ids:
return error_response(
code=ErrorCodes.VALIDATION_ERROR,
message='缺少必填参数: engine_ids',
status_code=status.HTTP_400_BAD_REQUEST
)
if not isinstance(engine_ids, list):
return error_response(
code=ErrorCodes.VALIDATION_ERROR,
message='engine_ids 必须是数组',
status_code=status.HTTP_400_BAD_REQUEST
)
try:
# 步骤1准备扫描所需的数据(验证参数、查询资源、返回目标列表和引擎)
# 步骤1准备多引擎扫描所需的数据
scan_service = ScanService()
targets, engine = scan_service.prepare_initiate_scan(
targets, merged_configuration, engine_names, engine_ids = scan_service.prepare_initiate_scan_multi_engine(
organization_id=organization_id,
target_id=target_id,
engine_id=engine_id
engine_ids=engine_ids
)
# 步骤2批量创建扫描记录并分发扫描任务
created_scans = scan_service.create_scans(
targets=targets,
engine=engine
engine_ids=engine_ids,
engine_names=engine_names,
merged_configuration=merged_configuration
)
# 检查是否成功创建扫描任务
@@ -259,6 +290,17 @@ class ScanViewSet(viewsets.ModelViewSet):
},
status_code=status.HTTP_201_CREATED
)
except ConfigConflictError as e:
return error_response(
code='CONFIG_CONFLICT',
message=str(e),
details=[
{'key': k, 'engines': [e1, e2]}
for k, e1, e2 in e.conflicts
],
status_code=status.HTTP_400_BAD_REQUEST
)
except ObjectDoesNotExist as e:
# 资源不存在错误(由 service 层抛出)

View File

@@ -17,6 +17,7 @@ from ..serializers import (
)
from ..services.scheduled_scan_service import ScheduledScanService
from ..repositories import ScheduledScanDTO
from ..utils.config_merger import ConfigConflictError
from apps.common.pagination import BasePagination
from apps.common.response_helpers import success_response, error_response
from apps.common.error_codes import ErrorCodes
@@ -67,7 +68,7 @@ class ScheduledScanViewSet(viewsets.ModelViewSet):
data = serializer.validated_data
dto = ScheduledScanDTO(
name=data['name'],
engine_id=data['engine_id'],
engine_ids=data['engine_ids'],
organization_id=data.get('organization_id'),
target_id=data.get('target_id'),
cron_expression=data.get('cron_expression', '0 2 * * *'),
@@ -81,6 +82,16 @@ class ScheduledScanViewSet(viewsets.ModelViewSet):
data=response_serializer.data,
status_code=status.HTTP_201_CREATED
)
except ConfigConflictError as e:
return error_response(
code='CONFIG_CONFLICT',
message=str(e),
details=[
{'key': k, 'engines': [e1, e2]}
for k, e1, e2 in e.conflicts
],
status_code=status.HTTP_400_BAD_REQUEST
)
except ValidationError as e:
return error_response(
code=ErrorCodes.VALIDATION_ERROR,
@@ -98,7 +109,7 @@ class ScheduledScanViewSet(viewsets.ModelViewSet):
data = serializer.validated_data
dto = ScheduledScanDTO(
name=data.get('name'),
engine_id=data.get('engine_id'),
engine_ids=data.get('engine_ids'),
organization_id=data.get('organization_id'),
target_id=data.get('target_id'),
cron_expression=data.get('cron_expression'),
@@ -109,6 +120,16 @@ class ScheduledScanViewSet(viewsets.ModelViewSet):
response_serializer = ScheduledScanSerializer(scheduled_scan)
return success_response(data=response_serializer.data)
except ConfigConflictError as e:
return error_response(
code='CONFIG_CONFLICT',
message=str(e),
details=[
{'key': k, 'engines': [e1, e2]}
for k, e1, e2 in e.conflicts
],
status_code=status.HTTP_400_BAD_REQUEST
)
except ValidationError as e:
return error_response(
code=ErrorCodes.VALIDATION_ERROR,

View File

@@ -41,6 +41,7 @@ python-dateutil==2.9.0
pytz==2024.1
validators==0.22.0
PyYAML==6.0.1
ruamel.yaml>=0.18.0 # 保留注释的 YAML 解析
colorlog==6.8.2 # 彩色日志输出
python-json-logger==2.0.7 # JSON 结构化日志
Jinja2>=3.1.6 # 命令模板引擎

View File

@@ -180,6 +180,28 @@ def get_db_config() -> dict:
}
def generate_raw_response_headers(headers_dict: dict) -> str:
"""
将响应头字典转换为原始 HTTP 响应头字符串格式
Args:
headers_dict: 响应头字典
Returns:
原始 HTTP 响应头字符串,格式如:
HTTP/1.1 200 OK
Server: nginx
Content-Type: text/html
...
"""
lines = ['HTTP/1.1 200 OK']
for key, value in headers_dict.items():
# 将下划线转换为连字符,并首字母大写
header_name = key.replace('_', '-').title()
lines.append(f'{header_name}: {value}')
return '\r\n'.join(lines)
DB_CONFIG = get_db_config()
@@ -548,6 +570,10 @@ class TestDataGenerator:
'Authentication failed for protected resources.',
]
# 获取引擎名称映射
cur.execute("SELECT id, name FROM scan_engine WHERE id = ANY(%s)", (engine_ids,))
engine_name_map = {row[0]: row[1] for row in cur.fetchall()}
ids = []
# 随机选择目标数量 - 增加到 80-120 个
num_targets = min(random.randint(80, 120), len(target_ids))
@@ -558,7 +584,10 @@ class TestDataGenerator:
num_scans = random.randint(3, 15)
for _ in range(num_scans):
status = random.choices(statuses, weights=status_weights)[0]
engine_id = random.choice(engine_ids)
# 随机选择 1-3 个引擎
num_engines = random.randint(1, min(3, len(engine_ids)))
selected_engine_ids = random.sample(engine_ids, num_engines)
selected_engine_names = [engine_name_map.get(eid, f'Engine-{eid}') for eid in selected_engine_ids]
worker_id = random.choice(worker_ids) if worker_ids else None
progress = random.randint(10, 95) if status == 'running' else (100 if status == 'completed' else random.randint(0, 50))
@@ -581,20 +610,20 @@ class TestDataGenerator:
cur.execute("""
INSERT INTO scan (
target_id, engine_id, status, worker_id, progress, current_stage,
target_id, engine_ids, engine_names, merged_configuration, status, worker_id, progress, current_stage,
results_dir, error_message, container_ids, stage_progress,
cached_subdomains_count, cached_websites_count, cached_endpoints_count,
cached_ips_count, cached_directories_count, cached_vulns_total,
cached_vulns_critical, cached_vulns_high, cached_vulns_medium, cached_vulns_low,
created_at, stopped_at, deleted_at
) VALUES (
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
NOW() - INTERVAL '%s days', %s, NULL
)
RETURNING id
""", (
target_id, engine_id, status, worker_id, progress, stage,
target_id, selected_engine_ids, json.dumps(selected_engine_names), '', status, worker_id, progress, stage,
f'/app/results/scan_{target_id}_{random.randint(1000, 9999)}', error_msg, '{}', '{}',
subdomains, websites, endpoints, ips, directories, vulns_total,
vulns_critical, vulns_high, vulns_medium, vulns_low,
@@ -651,6 +680,10 @@ class TestDataGenerator:
num_schedules = random.randint(40, 50)
selected = random.sample(schedule_templates, min(num_schedules, len(schedule_templates)))
# 获取引擎名称映射
cur.execute("SELECT id, name FROM scan_engine WHERE id = ANY(%s)", (engine_ids,))
engine_name_map = {row[0]: row[1] for row in cur.fetchall()}
count = 0
for name_base, cron_template in selected:
name = f'{name_base}-{suffix}-{count:02d}'
@@ -662,7 +695,11 @@ class TestDataGenerator:
)
enabled = random.random() > 0.3 # 70% 启用
engine_id = random.choice(engine_ids)
# 随机选择 1-3 个引擎
num_engines = random.randint(1, min(3, len(engine_ids)))
selected_engine_ids = random.sample(engine_ids, num_engines)
selected_engine_names = [engine_name_map.get(eid, f'Engine-{eid}') for eid in selected_engine_ids]
# 随机决定关联组织还是目标
if org_ids and target_ids:
if random.random() > 0.5:
@@ -686,12 +723,12 @@ class TestDataGenerator:
cur.execute("""
INSERT INTO scheduled_scan (
name, engine_id, organization_id, target_id, cron_expression, is_enabled,
name, engine_ids, engine_names, merged_configuration, organization_id, target_id, cron_expression, is_enabled,
run_count, last_run_time, next_run_time, created_at, updated_at
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, NOW() - INTERVAL '%s days', NOW())
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW() - INTERVAL '%s days', NOW())
ON CONFLICT DO NOTHING
""", (
name, engine_id, org_id, target_id, cron, enabled,
name, selected_engine_ids, json.dumps(selected_engine_names), '', org_id, target_id, cron, enabled,
run_count if has_run else 0,
datetime.now() - timedelta(days=random.randint(0, 14), hours=random.randint(0, 23)) if has_run else None,
datetime.now() + timedelta(hours=random.randint(1, 336)) # 最多 2 周后
@@ -812,7 +849,7 @@ class TestDataGenerator:
]
# 真实的 body preview 内容
body_previews = [
response_bodies = [
'<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Login - Enterprise Portal</title><link rel="stylesheet" href="/assets/css/main.css"></head><body><div id="app"></div><script src="/assets/js/bundle.js"></script></body></html>',
'<!DOCTYPE html><html><head><title>Dashboard</title><meta name="description" content="Enterprise management dashboard for monitoring and analytics"><link rel="icon" href="/favicon.ico"></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>',
'{"status":"ok","version":"2.4.1","environment":"production","timestamp":"2024-12-22T10:30:00Z","services":{"database":"healthy","cache":"healthy","queue":"healthy"},"uptime":864000}',
@@ -843,14 +880,27 @@ class TestDataGenerator:
# 生成固定 245 长度的 URL
url = generate_fixed_length_url(target_name, length=245, path_hint=f'website/{i:04d}')
# 生成模拟的响应头数据
response_headers = {
'server': random.choice(['nginx', 'Apache', 'cloudflare', 'Microsoft-IIS/10.0']),
'content_type': 'text/html; charset=utf-8',
'x_powered_by': random.choice(['PHP/8.2', 'ASP.NET', 'Express', None]),
'x_frame_options': random.choice(['DENY', 'SAMEORIGIN', None]),
'strict_transport_security': 'max-age=31536000; includeSubDomains' if random.choice([True, False]) else None,
'set_cookie': f'session={random.randint(100000, 999999)}; HttpOnly; Secure' if random.choice([True, False]) else None,
}
# 移除 None 值
response_headers = {k: v for k, v in response_headers.items() if v is not None}
batch_data.append((
url, target_id, target_name, random.choice(titles),
random.choice(webservers), random.choice(tech_stacks),
random.choice([200, 301, 302, 403, 404]),
random.randint(1000, 500000), 'text/html; charset=utf-8',
f'https://{target_name}/login' if random.choice([True, False]) else '',
random.choice(body_previews),
random.choice([True, False, None])
random.choice(response_bodies),
random.choice([True, False, None]),
generate_raw_response_headers(response_headers)
))
# 批量插入
@@ -859,12 +909,12 @@ class TestDataGenerator:
execute_values(cur, """
INSERT INTO website (
url, target_id, host, title, webserver, tech, status_code,
content_length, content_type, location, body_preview, vhost,
created_at
content_length, content_type, location, response_body, vhost,
response_headers, created_at
) VALUES %s
ON CONFLICT DO NOTHING
RETURNING id
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
ids = [row[0] for row in cur.fetchall()]
print(f" ✓ 创建了 {len(batch_data)} 个网站\n")
@@ -965,7 +1015,7 @@ class TestDataGenerator:
]
# 真实的 API 响应 body preview
body_previews = [
response_bodies = [
'{"status":"success","data":{"user_id":12345,"username":"john_doe","email":"john@example.com","role":"user","created_at":"2024-01-15T10:30:00Z","last_login":"2024-12-22T08:45:00Z"}}',
'{"success":true,"message":"Authentication successful","token":"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c","expires_in":3600}',
'{"error":"Unauthorized","code":"AUTH_FAILED","message":"Invalid credentials provided. Please check your username and password.","timestamp":"2024-12-22T15:30:45.123Z","request_id":"req_abc123xyz"}',
@@ -1017,14 +1067,27 @@ class TestDataGenerator:
# 生成 10-20 个 tags (gf_patterns)
tags = random.choice(gf_patterns)
# 生成模拟的响应头数据
response_headers = {
'server': random.choice(['nginx', 'gunicorn', 'uvicorn', 'Apache']),
'content_type': 'application/json',
'x_request_id': f'req_{random.randint(100000, 999999)}',
'x_ratelimit_limit': str(random.choice([100, 1000, 5000])),
'x_ratelimit_remaining': str(random.randint(0, 1000)),
'cache_control': random.choice(['no-cache', 'max-age=3600', 'private', None]),
}
# 移除 None 值
response_headers = {k: v for k, v in response_headers.items() if v is not None}
batch_data.append((
url, target_id, target_name, title,
random.choice(['nginx/1.24.0', 'gunicorn/21.2.0']),
random.choice([200, 201, 301, 400, 401, 403, 404, 500]),
random.randint(100, 50000), 'application/json',
tech_list,
'', random.choice(body_previews),
random.choice([True, False, None]), tags
'', random.choice(response_bodies),
random.choice([True, False, None]), tags,
generate_raw_response_headers(response_headers)
))
count += 1
@@ -1033,11 +1096,11 @@ class TestDataGenerator:
execute_values(cur, """
INSERT INTO endpoint (
url, target_id, host, title, webserver, status_code, content_length,
content_type, tech, location, body_preview, vhost, matched_gf_patterns,
created_at
content_type, tech, location, response_body, vhost, matched_gf_patterns,
response_headers, created_at
) VALUES %s
ON CONFLICT DO NOTHING
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
print(f" ✓ 创建了 {count} 个端点\n")
@@ -1401,13 +1464,23 @@ class TestDataGenerator:
# 生成固定 245 长度的 URL
url = generate_fixed_length_url(target_name, length=245, path_hint=f'website-snap/{i:04d}')
# 生成模拟的响应头数据
response_headers = {
'server': random.choice(['nginx', 'Apache', 'cloudflare']),
'content_type': 'text/html; charset=utf-8',
'x_frame_options': random.choice(['DENY', 'SAMEORIGIN', None]),
}
# 移除 None 值
response_headers = {k: v for k, v in response_headers.items() if v is not None}
batch_data.append((
scan_id, url, target_name, random.choice(titles),
random.choice(webservers), random.choice(tech_stacks),
random.choice([200, 301, 403]),
random.randint(1000, 50000), 'text/html; charset=utf-8',
'', # location 字段
'<!DOCTYPE html><html><head><title>Test</title></head><body>Content</body></html>'
'<!DOCTYPE html><html><head><title>Test</title></head><body>Content</body></html>',
generate_raw_response_headers(response_headers)
))
count += 1
@@ -1416,10 +1489,11 @@ class TestDataGenerator:
execute_values(cur, """
INSERT INTO website_snapshot (
scan_id, url, host, title, web_server, tech, status,
content_length, content_type, location, body_preview, created_at
content_length, content_type, location, response_body,
response_headers, created_at
) VALUES %s
ON CONFLICT DO NOTHING
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
print(f" ✓ 创建了 {count} 个网站快照\n")
@@ -1498,6 +1572,13 @@ class TestDataGenerator:
num_tags = random.randint(10, 20)
tags = random.sample(all_tags, min(num_tags, len(all_tags)))
# 生成模拟的响应头数据
response_headers = {
'server': 'nginx/1.24.0',
'content_type': 'application/json',
'x_request_id': f'req_{random.randint(100000, 999999)}',
}
batch_data.append((
scan_id, url, target_name, title,
random.choice([200, 201, 401, 403, 404]),
@@ -1506,7 +1587,8 @@ class TestDataGenerator:
'nginx/1.24.0',
'application/json', tech_list,
'{"status":"ok","data":{}}',
tags
tags,
generate_raw_response_headers(response_headers)
))
count += 1
@@ -1515,11 +1597,11 @@ class TestDataGenerator:
execute_values(cur, """
INSERT INTO endpoint_snapshot (
scan_id, url, host, title, status_code, content_length,
location, webserver, content_type, tech, body_preview,
matched_gf_patterns, created_at
location, webserver, content_type, tech, response_body,
matched_gf_patterns, response_headers, created_at
) VALUES %s
ON CONFLICT DO NOTHING
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
print(f" ✓ 创建了 {count} 个端点快照\n")
@@ -2543,9 +2625,10 @@ class MillionDataGenerator:
if len(batch_data) >= batch_size:
execute_values(cur, """
INSERT INTO website (url, target_id, host, title, webserver, tech,
status_code, content_length, content_type, location, body_preview, created_at)
status_code, content_length, content_type, location, response_body,
vhost, response_headers, created_at)
VALUES %s ON CONFLICT DO NOTHING
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NULL, '', NOW())")
self.conn.commit()
batch_data = []
print(f"{count:,} / {target_count:,}")
@@ -2555,9 +2638,10 @@ class MillionDataGenerator:
if batch_data:
execute_values(cur, """
INSERT INTO website (url, target_id, host, title, webserver, tech,
status_code, content_length, content_type, location, body_preview, created_at)
status_code, content_length, content_type, location, response_body,
vhost, response_headers, created_at)
VALUES %s ON CONFLICT DO NOTHING
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NULL, '', NOW())")
self.conn.commit()
print(f" ✓ 创建了 {count:,} 个网站\n")
@@ -2631,10 +2715,10 @@ class MillionDataGenerator:
if len(batch_data) >= batch_size:
execute_values(cur, """
INSERT INTO endpoint (url, target_id, host, title, webserver, status_code,
content_length, content_type, tech, location, body_preview, vhost,
matched_gf_patterns, created_at)
content_length, content_type, tech, location, response_body, vhost,
matched_gf_patterns, response_headers, created_at)
VALUES %s ON CONFLICT DO NOTHING
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, '', NOW())")
self.conn.commit()
batch_data = []
print(f"{count:,} / {target_count:,}")
@@ -2644,10 +2728,10 @@ class MillionDataGenerator:
if batch_data:
execute_values(cur, """
INSERT INTO endpoint (url, target_id, host, title, webserver, status_code,
content_length, content_type, tech, location, body_preview, vhost,
matched_gf_patterns, created_at)
content_length, content_type, tech, location, response_body, vhost,
matched_gf_patterns, response_headers, created_at)
VALUES %s ON CONFLICT DO NOTHING
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())")
""", batch_data, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, '', NOW())")
self.conn.commit()
print(f" ✓ 创建了 {count:,} 个端点\n")

View File

@@ -64,7 +64,7 @@ services:
restart: always
environment:
- SERVER_URL=http://server:8888
- WORKER_NAME=本地节点
- WORKER_NAME=Local-Worker
- IS_LOCAL=true
- IMAGE_TAG=${IMAGE_TAG:-dev}
- WORKER_API_KEY=${WORKER_API_KEY}

View File

@@ -68,7 +68,7 @@ services:
restart: always
environment:
- SERVER_URL=http://server:8888
- WORKER_NAME=本地节点
- WORKER_NAME=Local-Worker
- IS_LOCAL=true
- IMAGE_TAG=${IMAGE_TAG}
- WORKER_API_KEY=${WORKER_API_KEY}

View File

@@ -9,3 +9,12 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "postgres" <<-EOSQL
GRANT ALL PRIVILEGES ON DATABASE xingrin TO "$POSTGRES_USER";
GRANT ALL PRIVILEGES ON DATABASE xingrin_dev TO "$POSTGRES_USER";
EOSQL
# 启用 pg_trgm 扩展(用于文本模糊搜索索引)
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "xingrin" <<-EOSQL
CREATE EXTENSION IF NOT EXISTS pg_trgm;
EOSQL
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "xingrin_dev" <<-EOSQL
CREATE EXTENSION IF NOT EXISTS pg_trgm;
EOSQL

View File

@@ -64,6 +64,16 @@ export default function ScanHistoryLayout({
<div className="flex items-center justify-between px-4 lg:px-6">
<Tabs value={getActiveTab()} className="w-full">
<TabsList>
<TabsTrigger value="websites" asChild>
<Link href={tabPaths.websites} className="flex items-center gap-0.5">
Websites
{counts.websites > 0 && (
<Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
{counts.websites}
</Badge>
)}
</Link>
</TabsTrigger>
<TabsTrigger value="subdomain" asChild>
<Link href={tabPaths.subdomain} className="flex items-center gap-0.5">
Subdomains
@@ -74,12 +84,12 @@ export default function ScanHistoryLayout({
)}
</Link>
</TabsTrigger>
<TabsTrigger value="websites" asChild>
<Link href={tabPaths.websites} className="flex items-center gap-0.5">
Websites
{counts.websites > 0 && (
<TabsTrigger value="ip-addresses" asChild>
<Link href={tabPaths["ip-addresses"]} className="flex items-center gap-0.5">
IP Addresses
{counts["ip-addresses"] > 0 && (
<Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
{counts.websites}
{counts["ip-addresses"]}
</Badge>
)}
</Link>
@@ -104,16 +114,6 @@ export default function ScanHistoryLayout({
)}
</Link>
</TabsTrigger>
<TabsTrigger value="ip-addresses" asChild>
<Link href={tabPaths["ip-addresses"]} className="flex items-center gap-0.5">
IP Addresses
{counts["ip-addresses"] > 0 && (
<Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
{counts["ip-addresses"]}
</Badge>
)}
</Link>
</TabsTrigger>
<TabsTrigger value="vulnerabilities" asChild>
<Link href={tabPaths.vulnerabilities} className="flex items-center gap-0.5">
Vulnerabilities

View File

@@ -8,7 +8,7 @@ export default function ScanHistoryDetailPage() {
const router = useRouter()
useEffect(() => {
router.replace(`/scan/history/${id}/subdomain/`)
router.replace(`/scan/history/${id}/websites/`)
}, [id, router])
return null

View File

@@ -5,15 +5,15 @@ import { useEffect } from "react"
/**
* Target detail page (compatible with old routes)
* Automatically redirects to subdomain page
* Automatically redirects to websites page
*/
export default function TargetDetailsPage() {
const { id } = useParams<{ id: string }>()
const router = useRouter()
useEffect(() => {
// Redirect to subdomain page
router.replace(`/target/${id}/subdomain/`)
// Redirect to websites page
router.replace(`/target/${id}/websites/`)
}, [id, router])
return null

View File

@@ -138,6 +138,16 @@ export default function TargetLayout({
<div className="flex items-center justify-between px-4 lg:px-6">
<Tabs value={getActiveTab()} className="w-full">
<TabsList>
<TabsTrigger value="websites" asChild>
<Link href={tabPaths.websites} className="flex items-center gap-0.5">
Websites
{counts.websites > 0 && (
<Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
{counts.websites}
</Badge>
)}
</Link>
</TabsTrigger>
<TabsTrigger value="subdomain" asChild>
<Link href={tabPaths.subdomain} className="flex items-center gap-0.5">
Subdomains
@@ -148,12 +158,12 @@ export default function TargetLayout({
)}
</Link>
</TabsTrigger>
<TabsTrigger value="websites" asChild>
<Link href={tabPaths.websites} className="flex items-center gap-0.5">
Websites
{counts.websites > 0 && (
<TabsTrigger value="ip-addresses" asChild>
<Link href={tabPaths["ip-addresses"]} className="flex items-center gap-0.5">
IP Addresses
{counts["ip-addresses"] > 0 && (
<Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
{counts.websites}
{counts["ip-addresses"]}
</Badge>
)}
</Link>
@@ -178,16 +188,6 @@ export default function TargetLayout({
)}
</Link>
</TabsTrigger>
<TabsTrigger value="ip-addresses" asChild>
<Link href={tabPaths["ip-addresses"]} className="flex items-center gap-0.5">
IP Addresses
{counts["ip-addresses"] > 0 && (
<Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
{counts["ip-addresses"]}
</Badge>
)}
</Link>
</TabsTrigger>
<TabsTrigger value="vulnerabilities" asChild>
<Link href={tabPaths.vulnerabilities} className="flex items-center gap-0.5">
Vulnerabilities

View File

@@ -5,15 +5,15 @@ import { useEffect } from "react"
/**
* Target detail default page
* Automatically redirects to subdomain page
* Automatically redirects to websites page
*/
export default function TargetDetailPage() {
const { id } = useParams<{ id: string }>()
const router = useRouter()
useEffect(() => {
// Redirect to subdomain page
router.replace(`/target/${id}/subdomain/`)
// Redirect to websites page
router.replace(`/target/${id}/websites/`)
}, [id, router])
return null

View File

@@ -15,6 +15,7 @@ import {
IconServer, // Server icon
IconTerminal2, // Terminal icon
IconBug, // Vulnerability icon
IconMessageReport, // Feedback icon
} from "@tabler/icons-react"
// Import internationalization hook
import { useTranslations } from 'next-intl'
@@ -132,6 +133,11 @@ export function AppSidebar({ ...props }: React.ComponentProps<typeof Sidebar>) {
// Secondary navigation menu items
const navSecondary = [
{
title: t('feedback'),
url: "https://github.com/yyhuni/xingrin/issues",
icon: IconMessageReport,
},
{
title: t('help'),
url: "https://github.com/yyhuni/xingrin",

View File

@@ -209,6 +209,7 @@ export function DashboardDataTable() {
target: t('columns.scanHistory.target'),
summary: t('columns.scanHistory.summary'),
engineName: t('columns.scanHistory.engineName'),
workerName: t('columns.scanHistory.workerName'),
createdAt: t('columns.common.createdAt'),
status: t('columns.common.status'),
progress: t('columns.scanHistory.progress'),

View File

@@ -28,6 +28,7 @@ export function DashboardScanHistory() {
target: tColumns("scanHistory.target"),
summary: tColumns("scanHistory.summary"),
engineName: tColumns("scanHistory.engineName"),
workerName: tColumns("scanHistory.workerName"),
createdAt: tColumns("common.createdAt"),
status: tColumns("common.status"),
progress: tColumns("scanHistory.progress"),

View File

@@ -12,6 +12,7 @@ import { ExpandableCell, ExpandableTagList } from "@/components/ui/data-table/ex
export interface EndpointTranslations {
columns: {
url: string
host: string
title: string
status: string
contentLength: string
@@ -19,9 +20,10 @@ export interface EndpointTranslations {
webServer: string
contentType: string
technologies: string
bodyPreview: string
responseBody: string
vhost: string
gfPatterns: string
responseHeaders: string
responseTime: string
createdAt: string
}
@@ -112,6 +114,19 @@ export function createEndpointColumns({
<ExpandableCell value={row.getValue("url")} />
),
},
{
accessorKey: "host",
meta: { title: t.columns.host },
header: ({ column }) => (
<DataTableColumnHeader column={column} title={t.columns.host} />
),
size: 200,
minSize: 100,
maxSize: 300,
cell: ({ row }) => (
<ExpandableCell value={row.getValue("host")} />
),
},
{
accessorKey: "title",
meta: { title: t.columns.title },
@@ -215,17 +230,32 @@ export function createEndpointColumns({
},
},
{
accessorKey: "bodyPreview",
meta: { title: t.columns.bodyPreview },
accessorKey: "responseBody",
meta: { title: t.columns.responseBody },
header: ({ column }) => (
<DataTableColumnHeader column={column} title={t.columns.bodyPreview} />
<DataTableColumnHeader column={column} title={t.columns.responseBody} />
),
size: 350,
minSize: 250,
cell: ({ row }) => (
<ExpandableCell value={row.getValue("bodyPreview")} />
<ExpandableCell value={row.getValue("responseBody")} />
),
},
{
accessorKey: "responseHeaders",
meta: { title: t.columns.responseHeaders },
header: ({ column }) => (
<DataTableColumnHeader column={column} title={t.columns.responseHeaders} />
),
size: 250,
minSize: 150,
maxSize: 400,
cell: ({ row }) => {
const headers = row.getValue("responseHeaders") as string | null | undefined
if (!headers) return <span className="text-muted-foreground text-sm">-</span>
return <ExpandableCell value={headers} maxLines={3} />
},
},
{
accessorKey: "vhost",
meta: { title: t.columns.vhost },

View File

@@ -15,6 +15,7 @@ const ENDPOINT_FILTER_FIELDS: FilterField[] = [
{ key: "title", label: "Title", description: "Page title" },
{ key: "status", label: "Status", description: "HTTP status code" },
{ key: "tech", label: "Tech", description: "Technologies" },
{ key: "responseHeaders", label: "Headers", description: "Response headers" },
]
// Endpoint page filter examples

View File

@@ -62,6 +62,7 @@ export function EndpointsDetailView({
const translations = useMemo(() => ({
columns: {
url: tColumns("common.url"),
host: tColumns("endpoint.host"),
title: tColumns("endpoint.title"),
status: tColumns("common.status"),
contentLength: tColumns("endpoint.contentLength"),
@@ -69,9 +70,10 @@ export function EndpointsDetailView({
webServer: tColumns("endpoint.webServer"),
contentType: tColumns("endpoint.contentType"),
technologies: tColumns("endpoint.technologies"),
bodyPreview: tColumns("endpoint.bodyPreview"),
responseBody: tColumns("endpoint.responseBody"),
vhost: tColumns("endpoint.vhost"),
gfPatterns: tColumns("endpoint.gfPatterns"),
responseHeaders: tColumns("endpoint.responseHeaders"),
responseTime: tColumns("endpoint.responseTime"),
createdAt: tColumns("common.createdAt"),
},
@@ -202,7 +204,7 @@ export function EndpointsDetailView({
const headers = [
'url', 'host', 'location', 'title', 'status_code',
'content_length', 'content_type', 'webserver', 'tech',
'body_preview', 'vhost', 'matched_gf_patterns', 'created_at'
'response_body', 'vhost', 'matched_gf_patterns', 'created_at'
]
const rows = items.map(item => [
@@ -215,7 +217,7 @@ export function EndpointsDetailView({
escapeCSV(item.contentType),
escapeCSV(item.webserver),
escapeCSV(formatArrayForCSV(item.tech)),
escapeCSV(item.bodyPreview),
escapeCSV(item.responseBody),
escapeCSV(item.vhost),
escapeCSV(formatDateForCSV(item.createdAt ?? ''))
].join(','))

View File

@@ -132,7 +132,7 @@ function TargetNameCell({
return (
<div className="group flex items-start gap-1 flex-1 min-w-0">
<button
onClick={() => navigate(`/target/${targetId}/subdomain/`)}
onClick={() => navigate(`/target/${targetId}/website/`)}
className="text-sm font-medium hover:text-primary hover:underline underline-offset-2 transition-colors cursor-pointer text-left break-all leading-relaxed whitespace-normal"
>
{name}
@@ -251,7 +251,7 @@ export const createTargetColumns = ({
cell: ({ row }) => (
<TargetRowActions
target={row.original}
onView={() => navigate(`/target/${row.original.id}/subdomain/`)}
onView={() => navigate(`/target/${row.original.id}/website/`)}
onDelete={() => handleDelete(row.original)}
t={t}
/>

View File

@@ -44,6 +44,7 @@ export interface ScanHistoryTranslations {
target: string
summary: string
engineName: string
workerName: string
createdAt: string
status: string
progress: string
@@ -240,7 +241,7 @@ export const createScanHistoryColumns = ({
header: ({ column }) => (
<DataTableColumnHeader column={column} title={t.columns.summary} />
),
size: 420,
size: 290,
minSize: 150,
cell: ({ row }) => {
const summary = (row.getValue("summary") as {
@@ -391,20 +392,46 @@ export const createScanHistoryColumns = ({
enableSorting: false,
},
{
accessorKey: "engineName",
size: 120,
minSize: 80,
maxSize: 180,
accessorKey: "engineNames",
size: 150,
minSize: 100,
maxSize: 200,
enableResizing: false,
meta: { title: t.columns.engineName },
header: ({ column }) => (
<DataTableColumnHeader column={column} title={t.columns.engineName} />
),
cell: ({ row }) => {
const engineName = row.getValue("engineName") as string
const engineNames = row.getValue("engineNames") as string[] | undefined
if (!engineNames || engineNames.length === 0) {
return <span className="text-muted-foreground text-sm">-</span>
}
return (
<Badge variant="secondary">
{engineName}
<div className="flex flex-wrap gap-1">
{engineNames.map((name, index) => (
<Badge key={index} variant="secondary">
{name}
</Badge>
))}
</div>
)
},
},
{
accessorKey: "workerName",
size: 120,
minSize: 80,
maxSize: 180,
enableResizing: false,
meta: { title: t.columns.workerName },
header: ({ column }) => (
<DataTableColumnHeader column={column} title={t.columns.workerName} />
),
cell: ({ row }) => {
const workerName = row.getValue("workerName") as string | null | undefined
return (
<Badge variant="outline">
{workerName || "-"}
</Badge>
)
},

View File

@@ -57,6 +57,7 @@ export function ScanHistoryList({ hideToolbar = false }: ScanHistoryListProps) {
target: tColumns("scanHistory.target"),
summary: tColumns("scanHistory.summary"),
engineName: tColumns("scanHistory.engineName"),
workerName: tColumns("scanHistory.workerName"),
createdAt: tColumns("common.createdAt"),
status: tColumns("common.status"),
progress: tColumns("scanHistory.progress"),
@@ -367,7 +368,7 @@ export function ScanHistoryList({ hideToolbar = false }: ScanHistoryListProps) {
{selectedScans.map((scan) => (
<li key={scan.id} className="flex items-center justify-between">
<span className="font-medium">{scan.targetName}</span>
<span className="text-muted-foreground text-xs">{scan.engineName}</span>
<span className="text-muted-foreground text-xs">{scan.engineNames?.join(", ") || "-"}</span>
</li>
))}
</ul>

View File

@@ -14,7 +14,7 @@ import {
DialogTitle,
} from "@/components/ui/dialog"
import { Badge } from "@/components/ui/badge"
import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"
import { Checkbox } from "@/components/ui/checkbox"
import { LoadingSpinner } from "@/components/loading-spinner"
import { cn } from "@/lib/utils"
import { CAPABILITY_CONFIG, getEngineIcon, parseEngineCapabilities } from "@/lib/engine-config"
@@ -47,23 +47,35 @@ export function InitiateScanDialog({
const t = useTranslations("scan.initiate")
const tToast = useTranslations("toast")
const tCommon = useTranslations("common.actions")
const [selectedEngineId, setSelectedEngineId] = useState<string>("")
const [selectedEngineIds, setSelectedEngineIds] = useState<number[]>([])
const [isSubmitting, setIsSubmitting] = useState(false)
const { data: engines, isLoading, error } = useEngines()
const selectedEngine = useMemo(() => {
if (!selectedEngineId || !engines) return null
return engines.find((e) => e.id.toString() === selectedEngineId) || null
}, [selectedEngineId, engines])
const selectedEngines = useMemo(() => {
if (!selectedEngineIds.length || !engines) return []
return engines.filter((e) => selectedEngineIds.includes(e.id))
}, [selectedEngineIds, engines])
const selectedCapabilities = useMemo(() => {
if (!selectedEngine) return []
return parseEngineCapabilities(selectedEngine.configuration || "")
}, [selectedEngine])
if (!selectedEngines.length) return []
const allCaps = new Set<string>()
selectedEngines.forEach((engine) => {
parseEngineCapabilities(engine.configuration || "").forEach((cap) => allCaps.add(cap))
})
return Array.from(allCaps)
}, [selectedEngines])
const handleEngineToggle = (engineId: number, checked: boolean) => {
if (checked) {
setSelectedEngineIds((prev) => [...prev, engineId])
} else {
setSelectedEngineIds((prev) => prev.filter((id) => id !== engineId))
}
}
const handleInitiate = async () => {
if (!selectedEngineId) return
if (!selectedEngineIds.length) return
if (!organizationId && !targetId) {
toast.error(tToast("paramError"), { description: tToast("paramErrorDesc") })
return
@@ -73,7 +85,7 @@ export function InitiateScanDialog({
const response = await initiateScan({
organizationId,
targetId,
engineId: Number(selectedEngineId),
engineIds: selectedEngineIds,
})
// 后端返回 201 说明成功创建扫描任务
@@ -83,12 +95,20 @@ export function InitiateScanDialog({
})
onSuccess?.()
onOpenChange(false)
setSelectedEngineId("")
} catch (err) {
setSelectedEngineIds([])
} catch (err: unknown) {
console.error("Failed to initiate scan:", err)
toast.error(tToast("initiateScanFailed"), {
description: err instanceof Error ? err.message : tToast("unknownError"),
})
// 处理配置冲突错误
const error = err as { response?: { data?: { error?: { code?: string; message?: string } } } }
if (error?.response?.data?.error?.code === 'CONFIG_CONFLICT') {
toast.error(tToast("configConflict"), {
description: error.response.data.error.message,
})
} else {
toast.error(tToast("initiateScanFailed"), {
description: err instanceof Error ? err.message : tToast("unknownError"),
})
}
} finally {
setIsSubmitting(false)
}
@@ -97,7 +117,7 @@ export function InitiateScanDialog({
const handleOpenChange = (newOpen: boolean) => {
if (!isSubmitting) {
onOpenChange(newOpen)
if (!newOpen) setSelectedEngineId("")
if (!newOpen) setSelectedEngineIds([])
}
}
@@ -108,25 +128,32 @@ export function InitiateScanDialog({
<DialogTitle className="flex items-center gap-2">
<Play className="h-5 w-5" />
{t("title")}
<span className="text-sm font-normal text-muted-foreground">
{targetName ? (
<>
{t("targetDesc")} <span className="font-medium text-foreground">{targetName}</span> {t("selectEngine")}
</>
) : (
<>
{t("orgDesc")} <span className="font-medium text-foreground">{organization?.name}</span> {t("selectEngine")}
</>
)}
</span>
</DialogTitle>
<DialogDescription>
{targetName ? (
<>
{t("targetDesc")} <span className="font-semibold text-foreground">{targetName}</span> {t("selectEngine")}
</>
) : (
<>
{t("orgDesc")} <span className="font-semibold text-foreground">{organization?.name}</span> {t("selectEngine")}
</>
)}
</DialogDescription>
</DialogHeader>
<div className="flex border-t h-[480px]">
{/* Left side engine list */}
<div className="w-[260px] border-r flex flex-col shrink-0">
<div className="px-4 py-3 border-b bg-muted/30 shrink-0">
<h3 className="text-sm font-medium">{t("selectEngineTitle")}</h3>
<h3 className="text-sm font-medium">
{t("selectEngineTitle")}
{selectedEngineIds.length > 0 && (
<span className="text-xs text-muted-foreground font-normal ml-2">
{t("selectedCount", { count: selectedEngineIds.length })}
</span>
)}
</h3>
</div>
<div className="flex-1 overflow-y-auto">
<div className="p-2">
@@ -140,18 +167,13 @@ export function InitiateScanDialog({
) : !engines?.length ? (
<div className="py-8 text-center text-sm text-muted-foreground">{t("noEngines")}</div>
) : (
<RadioGroup
value={selectedEngineId}
onValueChange={setSelectedEngineId}
disabled={isSubmitting}
className="space-y-1"
>
<div className="space-y-1">
{engines.map((engine) => {
const capabilities = parseEngineCapabilities(engine.configuration || "")
const EngineIcon = getEngineIcon(capabilities)
const primaryCap = capabilities[0]
const iconConfig = primaryCap ? CAPABILITY_CONFIG[primaryCap] : null
const isSelected = selectedEngineId === engine.id.toString()
const isSelected = selectedEngineIds.includes(engine.id)
return (
<label
@@ -164,10 +186,11 @@ export function InitiateScanDialog({
: "hover:bg-muted/50 border border-transparent"
)}
>
<RadioGroupItem
value={engine.id.toString()}
<Checkbox
id={`engine-${engine.id}`}
className="sr-only"
checked={isSelected}
onCheckedChange={(checked) => handleEngineToggle(engine.id, checked as boolean)}
disabled={isSubmitting}
/>
<div
className={cn(
@@ -183,40 +206,35 @@ export function InitiateScanDialog({
{capabilities.length > 0 ? t("capabilities", { count: capabilities.length }) : t("noConfig")}
</div>
</div>
{isSelected && <div className="w-2 h-2 rounded-full bg-primary shrink-0" />}
</label>
)
})}
</RadioGroup>
</div>
)}
</div>
</div>
</div>
{/* Right side engine details */}
<div className="flex-1 flex flex-col min-w-0 overflow-hidden">
{selectedEngine ? (
<div className="flex-1 flex flex-col min-w-0 overflow-hidden w-0">
{selectedEngines.length > 0 ? (
<>
<div className="px-4 py-3 border-b bg-muted/30 flex items-center gap-2 shrink-0">
<Settings2 className="h-4 w-4 text-muted-foreground" />
<h3 className="text-sm font-medium truncate">{selectedEngine.name}</h3>
<div className="px-4 py-3 border-b bg-muted/30 shrink-0 min-w-0">
<div className="flex flex-wrap gap-1.5">
{selectedCapabilities.map((capKey) => {
const config = CAPABILITY_CONFIG[capKey]
return (
<Badge key={capKey} variant="outline" className={cn("text-xs", config?.color)}>
{config?.label || capKey}
</Badge>
)
})}
</div>
</div>
<div className="flex-1 flex flex-col overflow-hidden p-4 gap-3">
{selectedCapabilities.length > 0 && (
<div className="flex flex-wrap gap-1.5 shrink-0">
{selectedCapabilities.map((capKey) => {
const config = CAPABILITY_CONFIG[capKey]
return (
<Badge key={capKey} variant="outline" className={cn("text-xs", config?.color)}>
{config?.label || capKey}
</Badge>
)
})}
</div>
)}
<div className="flex-1 bg-muted/50 rounded-lg border overflow-hidden min-h-0">
<div className="flex-1 flex flex-col overflow-hidden p-4 min-w-0">
<div className="flex-1 bg-muted/50 rounded-lg border overflow-hidden min-h-0 min-w-0">
<pre className="h-full p-3 text-xs font-mono overflow-auto whitespace-pre-wrap break-all">
{selectedEngine.configuration || `# ${t("noConfig")}`}
{selectedEngines.map((e) => e.configuration || `# ${t("noConfig")}`).join("\n\n")}
</pre>
</div>
</div>
@@ -236,7 +254,7 @@ export function InitiateScanDialog({
<Button variant="outline" onClick={() => handleOpenChange(false)} disabled={isSubmitting}>
{tCommon("cancel")}
</Button>
<Button onClick={handleInitiate} disabled={!selectedEngineId || isSubmitting}>
<Button onClick={handleInitiate} disabled={!selectedEngineIds.length || isSubmitting}>
{isSubmitting ? (
<>
<LoadingSpinner />

View File

@@ -5,6 +5,8 @@ import { useTranslations } from "next-intl"
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
DialogTrigger,
@@ -12,17 +14,15 @@ import {
import { Button } from "@/components/ui/button"
import { Textarea } from "@/components/ui/textarea"
import { Badge } from "@/components/ui/badge"
import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group"
import { Checkbox } from "@/components/ui/checkbox"
import { LoadingSpinner } from "@/components/loading-spinner"
import { cn } from "@/lib/utils"
import { toast } from "sonner"
import { Zap, Settings, ChevronRight, ChevronLeft, Loader2, AlertCircle } from "lucide-react"
import { getEngines } from "@/services/engine.service"
import { Zap, Settings2, AlertCircle, ChevronRight, ChevronLeft, Target, Server } from "lucide-react"
import { quickScan } from "@/services/scan.service"
import { CAPABILITY_CONFIG, getEngineIcon, parseEngineCapabilities } from "@/lib/engine-config"
import { TargetValidator } from "@/lib/target-validator"
import type { ScanEngine } from "@/types/engine.types"
const STEP_TITLES_KEYS = ["steps.enterTargets", "steps.selectEngine", "steps.confirmScan"]
import { useEngines } from "@/hooks/use-engines"
interface QuickScanDialogProps {
trigger?: React.ReactNode
@@ -31,13 +31,13 @@ interface QuickScanDialogProps {
export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
const t = useTranslations("quickScan")
const [open, setOpen] = React.useState(false)
const [step, setStep] = React.useState(1)
const [isLoading, setIsLoading] = React.useState(false)
const [isSubmitting, setIsSubmitting] = React.useState(false)
const [step, setStep] = React.useState(1)
const [targetInput, setTargetInput] = React.useState("")
const [selectedEngineId, setSelectedEngineId] = React.useState<string>("")
const [engines, setEngines] = React.useState<ScanEngine[]>([])
const [selectedEngineIds, setSelectedEngineIds] = React.useState<number[]>([])
const { data: engines, isLoading, error } = useEngines()
const lineNumbersRef = React.useRef<HTMLDivElement | null>(null)
@@ -56,20 +56,24 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
const invalidInputs = validationResults.filter(r => !r.isValid)
const hasErrors = invalidInputs.length > 0
React.useEffect(() => {
if (open && step === 2 && engines.length === 0) {
setIsLoading(true)
getEngines()
.then(setEngines)
.catch(() => toast.error(t("toast.getEnginesFailed")))
.finally(() => setIsLoading(false))
}
}, [open, step, engines.length, t])
const selectedEngines = React.useMemo(() => {
if (!selectedEngineIds.length || !engines) return []
return engines.filter(e => selectedEngineIds.includes(e.id))
}, [selectedEngineIds, engines])
const selectedCapabilities = React.useMemo(() => {
if (!selectedEngines.length) return []
const allCaps = new Set<string>()
selectedEngines.forEach((engine) => {
parseEngineCapabilities(engine.configuration || "").forEach((cap) => allCaps.add(cap))
})
return Array.from(allCaps)
}, [selectedEngines])
const resetForm = () => {
setStep(1)
setTargetInput("")
setSelectedEngineId("")
setSelectedEngineIds([])
setStep(1)
}
const handleClose = (isOpen: boolean) => {
@@ -77,38 +81,53 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
if (!isOpen) resetForm()
}
const handleNext = () => {
if (step === 1) {
if (validInputs.length === 0) {
toast.error(t("toast.noValidTarget"))
return
}
if (hasErrors) {
toast.error(t("toast.hasInvalidInputs", { count: invalidInputs.length }))
return
}
const handleEngineToggle = (engineId: number, checked: boolean) => {
if (checked) {
setSelectedEngineIds((prev) => [...prev, engineId])
} else {
setSelectedEngineIds((prev) => prev.filter((id) => id !== engineId))
}
if (step === 2 && !selectedEngineId) {
}
const canProceedToStep2 = validInputs.length > 0 && !hasErrors
const canSubmit = selectedEngineIds.length > 0
const handleNext = () => {
if (step === 1 && canProceedToStep2) setStep(2)
}
const handleBack = () => {
if (step > 1) setStep(step - 1)
}
const steps = [
{ id: 1, title: t("step1Title"), icon: Target },
{ id: 2, title: t("step2Title"), icon: Server },
]
const handleSubmit = async () => {
if (validInputs.length === 0) {
toast.error(t("toast.noValidTarget"))
return
}
if (hasErrors) {
toast.error(t("toast.hasInvalidInputs", { count: invalidInputs.length }))
return
}
if (selectedEngineIds.length === 0) {
toast.error(t("toast.selectEngine"))
return
}
setStep(step + 1)
}
const handlePrev = () => setStep(step - 1)
const handleSubmit = async () => {
const targets = validInputs.map(r => r.originalInput)
if (targets.length === 0) return
setIsSubmitting(true)
try {
const response = await quickScan({
targets: targets.map(name => ({ name })),
engineId: Number(selectedEngineId),
engineIds: selectedEngineIds,
})
// 后端返回 201 说明成功创建扫描任务
const { targetStats, scans, count } = response
const scanCount = scans?.length || count || 0
@@ -118,29 +137,29 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
: undefined
})
handleClose(false)
} catch (error: any) {
toast.error(error?.response?.data?.detail || error?.response?.data?.error || t("toast.createFailed"))
} catch (error: unknown) {
const err = error as { response?: { data?: { error?: { code?: string; message?: string }; detail?: string } } }
if (err?.response?.data?.error?.code === 'CONFIG_CONFLICT') {
toast.error(t("toast.configConflict"), {
description: err.response.data.error.message,
})
} else {
toast.error(err?.response?.data?.detail || err?.response?.data?.error?.message || t("toast.createFailed"))
}
} finally {
setIsSubmitting(false)
}
}
const selectedEngine = engines.find(e => String(e.id) === selectedEngineId)
return (
<Dialog open={open} onOpenChange={handleClose}>
<DialogTrigger asChild>
{trigger || (
<div className="relative group">
{/* Border glow effect */}
<div className="absolute -inset-[1px] rounded-md overflow-hidden">
<div className="absolute inset-0 bg-gradient-to-r from-transparent via-primary to-transparent animate-border-flow" />
</div>
<Button
variant="outline"
size="sm"
className="gap-1.5 relative bg-background border-primary/20"
>
<Button variant="outline" size="sm" className="gap-1.5 relative bg-background border-primary/20">
<Zap className="h-4 w-4 text-primary" />
{t("title")}
</Button>
@@ -148,140 +167,194 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
)}
</DialogTrigger>
<DialogContent className="max-w-[90vw] sm:max-w-[900px] p-0 gap-0">
<DialogHeader className="px-6 py-4 border-b">
<DialogTitle className="flex items-center gap-2">
<Zap className="h-5 w-5 text-primary" />
{t("title")}
<span className="text-muted-foreground font-normal text-sm ml-2">
{t("step", { current: step, total: 3, title: t(STEP_TITLES_KEYS[step - 1]) })}
</span>
</DialogTitle>
<DialogHeader className="px-6 pt-6 pb-4">
<div className="flex items-center justify-between">
<div>
<DialogTitle className="flex items-center gap-2">
<Zap className="h-5 w-5 text-primary" />
{t("title")}
</DialogTitle>
<DialogDescription className="mt-1">
{t("description")}
</DialogDescription>
</div>
{/* Step indicator */}
<div className="flex items-center gap-2 mr-8">
{steps.map((s, index) => (
<React.Fragment key={s.id}>
<button
type="button"
onClick={() => {
if (s.id < step) setStep(s.id)
else if (s.id === 2 && canProceedToStep2) setStep(2)
}}
className={cn(
"flex items-center gap-1.5 px-3 py-1.5 rounded-full text-sm transition-colors",
step === s.id
? "bg-primary text-primary-foreground"
: step > s.id
? "bg-primary/20 text-primary cursor-pointer hover:bg-primary/30"
: "bg-muted text-muted-foreground"
)}
disabled={s.id > step && !(s.id === 2 && canProceedToStep2)}
>
<s.icon className="h-4 w-4" />
<span className="hidden sm:inline">{s.title}</span>
</button>
{index < steps.length - 1 && (
<div className={cn(
"w-8 h-[2px]",
step > s.id ? "bg-primary/50" : "bg-muted"
)} />
)}
</React.Fragment>
))}
</div>
</div>
</DialogHeader>
<div className="h-[380px]">
{/* Step 1: Enter targets */}
<div className="border-t h-[480px] overflow-hidden">
{/* Step 1: Target input */}
{step === 1 && (
<div className="flex flex-col h-full">
<div className="flex-1 flex overflow-hidden border-t">
<div className="flex-shrink-0 w-12 border-r bg-muted/30">
<div
ref={lineNumbersRef}
className="py-3 px-2 text-right font-mono text-xs text-muted-foreground leading-[1.5] h-full overflow-y-auto scrollbar-hide"
>
{Array.from({ length: Math.max(targetInput.split('\n').length, 20) }, (_, i) => (
<div key={i + 1} className="h-[21px]">{i + 1}</div>
))}
<div className="px-4 py-3 border-b bg-muted/30 shrink-0">
<h3 className="text-sm leading-6">
<span className="font-medium">{t("scanTargets")}</span>
<span className="text-muted-foreground">{t("supportedFormats")}</span>
</h3>
</div>
<div className="flex-1 flex flex-col overflow-hidden">
<div className="flex-1 flex overflow-hidden">
<div className="flex-shrink-0 w-10 bg-muted/30">
<div ref={lineNumbersRef} className="py-3 px-2 text-right font-mono text-xs text-muted-foreground leading-[1.5] h-full overflow-y-auto scrollbar-hide">
{Array.from({ length: Math.max(targetInput.split('\n').length, 20) }, (_, i) => (
<div key={i + 1} className="h-[21px]">{i + 1}</div>
))}
</div>
</div>
<div className="flex-1 overflow-hidden">
<Textarea
value={targetInput}
onChange={(e) => setTargetInput(e.target.value)}
onScroll={handleTextareaScroll}
placeholder={t("targetPlaceholder")}
className="font-mono h-full overflow-y-auto resize-none border-0 rounded-none focus-visible:ring-0 focus-visible:ring-offset-0 text-sm py-3 px-3"
style={{ lineHeight: '21px' }}
autoFocus
/>
</div>
</div>
<div className="flex-1 overflow-hidden">
<Textarea
value={targetInput}
onChange={(e) => setTargetInput(e.target.value)}
onScroll={handleTextareaScroll}
placeholder={t("targetPlaceholder")}
className="font-mono h-full overflow-y-auto resize-none border-0 rounded-none focus-visible:ring-0 focus-visible:ring-offset-0 text-sm py-3 px-4"
style={{ lineHeight: '21px' }}
autoFocus
/>
</div>
{hasErrors && (
<div className="px-3 py-2 border-t bg-destructive/5 max-h-[60px] overflow-y-auto">
{invalidInputs.slice(0, 2).map((r) => (
<div key={r.lineNumber} className="flex items-center gap-1 text-xs text-destructive">
<AlertCircle className="h-3 w-3 shrink-0" />
<span>{t("lineError", { lineNumber: r.lineNumber, error: r.error || "" })}</span>
</div>
))}
{invalidInputs.length > 2 && <div className="text-xs text-muted-foreground">{t("moreErrors", { count: invalidInputs.length - 2 })}</div>}
</div>
)}
</div>
{hasErrors && (
<div className="px-4 py-2 border-t bg-destructive/5 max-h-[60px] overflow-y-auto">
{invalidInputs.slice(0, 3).map((r) => (
<div key={r.lineNumber} className="flex items-center gap-1 text-xs text-destructive">
<AlertCircle className="h-3 w-3 shrink-0" />
<span>{t("lineError", { lineNumber: r.lineNumber, error: r.error || "" })}</span>
</div>
))}
{invalidInputs.length > 3 && (
<div className="text-xs text-muted-foreground">{t("moreErrors", { count: invalidInputs.length - 3 })}</div>
)}
</div>
)}
</div>
)}
{/* Step 2: Select engine */}
{/* Step 2: Select engines */}
{step === 2 && (
<div className="flex h-full">
<div className="w-[260px] border-r flex flex-col shrink-0">
<div className="w-[320px] border-r flex flex-col shrink-0">
<div className="px-4 py-3 border-b bg-muted/30 shrink-0">
<h3 className="text-sm font-medium">{t("selectEngine")}</h3>
{selectedEngineIds.length > 0 && (
<p className="text-xs text-muted-foreground mt-1">
{t("selectedCount", { count: selectedEngineIds.length })}
</p>
)}
</div>
<div className="flex-1 overflow-y-auto">
{isLoading ? (
<div className="flex items-center justify-center py-8">
<Loader2 className="h-6 w-6 animate-spin text-muted-foreground" />
</div>
) : engines.length === 0 ? (
<div className="py-8 text-center text-sm text-muted-foreground">{t("noEngines")}</div>
) : (
<RadioGroup
value={selectedEngineId}
onValueChange={setSelectedEngineId}
disabled={isSubmitting}
className="p-2 space-y-1"
>
{engines.map((engine) => {
const capabilities = parseEngineCapabilities(engine.configuration || '')
const EngineIcon = getEngineIcon(capabilities)
const primaryCap = capabilities[0]
const iconConfig = primaryCap ? CAPABILITY_CONFIG[primaryCap] : null
const isSelected = selectedEngineId === engine.id.toString()
<div className="p-2">
{isLoading ? (
<div className="flex items-center justify-center py-8">
<LoadingSpinner />
<span className="ml-2 text-sm text-muted-foreground">{t("loading")}</span>
</div>
) : error ? (
<div className="py-8 text-center text-sm text-destructive">{t("loadFailed")}</div>
) : !engines?.length ? (
<div className="py-8 text-center text-sm text-muted-foreground">{t("noEngines")}</div>
) : (
<div className="space-y-1">
{engines.map((engine) => {
const capabilities = parseEngineCapabilities(engine.configuration || "")
const EngineIcon = getEngineIcon(capabilities)
const primaryCap = capabilities[0]
const iconConfig = primaryCap ? CAPABILITY_CONFIG[primaryCap] : null
const isSelected = selectedEngineIds.includes(engine.id)
return (
<label
key={engine.id}
htmlFor={`engine-${engine.id}`}
className={cn(
"flex items-center gap-3 p-3 rounded-lg cursor-pointer transition-all",
isSelected
? "bg-primary/10 border border-primary/30"
: "hover:bg-muted/50 border border-transparent"
)}
>
<RadioGroupItem value={engine.id.toString()} id={`engine-${engine.id}`} className="sr-only" />
<div className={cn("flex h-8 w-8 items-center justify-center rounded-md shrink-0", iconConfig?.color || "bg-muted text-muted-foreground")}>
<EngineIcon className="h-4 w-4" />
</div>
<div className="flex-1 min-w-0">
<div className="font-medium text-sm truncate">{engine.name}</div>
<div className="text-xs text-muted-foreground">{capabilities.length > 0 ? t("capabilities", { count: capabilities.length }) : t("noConfig")}</div>
</div>
{isSelected && <div className="w-2 h-2 rounded-full bg-primary shrink-0" />}
</label>
)
})}
</RadioGroup>
)}
return (
<label
key={engine.id}
htmlFor={`quick-engine-${engine.id}`}
className={cn(
"flex items-center gap-3 p-3 rounded-lg cursor-pointer transition-all",
isSelected
? "bg-primary/10 border border-primary/30"
: "hover:bg-muted/50 border border-transparent"
)}
>
<Checkbox
id={`quick-engine-${engine.id}`}
checked={isSelected}
onCheckedChange={(checked) => handleEngineToggle(engine.id, checked as boolean)}
disabled={isSubmitting}
/>
<div
className={cn(
"flex h-8 w-8 items-center justify-center rounded-md shrink-0",
iconConfig?.color || "bg-muted text-muted-foreground"
)}
>
<EngineIcon className="h-4 w-4" />
</div>
<div className="flex-1 min-w-0">
<div className="font-medium text-sm truncate">{engine.name}</div>
<div className="text-xs text-muted-foreground">
{capabilities.length > 0 ? t("capabilities", { count: capabilities.length }) : t("noConfig")}
</div>
</div>
</label>
)
})}
</div>
)}
</div>
</div>
</div>
<div className="flex-1 flex flex-col min-w-0 overflow-hidden">
{selectedEngine ? (
{selectedEngines.length > 0 ? (
<>
<div className="px-4 py-3 border-b bg-muted/30 flex items-center gap-2 shrink-0">
<Settings className="h-4 w-4 text-muted-foreground" />
<h3 className="text-sm font-medium truncate">{selectedEngine.name}</h3>
<Settings2 className="h-4 w-4 text-muted-foreground" />
<h3 className="text-sm font-medium truncate">
{selectedEngines.map((e) => e.name).join(", ")}
</h3>
</div>
<div className="flex-1 flex flex-col overflow-hidden p-4 gap-3">
{(() => {
const caps = parseEngineCapabilities(selectedEngine.configuration || '')
return caps.length > 0 && (
<div className="flex flex-wrap gap-1.5 shrink-0">
{caps.map((capKey) => {
const config = CAPABILITY_CONFIG[capKey]
return (
<Badge key={capKey} variant="outline" className={cn("text-xs", config?.color)}>
{config?.label || capKey}
</Badge>
)
})}
</div>
)
})()}
{selectedCapabilities.length > 0 && (
<div className="flex flex-wrap gap-1.5 shrink-0">
{selectedCapabilities.map((capKey) => {
const config = CAPABILITY_CONFIG[capKey]
return (
<Badge key={capKey} variant="outline" className={cn("text-xs", config?.color)}>
{config?.label || capKey}
</Badge>
)
})}
</div>
)}
<div className="flex-1 bg-muted/50 rounded-lg border overflow-hidden min-h-0">
<pre className="h-full p-3 text-xs font-mono overflow-auto whitespace-pre-wrap break-all">
{selectedEngine.configuration || `# ${t("noConfig")}`}
{selectedEngines.map((e) => e.configuration || `# ${t("noConfig")}`).join("\n\n")}
</pre>
</div>
</div>
@@ -289,7 +362,7 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
) : (
<div className="flex-1 flex items-center justify-center text-muted-foreground">
<div className="text-center">
<Settings className="h-10 w-10 mx-auto mb-3 opacity-30" />
<Settings2 className="h-10 w-10 mx-auto mb-3 opacity-30" />
<p className="text-sm">{t("selectEngineHint")}</p>
</div>
</div>
@@ -297,86 +370,41 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
</div>
</div>
)}
{/* Step 3: Confirm */}
{step === 3 && (
<div className="flex h-full">
<div className="w-[260px] border-r flex flex-col shrink-0">
<div className="px-4 py-3 border-b bg-muted/30 shrink-0">
<h3 className="text-sm font-medium">{t("scanTargets")}</h3>
</div>
<div className="flex-1 overflow-y-auto p-4">
<div className="space-y-1">
{validInputs.map((r, idx) => (
<div key={idx} className="font-mono text-xs truncate">{r.originalInput}</div>
))}
</div>
</div>
<div className="px-4 py-3 border-t bg-muted/30 text-xs text-muted-foreground">
{t("totalTargets", { count: validInputs.length })}
</div>
</div>
<div className="flex-1 flex flex-col min-w-0 overflow-hidden">
<div className="px-4 py-3 border-b bg-muted/30 flex items-center gap-2 shrink-0">
<Settings className="h-4 w-4 text-muted-foreground" />
<h3 className="text-sm font-medium truncate">{selectedEngine?.name}</h3>
</div>
<div className="flex-1 flex flex-col overflow-hidden p-4 gap-3">
{(() => {
const caps = parseEngineCapabilities(selectedEngine?.configuration || '')
return caps.length > 0 && (
<div className="flex flex-wrap gap-1.5 shrink-0">
{caps.map((capKey) => {
const config = CAPABILITY_CONFIG[capKey]
return (
<Badge key={capKey} variant="outline" className={cn("text-xs", config?.color)}>
{config?.label || capKey}
</Badge>
)
})}
</div>
)
})()}
<div className="flex-1 bg-muted/50 rounded-lg border overflow-hidden min-h-0">
<pre className="h-full p-3 text-xs font-mono overflow-auto whitespace-pre-wrap break-all">
{selectedEngine?.configuration || `# ${t("noConfig")}`}
</pre>
</div>
</div>
</div>
</div>
)}
</div>
<div className="border-t flex items-center justify-between px-4 py-3">
<span className="text-xs text-muted-foreground">
{step === 1 && (
<>
{t("supportedFormats")}
{validInputs.length > 0 && (
<span className="text-primary font-medium ml-2">{t("validTargets", { count: validInputs.length })}</span>
)}
{hasErrors && (
<span className="text-destructive ml-2">{t("invalidTargets", { count: invalidInputs.length })}</span>
)}
</>
<DialogFooter className="px-4 py-4 border-t !flex !items-center !justify-between">
<div className="text-sm">
{step === 1 && validInputs.length > 0 && (
<span className="text-primary">{t("validTargets", { count: validInputs.length })}</span>
)}
</span>
<div className="flex items-center gap-2">
<Button variant="outline" size="sm" onClick={handlePrev} disabled={step === 1} className={cn(step === 1 && "invisible")}>
<ChevronLeft className="h-4 w-4" />
{t("previous")}
</Button>
{step < 3 ? (
<Button size="sm" onClick={handleNext}>
{step === 1 && hasErrors && (
<span className="text-destructive ml-2">{t("invalidTargets", { count: invalidInputs.length })}</span>
)}
{step === 2 && selectedEngineIds.length > 0 && (
<span className="text-primary">{t("selectedCount", { count: selectedEngineIds.length })}</span>
)}
</div>
<div className="flex gap-2">
{step > 1 && (
<Button variant="outline" onClick={handleBack} disabled={isSubmitting}>
<ChevronLeft className="h-4 w-4 mr-1" />
{t("back")}
</Button>
)}
{step === 1 ? (
<Button
onClick={handleNext}
disabled={!canProceedToStep2}
>
{t("next")}
<ChevronRight className="h-4 w-4" />
<ChevronRight className="h-4 w-4 ml-1" />
</Button>
) : (
<Button size="sm" onClick={handleSubmit} disabled={isSubmitting}>
<Button onClick={handleSubmit} disabled={!canSubmit || isSubmitting}>
{isSubmitting ? (
<>
<Loader2 className="h-4 w-4 animate-spin" />
<LoadingSpinner />
{t("creating")}
</>
) : (
@@ -388,7 +416,7 @@ export function QuickScanDialog({ trigger }: QuickScanDialogProps) {
</Button>
)}
</div>
</div>
</DialogFooter>
</DialogContent>
</Dialog>
)

View File

@@ -37,7 +37,7 @@ interface StageDetail {
export interface ScanProgressData {
id: number
targetName: string
engineName: string
engineNames: string[]
status: string
progress: number
currentStage?: ScanStage
@@ -211,7 +211,7 @@ export function ScanProgressDialog({
</div>
<div className="flex items-center justify-between text-sm">
<span className="text-muted-foreground">{t("engine")}</span>
<Badge variant="secondary">{data.engineName}</Badge>
<Badge variant="secondary">{data.engineNames?.join(", ") || "-"}</Badge>
</div>
{data.startedAt && (
<div className="flex items-center justify-between text-sm">
@@ -359,7 +359,7 @@ export function buildScanProgressData(scan: ScanRecord): ScanProgressData {
return {
id: scan.id,
targetName: scan.targetName,
engineName: scan.engineName,
engineNames: scan.engineNames || [],
status: scan.status,
progress: scan.progress,
currentStage: scan.currentStage,

View File

@@ -12,13 +12,6 @@ import {
import { Button } from "@/components/ui/button"
import { Input } from "@/components/ui/input"
import { Label } from "@/components/ui/label"
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select"
import { Badge } from "@/components/ui/badge"
import { Checkbox } from "@/components/ui/checkbox"
import {
@@ -126,7 +119,7 @@ export function CreateScheduledScanDialog({
const [currentStep, { goToNextStep, goToPrevStep, reset: resetStep }] = useStep(totalSteps)
const [name, setName] = React.useState("")
const [engineId, setEngineId] = React.useState<number | null>(null)
const [engineIds, setEngineIds] = React.useState<number[]>([])
const [selectionMode, setSelectionMode] = React.useState<SelectionMode>("organization")
const [selectedOrgId, setSelectedOrgId] = React.useState<number | null>(null)
const [selectedTargetId, setSelectedTargetId] = React.useState<number | null>(null)
@@ -152,7 +145,7 @@ export function CreateScheduledScanDialog({
const resetForm = () => {
setName("")
setEngineId(null)
setEngineIds([])
setSelectionMode("organization")
setSelectedOrgId(null)
setSelectedTargetId(null)
@@ -160,6 +153,14 @@ export function CreateScheduledScanDialog({
resetStep()
}
const handleEngineToggle = (engineId: number, checked: boolean) => {
if (checked) {
setEngineIds((prev) => [...prev, engineId])
} else {
setEngineIds((prev) => prev.filter((id) => id !== engineId))
}
}
const handleOpenChange = (isOpen: boolean) => {
if (!isOpen) resetForm()
onOpenChange(isOpen)
@@ -178,7 +179,7 @@ export function CreateScheduledScanDialog({
switch (currentStep) {
case 1:
if (!name.trim()) { toast.error(t("form.taskNameRequired")); return false }
if (!engineId) { toast.error(t("form.scanEngineRequired")); return false }
if (engineIds.length === 0) { toast.error(t("form.scanEngineRequired")); return false }
return true
case 2:
const parts = cronExpression.trim().split(/\s+/)
@@ -191,7 +192,7 @@ export function CreateScheduledScanDialog({
switch (currentStep) {
case 1:
if (!name.trim()) { toast.error(t("form.taskNameRequired")); return false }
if (!engineId) { toast.error(t("form.scanEngineRequired")); return false }
if (engineIds.length === 0) { toast.error(t("form.scanEngineRequired")); return false }
return true
case 2: return true
case 3:
@@ -215,7 +216,7 @@ export function CreateScheduledScanDialog({
if (!validateCurrentStep()) return
const request: CreateScheduledScanRequest = {
name: name.trim(),
engineId: engineId!,
engineIds: engineIds,
cronExpression: cronExpression.trim(),
}
if (selectionMode === "organization" && selectedOrgId) {
@@ -225,6 +226,14 @@ export function CreateScheduledScanDialog({
}
createScheduledScan(request, {
onSuccess: () => { resetForm(); onOpenChange(false); onSuccess?.() },
onError: (err: unknown) => {
const error = err as { response?: { data?: { error?: { code?: string; message?: string } } } }
if (error?.response?.data?.error?.code === 'CONFIG_CONFLICT') {
toast.error(t("toast.configConflict"), {
description: error.response.data.error.message,
})
}
},
})
}
@@ -294,14 +303,34 @@ export function CreateScheduledScanDialog({
</div>
<div className="space-y-2">
<Label>{t("form.scanEngine")} *</Label>
<Select value={engineId?.toString() || ""} onValueChange={(v) => setEngineId(Number(v))}>
<SelectTrigger><SelectValue placeholder={t("form.scanEnginePlaceholder")} /></SelectTrigger>
<SelectContent>
{engines.map((engine) => (
<SelectItem key={engine.id} value={engine.id.toString()}>{engine.name}</SelectItem>
))}
</SelectContent>
</Select>
{engineIds.length > 0 && (
<p className="text-xs text-muted-foreground">{t("form.selectedEngines", { count: engineIds.length })}</p>
)}
<div className="border rounded-md p-3 max-h-[200px] overflow-y-auto space-y-2">
{engines.length === 0 ? (
<p className="text-sm text-muted-foreground">{t("form.noEngine")}</p>
) : (
engines.map((engine) => (
<label
key={engine.id}
htmlFor={`engine-${engine.id}`}
className={cn(
"flex items-center gap-3 p-2 rounded-lg cursor-pointer transition-all",
engineIds.includes(engine.id)
? "bg-primary/10 border border-primary/30"
: "hover:bg-muted/50 border border-transparent"
)}
>
<Checkbox
id={`engine-${engine.id}`}
checked={engineIds.includes(engine.id)}
onCheckedChange={(checked) => handleEngineToggle(engine.id, checked as boolean)}
/>
<span className="text-sm">{engine.name}</span>
</label>
))
)}
</div>
<p className="text-xs text-muted-foreground">{t("form.scanEngineDesc")}</p>
</div>
</div>

View File

@@ -13,15 +13,10 @@ import {
import { Button } from "@/components/ui/button"
import { Input } from "@/components/ui/input"
import { Label } from "@/components/ui/label"
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select"
import { Badge } from "@/components/ui/badge"
import { Checkbox } from "@/components/ui/checkbox"
import { IconX, IconLoader2 } from "@tabler/icons-react"
import { cn } from "@/lib/utils"
import { useUpdateScheduledScan } from "@/hooks/use-scheduled-scans"
import { useTargets } from "@/hooks/use-targets"
import { useEngines } from "@/hooks/use-engines"
@@ -61,19 +56,27 @@ export function EditScheduledScanDialog({
]
const [name, setName] = React.useState("")
const [engineId, setEngineId] = React.useState<number | null>(null)
const [engineIds, setEngineIds] = React.useState<number[]>([])
const [selectedTargetId, setSelectedTargetId] = React.useState<number | null>(null)
const [cronExpression, setCronExpression] = React.useState("")
React.useEffect(() => {
if (scheduledScan && open) {
setName(scheduledScan.name)
setEngineId(scheduledScan.engine)
setEngineIds(scheduledScan.engineIds || [])
setSelectedTargetId(scheduledScan.targetId || null)
setCronExpression(scheduledScan.cronExpression || "0 2 * * *")
}
}, [scheduledScan, open])
const handleEngineToggle = (engineId: number, checked: boolean) => {
if (checked) {
setEngineIds((prev) => [...prev, engineId])
} else {
setEngineIds((prev) => prev.filter((id) => id !== engineId))
}
}
const handleTargetSelect = (targetId: number) => {
setSelectedTargetId(selectedTargetId === targetId ? null : targetId)
}
@@ -90,7 +93,7 @@ export function EditScheduledScanDialog({
toast.error(t("form.taskNameRequired"))
return
}
if (!engineId) {
if (engineIds.length === 0) {
toast.error(t("form.scanEngineRequired"))
return
}
@@ -105,7 +108,7 @@ export function EditScheduledScanDialog({
const request: UpdateScheduledScanRequest = {
name: name.trim(),
engineId: engineId,
engineIds: engineIds,
cronExpression: cronExpression.trim(),
}
@@ -120,6 +123,14 @@ export function EditScheduledScanDialog({
onOpenChange(false)
onSuccess?.()
},
onError: (err: unknown) => {
const error = err as { response?: { data?: { error?: { code?: string; message?: string } } } }
if (error?.response?.data?.error?.code === 'CONFIG_CONFLICT') {
toast.error(t("toast.configConflict"), {
description: error.response.data.error.message,
})
}
},
}
)
}
@@ -152,21 +163,34 @@ export function EditScheduledScanDialog({
<div className="grid gap-2">
<Label>{t("form.scanEngine")} *</Label>
<Select
value={engineId?.toString() || ""}
onValueChange={(v) => setEngineId(Number(v))}
>
<SelectTrigger>
<SelectValue placeholder={t("form.scanEnginePlaceholder")} />
</SelectTrigger>
<SelectContent>
{engines.map((engine) => (
<SelectItem key={engine.id} value={engine.id.toString()}>
{engine.name}
</SelectItem>
))}
</SelectContent>
</Select>
{engineIds.length > 0 && (
<p className="text-xs text-muted-foreground">{t("form.selectedEngines", { count: engineIds.length })}</p>
)}
<div className="border rounded-md p-3 max-h-[150px] overflow-y-auto space-y-2">
{engines.length === 0 ? (
<p className="text-sm text-muted-foreground">{t("form.noEngine")}</p>
) : (
engines.map((engine) => (
<label
key={engine.id}
htmlFor={`edit-engine-${engine.id}`}
className={cn(
"flex items-center gap-3 p-2 rounded-lg cursor-pointer transition-all",
engineIds.includes(engine.id)
? "bg-primary/10 border border-primary/30"
: "hover:bg-muted/50 border border-transparent"
)}
>
<Checkbox
id={`edit-engine-${engine.id}`}
checked={engineIds.includes(engine.id)}
onCheckedChange={(checked) => handleEngineToggle(engine.id, checked as boolean)}
/>
<span className="text-sm">{engine.name}</span>
</label>
))
)}
</div>
</div>
<div className="grid gap-2">

View File

@@ -179,19 +179,26 @@ export const createScheduledScanColumns = ({
},
},
{
accessorKey: "engineName",
size: 120,
minSize: 80,
accessorKey: "engineNames",
size: 150,
minSize: 100,
meta: { title: t.columns.scanEngine },
header: ({ column }) => (
<DataTableColumnHeader column={column} title={t.columns.scanEngine} />
),
cell: ({ row }) => {
const engineName = row.getValue("engineName") as string
const engineNames = row.original.engineNames || []
if (engineNames.length === 0) {
return <span className="text-muted-foreground text-sm">-</span>
}
return (
<Badge variant="secondary">
{engineName}
</Badge>
<div className="flex flex-wrap gap-1">
{engineNames.map((name, index) => (
<Badge key={index} variant="secondary">
{name}
</Badge>
))}
</div>
)
},
},
@@ -255,8 +262,8 @@ export const createScheduledScanColumns = ({
},
{
accessorKey: "isEnabled",
size: 100,
minSize: 80,
size: 120,
minSize: 100,
meta: { title: t.columns.status },
header: ({ column }) => (
<DataTableColumnHeader column={column} title={t.columns.status} />

View File

@@ -130,7 +130,7 @@ export function ExpandableCell({
<div
ref={contentRef}
className={cn(
"text-sm break-all leading-relaxed whitespace-normal",
"text-sm break-all leading-relaxed whitespace-pre-wrap",
variant === "mono" && "font-mono text-xs text-muted-foreground",
variant === "url" && "text-muted-foreground",
variant === "muted" && "text-muted-foreground",

View File

@@ -20,8 +20,9 @@ export interface WebsiteTranslations {
location: string
webServer: string
contentType: string
bodyPreview: string
responseBody: string
vhost: string
responseHeaders: string
createdAt: string
}
actions: {
@@ -199,17 +200,32 @@ export function createWebSiteColumns({
),
},
{
accessorKey: "bodyPreview",
meta: { title: t.columns.bodyPreview },
accessorKey: "responseBody",
meta: { title: t.columns.responseBody },
header: ({ column }) => (
<DataTableColumnHeader column={column} title={t.columns.bodyPreview} />
<DataTableColumnHeader column={column} title={t.columns.responseBody} />
),
size: 350,
minSize: 250,
cell: ({ row }) => (
<ExpandableCell value={row.getValue("bodyPreview")} />
<ExpandableCell value={row.getValue("responseBody")} />
),
},
{
accessorKey: "responseHeaders",
meta: { title: t.columns.responseHeaders },
header: ({ column }) => (
<DataTableColumnHeader column={column} title={t.columns.responseHeaders} />
),
size: 250,
minSize: 150,
maxSize: 400,
cell: ({ row }) => {
const headers = row.getValue("responseHeaders") as string | null
if (!headers) return "-"
return <ExpandableCell value={headers} maxLines={3} />
},
},
{
accessorKey: "vhost",
meta: { title: t.columns.vhost },

View File

@@ -16,6 +16,7 @@ const WEBSITE_FILTER_FIELDS: FilterField[] = [
{ key: "title", label: "Title", description: "Page title" },
{ key: "status", label: "Status", description: "HTTP status code" },
{ key: "tech", label: "Tech", description: "Technologies" },
{ key: "responseHeaders", label: "Headers", description: "Response headers" },
]
// Website page filter examples

View File

@@ -52,8 +52,9 @@ export function WebSitesView({
location: tColumns("endpoint.location"),
webServer: tColumns("endpoint.webServer"),
contentType: tColumns("endpoint.contentType"),
bodyPreview: tColumns("endpoint.bodyPreview"),
responseBody: tColumns("endpoint.responseBody"),
vhost: tColumns("endpoint.vhost"),
responseHeaders: tColumns("website.responseHeaders"),
createdAt: tColumns("common.createdAt"),
},
actions: {
@@ -175,7 +176,7 @@ export function WebSitesView({
const headers = [
'url', 'host', 'location', 'title', 'status_code',
'content_length', 'content_type', 'webserver', 'tech',
'body_preview', 'vhost', 'created_at'
'response_body', 'vhost', 'created_at'
]
const rows = items.map(item => [
@@ -188,7 +189,7 @@ export function WebSitesView({
escapeCSV(item.contentType),
escapeCSV(item.webserver),
escapeCSV(formatArrayForCSV(item.tech)),
escapeCSV(item.bodyPreview),
escapeCSV(item.responseBody),
escapeCSV(item.vhost),
escapeCSV(formatDateForCSV(item.createdAt))
].join(','))

View File

@@ -14,6 +14,7 @@
"target": "Target",
"summary": "Summary",
"engineName": "Engine Name",
"workerName": "Worker Node",
"progress": "Progress",
"subdomains": "Subdomains",
"websites": "Websites",
@@ -48,18 +49,21 @@
},
"endpoint": {
"title": "Title",
"host": "Host",
"contentLength": "Content Length",
"location": "Location",
"webServer": "Web Server",
"contentType": "Content Type",
"technologies": "Technologies",
"bodyPreview": "Body Preview",
"responseBody": "Response Body",
"vhost": "VHost",
"gfPatterns": "GF Patterns",
"responseHeaders": "Response Headers",
"responseTime": "Response Time"
},
"website": {
"host": "Host"
"host": "Host",
"responseHeaders": "Response Headers"
},
"directory": {
"length": "Length",
@@ -307,7 +311,8 @@
"workers": "Workers",
"systemLogs": "System Logs",
"notifications": "Notifications",
"help": "Get Help"
"help": "Get Help",
"feedback": "Feedback"
},
"dashboard": {
"title": "Dashboard",
@@ -582,7 +587,8 @@
"capabilities": "{count} capabilities",
"noConfig": "No config",
"initiating": "Initiating...",
"startScan": "Start Scan"
"startScan": "Start Scan",
"selectedCount": "{count} engines selected"
},
"cron": {
"everyMinute": "Every minute",
@@ -709,7 +715,8 @@
"organizationMode": "Organization Scan",
"organizationModeHint": "In organization scan mode, all targets under this organization will be dynamically fetched at execution",
"noAvailableTarget": "No available targets",
"selected": "Selected"
"selected": "Selected",
"selectedEngines": "{count} engines selected"
},
"presets": {
"everyHour": "Every Hour",
@@ -1627,13 +1634,18 @@
},
"quickScan": {
"title": "Quick Scan",
"description": "Quickly create scan tasks for multiple targets",
"steps": {
"enterTargets": "Enter Targets",
"selectEngine": "Select Engine",
"confirmScan": "Confirm Scan"
},
"step1Title": "Enter Targets",
"step2Title": "Select Engines",
"step3Title": "Confirm",
"step1Hint": "Enter scan targets in the left input box, one per line",
"step": "Step {current}/{total} · {title}",
"targetPlaceholder": "Enter one target per line, supported formats:\n\nDomain: example.com, sub.example.com\nIP Address: 192.168.1.1, 10.0.0.1\nCIDR: 192.168.0.0/24, 10.0.0.0/8\nURL: https://example.com/api/v1",
"targetPlaceholder": "Enter one target per line, supported formats:\n\nDomain: example.com, sub.example.com\nIP Address: 192.168.1.1, 10.0.0.1\nCIDR: 192.168.1.0/24, 10.0.0.0/8\nURL: https://example.com/api/v1",
"supportedFormats": "Supported: Domain, IP, CIDR, URL",
"validTargets": "{count} valid targets",
"invalidTargets": "{count} invalid",
@@ -1644,12 +1656,20 @@
"scanTargets": "Scan Targets",
"totalTargets": "{count} targets total",
"previous": "Previous",
"back": "Back",
"next": "Next",
"startScan": "Start Scan",
"creating": "Creating...",
"selectEngineHint": "Select an engine on the left to view configuration",
"moreErrors": "{count} more errors...",
"lineError": "Line {lineNumber}: {error}",
"loading": "Loading...",
"loadFailed": "Failed to load",
"selectedCount": "{count} selected",
"confirmTargets": "{count} scan targets",
"andMore": "{count} more...",
"selectedEngines": "Selected Engines",
"confirmSummary": "Will scan {targetCount} targets with {engineCount} engines",
"toast": {
"noValidTarget": "Please enter at least one valid target",
"hasInvalidInputs": "{count} invalid inputs, please fix before continuing",
@@ -1658,7 +1678,8 @@
"createFailed": "Failed to create scan task",
"createSuccess": "Created {count} scan tasks",
"createSuccessDesc": "{created} targets succeeded, {failed} failed",
"targetsFailed": "{count} targets failed"
"targetsFailed": "{count} targets failed",
"configConflict": "Engine configuration conflict"
}
},
"notificationDrawer": {

View File

@@ -14,6 +14,7 @@
"target": "Target",
"summary": "Summary",
"engineName": "Engine Name",
"workerName": "Worker Node",
"progress": "Progress",
"subdomains": "Subdomains",
"websites": "Websites",
@@ -48,18 +49,21 @@
},
"endpoint": {
"title": "Title",
"host": "Host",
"contentLength": "Content Length",
"location": "Location",
"webServer": "Web Server",
"contentType": "Content Type",
"technologies": "Technologies",
"bodyPreview": "Body Preview",
"responseBody": "Response Body",
"vhost": "VHost",
"gfPatterns": "GF Patterns",
"responseHeaders": "Response Headers",
"responseTime": "Response Time"
},
"website": {
"host": "Host"
"host": "Host",
"responseHeaders": "Response Headers"
},
"directory": {
"length": "Length",
@@ -307,7 +311,8 @@
"workers": "扫描节点",
"systemLogs": "系统日志",
"notifications": "通知设置",
"help": "获取帮助"
"help": "获取帮助",
"feedback": "反馈建议"
},
"dashboard": {
"title": "仪表盘",
@@ -582,7 +587,8 @@
"capabilities": "{count} 项能力",
"noConfig": "无配置",
"initiating": "发起中...",
"startScan": "开始扫描"
"startScan": "开始扫描",
"selectedCount": "已选择 {count} 个引擎"
},
"cron": {
"everyMinute": "每分钟",
@@ -709,7 +715,8 @@
"organizationMode": "组织扫描",
"organizationModeHint": "组织扫描模式下,执行时将动态获取该组织下所有目标",
"noAvailableTarget": "暂无可用目标",
"selected": "已选择"
"selected": "已选择",
"selectedEngines": "已选择 {count} 个引擎"
},
"presets": {
"everyHour": "每小时",
@@ -1627,14 +1634,19 @@
},
"quickScan": {
"title": "快速扫描",
"description": "快速创建扫描任务,批量扫描多个目标",
"steps": {
"enterTargets": "输入目标",
"selectEngine": "选择引擎",
"confirmScan": "确认扫描"
},
"step1Title": "输入目标",
"step2Title": "选择引擎",
"step3Title": "确认",
"step1Hint": "在左侧输入框中输入扫描目标,每行一个",
"step": "步骤 {current}/{total} · {title}",
"targetPlaceholder": "每行输入一个目标,支持以下格式:\n\n域名: example.com, sub.example.com\nIP地址: 192.168.1.1, 10.0.0.1\nCIDR网段: 192.168.0.0/24, 10.0.0.0/8\nURL: https://example.com/api/v1",
"supportedFormats": "支持: 域名、IP、CIDR、URL",
"targetPlaceholder": "每行输入一个目标,支持以下格式:\n\n域名: example.com, sub.example.com\nIP地址: 192.168.1.1, 10.0.0.1\nCIDR网段: 192.168.1.0/24, 10.0.0.0/8\nURL: https://example.com/api/v1",
"supportedFormats": "支持 域名、IP、CIDR、URL",
"validTargets": "{count} 个有效目标",
"invalidTargets": "{count} 个无效",
"selectEngine": "选择引擎",
@@ -1644,12 +1656,20 @@
"scanTargets": "扫描目标",
"totalTargets": "共 {count} 个目标",
"previous": "上一步",
"back": "上一步",
"next": "下一步",
"startScan": "开始扫描",
"creating": "创建中...",
"selectEngineHint": "选择左侧引擎查看配置详情",
"moreErrors": "还有 {count} 个错误...",
"lineError": "行 {lineNumber}: {error}",
"loading": "加载中...",
"loadFailed": "加载失败",
"selectedCount": "已选择 {count} 个",
"confirmTargets": "共 {count} 个扫描目标",
"andMore": "还有 {count} 个...",
"selectedEngines": "已选引擎",
"confirmSummary": "将使用 {engineCount} 个引擎扫描 {targetCount} 个目标",
"toast": {
"noValidTarget": "请输入至少一个有效目标",
"hasInvalidInputs": "存在 {count} 个无效输入,请修正后继续",
@@ -1658,7 +1678,8 @@
"createFailed": "创建扫描任务失败",
"createSuccess": "已创建 {count} 个扫描任务",
"createSuccessDesc": "{created} 个目标成功,{failed} 个失败",
"targetsFailed": "{count} 个目标处理失败"
"targetsFailed": "{count} 个目标处理失败",
"configConflict": "引擎配置冲突"
}
},
"notificationDrawer": {

View File

@@ -20,9 +20,10 @@ export interface Endpoint {
host?: string
location?: string
webserver?: string
bodyPreview?: string
responseBody?: string
tech?: string[]
vhost?: boolean | null
responseHeaders?: string
createdAt?: string
// Legacy domain association fields (may not exist in some APIs)

View File

@@ -36,6 +36,7 @@ export interface ScanRecord {
id: number
target?: number // Target ID (corresponds to backend target)
targetName: string // Target name (corresponds to backend targetName)
workerName?: string | null // Worker node name (corresponds to backend worker_name)
summary: {
subdomains: number
websites: number
@@ -50,8 +51,8 @@ export interface ScanRecord {
low: number
}
}
engine?: number // Engine ID (corresponds to backend engine)
engineName: string // Engine name (corresponds to backend engineName)
engineIds: number[] // Engine ID list (corresponds to backend engine_ids)
engineNames: string[] // Engine name list (corresponds to backend engine_names)
createdAt: string // Creation time (corresponds to backend createdAt)
status: ScanStatus
errorMessage?: string // Error message (corresponds to backend errorMessage, has value when failed)
@@ -81,7 +82,7 @@ export interface GetScansResponse {
export interface InitiateScanRequest {
organizationId?: number // Organization ID (choose one)
targetId?: number // Target ID (choose one)
engineId: number // Scan engine ID (required)
engineIds: number[] // Scan engine ID list (required)
}
/**
@@ -89,7 +90,7 @@ export interface InitiateScanRequest {
*/
export interface QuickScanRequest {
targets: { name: string }[] // Target list
engineId: number // Scan engine ID (required)
engineIds: number[] // Scan engine ID list (required)
}
/**
@@ -116,7 +117,8 @@ export interface QuickScanResponse {
export interface ScanTask {
id: number
target: number // Target ID
engine: number // Engine ID
engineIds: number[] // Engine ID list
engineNames: string[] // Engine name list
status: ScanStatus
createdAt: string
updatedAt: string

View File

@@ -12,8 +12,8 @@ export type ScanMode = 'organization' | 'target'
export interface ScheduledScan {
id: number
name: string
engine: number // Associated scan engine ID
engineName: string // Associated scan engine name
engineIds: number[] // Associated scan engine ID list
engineNames: string[] // Associated scan engine name list
organizationId: number | null // Organization ID (organization scan mode)
organizationName: string | null // Organization name
targetId: number | null // Target ID (target scan mode)
@@ -31,7 +31,7 @@ export interface ScheduledScan {
// Create scheduled scan request (organizationId and targetId are mutually exclusive)
export interface CreateScheduledScanRequest {
name: string
engineId: number
engineIds: number[] // Engine ID list
organizationId?: number // Organization scan mode
targetId?: number // Target scan mode
cronExpression: string // Cron expression, format: minute hour day month weekday
@@ -41,7 +41,7 @@ export interface CreateScheduledScanRequest {
// Update scheduled scan request (organizationId and targetId are mutually exclusive)
export interface UpdateScheduledScanRequest {
name?: string
engineId?: number
engineIds?: number[] // Engine ID list
organizationId?: number // Organization scan mode (clears targetId when set)
targetId?: number // Target scan mode (clears organizationId when set)
cronExpression?: string

View File

@@ -14,10 +14,11 @@ export interface WebSite {
contentType: string
statusCode: number
contentLength: number
bodyPreview: string
responseBody: string
tech: string[]
vhost: boolean | null
subdomain: string
responseHeaders?: string
createdAt: string
}