diff --git a/backend/apps/scan/configs/command_templates.py b/backend/apps/scan/configs/command_templates.py index d3252214..838401d5 100644 --- a/backend/apps/scan/configs/command_templates.py +++ b/backend/apps/scan/configs/command_templates.py @@ -13,12 +13,14 @@ SCAN_TOOLS_BASE_PATH = getattr(settings, 'SCAN_TOOLS_BASE_PATH', '/usr/local/bin SUBDOMAIN_DISCOVERY_COMMANDS = { 'subfinder': { - # 默认使用所有数据源（更全面，略慢），并始终开启递归 - # -all 使用所有数据源 - # -recursive 对支持递归的源启用递归枚举（默认开启） - 'base': "subfinder -d {domain} -all -recursive -o '{output_file}' -silent", + # 使用所有数据源（包括付费源，只要配置了 API key） + # -all 使用所有数据源（slow 但全面） + # -v 显示详细输出，包括使用的数据源（调试用） + # 注意：不要加 -recursive，它会排除不支持递归的源（如 fofa） + 'base': "subfinder -d {domain} -all -o '{output_file}' -v", 'optional': { 'threads': '-t {threads}', # 控制并发 goroutine 数 + 'provider_config': "-pc '{provider_config}'", # Provider 配置文件路径 } }, diff --git a/backend/apps/scan/flows/subdomain_discovery_flow.py b/backend/apps/scan/flows/subdomain_discovery_flow.py index f34a80f6..4a6c81e0 100644 --- a/backend/apps/scan/flows/subdomain_discovery_flow.py +++ b/backend/apps/scan/flows/subdomain_discovery_flow.py @@ -78,7 +78,8 @@ def _run_scans_parallel( enabled_tools: dict, domain_name: str, result_dir: Path, - scan_id: int + scan_id: int, + provider_config_path: str = None ) -> tuple[list, list, list]: """ 并行运行所有启用的子域名扫描工具 @@ -88,6 +89,7 @@ def _run_scans_parallel( domain_name: 目标域名 result_dir: 结果输出目录 scan_id: 扫描任务 ID（用于记录日志） + provider_config_path: Provider 配置文件路径（可选，用于 subfinder） Returns: tuple: (result_files, failed_tools, successful_tool_names) @@ -112,13 +114,19 @@ def _run_scans_parallel( # 1.2 构建完整命令（变量替换） try: + command_params = { + 'domain': domain_name, # 对应 {domain} + 'output_file': output_file # 对应 {output_file} + } + + # 如果是 subfinder 且有 provider_config，添加到参数 + if tool_name == 'subfinder' and provider_config_path: + command_params['provider_config'] = provider_config_path + command = build_scan_command( tool_name=tool_name, scan_type='subdomain_discovery', - command_params={ - 'domain': domain_name, # 对应 {domain} - 'output_file': output_file # 对应 {output_file} - }, + command_params=command_params, tool_config=tool_config ) except Exception as e: @@ -440,6 +448,19 @@ def subdomain_discovery_flow( failed_tools = [] successful_tool_names = [] + # ==================== 生成 Provider 配置文件 ==================== + # 为 subfinder 生成第三方数据源配置 + provider_config_path = None + try: + from apps.scan.services.subfinder_provider_config_service import SubfinderProviderConfigService + provider_config_service = SubfinderProviderConfigService() + provider_config_path = provider_config_service.generate(str(result_dir)) + if provider_config_path: + logger.info(f"Provider 配置文件已生成: {provider_config_path}") + user_log(scan_id, "subdomain_discovery", "Provider config generated for subfinder") + except Exception as e: + logger.warning(f"生成 Provider 配置文件失败: {e}") + # ==================== Stage 1: 被动收集（并行）==================== if enabled_passive_tools: logger.info("=" * 40) @@ -451,7 +472,8 @@ def subdomain_discovery_flow( enabled_tools=enabled_passive_tools, domain_name=domain_name, result_dir=result_dir, - scan_id=scan_id + scan_id=scan_id, + provider_config_path=provider_config_path ) all_result_files.extend(result_files) failed_tools.extend(stage1_failed) diff --git a/backend/apps/scan/migrations/0001_initial.py b/backend/apps/scan/migrations/0001_initial.py index 8865d6a1..0cc74351 100644 --- a/backend/apps/scan/migrations/0001_initial.py +++ b/backend/apps/scan/migrations/0001_initial.py @@ -133,4 +133,18 @@ class Migration(migrations.Migration): 'indexes': [models.Index(fields=['scan', 'created_at'], name='scan_log_scan_id_e8c8f5_idx')], }, ), + migrations.CreateModel( + name='SubfinderProviderSettings', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('providers', models.JSONField(default=dict, help_text='各 Provider 的 API Key 配置')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ], + options={ + 'verbose_name': 'Subfinder Provider 配置', + 'verbose_name_plural': 'Subfinder Provider 配置', + 'db_table': 'subfinder_provider_settings', + }, + ), ] diff --git a/backend/apps/scan/models.py b/backend/apps/scan/models.py deleted file mode 100644 index 61e9648b..00000000 --- a/backend/apps/scan/models.py +++ /dev/null @@ -1,244 +0,0 @@ -from django.db import models -from django.contrib.postgres.fields import ArrayField - -from ..common.definitions import ScanStatus - - - - -class SoftDeleteManager(models.Manager): - """软删除管理器：默认只返回未删除的记录""" - - def get_queryset(self): - return super().get_queryset().filter(deleted_at__isnull=True) - - -class Scan(models.Model): - """扫描任务模型""" - - id = models.AutoField(primary_key=True) - - target = models.ForeignKey('targets.Target', on_delete=models.CASCADE, related_name='scans', help_text='扫描目标') - - # 多引擎支持字段 - engine_ids = ArrayField( - models.IntegerField(), - default=list, - help_text='引擎 ID 列表' - ) - engine_names = models.JSONField( - default=list, - help_text='引擎名称列表，如 ["引擎A", "引擎B"]' - ) - yaml_configuration = models.TextField( - default='', - help_text='YAML 格式的扫描配置' - ) - - created_at = models.DateTimeField(auto_now_add=True, help_text='任务创建时间') - stopped_at = models.DateTimeField(null=True, blank=True, help_text='扫描结束时间') - - status = models.CharField( - max_length=20, - choices=ScanStatus.choices, - default=ScanStatus.INITIATED, - db_index=True, - help_text='任务状态' - ) - - results_dir = models.CharField(max_length=100, blank=True, default='', help_text='结果存储目录') - - container_ids = ArrayField( - models.CharField(max_length=100), - blank=True, - default=list, - help_text='容器 ID 列表（Docker Container ID）' - ) - - worker = models.ForeignKey( - 'engine.WorkerNode', - on_delete=models.SET_NULL, - related_name='scans', - null=True, - blank=True, - help_text='执行扫描的 Worker 节点' - ) - - error_message = models.CharField(max_length=2000, blank=True, default='', help_text='错误信息') - - # ==================== 软删除字段 ==================== - deleted_at = models.DateTimeField(null=True, blank=True, db_index=True, help_text='删除时间（NULL表示未删除）') - - # ==================== 管理器 ==================== - objects = SoftDeleteManager() # 默认管理器：只返回未删除的记录 - all_objects = models.Manager() # 全量管理器：包括已删除的记录（用于硬删除） - - # ==================== 进度跟踪字段 ==================== - progress = models.IntegerField(default=0, help_text='扫描进度 0-100') - current_stage = models.CharField(max_length=50, blank=True, default='', help_text='当前扫描阶段') - stage_progress = models.JSONField(default=dict, help_text='各阶段进度详情') - - # ==================== 缓存统计字段 ==================== - cached_subdomains_count = models.IntegerField(default=0, help_text='缓存的子域名数量') - cached_websites_count = models.IntegerField(default=0, help_text='缓存的网站数量') - cached_endpoints_count = models.IntegerField(default=0, help_text='缓存的端点数量') - cached_ips_count = models.IntegerField(default=0, help_text='缓存的IP地址数量') - cached_directories_count = models.IntegerField(default=0, help_text='缓存的目录数量') - cached_vulns_total = models.IntegerField(default=0, help_text='缓存的漏洞总数') - cached_vulns_critical = models.IntegerField(default=0, help_text='缓存的严重漏洞数量') - cached_vulns_high = models.IntegerField(default=0, help_text='缓存的高危漏洞数量') - cached_vulns_medium = models.IntegerField(default=0, help_text='缓存的中危漏洞数量') - cached_vulns_low = models.IntegerField(default=0, help_text='缓存的低危漏洞数量') - stats_updated_at = models.DateTimeField(null=True, blank=True, help_text='统计数据最后更新时间') - - class Meta: - db_table = 'scan' - verbose_name = '扫描任务' - verbose_name_plural = '扫描任务' - ordering = ['-created_at'] - indexes = [ - models.Index(fields=['-created_at']), # 优化按创建时间降序排序（list 查询的默认排序） - models.Index(fields=['target']), # 优化按目标查询扫描任务 - models.Index(fields=['deleted_at', '-created_at']), # 软删除 + 时间索引 - ] - - def __str__(self): - return f"Scan #{self.id} - {self.target.name}" - - -class ScanLog(models.Model): - """扫描日志模型 - - 存储扫描过程中的关键处理日志，用于前端实时查看扫描进度。 - - 日志类型： - - 阶段开始/完成/失败 - - 处理进度（如 "Progress: 50/120"） - - 发现结果统计（如 "Found 120 subdomains"） - - 错误信息 - - 日志格式：[stage_name] message - """ - - class Level(models.TextChoices): - INFO = 'info', 'Info' - WARNING = 'warning', 'Warning' - ERROR = 'error', 'Error' - - id = models.BigAutoField(primary_key=True) - scan = models.ForeignKey( - 'Scan', - on_delete=models.CASCADE, - related_name='logs', - db_index=True, - help_text='关联的扫描任务' - ) - level = models.CharField( - max_length=10, - choices=Level.choices, - default=Level.INFO, - help_text='日志级别' - ) - content = models.TextField(help_text='日志内容') - created_at = models.DateTimeField(auto_now_add=True, db_index=True, help_text='创建时间') - - class Meta: - db_table = 'scan_log' - verbose_name = '扫描日志' - verbose_name_plural = '扫描日志' - ordering = ['created_at'] - indexes = [ - models.Index(fields=['scan', 'created_at']), - ] - - def __str__(self): - return f"[{self.level}] {self.content[:50]}" - - -class ScheduledScan(models.Model): - """ - 定时扫描任务模型 - - 调度机制： - - APScheduler 每分钟检查 next_run_time - - 到期任务通过 task_distributor 分发到 Worker 执行 - - 支持 cron 表达式进行灵活调度 - - 扫描模式（二选一）： - - 组织扫描：设置 organization，执行时动态获取组织下所有目标 - - 目标扫描：设置 target，扫描单个目标 - - organization 优先级高于 target - """ - - id = models.AutoField(primary_key=True) - - # 基本信息 - name = models.CharField(max_length=200, help_text='任务名称') - - # 多引擎支持字段 - engine_ids = ArrayField( - models.IntegerField(), - default=list, - help_text='引擎 ID 列表' - ) - engine_names = models.JSONField( - default=list, - help_text='引擎名称列表，如 ["引擎A", "引擎B"]' - ) - yaml_configuration = models.TextField( - default='', - help_text='YAML 格式的扫描配置' - ) - - # 关联的组织（组织扫描模式：执行时动态获取组织下所有目标） - organization = models.ForeignKey( - 'targets.Organization', - on_delete=models.CASCADE, - related_name='scheduled_scans', - null=True, - blank=True, - help_text='扫描组织（设置后执行时动态获取组织下所有目标）' - ) - - # 关联的目标（目标扫描模式：扫描单个目标） - target = models.ForeignKey( - 'targets.Target', - on_delete=models.CASCADE, - related_name='scheduled_scans', - null=True, - blank=True, - help_text='扫描单个目标（与 organization 二选一）' - ) - - # 调度配置 - 直接使用 Cron 表达式 - cron_expression = models.CharField( - max_length=100, - default='0 2 * * *', - help_text='Cron 表达式，格式：分 时 日 月 周' - ) - - # 状态 - is_enabled = models.BooleanField(default=True, db_index=True, help_text='是否启用') - - # 执行统计 - run_count = models.IntegerField(default=0, help_text='已执行次数') - last_run_time = models.DateTimeField(null=True, blank=True, help_text='上次执行时间') - next_run_time = models.DateTimeField(null=True, blank=True, help_text='下次执行时间') - - # 时间戳 - created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间') - updated_at = models.DateTimeField(auto_now=True, help_text='更新时间') - - class Meta: - db_table = 'scheduled_scan' - verbose_name = '定时扫描任务' - verbose_name_plural = '定时扫描任务' - ordering = ['-created_at'] - indexes = [ - models.Index(fields=['-created_at']), - models.Index(fields=['is_enabled', '-created_at']), - models.Index(fields=['name']), # 优化 name 搜索 - ] - - def __str__(self): - return f"ScheduledScan #{self.id} - {self.name}" \ No newline at end of file diff --git a/backend/apps/scan/models/__init__.py b/backend/apps/scan/models/__init__.py new file mode 100644 index 00000000..d087242c --- /dev/null +++ b/backend/apps/scan/models/__init__.py @@ -0,0 +1,18 @@ +"""Scan Models - 统一导出""" + +from .scan_models import Scan, SoftDeleteManager +from .scan_log_model import ScanLog +from .scheduled_scan_model import ScheduledScan +from .subfinder_provider_settings_model import SubfinderProviderSettings + +# 兼容旧名称（已废弃，请使用 SubfinderProviderSettings） +ProviderSettings = SubfinderProviderSettings + +__all__ = [ + 'Scan', + 'ScanLog', + 'ScheduledScan', + 'SoftDeleteManager', + 'SubfinderProviderSettings', + 'ProviderSettings', # 兼容旧名称 +] diff --git a/backend/apps/scan/models/scan_log_model.py b/backend/apps/scan/models/scan_log_model.py new file mode 100644 index 00000000..ebac469b --- /dev/null +++ b/backend/apps/scan/models/scan_log_model.py @@ -0,0 +1,41 @@ +"""扫描日志模型""" + +from django.db import models + + +class ScanLog(models.Model): + """扫描日志模型""" + + class Level(models.TextChoices): + INFO = 'info', 'Info' + WARNING = 'warning', 'Warning' + ERROR = 'error', 'Error' + + id = models.BigAutoField(primary_key=True) + scan = models.ForeignKey( + 'Scan', + on_delete=models.CASCADE, + related_name='logs', + db_index=True, + help_text='关联的扫描任务' + ) + level = models.CharField( + max_length=10, + choices=Level.choices, + default=Level.INFO, + help_text='日志级别' + ) + content = models.TextField(help_text='日志内容') + created_at = models.DateTimeField(auto_now_add=True, db_index=True, help_text='创建时间') + + class Meta: + db_table = 'scan_log' + verbose_name = '扫描日志' + verbose_name_plural = '扫描日志' + ordering = ['created_at'] + indexes = [ + models.Index(fields=['scan', 'created_at']), + ] + + def __str__(self): + return f"[{self.level}] {self.content[:50]}" diff --git a/backend/apps/scan/models/scan_models.py b/backend/apps/scan/models/scan_models.py new file mode 100644 index 00000000..f8801860 --- /dev/null +++ b/backend/apps/scan/models/scan_models.py @@ -0,0 +1,106 @@ +"""扫描相关模型""" + +from django.db import models +from django.contrib.postgres.fields import ArrayField + +from apps.common.definitions import ScanStatus + + +class SoftDeleteManager(models.Manager): + """软删除管理器：默认只返回未删除的记录""" + + def get_queryset(self): + return super().get_queryset().filter(deleted_at__isnull=True) + + +class Scan(models.Model): + """扫描任务模型""" + + id = models.AutoField(primary_key=True) + + target = models.ForeignKey('targets.Target', on_delete=models.CASCADE, related_name='scans', help_text='扫描目标') + + # 多引擎支持字段 + engine_ids = ArrayField( + models.IntegerField(), + default=list, + help_text='引擎 ID 列表' + ) + engine_names = models.JSONField( + default=list, + help_text='引擎名称列表，如 ["引擎A", "引擎B"]' + ) + yaml_configuration = models.TextField( + default='', + help_text='YAML 格式的扫描配置' + ) + + created_at = models.DateTimeField(auto_now_add=True, help_text='任务创建时间') + stopped_at = models.DateTimeField(null=True, blank=True, help_text='扫描结束时间') + + status = models.CharField( + max_length=20, + choices=ScanStatus.choices, + default=ScanStatus.INITIATED, + db_index=True, + help_text='任务状态' + ) + + results_dir = models.CharField(max_length=100, blank=True, default='', help_text='结果存储目录') + + container_ids = ArrayField( + models.CharField(max_length=100), + blank=True, + default=list, + help_text='容器 ID 列表（Docker Container ID）' + ) + + worker = models.ForeignKey( + 'engine.WorkerNode', + on_delete=models.SET_NULL, + related_name='scans', + null=True, + blank=True, + help_text='执行扫描的 Worker 节点' + ) + + error_message = models.CharField(max_length=2000, blank=True, default='', help_text='错误信息') + + # ==================== 软删除字段 ==================== + deleted_at = models.DateTimeField(null=True, blank=True, db_index=True, help_text='删除时间（NULL表示未删除）') + + # ==================== 管理器 ==================== + objects = SoftDeleteManager() # 默认管理器：只返回未删除的记录 + all_objects = models.Manager() # 全量管理器：包括已删除的记录（用于硬删除） + + # ==================== 进度跟踪字段 ==================== + progress = models.IntegerField(default=0, help_text='扫描进度 0-100') + current_stage = models.CharField(max_length=50, blank=True, default='', help_text='当前扫描阶段') + stage_progress = models.JSONField(default=dict, help_text='各阶段进度详情') + + # ==================== 缓存统计字段 ==================== + cached_subdomains_count = models.IntegerField(default=0, help_text='缓存的子域名数量') + cached_websites_count = models.IntegerField(default=0, help_text='缓存的网站数量') + cached_endpoints_count = models.IntegerField(default=0, help_text='缓存的端点数量') + cached_ips_count = models.IntegerField(default=0, help_text='缓存的IP地址数量') + cached_directories_count = models.IntegerField(default=0, help_text='缓存的目录数量') + cached_vulns_total = models.IntegerField(default=0, help_text='缓存的漏洞总数') + cached_vulns_critical = models.IntegerField(default=0, help_text='缓存的严重漏洞数量') + cached_vulns_high = models.IntegerField(default=0, help_text='缓存的高危漏洞数量') + cached_vulns_medium = models.IntegerField(default=0, help_text='缓存的中危漏洞数量') + cached_vulns_low = models.IntegerField(default=0, help_text='缓存的低危漏洞数量') + stats_updated_at = models.DateTimeField(null=True, blank=True, help_text='统计数据最后更新时间') + + class Meta: + db_table = 'scan' + verbose_name = '扫描任务' + verbose_name_plural = '扫描任务' + ordering = ['-created_at'] + indexes = [ + models.Index(fields=['-created_at']), + models.Index(fields=['target']), + models.Index(fields=['deleted_at', '-created_at']), + ] + + def __str__(self): + return f"Scan #{self.id} - {self.target.name}" diff --git a/backend/apps/scan/models/scheduled_scan_model.py b/backend/apps/scan/models/scheduled_scan_model.py new file mode 100644 index 00000000..7b946e55 --- /dev/null +++ b/backend/apps/scan/models/scheduled_scan_model.py @@ -0,0 +1,73 @@ +"""定时扫描任务模型""" + +from django.db import models +from django.contrib.postgres.fields import ArrayField + + +class ScheduledScan(models.Model): + """定时扫描任务模型""" + + id = models.AutoField(primary_key=True) + + name = models.CharField(max_length=200, help_text='任务名称') + + engine_ids = ArrayField( + models.IntegerField(), + default=list, + help_text='引擎 ID 列表' + ) + engine_names = models.JSONField( + default=list, + help_text='引擎名称列表，如 ["引擎A", "引擎B"]' + ) + yaml_configuration = models.TextField( + default='', + help_text='YAML 格式的扫描配置' + ) + + organization = models.ForeignKey( + 'targets.Organization', + on_delete=models.CASCADE, + related_name='scheduled_scans', + null=True, + blank=True, + help_text='扫描组织（设置后执行时动态获取组织下所有目标）' + ) + + target = models.ForeignKey( + 'targets.Target', + on_delete=models.CASCADE, + related_name='scheduled_scans', + null=True, + blank=True, + help_text='扫描单个目标（与 organization 二选一）' + ) + + cron_expression = models.CharField( + max_length=100, + default='0 2 * * *', + help_text='Cron 表达式，格式：分 时 日 月 周' + ) + + is_enabled = models.BooleanField(default=True, db_index=True, help_text='是否启用') + + run_count = models.IntegerField(default=0, help_text='已执行次数') + last_run_time = models.DateTimeField(null=True, blank=True, help_text='上次执行时间') + next_run_time = models.DateTimeField(null=True, blank=True, help_text='下次执行时间') + + created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间') + updated_at = models.DateTimeField(auto_now=True, help_text='更新时间') + + class Meta: + db_table = 'scheduled_scan' + verbose_name = '定时扫描任务' + verbose_name_plural = '定时扫描任务' + ordering = ['-created_at'] + indexes = [ + models.Index(fields=['-created_at']), + models.Index(fields=['is_enabled', '-created_at']), + models.Index(fields=['name']), + ] + + def __str__(self): + return f"ScheduledScan #{self.id} - {self.name}" diff --git a/backend/apps/scan/models/subfinder_provider_settings_model.py b/backend/apps/scan/models/subfinder_provider_settings_model.py new file mode 100644 index 00000000..740e62b7 --- /dev/null +++ b/backend/apps/scan/models/subfinder_provider_settings_model.py @@ -0,0 +1,64 @@ +"""Subfinder Provider 配置模型（单例模式） + +用于存储 subfinder 第三方数据源的 API Key 配置 +""" + +from django.db import models + + +class SubfinderProviderSettings(models.Model): + """ + Subfinder Provider 配置（单例模式） + 存储第三方数据源的 API Key 配置，用于 subfinder 子域名发现 + + 支持的 Provider: + - fofa: email + api_key (composite) + - censys: api_id + api_secret (composite) + - hunter, shodan, zoomeye, securitytrails, threatbook, quake: api_key (single) + """ + + providers = models.JSONField( + default=dict, + help_text='各 Provider 的 API Key 配置' + ) + + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + db_table = 'subfinder_provider_settings' + verbose_name = 'Subfinder Provider 配置' + verbose_name_plural = 'Subfinder Provider 配置' + + DEFAULT_PROVIDERS = { + 'fofa': {'enabled': False, 'email': '', 'api_key': ''}, + 'hunter': {'enabled': False, 'api_key': ''}, + 'shodan': {'enabled': False, 'api_key': ''}, + 'censys': {'enabled': False, 'api_id': '', 'api_secret': ''}, + 'zoomeye': {'enabled': False, 'api_key': ''}, + 'securitytrails': {'enabled': False, 'api_key': ''}, + 'threatbook': {'enabled': False, 'api_key': ''}, + 'quake': {'enabled': False, 'api_key': ''}, + } + + def save(self, *args, **kwargs): + self.pk = 1 + super().save(*args, **kwargs) + + @classmethod + def get_instance(cls) -> 'SubfinderProviderSettings': + """获取或创建单例实例""" + obj, _ = cls.objects.get_or_create( + pk=1, + defaults={'providers': cls.DEFAULT_PROVIDERS.copy()} + ) + return obj + + def get_provider_config(self, provider: str) -> dict: + """获取指定 Provider 的配置""" + return self.providers.get(provider, self.DEFAULT_PROVIDERS.get(provider, {})) + + def is_provider_enabled(self, provider: str) -> bool: + """检查指定 Provider 是否启用""" + config = self.get_provider_config(provider) + return config.get('enabled', False) diff --git a/backend/apps/scan/serializers.py b/backend/apps/scan/serializers.py deleted file mode 100644 index eeeada1b..00000000 --- a/backend/apps/scan/serializers.py +++ /dev/null @@ -1,411 +0,0 @@ -from rest_framework import serializers -from django.db.models import Count -import yaml - -from .models import Scan, ScheduledScan, ScanLog - - -# ==================== 扫描日志序列化器 ==================== - -class ScanLogSerializer(serializers.ModelSerializer): - """扫描日志序列化器""" - - class Meta: - model = ScanLog - fields = ['id', 'level', 'content', 'created_at'] - - -# ==================== 通用验证 Mixin ==================== - -class DuplicateKeyLoader(yaml.SafeLoader): - """自定义 YAML Loader，检测重复 key""" - pass - - -def _check_duplicate_keys(loader, node, deep=False): - """检测 YAML mapping 中的重复 key""" - mapping = {} - for key_node, value_node in node.value: - key = loader.construct_object(key_node, deep=deep) - if key in mapping: - raise yaml.constructor.ConstructorError( - "while constructing a mapping", node.start_mark, - f"发现重复的配置项 '{key}'，后面的配置会覆盖前面的配置，请删除重复项", key_node.start_mark - ) - mapping[key] = loader.construct_object(value_node, deep=deep) - return mapping - - -DuplicateKeyLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - _check_duplicate_keys -) - - -class ScanConfigValidationMixin: - """扫描配置验证 Mixin，提供通用的验证方法""" - - def validate_configuration(self, value): - """验证 YAML 配置格式，包括检测重复 key""" - import yaml - - if not value or not value.strip(): - raise serializers.ValidationError("configuration 不能为空") - - try: - # 使用自定义 Loader 检测重复 key - yaml.load(value, Loader=DuplicateKeyLoader) - except yaml.YAMLError as e: - raise serializers.ValidationError(f"无效的 YAML 格式: {str(e)}") - - return value - - def validate_engine_ids(self, value): - """验证引擎 ID 列表""" - if not value: - raise serializers.ValidationError("engine_ids 不能为空，请至少选择一个扫描引擎") - return value - - def validate_engine_names(self, value): - """验证引擎名称列表""" - if not value: - raise serializers.ValidationError("engine_names 不能为空") - return value - - -# ==================== 扫描任务序列化器 ==================== - - -class ScanSerializer(serializers.ModelSerializer): - """扫描任务序列化器""" - target_name = serializers.SerializerMethodField() - - class Meta: - model = Scan - fields = [ - 'id', 'target', 'target_name', 'engine_ids', 'engine_names', - 'created_at', 'stopped_at', 'status', 'results_dir', - 'container_ids', 'error_message' - ] - read_only_fields = [ - 'id', 'created_at', 'stopped_at', 'results_dir', - 'container_ids', 'error_message', 'status' - ] - - def get_target_name(self, obj): - """获取目标名称""" - return obj.target.name if obj.target else None - - -class ScanHistorySerializer(serializers.ModelSerializer): - """扫描历史列表专用序列化器 - - 为前端扫描历史页面提供优化的数据格式，包括： - - 扫描汇总统计（子域名、端点、漏洞数量） - - 进度百分比和当前阶段 - - 执行节点信息 - """ - - # 字段映射 - target_name = serializers.CharField(source='target.name', read_only=True) - worker_name = serializers.CharField(source='worker.name', read_only=True, allow_null=True) - - # 计算字段 - summary = serializers.SerializerMethodField() - - # 进度跟踪字段（直接从模型读取） - progress = serializers.IntegerField(read_only=True) - current_stage = serializers.CharField(read_only=True) - stage_progress = serializers.JSONField(read_only=True) - - class Meta: - model = Scan - fields = [ - 'id', 'target', 'target_name', 'engine_ids', 'engine_names', - 'worker_name', 'created_at', 'status', 'error_message', 'summary', - 'progress', 'current_stage', 'stage_progress' - ] - - def get_summary(self, obj): - """获取扫描汇总数据。 - - 设计原则： - - 子域名/网站/端点/IP/目录使用缓存字段（避免实时 COUNT） - - 漏洞统计使用 Scan 上的缓存字段，在扫描结束时统一聚合 - """ - # 1. 使用缓存字段构建基础统计（子域名、网站、端点、IP、目录） - summary = { - 'subdomains': obj.cached_subdomains_count or 0, - 'websites': obj.cached_websites_count or 0, - 'endpoints': obj.cached_endpoints_count or 0, - 'ips': obj.cached_ips_count or 0, - 'directories': obj.cached_directories_count or 0, - } - - # 2. 使用 Scan 模型上的缓存漏洞统计（按严重性聚合） - summary['vulnerabilities'] = { - 'total': obj.cached_vulns_total or 0, - 'critical': obj.cached_vulns_critical or 0, - 'high': obj.cached_vulns_high or 0, - 'medium': obj.cached_vulns_medium or 0, - 'low': obj.cached_vulns_low or 0, - } - - return summary - - -class QuickScanSerializer(ScanConfigValidationMixin, serializers.Serializer): - """ - 快速扫描序列化器 - - 功能： - - 接收目标列表和 YAML 配置 - - 自动创建/获取目标 - - 立即发起扫描 - """ - - # 批量创建的最大数量限制 - MAX_BATCH_SIZE = 1000 - - # 目标列表 - targets = serializers.ListField( - child=serializers.DictField(), - help_text='目标列表，每个目标包含 name 字段' - ) - - # YAML 配置（必填） - configuration = serializers.CharField( - required=True, - help_text='YAML 格式的扫描配置（必填）' - ) - - # 扫描引擎 ID 列表（必填，用于记录和显示） - engine_ids = serializers.ListField( - child=serializers.IntegerField(), - required=True, - help_text='使用的扫描引擎 ID 列表（必填）' - ) - - # 引擎名称列表（必填，用于记录和显示） - engine_names = serializers.ListField( - child=serializers.CharField(), - required=True, - help_text='引擎名称列表（必填）' - ) - - def validate_targets(self, value): - """验证目标列表""" - if not value: - raise serializers.ValidationError("目标列表不能为空") - - # 检查数量限制，防止服务器过载 - if len(value) > self.MAX_BATCH_SIZE: - raise serializers.ValidationError( - f"快速扫描最多支持 {self.MAX_BATCH_SIZE} 个目标，当前提交了 {len(value)} 个" - ) - - # 验证每个目标的必填字段 - for idx, target in enumerate(value): - if 'name' not in target: - raise serializers.ValidationError(f"第 {idx + 1} 个目标缺少 name 字段") - if not target['name']: - raise serializers.ValidationError(f"第 {idx + 1} 个目标的 name 不能为空") - - return value - - -# ==================== 定时扫描序列化器 ==================== - -class ScheduledScanSerializer(serializers.ModelSerializer): - """定时扫描任务序列化器（用于列表和详情）""" - - # 关联字段 - organization_id = serializers.IntegerField(source='organization.id', read_only=True, allow_null=True) - organization_name = serializers.CharField(source='organization.name', read_only=True, allow_null=True) - target_id = serializers.IntegerField(source='target.id', read_only=True, allow_null=True) - target_name = serializers.CharField(source='target.name', read_only=True, allow_null=True) - scan_mode = serializers.SerializerMethodField() - - class Meta: - model = ScheduledScan - fields = [ - 'id', 'name', - 'engine_ids', 'engine_names', - 'organization_id', 'organization_name', - 'target_id', 'target_name', - 'scan_mode', - 'cron_expression', - 'is_enabled', - 'run_count', 'last_run_time', 'next_run_time', - 'created_at', 'updated_at' - ] - read_only_fields = [ - 'id', 'run_count', - 'last_run_time', 'next_run_time', - 'created_at', 'updated_at' - ] - - def get_scan_mode(self, obj): - """获取扫描模式：organization 或 target""" - return 'organization' if obj.organization_id else 'target' - - -class CreateScheduledScanSerializer(ScanConfigValidationMixin, serializers.Serializer): - """创建定时扫描任务序列化器 - - 扫描模式（二选一）： - - 组织扫描：提供 organization_id，执行时动态获取组织下所有目标 - - 目标扫描：提供 target_id，扫描单个目标 - """ - - name = serializers.CharField(max_length=200, help_text='任务名称') - - # YAML 配置（必填） - configuration = serializers.CharField( - required=True, - help_text='YAML 格式的扫描配置（必填）' - ) - - # 扫描引擎 ID 列表（必填，用于记录和显示） - engine_ids = serializers.ListField( - child=serializers.IntegerField(), - required=True, - help_text='扫描引擎 ID 列表（必填）' - ) - - # 引擎名称列表（必填，用于记录和显示） - engine_names = serializers.ListField( - child=serializers.CharField(), - required=True, - help_text='引擎名称列表（必填）' - ) - - # 组织扫描模式 - organization_id = serializers.IntegerField( - required=False, - allow_null=True, - help_text='组织 ID（组织扫描模式：执行时动态获取组织下所有目标）' - ) - - # 目标扫描模式 - target_id = serializers.IntegerField( - required=False, - allow_null=True, - help_text='目标 ID（目标扫描模式：扫描单个目标）' - ) - - cron_expression = serializers.CharField( - max_length=100, - default='0 2 * * *', - help_text='Cron 表达式，格式：分 时 日 月 周' - ) - is_enabled = serializers.BooleanField(default=True, help_text='是否立即启用') - - def validate(self, data): - """验证 organization_id 和 target_id 互斥""" - organization_id = data.get('organization_id') - target_id = data.get('target_id') - - if not organization_id and not target_id: - raise serializers.ValidationError('必须提供 organization_id 或 target_id 其中之一') - - if organization_id and target_id: - raise serializers.ValidationError('organization_id 和 target_id 只能提供其中之一') - - return data - - -class InitiateScanSerializer(ScanConfigValidationMixin, serializers.Serializer): - """发起扫描任务序列化器 - - 扫描模式（二选一）： - - 组织扫描：提供 organization_id，扫描组织下所有目标 - - 目标扫描：提供 target_id，扫描单个目标 - """ - - # YAML 配置（必填） - configuration = serializers.CharField( - required=True, - help_text='YAML 格式的扫描配置（必填）' - ) - - # 扫描引擎 ID 列表（必填） - engine_ids = serializers.ListField( - child=serializers.IntegerField(), - required=True, - help_text='扫描引擎 ID 列表（必填）' - ) - - # 引擎名称列表（必填） - engine_names = serializers.ListField( - child=serializers.CharField(), - required=True, - help_text='引擎名称列表（必填）' - ) - - # 组织扫描模式 - organization_id = serializers.IntegerField( - required=False, - allow_null=True, - help_text='组织 ID（组织扫描模式）' - ) - - # 目标扫描模式 - target_id = serializers.IntegerField( - required=False, - allow_null=True, - help_text='目标 ID（目标扫描模式）' - ) - - def validate(self, data): - """验证 organization_id 和 target_id 互斥""" - organization_id = data.get('organization_id') - target_id = data.get('target_id') - - if not organization_id and not target_id: - raise serializers.ValidationError('必须提供 organization_id 或 target_id 其中之一') - - if organization_id and target_id: - raise serializers.ValidationError('organization_id 和 target_id 只能提供其中之一') - - return data - - -class UpdateScheduledScanSerializer(serializers.Serializer): - """更新定时扫描任务序列化器""" - - name = serializers.CharField(max_length=200, required=False, help_text='任务名称') - engine_ids = serializers.ListField( - child=serializers.IntegerField(), - required=False, - help_text='扫描引擎 ID 列表' - ) - - # 组织扫描模式 - organization_id = serializers.IntegerField( - required=False, - allow_null=True, - help_text='组织 ID（设置后清空 target_id）' - ) - - # 目标扫描模式 - target_id = serializers.IntegerField( - required=False, - allow_null=True, - help_text='目标 ID（设置后清空 organization_id）' - ) - - cron_expression = serializers.CharField(max_length=100, required=False, help_text='Cron 表达式') - is_enabled = serializers.BooleanField(required=False, help_text='是否启用') - - def validate_engine_ids(self, value): - """验证引擎 ID 列表""" - if value is not None and not value: - raise serializers.ValidationError("engine_ids 不能为空") - return value - - -class ToggleScheduledScanSerializer(serializers.Serializer): - """切换定时扫描启用状态序列化器""" - - is_enabled = serializers.BooleanField(help_text='是否启用') diff --git a/backend/apps/scan/serializers/__init__.py b/backend/apps/scan/serializers/__init__.py new file mode 100644 index 00000000..a71bae9d --- /dev/null +++ b/backend/apps/scan/serializers/__init__.py @@ -0,0 +1,40 @@ +"""Scan Serializers - 统一导出""" + +from .mixins import ScanConfigValidationMixin +from .scan_serializers import ( + ScanSerializer, + ScanHistorySerializer, + QuickScanSerializer, + InitiateScanSerializer, +) +from .scan_log_serializers import ScanLogSerializer +from .scheduled_scan_serializers import ( + ScheduledScanSerializer, + CreateScheduledScanSerializer, + UpdateScheduledScanSerializer, + ToggleScheduledScanSerializer, +) +from .subfinder_provider_settings_serializers import SubfinderProviderSettingsSerializer + +# 兼容旧名称 +ProviderSettingsSerializer = SubfinderProviderSettingsSerializer + +__all__ = [ + # Mixins + 'ScanConfigValidationMixin', + # Scan + 'ScanSerializer', + 'ScanHistorySerializer', + 'QuickScanSerializer', + 'InitiateScanSerializer', + # ScanLog + 'ScanLogSerializer', + # Scheduled Scan + 'ScheduledScanSerializer', + 'CreateScheduledScanSerializer', + 'UpdateScheduledScanSerializer', + 'ToggleScheduledScanSerializer', + # Subfinder Provider Settings + 'SubfinderProviderSettingsSerializer', + 'ProviderSettingsSerializer', # 兼容旧名称 +] diff --git a/backend/apps/scan/serializers/mixins.py b/backend/apps/scan/serializers/mixins.py new file mode 100644 index 00000000..1153c708 --- /dev/null +++ b/backend/apps/scan/serializers/mixins.py @@ -0,0 +1,57 @@ +"""序列化器通用 Mixin 和工具类""" + +from rest_framework import serializers +import yaml + + +class DuplicateKeyLoader(yaml.SafeLoader): + """自定义 YAML Loader，检测重复 key""" + pass + + +def _check_duplicate_keys(loader, node, deep=False): + """检测 YAML mapping 中的重复 key""" + mapping = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if key in mapping: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", node.start_mark, + f"发现重复的配置项 '{key}'，后面的配置会覆盖前面的配置，请删除重复项", key_node.start_mark + ) + mapping[key] = loader.construct_object(value_node, deep=deep) + return mapping + + +DuplicateKeyLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + _check_duplicate_keys +) + + +class ScanConfigValidationMixin: + """扫描配置验证 Mixin""" + + def validate_configuration(self, value): + """验证 YAML 配置格式""" + if not value or not value.strip(): + raise serializers.ValidationError("configuration 不能为空") + + try: + yaml.load(value, Loader=DuplicateKeyLoader) + except yaml.YAMLError as e: + raise serializers.ValidationError(f"无效的 YAML 格式: {str(e)}") + + return value + + def validate_engine_ids(self, value): + """验证引擎 ID 列表""" + if not value: + raise serializers.ValidationError("engine_ids 不能为空，请至少选择一个扫描引擎") + return value + + def validate_engine_names(self, value): + """验证引擎名称列表""" + if not value: + raise serializers.ValidationError("engine_names 不能为空") + return value diff --git a/backend/apps/scan/serializers/scan_log_serializers.py b/backend/apps/scan/serializers/scan_log_serializers.py new file mode 100644 index 00000000..bd6afdde --- /dev/null +++ b/backend/apps/scan/serializers/scan_log_serializers.py @@ -0,0 +1,13 @@ +"""扫描日志序列化器""" + +from rest_framework import serializers + +from ..models import ScanLog + + +class ScanLogSerializer(serializers.ModelSerializer): + """扫描日志序列化器""" + + class Meta: + model = ScanLog + fields = ['id', 'level', 'content', 'created_at'] diff --git a/backend/apps/scan/serializers/scan_serializers.py b/backend/apps/scan/serializers/scan_serializers.py new file mode 100644 index 00000000..2e7991e4 --- /dev/null +++ b/backend/apps/scan/serializers/scan_serializers.py @@ -0,0 +1,111 @@ +"""扫描任务序列化器""" + +from rest_framework import serializers + +from ..models import Scan +from .mixins import ScanConfigValidationMixin + + +class ScanSerializer(serializers.ModelSerializer): + """扫描任务序列化器""" + target_name = serializers.SerializerMethodField() + + class Meta: + model = Scan + fields = [ + 'id', 'target', 'target_name', 'engine_ids', 'engine_names', + 'created_at', 'stopped_at', 'status', 'results_dir', + 'container_ids', 'error_message' + ] + read_only_fields = [ + 'id', 'created_at', 'stopped_at', 'results_dir', + 'container_ids', 'error_message', 'status' + ] + + def get_target_name(self, obj): + return obj.target.name if obj.target else None + + +class ScanHistorySerializer(serializers.ModelSerializer): + """扫描历史列表序列化器""" + + target_name = serializers.CharField(source='target.name', read_only=True) + worker_name = serializers.CharField(source='worker.name', read_only=True, allow_null=True) + summary = serializers.SerializerMethodField() + progress = serializers.IntegerField(read_only=True) + current_stage = serializers.CharField(read_only=True) + stage_progress = serializers.JSONField(read_only=True) + + class Meta: + model = Scan + fields = [ + 'id', 'target', 'target_name', 'engine_ids', 'engine_names', + 'worker_name', 'created_at', 'status', 'error_message', 'summary', + 'progress', 'current_stage', 'stage_progress' + ] + + def get_summary(self, obj): + summary = { + 'subdomains': obj.cached_subdomains_count or 0, + 'websites': obj.cached_websites_count or 0, + 'endpoints': obj.cached_endpoints_count or 0, + 'ips': obj.cached_ips_count or 0, + 'directories': obj.cached_directories_count or 0, + } + summary['vulnerabilities'] = { + 'total': obj.cached_vulns_total or 0, + 'critical': obj.cached_vulns_critical or 0, + 'high': obj.cached_vulns_high or 0, + 'medium': obj.cached_vulns_medium or 0, + 'low': obj.cached_vulns_low or 0, + } + return summary + + +class QuickScanSerializer(ScanConfigValidationMixin, serializers.Serializer): + """快速扫描序列化器""" + + MAX_BATCH_SIZE = 1000 + + targets = serializers.ListField( + child=serializers.DictField(), + help_text='目标列表，每个目标包含 name 字段' + ) + configuration = serializers.CharField(required=True, help_text='YAML 格式的扫描配置') + engine_ids = serializers.ListField(child=serializers.IntegerField(), required=True) + engine_names = serializers.ListField(child=serializers.CharField(), required=True) + + def validate_targets(self, value): + if not value: + raise serializers.ValidationError("目标列表不能为空") + if len(value) > self.MAX_BATCH_SIZE: + raise serializers.ValidationError( + f"快速扫描最多支持 {self.MAX_BATCH_SIZE} 个目标，当前提交了 {len(value)} 个" + ) + for idx, target in enumerate(value): + if 'name' not in target: + raise serializers.ValidationError(f"第 {idx + 1} 个目标缺少 name 字段") + if not target['name']: + raise serializers.ValidationError(f"第 {idx + 1} 个目标的 name 不能为空") + return value + + +class InitiateScanSerializer(ScanConfigValidationMixin, serializers.Serializer): + """发起扫描任务序列化器""" + + configuration = serializers.CharField(required=True, help_text='YAML 格式的扫描配置') + engine_ids = serializers.ListField(child=serializers.IntegerField(), required=True) + engine_names = serializers.ListField(child=serializers.CharField(), required=True) + organization_id = serializers.IntegerField(required=False, allow_null=True) + target_id = serializers.IntegerField(required=False, allow_null=True) + + def validate(self, data): + organization_id = data.get('organization_id') + target_id = data.get('target_id') + + if not organization_id and not target_id: + raise serializers.ValidationError('必须提供 organization_id 或 target_id 其中之一') + if organization_id and target_id: + raise serializers.ValidationError('organization_id 和 target_id 只能提供其中之一') + + return data diff --git a/backend/apps/scan/serializers/scheduled_scan_serializers.py b/backend/apps/scan/serializers/scheduled_scan_serializers.py new file mode 100644 index 00000000..c297d554 --- /dev/null +++ b/backend/apps/scan/serializers/scheduled_scan_serializers.py @@ -0,0 +1,84 @@ +"""定时扫描序列化器""" + +from rest_framework import serializers + +from ..models import ScheduledScan +from .mixins import ScanConfigValidationMixin + + +class ScheduledScanSerializer(serializers.ModelSerializer): + """定时扫描任务序列化器（用于列表和详情）""" + + organization_id = serializers.IntegerField(source='organization.id', read_only=True, allow_null=True) + organization_name = serializers.CharField(source='organization.name', read_only=True, allow_null=True) + target_id = serializers.IntegerField(source='target.id', read_only=True, allow_null=True) + target_name = serializers.CharField(source='target.name', read_only=True, allow_null=True) + scan_mode = serializers.SerializerMethodField() + + class Meta: + model = ScheduledScan + fields = [ + 'id', 'name', + 'engine_ids', 'engine_names', + 'organization_id', 'organization_name', + 'target_id', 'target_name', + 'scan_mode', + 'cron_expression', + 'is_enabled', + 'run_count', 'last_run_time', 'next_run_time', + 'created_at', 'updated_at' + ] + read_only_fields = [ + 'id', 'run_count', + 'last_run_time', 'next_run_time', + 'created_at', 'updated_at' + ] + + def get_scan_mode(self, obj): + return 'organization' if obj.organization_id else 'target' + + +class CreateScheduledScanSerializer(ScanConfigValidationMixin, serializers.Serializer): + """创建定时扫描任务序列化器""" + + name = serializers.CharField(max_length=200, help_text='任务名称') + configuration = serializers.CharField(required=True, help_text='YAML 格式的扫描配置') + engine_ids = serializers.ListField(child=serializers.IntegerField(), required=True) + engine_names = serializers.ListField(child=serializers.CharField(), required=True) + organization_id = serializers.IntegerField(required=False, allow_null=True) + target_id = serializers.IntegerField(required=False, allow_null=True) + cron_expression = serializers.CharField(max_length=100, default='0 2 * * *') + is_enabled = serializers.BooleanField(default=True) + + def validate(self, data): + organization_id = data.get('organization_id') + target_id = data.get('target_id') + + if not organization_id and not target_id: + raise serializers.ValidationError('必须提供 organization_id 或 target_id 其中之一') + if organization_id and target_id: + raise serializers.ValidationError('organization_id 和 target_id 只能提供其中之一') + + return data + + +class UpdateScheduledScanSerializer(serializers.Serializer): + """更新定时扫描任务序列化器""" + + name = serializers.CharField(max_length=200, required=False) + engine_ids = serializers.ListField(child=serializers.IntegerField(), required=False) + organization_id = serializers.IntegerField(required=False, allow_null=True) + target_id = serializers.IntegerField(required=False, allow_null=True) + cron_expression = serializers.CharField(max_length=100, required=False) + is_enabled = serializers.BooleanField(required=False) + + def validate_engine_ids(self, value): + if value is not None and not value: + raise serializers.ValidationError("engine_ids 不能为空") + return value + + +class ToggleScheduledScanSerializer(serializers.Serializer): + """切换定时扫描启用状态序列化器""" + + is_enabled = serializers.BooleanField(help_text='是否启用') diff --git a/backend/apps/scan/serializers/subfinder_provider_settings_serializers.py b/backend/apps/scan/serializers/subfinder_provider_settings_serializers.py new file mode 100644 index 00000000..741f7a4e --- /dev/null +++ b/backend/apps/scan/serializers/subfinder_provider_settings_serializers.py @@ -0,0 +1,55 @@ +"""Subfinder Provider 配置序列化器""" + +from rest_framework import serializers + + +class SubfinderProviderSettingsSerializer(serializers.Serializer): + """Subfinder Provider 配置序列化器 + + 支持的 Provider: + - fofa: email + api_key (composite) + - censys: api_id + api_secret (composite) + - hunter, shodan, zoomeye, securitytrails, threatbook, quake: api_key (single) + + 注意：djangorestframework-camel-case 会自动处理 camelCase <-> snake_case 转换 + 所以这里统一使用 snake_case + """ + + VALID_PROVIDERS = { + 'fofa', 'hunter', 'shodan', 'censys', + 'zoomeye', 'securitytrails', 'threatbook', 'quake' + } + + def to_internal_value(self, data): + """验证并转换输入数据""" + if not isinstance(data, dict): + raise serializers.ValidationError('Expected a dictionary') + + result = {} + for provider, config in data.items(): + if provider not in self.VALID_PROVIDERS: + continue + + if not isinstance(config, dict): + continue + + db_config = {'enabled': bool(config.get('enabled', False))} + + if provider == 'fofa': + db_config['email'] = str(config.get('email', '')) + db_config['api_key'] = str(config.get('api_key', '')) + elif provider == 'censys': + db_config['api_id'] = str(config.get('api_id', '')) + db_config['api_secret'] = str(config.get('api_secret', '')) + else: + db_config['api_key'] = str(config.get('api_key', '')) + + result[provider] = db_config + + return result + + def to_representation(self, instance): + """输出数据（数据库格式，camel-case 中间件会自动转换）""" + if isinstance(instance, dict): + return instance + return instance.providers if hasattr(instance, 'providers') else {} diff --git a/backend/apps/scan/services/subfinder_provider_config_service.py b/backend/apps/scan/services/subfinder_provider_config_service.py new file mode 100644 index 00000000..a4af8b5a --- /dev/null +++ b/backend/apps/scan/services/subfinder_provider_config_service.py @@ -0,0 +1,138 @@ +"""Subfinder Provider 配置文件生成服务 + +负责生成 subfinder 的 provider-config.yaml 配置文件 +""" + +import logging +import os +from pathlib import Path +from typing import Optional + +import yaml + +from ..models import SubfinderProviderSettings + +logger = logging.getLogger(__name__) + + +class SubfinderProviderConfigService: + """Subfinder Provider 配置文件生成服务""" + + # Provider 格式定义 + PROVIDER_FORMATS = { + 'fofa': {'type': 'composite', 'format': '{email}:{api_key}'}, + 'censys': {'type': 'composite', 'format': '{api_id}:{api_secret}'}, + 'hunter': {'type': 'single', 'field': 'api_key'}, + 'shodan': {'type': 'single', 'field': 'api_key'}, + 'zoomeye': {'type': 'single', 'field': 'api_key'}, + 'securitytrails': {'type': 'single', 'field': 'api_key'}, + 'threatbook': {'type': 'single', 'field': 'api_key'}, + 'quake': {'type': 'single', 'field': 'api_key'}, + } + + def generate(self, output_dir: str) -> Optional[str]: + """ + 生成 provider-config.yaml 文件 + + Args: + output_dir: 输出目录路径 + + Returns: + 生成的配置文件路径，如果没有启用的 provider 则返回 None + """ + settings = SubfinderProviderSettings.get_instance() + + config = {} + has_enabled = False + + for provider, format_info in self.PROVIDER_FORMATS.items(): + provider_config = settings.providers.get(provider, {}) + + if not provider_config.get('enabled'): + config[provider] = [] + continue + + value = self._build_provider_value(provider, provider_config) + if value: + config[provider] = [value] # 单个 key 放入数组 + has_enabled = True + logger.debug(f"Provider {provider} 已启用") + else: + config[provider] = [] + + # 检查是否有任何启用的 provider + if not has_enabled: + logger.info("没有启用的 Provider，跳过配置文件生成") + return None + + # 确保输出目录存在 + output_path = Path(output_dir) / 'provider-config.yaml' + output_path.parent.mkdir(parents=True, exist_ok=True) + + # 写入 YAML 文件（使用默认列表格式，和 subfinder 一致） + with open(output_path, 'w', encoding='utf-8') as f: + yaml.dump(config, f, default_flow_style=False, allow_unicode=True) + + # 设置文件权限为 600（仅所有者可读写） + os.chmod(output_path, 0o600) + + logger.info(f"Provider 配置文件已生成: {output_path}") + return str(output_path) + + def _build_provider_value(self, provider: str, config: dict) -> Optional[str]: + """根据 provider 格式规则构建配置值 + + Args: + provider: provider 名称 + config: provider 配置字典 + + Returns: + 构建的配置值字符串，如果配置不完整则返回 None + """ + format_info = self.PROVIDER_FORMATS.get(provider) + if not format_info: + return None + + if format_info['type'] == 'composite': + # 复合格式：需要多个字段 + format_str = format_info['format'] + try: + # 提取格式字符串中的字段名 + # 例如 '{email}:{api_key}' -> ['email', 'api_key'] + import re + fields = re.findall(r'\{(\w+)\}', format_str) + + # 检查所有字段是否都有值 + values = {} + for field in fields: + value = config.get(field, '').strip() + if not value: + logger.debug(f"Provider {provider} 缺少字段 {field}") + return None + values[field] = value + + return format_str.format(**values) + except (KeyError, ValueError) as e: + logger.warning(f"构建 {provider} 配置值失败: {e}") + return None + else: + # 单字段格式 + field = format_info['field'] + value = config.get(field, '').strip() + if not value: + logger.debug(f"Provider {provider} 缺少字段 {field}") + return None + return value + + def cleanup(self, config_path: str) -> None: + """清理配置文件 + + Args: + config_path: 配置文件路径 + """ + try: + if config_path and Path(config_path).exists(): + Path(config_path).unlink() + logger.debug(f"已清理配置文件: {config_path}") + except Exception as e: + logger.warning(f"清理配置文件失败: {config_path} - {e}") diff --git a/backend/apps/scan/urls.py b/backend/apps/scan/urls.py index 1254161b..abdeb938 100644 --- a/backend/apps/scan/urls.py +++ b/backend/apps/scan/urls.py @@ -1,6 +1,6 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter -from .views import ScanViewSet, ScheduledScanViewSet, ScanLogListView +from .views import ScanViewSet, ScheduledScanViewSet, ScanLogListView, SubfinderProviderSettingsView from .notifications.views import notification_callback from apps.asset.views import ( SubdomainSnapshotViewSet, WebsiteSnapshotViewSet, DirectorySnapshotViewSet, @@ -31,6 +31,8 @@ urlpatterns = [ path('', include(router.urls)), # Worker 回调 API path('callbacks/notification/', notification_callback, name='notification-callback'), + # API Key 配置 + path('settings/api-keys/', SubfinderProviderSettingsView.as_view(), name='subfinder-provider-settings'), # 扫描日志 API path('scans//logs/', ScanLogListView.as_view(), name='scan-logs-list'), # 嵌套路由：/api/scans/{scan_pk}/xxx/ diff --git a/backend/apps/scan/views/__init__.py b/backend/apps/scan/views/__init__.py index db1c9186..393d7265 100644 --- a/backend/apps/scan/views/__init__.py +++ b/backend/apps/scan/views/__init__.py @@ -3,9 +3,11 @@ from .scan_views import ScanViewSet from .scheduled_scan_views import ScheduledScanViewSet from .scan_log_views import ScanLogListView +from .subfinder_provider_settings_views import SubfinderProviderSettingsView __all__ = [ 'ScanViewSet', 'ScheduledScanViewSet', 'ScanLogListView', + 'SubfinderProviderSettingsView', ] diff --git a/backend/apps/scan/views/subfinder_provider_settings_views.py b/backend/apps/scan/views/subfinder_provider_settings_views.py new file mode 100644 index 00000000..4772a3a3 --- /dev/null +++ b/backend/apps/scan/views/subfinder_provider_settings_views.py @@ -0,0 +1,38 @@ +"""Subfinder Provider 配置视图""" + +import logging +from rest_framework import status +from rest_framework.views import APIView +from rest_framework.response import Response + +from ..models import SubfinderProviderSettings +from ..serializers import SubfinderProviderSettingsSerializer + +logger = logging.getLogger(__name__) + + +class SubfinderProviderSettingsView(APIView): + """Subfinder Provider 配置视图 + + GET /api/settings/api-keys/ - 获取配置 + PUT /api/settings/api-keys/ - 更新配置 + """ + + def get(self, request): + """获取 Subfinder Provider 配置""" + settings = SubfinderProviderSettings.get_instance() + serializer = SubfinderProviderSettingsSerializer(settings.providers) + return Response(serializer.data) + + def put(self, request): + """更新 Subfinder Provider 配置""" + serializer = SubfinderProviderSettingsSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + settings = SubfinderProviderSettings.get_instance() + settings.providers.update(serializer.validated_data) + settings.save() + + logger.info("Subfinder Provider 配置已更新") + + return Response(SubfinderProviderSettingsSerializer(settings.providers).data) diff --git a/frontend/app/[locale]/settings/api-keys/page.tsx b/frontend/app/[locale]/settings/api-keys/page.tsx new file mode 100644 index 00000000..d021638a --- /dev/null +++ b/frontend/app/[locale]/settings/api-keys/page.tsx @@ -0,0 +1,306 @@ +"use client" + +import React, { useState, useEffect } from 'react' +import { IconEye, IconEyeOff, IconWorldSearch, IconRadar2 } from '@tabler/icons-react' + +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card' +import { Button } from '@/components/ui/button' +import { Input } from '@/components/ui/input' +import { Switch } from '@/components/ui/switch' +import { Separator } from '@/components/ui/separator' +import { Badge } from '@/components/ui/badge' +import { Skeleton } from '@/components/ui/skeleton' +import { useApiKeySettings, useUpdateApiKeySettings } from '@/hooks/use-api-key-settings' +import type { ApiKeySettings } from '@/types/api-key-settings.types' + +// 密码输入框组件（带显示/隐藏切换） +function PasswordInput({ value, onChange, placeholder, disabled }: { + value: string + onChange: (value: string) => void + placeholder?: string + disabled?: boolean +}) { + const [show, setShow] = useState(false) + return ( +
+ onChange(e.target.value)} + placeholder={placeholder} + disabled={disabled} + className="pr-10" + /> + +
+ ) +} + +// Provider 配置定义 +const PROVIDERS = [ + { + key: 'fofa', + name: 'FOFA', + description: '网络空间测绘平台，提供全球互联网资产搜索', + icon: IconWorldSearch, + color: 'text-blue-500', + bgColor: 'bg-blue-500/10', + fields: [ + { name: 'email', label: '邮箱', type: 'text', placeholder: 'your@email.com' }, + { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 FOFA API Key' }, + ], + docUrl: 'https://fofa.info/api', + }, + { + key: 'hunter', + name: 'Hunter (鹰图)', + description: '奇安信威胁情报平台，提供网络空间资产测绘', + icon: IconRadar2, + color: 'text-orange-500', + bgColor: 'bg-orange-500/10', + fields: [ + { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 Hunter API Key' }, + ], + docUrl: 'https://hunter.qianxin.com/', + }, + { + key: 'shodan', + name: 'Shodan', + description: '全球最大的互联网设备搜索引擎', + icon: IconWorldSearch, + color: 'text-red-500', + bgColor: 'bg-red-500/10', + fields: [ + { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 Shodan API Key' }, + ], + docUrl: 'https://developer.shodan.io/', + }, + { + key: 'censys', + name: 'Censys', + description: '互联网资产搜索和监控平台', + icon: IconWorldSearch, + color: 'text-purple-500', + bgColor: 'bg-purple-500/10', + fields: [ + { name: 'apiId', label: 'API ID', type: 'text', placeholder: '输入 Censys API ID' }, + { name: 'apiSecret', label: 'API Secret', type: 'password', placeholder: '输入 Censys API Secret' }, + ], + docUrl: 'https://search.censys.io/api', + }, + { + key: 'zoomeye', + name: 'ZoomEye (钟馗之眼)', + description: '知道创宇网络空间搜索引擎', + icon: IconWorldSearch, + color: 'text-green-500', + bgColor: 'bg-green-500/10', + fields: [ + { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 ZoomEye API Key' }, + ], + docUrl: 'https://www.zoomeye.org/doc', + }, + { + key: 'securitytrails', + name: 'SecurityTrails', + description: 'DNS 历史记录和子域名数据平台', + icon: IconWorldSearch, + color: 'text-cyan-500', + bgColor: 'bg-cyan-500/10', + fields: [ + { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 SecurityTrails API Key' }, + ], + docUrl: 'https://securitytrails.com/corp/api', + }, + { + key: 'threatbook', + name: 'ThreatBook (微步在线)', + description: '威胁情报平台，提供域名和 IP 情报查询', + icon: IconWorldSearch, + color: 'text-indigo-500', + bgColor: 'bg-indigo-500/10', + fields: [ + { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 ThreatBook API Key' }, + ], + docUrl: 'https://x.threatbook.com/api', + }, + { + key: 'quake', + name: 'Quake (360)', + description: '360 网络空间测绘系统', + icon: IconWorldSearch, + color: 'text-teal-500', + bgColor: 'bg-teal-500/10', + fields: [ + { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 Quake API Key' }, + ], + docUrl: 'https://quake.360.net/quake/#/help', + }, +] + +// 默认配置 +const DEFAULT_SETTINGS: ApiKeySettings = { + fofa: { enabled: false, email: '', apiKey: '' }, + hunter: { enabled: false, apiKey: '' }, + shodan: { enabled: false, apiKey: '' }, + censys: { enabled: false, apiId: '', apiSecret: '' }, + zoomeye: { enabled: false, apiKey: '' }, + securitytrails: { enabled: false, apiKey: '' }, + threatbook: { enabled: false, apiKey: '' }, + quake: { enabled: false, apiKey: '' }, +} + +export default function ApiKeysSettingsPage() { + const { data: settings, isLoading } = useApiKeySettings() + const updateMutation = useUpdateApiKeySettings() + + const [formData, setFormData] = useState(DEFAULT_SETTINGS) + const [hasChanges, setHasChanges] = useState(false) + + // 当数据加载完成后，更新表单数据 + useEffect(() => { + if (settings) { + setFormData({ ...DEFAULT_SETTINGS, ...settings }) + setHasChanges(false) + } + }, [settings]) + + const updateProvider = (providerKey: string, field: string, value: any) => { + setFormData(prev => ({ + ...prev, + [providerKey]: { + ...prev[providerKey as keyof ApiKeySettings], + [field]: value, + } + })) + setHasChanges(true) + } + + const handleSave = async () => { + updateMutation.mutate(formData) + setHasChanges(false) + } + + const enabledCount = Object.values(formData).filter((p: any) => p?.enabled).length + + if (isLoading) { + return ( +
+
+ + +
+
+ {[1, 2, 3].map((i) => ( + + ))} +
+
+ ) + } + + return ( +
+ {/* 页面标题 */} +
+
+

API 密钥配置

+ {enabledCount > 0 && ( + {enabledCount} 个已启用 + )} +
+

+ 配置第三方数据源的 API 密钥，用于增强子域名发现能力。启用后将在 subfinder 扫描时自动使用。 +

+
+ + {/* Provider 卡片列表 */} +
+ {PROVIDERS.map((provider) => { + const data = formData[provider.key as keyof ApiKeySettings] || {} + const isEnabled = (data as any)?.enabled || false + + return ( + + +
+
+
+ +
+
+
+ {provider.name} + {isEnabled && 已启用} +
+ {provider.description} +
+
+ updateProvider(provider.key, 'enabled', checked)} + /> +
+ + + {/* 展开的配置表单 */} + {isEnabled && ( + + +
+ {provider.fields.map((field) => ( +
+ + {field.type === 'password' ? ( + updateProvider(provider.key, field.name, value)} + placeholder={field.placeholder} + /> + ) : ( + updateProvider(provider.key, field.name, e.target.value)} + placeholder={field.placeholder} + /> + )} +
+ ))} +

+ 获取 API Key： + + {provider.docUrl} + +

+
+ + )} + + ) + })} +
+ + {/* 保存按钮 */} +
+ +
+
+ ) +} diff --git a/frontend/components/app-sidebar.tsx b/frontend/components/app-sidebar.tsx index 6d4c2f5b..e3bcc562 100644 --- a/frontend/components/app-sidebar.tsx +++ b/frontend/components/app-sidebar.tsx @@ -17,6 +17,7 @@ import { IconBug, // Vulnerability icon IconMessageReport, // Feedback icon IconSearch, // Search icon + IconKey, // API Key icon } from "@tabler/icons-react" // Import internationalization hook import { useTranslations } from 'next-intl' @@ -168,6 +169,11 @@ export function AppSidebar({ ...props }: React.ComponentProps) { url: "/settings/notifications/", icon: IconSettings, }, + { + name: t('apiKeys'), + url: "/settings/api-keys/", + icon: IconKey, + }, ] return ( diff --git a/frontend/hooks/use-api-key-settings.ts b/frontend/hooks/use-api-key-settings.ts new file mode 100644 index 00000000..87de5ada --- /dev/null +++ b/frontend/hooks/use-api-key-settings.ts @@ -0,0 +1,29 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { ApiKeySettingsService } from '@/services/api-key-settings.service' +import type { ApiKeySettings } from '@/types/api-key-settings.types' +import { useToastMessages } from '@/lib/toast-helpers' +import { getErrorCode } from '@/lib/response-parser' + +export function useApiKeySettings() { + return useQuery({ + queryKey: ['api-key-settings'], + queryFn: () => ApiKeySettingsService.getSettings(), + }) +} + +export function useUpdateApiKeySettings() { + const qc = useQueryClient() + const toastMessages = useToastMessages() + + return useMutation({ + mutationFn: (data: Partial) => + ApiKeySettingsService.updateSettings(data), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ['api-key-settings'] }) + toastMessages.success('toast.apiKeys.settings.success') + }, + onError: (error: any) => { + toastMessages.errorFromCode(getErrorCode(error?.response?.data), 'toast.apiKeys.settings.error') + }, + }) +} diff --git a/frontend/messages/en.json b/frontend/messages/en.json index f7bbd67b..7f18f7e0 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -319,6 +319,7 @@ "workers": "Workers", "systemLogs": "System Logs", "notifications": "Notifications", + "apiKeys": "API Keys", "help": "Get Help", "feedback": "Feedback" }, @@ -1690,6 +1691,12 @@ "error": "Notification connection error: {message}" } }, + "apiKeys": { + "settings": { + "success": "API key settings saved", + "error": "Failed to save API key settings" + } + }, "tool": { "create": { "success": "Tool created successfully", diff --git a/frontend/messages/zh.json b/frontend/messages/zh.json index 9764d0e8..b4d5bd0a 100644 --- a/frontend/messages/zh.json +++ b/frontend/messages/zh.json @@ -319,6 +319,7 @@ "workers": "扫描节点", "systemLogs": "系统日志", "notifications": "通知设置", + "apiKeys": "API 密钥", "help": "获取帮助", "feedback": "反馈建议" }, @@ -1690,6 +1691,12 @@ "error": "通知连接错误: {message}" } }, + "apiKeys": { + "settings": { + "success": "API 密钥配置已保存", + "error": "保存 API 密钥配置失败" + } + }, "tool": { "create": { "success": "工具创建成功", diff --git a/frontend/services/api-key-settings.service.ts b/frontend/services/api-key-settings.service.ts new file mode 100644 index 00000000..f5e5f9cc --- /dev/null +++ b/frontend/services/api-key-settings.service.ts @@ -0,0 +1,14 @@ +import { api } from '@/lib/api-client' +import type { ApiKeySettings } from '@/types/api-key-settings.types' + +export class ApiKeySettingsService { + static async getSettings(): Promise { + const res = await api.get('/settings/api-keys/') + return res.data + } + + static async updateSettings(data: Partial): Promise { + const res = await api.put('/settings/api-keys/', data) + return res.data + } +} diff --git a/frontend/types/api-key-settings.types.ts b/frontend/types/api-key-settings.types.ts new file mode 100644 index 00000000..aceb9465 --- /dev/null +++ b/frontend/types/api-key-settings.types.ts @@ -0,0 +1,42 @@ +/** + * API Key 配置类型定义 + * 用于 subfinder 第三方数据源配置 + */ + +// 单字段 Provider 配置（hunter, shodan, zoomeye, securitytrails, threatbook, quake） +export interface SingleFieldProviderConfig { + enabled: boolean + apiKey: string +} + +// FOFA Provider 配置（email + apiKey） +export interface FofaProviderConfig { + enabled: boolean + email: string + apiKey: string +} + +// Censys Provider 配置（apiId + apiSecret） +export interface CensysProviderConfig { + enabled: boolean + apiId: string + apiSecret: string +} + +// 完整的 API Key 配置 +export interface ApiKeySettings { + fofa: FofaProviderConfig + hunter: SingleFieldProviderConfig + shodan: SingleFieldProviderConfig + censys: CensysProviderConfig + zoomeye: SingleFieldProviderConfig + securitytrails: SingleFieldProviderConfig + threatbook: SingleFieldProviderConfig + quake: SingleFieldProviderConfig +} + +// Provider 类型 +export type ProviderKey = keyof ApiKeySettings + +// Provider 配置联合类型 +export type ProviderConfig = FofaProviderConfig | CensysProviderConfig | SingleFieldProviderConfig