Merge branch 'dev'

- Create new blacklist mock data module with global and target-specific patterns
- Add mock functions for getting and updating global blacklist rules
- Add mock functions for getting and updating target-specific blacklist rules
- Integrate mock blacklist endpoints into global-blacklist.service.ts
- Integrate mock blacklist endpoints into target.service.ts
- Export blacklist mock functions from main mock index
- Enable testing of blacklist management UI without backend API

.github/workflows/docker-build.yml

@@ -19,7 +19,8 @@ permissions:
   contents: write
 
 jobs:
-  build:
+  # AMD64 构建（原生 x64 runner）
+  build-amd64:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -27,43 +28,30 @@ jobs:
           - image: xingrin-server
             dockerfile: docker/server/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
           - image: xingrin-frontend
             dockerfile: docker/frontend/Dockerfile
             context: .
-            platforms: linux/amd64  # ARM64 构建时 Next.js 在 QEMU 下会崩溃
           - image: xingrin-worker
             dockerfile: docker/worker/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
           - image: xingrin-nginx
             dockerfile: docker/nginx/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
           - image: xingrin-agent
             dockerfile: docker/agent/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
           - image: xingrin-postgres
             dockerfile: docker/postgres/Dockerfile
             context: docker/postgres
-            platforms: linux/amd64,linux/arm64
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Free disk space (for large builds like worker)
+      - name: Free disk space
         run: |
-          echo "=== Before cleanup ==="
-          df -h
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /usr/local/lib/android
-          sudo rm -rf /opt/ghc
-          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
           sudo docker image prune -af
-          echo "=== After cleanup ==="
-          df -h
 
       - name: Generate SSL certificates for nginx build
         if: matrix.image == 'xingrin-nginx'
@@ -73,10 +61,6 @@ jobs:
             -keyout docker/nginx/ssl/privkey.pem \
             -out docker/nginx/ssl/fullchain.pem \
             -subj "/CN=localhost"
-          echo "SSL certificates generated for CI build"
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -87,7 +71,120 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Get version from git tag
+      - name: Get version
+        id: version
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+          else
+            echo "VERSION=dev-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build and push AMD64
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
+          platforms: linux/amd64
+          push: true
+          tags: ${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}-amd64
+          build-args: IMAGE_TAG=${{ steps.version.outputs.VERSION }}
+          cache-from: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-amd64
+          cache-to: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-amd64,mode=max
+          provenance: false
+          sbom: false
+
+  # ARM64 构建（原生 ARM64 runner）
+  build-arm64:
+    runs-on: ubuntu-22.04-arm
+    strategy:
+      matrix:
+        include:
+          - image: xingrin-server
+            dockerfile: docker/server/Dockerfile
+            context: .
+          - image: xingrin-frontend
+            dockerfile: docker/frontend/Dockerfile
+            context: .
+          - image: xingrin-worker
+            dockerfile: docker/worker/Dockerfile
+            context: .
+          - image: xingrin-nginx
+            dockerfile: docker/nginx/Dockerfile
+            context: .
+          - image: xingrin-agent
+            dockerfile: docker/agent/Dockerfile
+            context: .
+          - image: xingrin-postgres
+            dockerfile: docker/postgres/Dockerfile
+            context: docker/postgres
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Generate SSL certificates for nginx build
+        if: matrix.image == 'xingrin-nginx'
+        run: |
+          mkdir -p docker/nginx/ssl
+          openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
+            -keyout docker/nginx/ssl/privkey.pem \
+            -out docker/nginx/ssl/fullchain.pem \
+            -subj "/CN=localhost"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get version
+        id: version
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+          else
+            echo "VERSION=dev-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build and push ARM64
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
+          platforms: linux/arm64
+          push: true
+          tags: ${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}-arm64
+          build-args: IMAGE_TAG=${{ steps.version.outputs.VERSION }}
+          cache-from: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-arm64
+          cache-to: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-arm64,mode=max
+          provenance: false
+          sbom: false
+
+  # 合并多架构 manifest
+  merge-manifests:
+    runs-on: ubuntu-latest
+    needs: [build-amd64, build-arm64]
+    strategy:
+      matrix:
+        image:
+          - xingrin-server
+          - xingrin-frontend
+          - xingrin-worker
+          - xingrin-nginx
+          - xingrin-agent
+          - xingrin-postgres
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get version
         id: version
         run: |
           if [[ $GITHUB_REF == refs/tags/* ]]; then
@@ -98,28 +195,27 @@ jobs:
             echo "IS_RELEASE=false" >> $GITHUB_OUTPUT
           fi
 
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: ${{ matrix.context }}
-          file: ${{ matrix.dockerfile }}
-          platforms: ${{ matrix.platforms }}
-          push: true
-          tags: |
-            ${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}
-            ${{ steps.version.outputs.IS_RELEASE == 'true' && format('{0}/{1}:latest', env.IMAGE_PREFIX, matrix.image) || '' }}
-          build-args: |
-            IMAGE_TAG=${{ steps.version.outputs.VERSION }}
-          cache-from: type=gha,scope=${{ matrix.image }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.image }}
-          provenance: false
-          sbom: false
+      - name: Create and push multi-arch manifest
+        run: |
+          VERSION=${{ steps.version.outputs.VERSION }}
+          IMAGE=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}
+          
+          docker manifest create ${IMAGE}:${VERSION} \
+            ${IMAGE}:${VERSION}-amd64 \
+            ${IMAGE}:${VERSION}-arm64
+          docker manifest push ${IMAGE}:${VERSION}
+          
+          if [[ "${{ steps.version.outputs.IS_RELEASE }}" == "true" ]]; then
+            docker manifest create ${IMAGE}:latest \
+              ${IMAGE}:${VERSION}-amd64 \
+              ${IMAGE}:${VERSION}-arm64
+            docker manifest push ${IMAGE}:latest
+          fi
 
-  # 所有镜像构建成功后，更新 VERSION 文件
-  # 根据 tag 所在的分支更新对应分支的 VERSION 文件
+  # 更新 VERSION 文件
   update-version:
     runs-on: ubuntu-latest
-    needs: build
+    needs: merge-manifests
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
       - name: Checkout repository

README.md

@@ -25,6 +25,13 @@
 
 ---
 
+## 🌐 在线 Demo
+
+👉 **[https://xingrin.vercel.app/](https://xingrin.vercel.app/)**
+
+> ⚠️ 仅用于 UI 展示，未接入后端数据库
+
+---
 
 <p align="center">
   <b>🎨 现代化 UI </b>
@@ -62,6 +69,12 @@
 - **自定义流程** - YAML 配置扫描流程，灵活编排
 - **定时扫描** - Cron 表达式配置，自动化周期扫描
 
+### 🚫 黑名单过滤
+- **两层黑名单** - 全局黑名单 + Target 级黑名单，灵活控制扫描范围
+- **智能规则识别** - 自动识别域名通配符（`*.gov`）、IP、CIDR 网段
+- **敏感目标保护** - 过滤政府、军事、教育等敏感域名，防止误扫
+- **内网过滤** - 支持 `10.0.0.0/8`、`172.16.0.0/12`、`192.168.0.0/16` 等私有网段
+
 ### 🔖 指纹识别
 - **多源指纹库** - 内置 EHole、Goby、Wappalyzer、Fingers、FingerPrintHub、ARL 等 2.7W+ 指纹规则
 - **自动识别** - 扫描流程自动执行，识别 Web 应用技术栈
@@ -198,6 +211,7 @@ url="/api/v1" && status!="404"
 ### 环境要求
 
 - **操作系统**: Ubuntu 20.04+ / Debian 11+ 
+- **系统架构**: AMD64 (x86_64) / ARM64 (aarch64)
 - **硬件**: 2核 4G 内存起步，20GB+ 磁盘空间
 
 ### 一键安装
@@ -242,14 +256,11 @@ sudo ./uninstall.sh
 
 ## 🤝 反馈与贡献
 
-- 🐛 **如果发现 Bug** 可以点击右边链接进行提交 [Issue](https://github.com/yyhuni/xingrin/issues)
-- 💡 **有新想法，比如UI设计，功能设计等** 欢迎点击右边链接进行提交建议 [Issue](https://github.com/yyhuni/xingrin/issues)
+- 💡 **发现 Bug，有新想法，比如UI设计，功能设计等** 欢迎点击右边链接进行提交建议 [Issue](https://github.com/yyhuni/xingrin/issues) 或者公众号私信
 
 ## 📧 联系
-- 目前版本就我个人使用，可能会有很多边界问题
-- 如有问题，建议，其他，优先提交[Issue](https://github.com/yyhuni/xingrin/issues)，也可以直接给我的公众号发消息，我都会回复的
-
 - 微信公众号: **塔罗安全学苑**
+- 微信群去公众号底下的菜单，有个交流群，点击就可以看到了，链接过期可以私信我拉你
 
 <img src="docs/wechat-qrcode.png" alt="微信公众号" width="200">
 

VERSION

@@ -1 +1 @@
-v1.3.5-dev
+v1.4.0-dev

backend/apps/asset/apps.py

@@ -1,106 +1,6 @@
-import logging
-import sys
-
 from django.apps import AppConfig
 
-logger = logging.getLogger(__name__)
-
 
 class AssetConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'apps.asset'
-    
-    def ready(self):
-        # 导入所有模型以确保Django发现并注册
-        from . import models
-        
-        # 启用 pg_trgm 扩展（用于文本模糊搜索索引）
-        # 用于已有数据库升级场景
-        self._ensure_pg_trgm_extension()
-        
-        # 验证 pg_ivm 扩展是否可用（用于 IMMV 增量维护）
-        self._verify_pg_ivm_extension()
-    
-    def _ensure_pg_trgm_extension(self):
-        """
-        确保 pg_trgm 扩展已启用。
-        该扩展用于 response_body 和 response_headers 字段的 GIN 索引，
-        支持高效的文本模糊搜索。
-        """
-        from django.db import connection
-        
-        # 检查是否为 PostgreSQL 数据库
-        if connection.vendor != 'postgresql':
-            logger.debug("跳过 pg_trgm 扩展：当前数据库不是 PostgreSQL")
-            return
-        
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
-                logger.debug("pg_trgm 扩展已启用")
-        except Exception as e:
-            # 记录错误但不阻止应用启动
-            # 常见原因：权限不足（需要超级用户权限）
-            logger.warning(
-                "无法创建 pg_trgm 扩展: %s。"
-                "这可能导致 response_body 和 response_headers 字段的 GIN 索引无法正常工作。"
-                "请手动执行: CREATE EXTENSION IF NOT EXISTS pg_trgm;",
-                str(e)
-            )
-    
-    def _verify_pg_ivm_extension(self):
-        """
-        验证 pg_ivm 扩展是否可用。
-        pg_ivm 用于 IMMV（增量维护物化视图），是系统必需的扩展。
-        如果不可用，将记录错误并退出。
-        """
-        from django.db import connection
-        
-        # 检查是否为 PostgreSQL 数据库
-        if connection.vendor != 'postgresql':
-            logger.debug("跳过 pg_ivm 验证：当前数据库不是 PostgreSQL")
-            return
-        
-        # 跳过某些管理命令（如 migrate、makemigrations）
-        import sys
-        if len(sys.argv) > 1 and sys.argv[1] in ('migrate', 'makemigrations', 'collectstatic', 'check'):
-            logger.debug("跳过 pg_ivm 验证：当前为管理命令")
-            return
-        
-        try:
-            with connection.cursor() as cursor:
-                # 检查 pg_ivm 扩展是否已安装
-                cursor.execute("""
-                    SELECT COUNT(*) FROM pg_extension WHERE extname = 'pg_ivm'
-                """)
-                count = cursor.fetchone()[0]
-                
-                if count > 0:
-                    logger.info("✓ pg_ivm 扩展已启用")
-                else:
-                    # 尝试创建扩展
-                    try:
-                        cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_ivm;")
-                        logger.info("✓ pg_ivm 扩展已创建并启用")
-                    except Exception as create_error:
-                        logger.error(
-                            "=" * 60 + "\n"
-                            "错误: pg_ivm 扩展未安装\n"
-                            "=" * 60 + "\n"
-                            "pg_ivm 是系统必需的扩展，用于增量维护物化视图。\n\n"
-                            "请在 PostgreSQL 服务器上安装 pg_ivm：\n"
-                            "  curl -sSL https://raw.githubusercontent.com/yyhuni/xingrin/main/docker/scripts/install-pg-ivm.sh | sudo bash\n\n"
-                            "或手动安装：\n"
-                            "  1. apt install build-essential postgresql-server-dev-15 git\n"
-                            "  2. git clone https://github.com/sraoss/pg_ivm.git && cd pg_ivm && make && make install\n"
-                            "  3. 在 postgresql.conf 中添加: shared_preload_libraries = 'pg_ivm'\n"
-                            "  4. 重启 PostgreSQL\n"
-                            "=" * 60
-                        )
-                        # 在生产环境中退出，开发环境中仅警告
-                        from django.conf import settings
-                        if not settings.DEBUG:
-                            sys.exit(1)
-                        
-        except Exception as e:
-            logger.error(f"pg_ivm 扩展验证失败: {e}")

backend/apps/asset/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.2.7 on 2026-01-02 04:45
+# Generated by Django 5.2.7 on 2026-01-06 00:55
 
 import django.contrib.postgres.fields
 import django.contrib.postgres.indexes

backend/apps/asset/migrations/0002_create_search_materialized_view.py

@@ -1,187 +0,0 @@
-"""
-创建资产搜索 IMMV（增量维护物化视图）
-
-使用 pg_ivm 扩展创建 IMMV，数据变更时自动增量更新，无需手动刷新。
-
-包含：
-1. asset_search_view - Website 搜索视图
-2. endpoint_search_view - Endpoint 搜索视图
-"""
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('asset', '0001_initial'),
-    ]
-
-    operations = [
-        # 1. 确保 pg_ivm 扩展已启用
-        migrations.RunSQL(
-            sql="CREATE EXTENSION IF NOT EXISTS pg_ivm;",
-            reverse_sql="-- pg_ivm extension kept for other uses"
-        ),
-        
-        # ==================== Website IMMV ====================
-        
-        # 2. 创建 asset_search_view IMMV
-        migrations.RunSQL(
-            sql="""
-                SELECT pgivm.create_immv('asset_search_view', $$
-                    SELECT 
-                        w.id,
-                        w.url,
-                        w.host,
-                        w.title,
-                        w.tech,
-                        w.status_code,
-                        w.response_headers,
-                        w.response_body,
-                        w.content_type,
-                        w.content_length,
-                        w.webserver,
-                        w.location,
-                        w.vhost,
-                        w.created_at,
-                        w.target_id
-                    FROM website w
-                $$);
-            """,
-            reverse_sql="SELECT pgivm.drop_immv('asset_search_view');"
-        ),
-        
-        # 3. 创建 asset_search_view 索引
-        migrations.RunSQL(
-            sql="""
-                -- 唯一索引
-                CREATE UNIQUE INDEX IF NOT EXISTS asset_search_view_id_idx 
-                ON asset_search_view (id);
-                
-                -- host 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_host_trgm_idx 
-                ON asset_search_view USING gin (host gin_trgm_ops);
-                
-                -- title 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_title_trgm_idx 
-                ON asset_search_view USING gin (title gin_trgm_ops);
-                
-                -- url 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_url_trgm_idx 
-                ON asset_search_view USING gin (url gin_trgm_ops);
-                
-                -- response_headers 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_headers_trgm_idx 
-                ON asset_search_view USING gin (response_headers gin_trgm_ops);
-                
-                -- response_body 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_body_trgm_idx 
-                ON asset_search_view USING gin (response_body gin_trgm_ops);
-                
-                -- tech 数组索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_tech_idx 
-                ON asset_search_view USING gin (tech);
-                
-                -- status_code 索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_status_idx 
-                ON asset_search_view (status_code);
-                
-                -- created_at 排序索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_created_idx 
-                ON asset_search_view (created_at DESC);
-            """,
-            reverse_sql="""
-                DROP INDEX IF EXISTS asset_search_view_id_idx;
-                DROP INDEX IF EXISTS asset_search_view_host_trgm_idx;
-                DROP INDEX IF EXISTS asset_search_view_title_trgm_idx;
-                DROP INDEX IF EXISTS asset_search_view_url_trgm_idx;
-                DROP INDEX IF EXISTS asset_search_view_headers_trgm_idx;
-                DROP INDEX IF EXISTS asset_search_view_body_trgm_idx;
-                DROP INDEX IF EXISTS asset_search_view_tech_idx;
-                DROP INDEX IF EXISTS asset_search_view_status_idx;
-                DROP INDEX IF EXISTS asset_search_view_created_idx;
-            """
-        ),
-
-        # ==================== Endpoint IMMV ====================
-        
-        # 4. 创建 endpoint_search_view IMMV
-        migrations.RunSQL(
-            sql="""
-                SELECT pgivm.create_immv('endpoint_search_view', $$
-                    SELECT 
-                        e.id,
-                        e.url,
-                        e.host,
-                        e.title,
-                        e.tech,
-                        e.status_code,
-                        e.response_headers,
-                        e.response_body,
-                        e.content_type,
-                        e.content_length,
-                        e.webserver,
-                        e.location,
-                        e.vhost,
-                        e.matched_gf_patterns,
-                        e.created_at,
-                        e.target_id
-                    FROM endpoint e
-                $$);
-            """,
-            reverse_sql="SELECT pgivm.drop_immv('endpoint_search_view');"
-        ),
-        
-        # 5. 创建 endpoint_search_view 索引
-        migrations.RunSQL(
-            sql="""
-                -- 唯一索引
-                CREATE UNIQUE INDEX IF NOT EXISTS endpoint_search_view_id_idx 
-                ON endpoint_search_view (id);
-                
-                -- host 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_host_trgm_idx 
-                ON endpoint_search_view USING gin (host gin_trgm_ops);
-                
-                -- title 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_title_trgm_idx 
-                ON endpoint_search_view USING gin (title gin_trgm_ops);
-                
-                -- url 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_url_trgm_idx 
-                ON endpoint_search_view USING gin (url gin_trgm_ops);
-                
-                -- response_headers 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_headers_trgm_idx 
-                ON endpoint_search_view USING gin (response_headers gin_trgm_ops);
-                
-                -- response_body 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_body_trgm_idx 
-                ON endpoint_search_view USING gin (response_body gin_trgm_ops);
-                
-                -- tech 数组索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_tech_idx 
-                ON endpoint_search_view USING gin (tech);
-                
-                -- status_code 索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_status_idx 
-                ON endpoint_search_view (status_code);
-                
-                -- created_at 排序索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_created_idx 
-                ON endpoint_search_view (created_at DESC);
-            """,
-            reverse_sql="""
-                DROP INDEX IF EXISTS endpoint_search_view_id_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_host_trgm_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_title_trgm_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_url_trgm_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_headers_trgm_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_body_trgm_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_tech_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_status_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_created_idx;
-            """
-        ),
-    ]

backend/apps/asset/migrations/0002_create_search_views.py

@@ -0,0 +1,104 @@
+"""
+创建资产搜索物化视图（使用 pg_ivm 增量维护）
+
+这些视图用于资产搜索功能，提供高性能的全文搜索能力。
+"""
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    """创建资产搜索所需的增量物化视图"""
+
+    dependencies = [
+        ('asset', '0001_initial'),
+    ]
+
+    operations = [
+        # 1. 确保 pg_ivm 扩展已安装
+        migrations.RunSQL(
+            sql="CREATE EXTENSION IF NOT EXISTS pg_ivm;",
+            reverse_sql="DROP EXTENSION IF EXISTS pg_ivm;",
+        ),
+        
+        # 2. 创建 Website 搜索视图
+        # 注意：pg_ivm 不支持 ArrayField，所以 tech 字段需要从原表 JOIN 获取
+        migrations.RunSQL(
+            sql="""
+                SELECT pgivm.create_immv('asset_search_view', $$
+                    SELECT 
+                        w.id,
+                        w.url,
+                        w.host,
+                        w.title,
+                        w.status_code,
+                        w.response_headers,
+                        w.response_body,
+                        w.content_type,
+                        w.content_length,
+                        w.webserver,
+                        w.location,
+                        w.vhost,
+                        w.created_at,
+                        w.target_id
+                    FROM website w
+                $$);
+            """,
+            reverse_sql="DROP TABLE IF EXISTS asset_search_view CASCADE;",
+        ),
+        
+        # 3. 创建 Endpoint 搜索视图
+        migrations.RunSQL(
+            sql="""
+                SELECT pgivm.create_immv('endpoint_search_view', $$
+                    SELECT 
+                        e.id,
+                        e.url,
+                        e.host,
+                        e.title,
+                        e.status_code,
+                        e.response_headers,
+                        e.response_body,
+                        e.content_type,
+                        e.content_length,
+                        e.webserver,
+                        e.location,
+                        e.vhost,
+                        e.created_at,
+                        e.target_id
+                    FROM endpoint e
+                $$);
+            """,
+            reverse_sql="DROP TABLE IF EXISTS endpoint_search_view CASCADE;",
+        ),
+        
+        # 4. 为搜索视图创建索引（加速查询）
+        migrations.RunSQL(
+            sql=[
+                # Website 搜索视图索引
+                "CREATE INDEX IF NOT EXISTS asset_search_view_host_idx ON asset_search_view (host);",
+                "CREATE INDEX IF NOT EXISTS asset_search_view_url_idx ON asset_search_view (url);",
+                "CREATE INDEX IF NOT EXISTS asset_search_view_title_idx ON asset_search_view (title);",
+                "CREATE INDEX IF NOT EXISTS asset_search_view_status_idx ON asset_search_view (status_code);",
+                "CREATE INDEX IF NOT EXISTS asset_search_view_created_idx ON asset_search_view (created_at DESC);",
+                # Endpoint 搜索视图索引
+                "CREATE INDEX IF NOT EXISTS endpoint_search_view_host_idx ON endpoint_search_view (host);",
+                "CREATE INDEX IF NOT EXISTS endpoint_search_view_url_idx ON endpoint_search_view (url);",
+                "CREATE INDEX IF NOT EXISTS endpoint_search_view_title_idx ON endpoint_search_view (title);",
+                "CREATE INDEX IF NOT EXISTS endpoint_search_view_status_idx ON endpoint_search_view (status_code);",
+                "CREATE INDEX IF NOT EXISTS endpoint_search_view_created_idx ON endpoint_search_view (created_at DESC);",
+            ],
+            reverse_sql=[
+                "DROP INDEX IF EXISTS asset_search_view_host_idx;",
+                "DROP INDEX IF EXISTS asset_search_view_url_idx;",
+                "DROP INDEX IF EXISTS asset_search_view_title_idx;",
+                "DROP INDEX IF EXISTS asset_search_view_status_idx;",
+                "DROP INDEX IF EXISTS asset_search_view_created_idx;",
+                "DROP INDEX IF EXISTS endpoint_search_view_host_idx;",
+                "DROP INDEX IF EXISTS endpoint_search_view_url_idx;",
+                "DROP INDEX IF EXISTS endpoint_search_view_title_idx;",
+                "DROP INDEX IF EXISTS endpoint_search_view_status_idx;",
+                "DROP INDEX IF EXISTS endpoint_search_view_created_idx;",
+            ],
+        ),
+    ]

backend/apps/asset/services/search_service.py

@@ -11,7 +11,7 @@
 
 import logging
 import re
-from typing import Optional, List, Dict, Any, Tuple, Literal
+from typing import Optional, List, Dict, Any, Tuple, Literal, Iterator
 
 from django.db import connection
 
@@ -37,46 +37,55 @@ VIEW_MAPPING = {
     'endpoint': 'endpoint_search_view',
 }
 
+# 资产类型到原表名的映射（用于 JOIN 获取数组字段）
+# ⚠️ 重要：pg_ivm 不支持 ArrayField，所有数组字段必须从原表 JOIN 获取
+TABLE_MAPPING = {
+    'website': 'website',
+    'endpoint': 'endpoint',
+}
+
 # 有效的资产类型
 VALID_ASSET_TYPES = {'website', 'endpoint'}
 
-# Website 查询字段
+# Website 查询字段（v=视图，t=原表）
+# ⚠️ 注意：t.tech 从原表获取，因为 pg_ivm 不支持 ArrayField
 WEBSITE_SELECT_FIELDS = """
-    id,
-    url,
-    host,
-    title,
-    tech,
-    status_code,
-    response_headers,
-    response_body,
-    content_type,
-    content_length,
-    webserver,
-    location,
-    vhost,
-    created_at,
-    target_id
+    v.id,
+    v.url,
+    v.host,
+    v.title,
+    t.tech,  -- ArrayField，从 website 表 JOIN 获取
+    v.status_code,
+    v.response_headers,
+    v.response_body,
+    v.content_type,
+    v.content_length,
+    v.webserver,
+    v.location,
+    v.vhost,
+    v.created_at,
+    v.target_id
 """
 
-# Endpoint 查询字段（包含 matched_gf_patterns）
+# Endpoint 查询字段
+# ⚠️ 注意：t.tech 和 t.matched_gf_patterns 从原表获取，因为 pg_ivm 不支持 ArrayField
 ENDPOINT_SELECT_FIELDS = """
-    id,
-    url,
-    host,
-    title,
-    tech,
-    status_code,
-    response_headers,
-    response_body,
-    content_type,
-    content_length,
-    webserver,
-    location,
-    vhost,
-    matched_gf_patterns,
-    created_at,
-    target_id
+    v.id,
+    v.url,
+    v.host,
+    v.title,
+    t.tech,  -- ArrayField，从 endpoint 表 JOIN 获取
+    v.status_code,
+    v.response_headers,
+    v.response_body,
+    v.content_type,
+    v.content_length,
+    v.webserver,
+    v.location,
+    v.vhost,
+    t.matched_gf_patterns,  -- ArrayField，从 endpoint 表 JOIN 获取
+    v.created_at,
+    v.target_id
 """
 
 
@@ -119,8 +128,8 @@ class SearchQueryParser:
         
         # 检查是否包含操作符语法，如果不包含则作为 host 模糊搜索
         if not cls.CONDITION_PATTERN.search(query):
-            # 裸文本，默认作为 host 模糊搜索
-            return "host ILIKE %s", [f"%{query}%"]
+            # 裸文本，默认作为 host 模糊搜索（v 是视图别名）
+            return "v.host ILIKE %s", [f"%{query}%"]
         
         # 按 || 分割为 OR 组
         or_groups = cls._split_by_or(query)
@@ -273,45 +282,45 @@ class SearchQueryParser:
     def _build_like_condition(cls, field: str, value: str, is_array: bool) -> Tuple[str, List[Any]]:
         """构建模糊匹配条件"""
         if is_array:
-            # 数组字段：检查数组中是否有元素包含该值
-            return f"EXISTS (SELECT 1 FROM unnest({field}) AS t WHERE t ILIKE %s)", [f"%{value}%"]
+            # 数组字段：检查数组中是否有元素包含该值（从原表 t 获取）
+            return f"EXISTS (SELECT 1 FROM unnest(t.{field}) AS elem WHERE elem ILIKE %s)", [f"%{value}%"]
         elif field == 'status_code':
             # 状态码是整数，模糊匹配转为精确匹配
             try:
-                return f"{field} = %s", [int(value)]
+                return f"v.{field} = %s", [int(value)]
             except ValueError:
-                return f"{field}::text ILIKE %s", [f"%{value}%"]
+                return f"v.{field}::text ILIKE %s", [f"%{value}%"]
         else:
-            return f"{field} ILIKE %s", [f"%{value}%"]
+            return f"v.{field} ILIKE %s", [f"%{value}%"]
     
     @classmethod
     def _build_exact_condition(cls, field: str, value: str, is_array: bool) -> Tuple[str, List[Any]]:
         """构建精确匹配条件"""
         if is_array:
-            # 数组字段：检查数组中是否包含该精确值
-            return f"%s = ANY({field})", [value]
+            # 数组字段：检查数组中是否包含该精确值（从原表 t 获取）
+            return f"%s = ANY(t.{field})", [value]
         elif field == 'status_code':
             # 状态码是整数
             try:
-                return f"{field} = %s", [int(value)]
+                return f"v.{field} = %s", [int(value)]
             except ValueError:
-                return f"{field}::text = %s", [value]
+                return f"v.{field}::text = %s", [value]
         else:
-            return f"{field} = %s", [value]
+            return f"v.{field} = %s", [value]
     
     @classmethod
     def _build_not_equal_condition(cls, field: str, value: str, is_array: bool) -> Tuple[str, List[Any]]:
         """构建不等于条件"""
         if is_array:
-            # 数组字段：检查数组中不包含该值
-            return f"NOT (%s = ANY({field}))", [value]
+            # 数组字段：检查数组中不包含该值（从原表 t 获取）
+            return f"NOT (%s = ANY(t.{field}))", [value]
         elif field == 'status_code':
             try:
-                return f"({field} IS NULL OR {field} != %s)", [int(value)]
+                return f"(v.{field} IS NULL OR v.{field} != %s)", [int(value)]
             except ValueError:
-                return f"({field} IS NULL OR {field}::text != %s)", [value]
+                return f"(v.{field} IS NULL OR v.{field}::text != %s)", [value]
         else:
-            return f"({field} IS NULL OR {field} != %s)", [value]
+            return f"(v.{field} IS NULL OR v.{field} != %s)", [value]
 
 
 AssetType = Literal['website', 'endpoint']
@@ -339,15 +348,18 @@ class AssetSearchService:
         """
         where_clause, params = SearchQueryParser.parse(query)
         
-        # 根据资产类型选择视图和字段
+        # 根据资产类型选择视图、原表和字段
         view_name = VIEW_MAPPING.get(asset_type, 'asset_search_view')
+        table_name = TABLE_MAPPING.get(asset_type, 'website')
         select_fields = ENDPOINT_SELECT_FIELDS if asset_type == 'endpoint' else WEBSITE_SELECT_FIELDS
         
+        # JOIN 原表获取数组字段（tech, matched_gf_patterns）
         sql = f"""
             SELECT {select_fields}
-            FROM {view_name}
+            FROM {view_name} v
+            JOIN {table_name} t ON v.id = t.id
             WHERE {where_clause}
-            ORDER BY created_at DESC
+            ORDER BY v.created_at DESC
         """
         
         # 添加 LIMIT
@@ -369,26 +381,31 @@ class AssetSearchService:
             logger.error(f"搜索查询失败: {e}, SQL: {sql}, params: {params}")
             raise
     
-    def count(self, query: str, asset_type: AssetType = 'website') -> int:
+    def count(self, query: str, asset_type: AssetType = 'website', statement_timeout_ms: int = 300000) -> int:
         """
         统计搜索结果数量
         
         Args:
             query: 搜索查询字符串
             asset_type: 资产类型 ('website' 或 'endpoint')
+            statement_timeout_ms: SQL 语句超时时间（毫秒），默认 5 分钟
         
         Returns:
             int: 结果总数
         """
         where_clause, params = SearchQueryParser.parse(query)
         
-        # 根据资产类型选择视图
+        # 根据资产类型选择视图和原表
         view_name = VIEW_MAPPING.get(asset_type, 'asset_search_view')
+        table_name = TABLE_MAPPING.get(asset_type, 'website')
         
-        sql = f"SELECT COUNT(*) FROM {view_name} WHERE {where_clause}"
+        # JOIN 原表以支持数组字段查询
+        sql = f"SELECT COUNT(*) FROM {view_name} v JOIN {table_name} t ON v.id = t.id WHERE {where_clause}"
         
         try:
             with connection.cursor() as cursor:
+                # 为导出设置更长的超时时间（仅影响当前会话）
+                cursor.execute(f"SET LOCAL statement_timeout = {statement_timeout_ms}")
                 cursor.execute(sql, params)
                 return cursor.fetchone()[0]
         except Exception as e:
@@ -399,41 +416,62 @@ class AssetSearchService:
         self, 
         query: str, 
         asset_type: AssetType = 'website',
-        batch_size: int = 1000
-    ):
+        batch_size: int = 1000,
+        statement_timeout_ms: int = 300000
+    ) -> Iterator[Dict[str, Any]]:
         """
-        流式搜索资产（使用服务端游标，内存友好）
+        流式搜索资产（使用分批查询，内存友好）
         
         Args:
             query: 搜索查询字符串
             asset_type: 资产类型 ('website' 或 'endpoint')
             batch_size: 每批获取的数量
+            statement_timeout_ms: SQL 语句超时时间（毫秒），默认 5 分钟
         
         Yields:
             Dict: 单条搜索结果
         """
         where_clause, params = SearchQueryParser.parse(query)
         
-        # 根据资产类型选择视图和字段
+        # 根据资产类型选择视图、原表和字段
         view_name = VIEW_MAPPING.get(asset_type, 'asset_search_view')
+        table_name = TABLE_MAPPING.get(asset_type, 'website')
         select_fields = ENDPOINT_SELECT_FIELDS if asset_type == 'endpoint' else WEBSITE_SELECT_FIELDS
         
-        sql = f"""
-            SELECT {select_fields}
-            FROM {view_name}
-            WHERE {where_clause}
-            ORDER BY created_at DESC
-        """
+        # 使用 OFFSET/LIMIT 分批查询（Django 不支持命名游标）
+        offset = 0
         
         try:
-            # 使用服务端游标，避免一次性加载所有数据到内存
-            with connection.cursor(name='export_cursor') as cursor:
-                cursor.itersize = batch_size
-                cursor.execute(sql, params)
-                columns = [col[0] for col in cursor.description]
+            while True:
+                # JOIN 原表获取数组字段
+                sql = f"""
+                    SELECT {select_fields}
+                    FROM {view_name} v
+                    JOIN {table_name} t ON v.id = t.id
+                    WHERE {where_clause}
+                    ORDER BY v.created_at DESC
+                    LIMIT {batch_size} OFFSET {offset}
+                """
                 
-                for row in cursor:
+                with connection.cursor() as cursor:
+                    # 为导出设置更长的超时时间（仅影响当前会话）
+                    cursor.execute(f"SET LOCAL statement_timeout = {statement_timeout_ms}")
+                    cursor.execute(sql, params)
+                    columns = [col[0] for col in cursor.description]
+                    rows = cursor.fetchall()
+                
+                if not rows:
+                    break
+                
+                for row in rows:
                     yield dict(zip(columns, row))
+                
+                # 如果返回的行数少于 batch_size，说明已经是最后一批
+                if len(rows) < batch_size:
+                    break
+                
+                offset += batch_size
+                
         except Exception as e:
             logger.error(f"流式搜索查询失败: {e}, SQL: {sql}, params: {params}")
             raise

backend/apps/asset/tasks/__init__.py

@@ -1,7 +0,0 @@
-"""
-Asset 应用的任务模块
-
-注意：物化视图刷新已移至 APScheduler 定时任务（apps.engine.scheduler）
-"""
-
-__all__ = []

backend/apps/asset/views/asset_views.py

@@ -8,7 +8,6 @@ from rest_framework.request import Request
 from rest_framework.exceptions import NotFound, ValidationError as DRFValidationError
 from django.core.exceptions import ValidationError, ObjectDoesNotExist
 from django.db import DatabaseError, IntegrityError, OperationalError
-from django.http import StreamingHttpResponse
 
 from ..serializers import (
     SubdomainListSerializer, WebSiteSerializer, DirectorySerializer, 
@@ -243,7 +242,7 @@ class SubdomainViewSet(viewsets.ModelViewSet):
         
         CSV 列：name, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         target_pk = self.kwargs.get('target_pk')
         if not target_pk:
@@ -254,12 +253,12 @@ class SubdomainViewSet(viewsets.ModelViewSet):
         headers = ['name', 'created_at']
         formatters = {'created_at': format_datetime}
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"target-{target_pk}-subdomains.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-subdomains.csv"'
-        return response
 
 
 class WebSiteViewSet(viewsets.ModelViewSet):
@@ -369,7 +368,7 @@ class WebSiteViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
+        from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
         
         target_pk = self.kwargs.get('target_pk')
         if not target_pk:
@@ -387,12 +386,12 @@ class WebSiteViewSet(viewsets.ModelViewSet):
             'tech': lambda x: format_list_field(x, separator=','),
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"target-{target_pk}-websites.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-websites.csv"'
-        return response
 
 
 class DirectoryViewSet(viewsets.ModelViewSet):
@@ -499,7 +498,7 @@ class DirectoryViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, status, content_length, words, lines, content_type, duration, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         target_pk = self.kwargs.get('target_pk')
         if not target_pk:
@@ -515,12 +514,12 @@ class DirectoryViewSet(viewsets.ModelViewSet):
             'created_at': format_datetime,
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"target-{target_pk}-directories.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-directories.csv"'
-        return response
 
 
 class EndpointViewSet(viewsets.ModelViewSet):
@@ -630,7 +629,7 @@ class EndpointViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, matched_gf_patterns, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
+        from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
         
         target_pk = self.kwargs.get('target_pk')
         if not target_pk:
@@ -649,12 +648,12 @@ class EndpointViewSet(viewsets.ModelViewSet):
             'matched_gf_patterns': lambda x: format_list_field(x, separator=','),
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"target-{target_pk}-endpoints.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-endpoints.csv"'
-        return response
 
 
 class HostPortMappingViewSet(viewsets.ModelViewSet):
@@ -707,7 +706,7 @@ class HostPortMappingViewSet(viewsets.ModelViewSet):
         
         CSV 列：ip, host, port, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         target_pk = self.kwargs.get('target_pk')
         if not target_pk:
@@ -722,14 +721,12 @@ class HostPortMappingViewSet(viewsets.ModelViewSet):
             'created_at': format_datetime
         }
         
-        # 生成流式响应
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"target-{target_pk}-ip-addresses.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-ip-addresses.csv"'
-        
-        return response
 
 
 class VulnerabilityViewSet(viewsets.ModelViewSet):
@@ -801,7 +798,7 @@ class SubdomainSnapshotViewSet(viewsets.ModelViewSet):
         
         CSV 列：name, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         scan_pk = self.kwargs.get('scan_pk')
         if not scan_pk:
@@ -812,12 +809,12 @@ class SubdomainSnapshotViewSet(viewsets.ModelViewSet):
         headers = ['name', 'created_at']
         formatters = {'created_at': format_datetime}
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"scan-{scan_pk}-subdomains.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-subdomains.csv"'
-        return response
 
 
 class WebsiteSnapshotViewSet(viewsets.ModelViewSet):
@@ -855,7 +852,7 @@ class WebsiteSnapshotViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
+        from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
         
         scan_pk = self.kwargs.get('scan_pk')
         if not scan_pk:
@@ -873,12 +870,12 @@ class WebsiteSnapshotViewSet(viewsets.ModelViewSet):
             'tech': lambda x: format_list_field(x, separator=','),
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"scan-{scan_pk}-websites.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-websites.csv"'
-        return response
 
 
 class DirectorySnapshotViewSet(viewsets.ModelViewSet):
@@ -913,7 +910,7 @@ class DirectorySnapshotViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, status, content_length, words, lines, content_type, duration, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         scan_pk = self.kwargs.get('scan_pk')
         if not scan_pk:
@@ -929,12 +926,12 @@ class DirectorySnapshotViewSet(viewsets.ModelViewSet):
             'created_at': format_datetime,
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"scan-{scan_pk}-directories.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-directories.csv"'
-        return response
 
 
 class EndpointSnapshotViewSet(viewsets.ModelViewSet):
@@ -972,7 +969,7 @@ class EndpointSnapshotViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, matched_gf_patterns, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
+        from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
         
         scan_pk = self.kwargs.get('scan_pk')
         if not scan_pk:
@@ -991,12 +988,12 @@ class EndpointSnapshotViewSet(viewsets.ModelViewSet):
             'matched_gf_patterns': lambda x: format_list_field(x, separator=','),
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"scan-{scan_pk}-endpoints.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-endpoints.csv"'
-        return response
 
 
 class HostPortMappingSnapshotViewSet(viewsets.ModelViewSet):
@@ -1031,7 +1028,7 @@ class HostPortMappingSnapshotViewSet(viewsets.ModelViewSet):
         
         CSV 列：ip, host, port, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         scan_pk = self.kwargs.get('scan_pk')
         if not scan_pk:
@@ -1046,14 +1043,12 @@ class HostPortMappingSnapshotViewSet(viewsets.ModelViewSet):
             'created_at': format_datetime
         }
         
-        # 生成流式响应
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"scan-{scan_pk}-ip-addresses.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-ip-addresses.csv"'
-        
-        return response
 
 
 class VulnerabilitySnapshotViewSet(viewsets.ModelViewSet):

backend/apps/asset/views/search_views.py

@@ -33,7 +33,6 @@ from urllib.parse import urlparse, urlunparse
 from rest_framework import status
 from rest_framework.views import APIView
 from rest_framework.request import Request
-from django.http import StreamingHttpResponse
 from django.db import connection
 
 from apps.common.response_helpers import success_response, error_response
@@ -285,7 +284,7 @@ class AssetSearchExportView(APIView):
         asset_type: 资产类型 ('website' 或 'endpoint'，默认 'website')
     
     Response:
-        CSV 文件流（使用服务端游标，支持大数据量导出）
+        CSV 文件（带 Content-Length，支持浏览器显示下载进度）
     """
     
     def __init__(self, **kwargs):
@@ -313,8 +312,8 @@ class AssetSearchExportView(APIView):
         return headers, formatters
     
     def get(self, request: Request):
-        """导出搜索结果为 CSV（流式导出，无数量限制）"""
-        from apps.common.utils import generate_csv_rows
+        """导出搜索结果为 CSV（带 Content-Length，支持下载进度显示）"""
+        from apps.common.utils import create_csv_export_response
         
         # 获取搜索查询
         query = request.query_params.get('q', '').strip()
@@ -347,18 +346,16 @@ class AssetSearchExportView(APIView):
         # 获取表头和格式化器
         headers, formatters = self._get_headers_and_formatters(asset_type)
         
-        # 获取流式数据迭代器
-        data_iterator = self.service.search_iter(query, asset_type)
-        
         # 生成文件名
         timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
         filename = f'search_{asset_type}_{timestamp}.csv'
         
-        # 返回流式响应
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        # 使用通用导出工具
+        data_iterator = self.service.search_iter(query, asset_type)
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=filename,
+            field_formatters=formatters,
+            show_progress=True  # 显示下载进度
         )
-        response['Content-Disposition'] = f'attachment; filename="{filename}"'
-        
-        return response

backend/apps/common/migrations/0001_initial.py

@@ -0,0 +1,34 @@
+# Generated by Django 5.2.7 on 2026-01-06 00:55
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('targets', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='BlacklistRule',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('pattern', models.CharField(help_text='规则模式，如 *.gov, 10.0.0.0/8, 192.168.1.1', max_length=255)),
+                ('rule_type', models.CharField(choices=[('domain', '域名'), ('ip', 'IP地址'), ('cidr', 'CIDR范围'), ('keyword', '关键词')], help_text='规则类型：domain, ip, cidr', max_length=20)),
+                ('scope', models.CharField(choices=[('global', '全局规则'), ('target', 'Target规则')], db_index=True, help_text='作用域：global 或 target', max_length=20)),
+                ('description', models.CharField(blank=True, default='', help_text='规则描述', max_length=500)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('target', models.ForeignKey(blank=True, help_text='关联的 Target（仅 scope=target 时有值）', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='blacklist_rules', to='targets.target')),
+            ],
+            options={
+                'db_table': 'blacklist_rule',
+                'ordering': ['-created_at'],
+                'indexes': [models.Index(fields=['scope', 'rule_type'], name='blacklist_r_scope_6ff77f_idx'), models.Index(fields=['target', 'scope'], name='blacklist_r_target__191441_idx')],
+                'constraints': [models.UniqueConstraint(fields=('pattern', 'scope', 'target'), name='unique_blacklist_rule')],
+            },
+        ),
+    ]

backend/apps/common/models/__init__.py

@@ -0,0 +1,4 @@
+"""Common models"""
+from apps.common.models.blacklist import BlacklistRule
+
+__all__ = ['BlacklistRule']

backend/apps/common/models/blacklist.py

@@ -0,0 +1,71 @@
+"""黑名单规则模型"""
+from django.db import models
+
+
+class BlacklistRule(models.Model):
+    """黑名单规则模型
+    
+    用于存储黑名单过滤规则，支持域名、IP、CIDR 三种类型。
+    支持两层作用域：全局规则和 Target 级规则。
+    """
+    
+    class RuleType(models.TextChoices):
+        DOMAIN = 'domain', '域名'
+        IP = 'ip', 'IP地址'
+        CIDR = 'cidr', 'CIDR范围'
+        KEYWORD = 'keyword', '关键词'
+    
+    class Scope(models.TextChoices):
+        GLOBAL = 'global', '全局规则'
+        TARGET = 'target', 'Target规则'
+    
+    id = models.AutoField(primary_key=True)
+    pattern = models.CharField(
+        max_length=255, 
+        help_text='规则模式，如 *.gov, 10.0.0.0/8, 192.168.1.1'
+    )
+    rule_type = models.CharField(
+        max_length=20, 
+        choices=RuleType.choices,
+        help_text='规则类型：domain, ip, cidr'
+    )
+    scope = models.CharField(
+        max_length=20, 
+        choices=Scope.choices, 
+        db_index=True,
+        help_text='作用域：global 或 target'
+    )
+    target = models.ForeignKey(
+        'targets.Target',
+        on_delete=models.CASCADE,
+        null=True, 
+        blank=True,
+        related_name='blacklist_rules',
+        help_text='关联的 Target（仅 scope=target 时有值）'
+    )
+    description = models.CharField(
+        max_length=500, 
+        blank=True, 
+        default='', 
+        help_text='规则描述'
+    )
+    created_at = models.DateTimeField(auto_now_add=True)
+    
+    class Meta:
+        db_table = 'blacklist_rule'
+        indexes = [
+            models.Index(fields=['scope', 'rule_type']),
+            models.Index(fields=['target', 'scope']),
+        ]
+        constraints = [
+            models.UniqueConstraint(
+                fields=['pattern', 'scope', 'target'],
+                name='unique_blacklist_rule'
+            ),
+        ]
+        ordering = ['-created_at']
+    
+    def __str__(self):
+        if self.scope == self.Scope.TARGET and self.target:
+            return f"[{self.scope}:{self.target_id}] {self.pattern}"
+        return f"[{self.scope}] {self.pattern}"

backend/apps/common/serializers/__init__.py

@@ -0,0 +1,12 @@
+"""Common serializers"""
+from .blacklist_serializers import (
+    BlacklistRuleSerializer,
+    GlobalBlacklistRuleSerializer,
+    TargetBlacklistRuleSerializer,
+)
+
+__all__ = [
+    'BlacklistRuleSerializer',
+    'GlobalBlacklistRuleSerializer',
+    'TargetBlacklistRuleSerializer',
+]

backend/apps/common/serializers/blacklist_serializers.py

@@ -0,0 +1,68 @@
+"""黑名单规则序列化器"""
+from rest_framework import serializers
+
+from apps.common.models import BlacklistRule
+from apps.common.utils import detect_rule_type
+
+
+class BlacklistRuleSerializer(serializers.ModelSerializer):
+    """黑名单规则序列化器"""
+    
+    class Meta:
+        model = BlacklistRule
+        fields = [
+            'id',
+            'pattern',
+            'rule_type',
+            'scope',
+            'target',
+            'description',
+            'created_at',
+        ]
+        read_only_fields = ['id', 'rule_type', 'created_at']
+    
+    def validate_pattern(self, value):
+        """验证规则模式"""
+        if not value or not value.strip():
+            raise serializers.ValidationError("规则模式不能为空")
+        return value.strip()
+    
+    def create(self, validated_data):
+        """创建规则时自动识别规则类型"""
+        pattern = validated_data.get('pattern', '')
+        validated_data['rule_type'] = detect_rule_type(pattern)
+        return super().create(validated_data)
+    
+    def update(self, instance, validated_data):
+        """更新规则时重新识别规则类型"""
+        if 'pattern' in validated_data:
+            pattern = validated_data['pattern']
+            validated_data['rule_type'] = detect_rule_type(pattern)
+        return super().update(instance, validated_data)
+
+
+class GlobalBlacklistRuleSerializer(BlacklistRuleSerializer):
+    """全局黑名单规则序列化器"""
+    
+    class Meta(BlacklistRuleSerializer.Meta):
+        fields = ['id', 'pattern', 'rule_type', 'description', 'created_at']
+        read_only_fields = ['id', 'rule_type', 'created_at']
+    
+    def create(self, validated_data):
+        """创建全局规则"""
+        validated_data['scope'] = BlacklistRule.Scope.GLOBAL
+        validated_data['target'] = None
+        return super().create(validated_data)
+
+
+class TargetBlacklistRuleSerializer(BlacklistRuleSerializer):
+    """Target 黑名单规则序列化器"""
+    
+    class Meta(BlacklistRuleSerializer.Meta):
+        fields = ['id', 'pattern', 'rule_type', 'description', 'created_at']
+        read_only_fields = ['id', 'rule_type', 'created_at']
+    
+    def create(self, validated_data):
+        """创建 Target 规则（target_id 由 view 设置）"""
+        validated_data['scope'] = BlacklistRule.Scope.TARGET
+        return super().create(validated_data)

backend/apps/common/services/__init__.py

@@ -3,13 +3,16 @@
 
 提供系统级别的公共服务，包括：
 - SystemLogService: 系统日志读取服务
+- BlacklistService: 黑名单过滤服务
 
 注意：FilterService 已移至 apps.common.utils.filter_utils
 推荐使用: from apps.common.utils.filter_utils import apply_filters
 """
 
 from .system_log_service import SystemLogService
+from .blacklist_service import BlacklistService
 
 __all__ = [
     'SystemLogService',
+    'BlacklistService',
 ]

backend/apps/common/services/blacklist_service.py

@@ -0,0 +1,176 @@
+"""
+黑名单规则管理服务
+
+负责黑名单规则的 CRUD 操作（数据库层面）。
+过滤逻辑请使用 apps.common.utils.BlacklistFilter。
+
+架构说明：
+- Model: BlacklistRule (apps.common.models.blacklist)
+- Service: BlacklistService (本文件) - 规则 CRUD
+- Utils: BlacklistFilter (apps.common.utils.blacklist_filter) - 过滤逻辑
+- View: GlobalBlacklistView, TargetViewSet.blacklist
+"""
+
+import logging
+from typing import List, Dict, Any, Optional
+
+from django.db.models import QuerySet
+
+from apps.common.utils import detect_rule_type
+
+logger = logging.getLogger(__name__)
+
+
+def _normalize_patterns(patterns: List[str]) -> List[str]:
+    """
+    规范化规则列表：去重 + 过滤空行
+    
+    Args:
+        patterns: 原始规则列表
+        
+    Returns:
+        List[str]: 去重后的规则列表（保持顺序）
+    """
+    return list(dict.fromkeys(filter(None, (p.strip() for p in patterns))))
+
+
+class BlacklistService:
+    """
+    黑名单规则管理服务
+    
+    只负责规则的 CRUD 操作，不包含过滤逻辑。
+    过滤逻辑请使用 BlacklistFilter 工具类。
+    """
+    
+    def get_global_rules(self) -> QuerySet:
+        """
+        获取全局黑名单规则列表
+        
+        Returns:
+            QuerySet: 全局规则查询集
+        """
+        from apps.common.models import BlacklistRule
+        return BlacklistRule.objects.filter(scope=BlacklistRule.Scope.GLOBAL)
+    
+    def get_target_rules(self, target_id: int) -> QuerySet:
+        """
+        获取 Target 级黑名单规则列表
+        
+        Args:
+            target_id: Target ID
+            
+        Returns:
+            QuerySet: Target 级规则查询集
+        """
+        from apps.common.models import BlacklistRule
+        return BlacklistRule.objects.filter(
+            scope=BlacklistRule.Scope.TARGET,
+            target_id=target_id
+        )
+    
+    def get_rules(self, target_id: Optional[int] = None) -> List:
+        """
+        获取黑名单规则（全局 + Target 级）
+        
+        Args:
+            target_id: Target ID，用于加载 Target 级规则
+            
+        Returns:
+            List[BlacklistRule]: 规则列表
+        """
+        from apps.common.models import BlacklistRule
+        
+        # 加载全局规则
+        rules = list(BlacklistRule.objects.filter(scope=BlacklistRule.Scope.GLOBAL))
+        
+        # 加载 Target 级规则
+        if target_id:
+            target_rules = BlacklistRule.objects.filter(
+                scope=BlacklistRule.Scope.TARGET,
+                target_id=target_id
+            )
+            rules.extend(target_rules)
+        
+        return rules
+    
+    def replace_global_rules(self, patterns: List[str]) -> Dict[str, Any]:
+        """
+        全量替换全局黑名单规则（PUT 语义）
+        
+        Args:
+            patterns: 新的规则模式列表
+            
+        Returns:
+            Dict: {'count': int} 最终规则数量
+        """
+        from apps.common.models import BlacklistRule
+        
+        count = self._replace_rules(
+            patterns=patterns,
+            scope=BlacklistRule.Scope.GLOBAL,
+            target=None
+        )
+        
+        logger.info("全量替换全局黑名单规则: %d 条", count)
+        return {'count': count}
+    
+    def replace_target_rules(self, target, patterns: List[str]) -> Dict[str, Any]:
+        """
+        全量替换 Target 级黑名单规则（PUT 语义）
+        
+        Args:
+            target: Target 对象
+            patterns: 新的规则模式列表
+            
+        Returns:
+            Dict: {'count': int} 最终规则数量
+        """
+        from apps.common.models import BlacklistRule
+        
+        count = self._replace_rules(
+            patterns=patterns,
+            scope=BlacklistRule.Scope.TARGET,
+            target=target
+        )
+        
+        logger.info("全量替换 Target 黑名单规则: %d 条 (Target: %s)", count, target.name)
+        return {'count': count}
+    
+    def _replace_rules(self, patterns: List[str], scope: str, target=None) -> int:
+        """
+        内部方法：全量替换规则
+        
+        Args:
+            patterns: 规则模式列表
+            scope: 规则作用域 (GLOBAL/TARGET)
+            target: Target 对象（仅 TARGET 作用域需要）
+            
+        Returns:
+            int: 最终规则数量
+        """
+        from apps.common.models import BlacklistRule
+        from django.db import transaction
+        
+        patterns = _normalize_patterns(patterns)
+        
+        with transaction.atomic():
+            # 1. 删除旧规则
+            delete_filter = {'scope': scope}
+            if target:
+                delete_filter['target'] = target
+            BlacklistRule.objects.filter(**delete_filter).delete()
+            
+            # 2. 创建新规则
+            if patterns:
+                rules = [
+                    BlacklistRule(
+                        pattern=pattern,
+                        rule_type=detect_rule_type(pattern),
+                        scope=scope,
+                        target=target
+                    )
+                    for pattern in patterns
+                ]
+                BlacklistRule.objects.bulk_create(rules)
+        
+        return len(patterns)

backend/apps/common/urls.py

@@ -2,13 +2,19 @@
 通用模块 URL 配置
 
 路由说明：
-- /api/health/    健康检查接口（无需认证）
-- /api/auth/*     认证相关接口（登录、登出、用户信息）
-- /api/system/*   系统管理接口（日志查看等）
+- /api/health/       健康检查接口（无需认证）
+- /api/auth/*        认证相关接口（登录、登出、用户信息）
+- /api/system/*      系统管理接口（日志查看等）
+- /api/blacklist/*   黑名单管理接口
 """
 
 from django.urls import path
-from .views import LoginView, LogoutView, MeView, ChangePasswordView, SystemLogsView, SystemLogFilesView, HealthCheckView
+
+from .views import (
+    LoginView, LogoutView, MeView, ChangePasswordView,
+    SystemLogsView, SystemLogFilesView, HealthCheckView,
+    GlobalBlacklistView,
+)
 
 urlpatterns = [
     # 健康检查（无需认证）
@@ -23,4 +29,7 @@ urlpatterns = [
     # 系统管理
     path('system/logs/', SystemLogsView.as_view(), name='system-logs'),
     path('system/logs/files/', SystemLogFilesView.as_view(), name='system-log-files'),
+    
+    # 黑名单管理（PUT 全量替换模式）
+    path('blacklist/rules/', GlobalBlacklistView.as_view(), name='blacklist-rules'),
 ]

backend/apps/common/utils/__init__.py

@@ -11,8 +11,14 @@ from .csv_utils import (
     generate_csv_rows,
     format_list_field,
     format_datetime,
+    create_csv_export_response,
     UTF8_BOM,
 )
+from .blacklist_filter import (
+    BlacklistFilter,
+    detect_rule_type,
+    extract_host,
+)
 
 __all__ = [
     'deduplicate_for_bulk',
@@ -24,5 +30,9 @@ __all__ = [
     'generate_csv_rows',
     'format_list_field',
     'format_datetime',
+    'create_csv_export_response',
     'UTF8_BOM',
+    'BlacklistFilter',
+    'detect_rule_type',
+    'extract_host',
 ]

backend/apps/common/utils/blacklist_filter.py

@@ -0,0 +1,246 @@
+"""
+黑名单过滤工具
+
+提供域名、IP、CIDR、关键词的黑名单匹配功能。
+纯工具类，不涉及数据库操作。
+
+支持的规则类型：
+    1. 域名精确匹配: example.com
+       - 规则: example.com
+       - 匹配: example.com
+       - 不匹配: sub.example.com, other.com
+    
+    2. 域名后缀匹配: *.example.com
+       - 规则: *.example.com
+       - 匹配: sub.example.com, a.b.example.com, example.com
+       - 不匹配: other.com, example.com.cn
+    
+    3. 关键词匹配: *cdn*
+       - 规则: *cdn*
+       - 匹配: cdn.example.com, a.cdn.b.com, mycdn123.com
+       - 不匹配: example.com (不包含 cdn)
+    
+    4. IP 精确匹配: 192.168.1.1
+       - 规则: 192.168.1.1
+       - 匹配: 192.168.1.1
+       - 不匹配: 192.168.1.2
+    
+    5. CIDR 范围匹配: 192.168.0.0/24
+       - 规则: 192.168.0.0/24
+       - 匹配: 192.168.0.1, 192.168.0.255
+       - 不匹配: 192.168.1.1
+
+使用方式：
+    from apps.common.utils import BlacklistFilter
+    
+    # 创建过滤器（传入规则列表）
+    rules = BlacklistRule.objects.filter(...)
+    filter = BlacklistFilter(rules)
+    
+    # 检查单个目标
+    if filter.is_allowed('http://example.com'):
+        process(url)
+    
+    # 流式处理
+    for url in urls:
+        if filter.is_allowed(url):
+            process(url)
+"""
+
+import ipaddress
+import logging
+from typing import List, Optional
+from urllib.parse import urlparse
+
+from apps.common.validators import is_valid_ip, validate_cidr
+
+logger = logging.getLogger(__name__)
+
+
+def detect_rule_type(pattern: str) -> str:
+    """
+    自动识别规则类型
+    
+    支持的模式：
+    - 域名精确匹配: example.com
+    - 域名后缀匹配: *.example.com
+    - 关键词匹配: *cdn* (匹配包含 cdn 的域名)
+    - IP 精确匹配: 192.168.1.1
+    - CIDR 范围: 192.168.0.0/24
+    
+    Args:
+        pattern: 规则模式字符串
+        
+    Returns:
+        str: 规则类型 ('domain', 'ip', 'cidr', 'keyword')
+    """
+    if not pattern:
+        return 'domain'
+    
+    pattern = pattern.strip()
+    
+    # 检查关键词模式: *keyword* (前后都有星号，中间无点)
+    if pattern.startswith('*') and pattern.endswith('*') and len(pattern) > 2:
+        keyword = pattern[1:-1]
+        # 关键词中不能有点（否则可能是域名模式）
+        if '.' not in keyword:
+            return 'keyword'
+    
+    # 检查 CIDR（包含 /）
+    if '/' in pattern:
+        try:
+            validate_cidr(pattern)
+            return 'cidr'
+        except ValueError:
+            pass
+    
+    # 检查 IP（去掉通配符前缀后验证）
+    clean_pattern = pattern.lstrip('*').lstrip('.')
+    if is_valid_ip(clean_pattern):
+        return 'ip'
+    
+    # 默认为域名
+    return 'domain'
+
+
+def extract_host(target: str) -> str:
+    """
+    从目标字符串中提取主机名
+    
+    支持：
+    - 纯域名：example.com
+    - 纯 IP：192.168.1.1
+    - URL：http://example.com/path
+    
+    Args:
+        target: 目标字符串
+        
+    Returns:
+        str: 提取的主机名
+    """
+    if not target:
+        return ''
+    
+    target = target.strip()
+    
+    # 如果是 URL，提取 hostname
+    if '://' in target:
+        try:
+            parsed = urlparse(target)
+            return parsed.hostname or target
+        except Exception:
+            return target
+    
+    return target
+
+
+class BlacklistFilter:
+    """
+    黑名单过滤器
+    
+    预编译规则，提供高效的匹配功能。
+    """
+    
+    def __init__(self, rules: List):
+        """
+        初始化过滤器
+        
+        Args:
+            rules: BlacklistRule 对象列表
+        """
+        from apps.common.models import BlacklistRule
+        
+        # 预解析：按类型分类 + CIDR 预编译
+        self._domain_rules = []  # (pattern, is_wildcard, suffix)
+        self._ip_rules = set()   # 精确 IP 用 set，O(1) 查找
+        self._cidr_rules = []    # (pattern, network_obj)
+        self._keyword_rules = [] # 关键词列表（小写）
+        
+        # 去重：跨 scope 可能有重复规则
+        seen_patterns = set()
+        
+        for rule in rules:
+            if rule.pattern in seen_patterns:
+                continue
+            seen_patterns.add(rule.pattern)
+            if rule.rule_type == BlacklistRule.RuleType.DOMAIN:
+                pattern = rule.pattern.lower()
+                if pattern.startswith('*.'):
+                    self._domain_rules.append((pattern, True, pattern[1:]))
+                else:
+                    self._domain_rules.append((pattern, False, None))
+            elif rule.rule_type == BlacklistRule.RuleType.IP:
+                self._ip_rules.add(rule.pattern)
+            elif rule.rule_type == BlacklistRule.RuleType.CIDR:
+                try:
+                    network = ipaddress.ip_network(rule.pattern, strict=False)
+                    self._cidr_rules.append((rule.pattern, network))
+                except ValueError:
+                    pass
+            elif rule.rule_type == BlacklistRule.RuleType.KEYWORD:
+                # *cdn* -> cdn
+                keyword = rule.pattern[1:-1].lower()
+                self._keyword_rules.append(keyword)
+    
+    def is_allowed(self, target: str) -> bool:
+        """
+        检查目标是否通过过滤
+        
+        Args:
+            target: 要检查的目标（域名/IP/URL）
+            
+        Returns:
+            bool: True 表示通过（不在黑名单），False 表示被过滤
+        """
+        if not target:
+            return True
+        
+        host = extract_host(target)
+        if not host:
+            return True
+        
+        # 先判断输入类型，再走对应分支
+        if is_valid_ip(host):
+            return self._check_ip_rules(host)
+        else:
+            return self._check_domain_rules(host)
+    
+    def _check_domain_rules(self, host: str) -> bool:
+        """检查域名规则（精确匹配 + 后缀匹配 + 关键词匹配）"""
+        host_lower = host.lower()
+        
+        # 1. 域名规则（精确 + 后缀）
+        for pattern, is_wildcard, suffix in self._domain_rules:
+            if is_wildcard:
+                if host_lower.endswith(suffix) or host_lower == pattern[2:]:
+                    return False
+            else:
+                if host_lower == pattern:
+                    return False
+        
+        # 2. 关键词匹配（字符串 in 操作，O(n*m)）
+        for keyword in self._keyword_rules:
+            if keyword in host_lower:
+                return False
+        
+        return True
+    
+    def _check_ip_rules(self, host: str) -> bool:
+        """检查 IP 规则（精确匹配 + CIDR）"""
+        # 1. IP 精确匹配（O(1)）
+        if host in self._ip_rules:
+            return False
+        
+        # 2. CIDR 匹配
+        if self._cidr_rules:
+            try:
+                ip_obj = ipaddress.ip_address(host)
+                for _, network in self._cidr_rules:
+                    if ip_obj in network:
+                        return False
+            except ValueError:
+                pass
+        
+        return True
+    
+

backend/apps/common/utils/csv_utils.py

@@ -4,13 +4,21 @@
 - UTF-8 BOM（Excel 兼容）
 - RFC 4180 规范转义
 - 流式生成（内存友好）
+- 带 Content-Length 的文件响应（支持浏览器下载进度显示）
 """
 
 import csv
 import io
+import os
+import tempfile
+import logging
 from datetime import datetime
 from typing import Iterator, Dict, Any, List, Callable, Optional
 
+from django.http import FileResponse, StreamingHttpResponse
+
+logger = logging.getLogger(__name__)
+
 # UTF-8 BOM，确保 Excel 正确识别编码
 UTF8_BOM = '\ufeff'
 
@@ -114,3 +122,123 @@ def format_datetime(dt: Optional[datetime]) -> str:
         dt = timezone.localtime(dt)
     
     return dt.strftime('%Y-%m-%d %H:%M:%S')
+
+
+def create_csv_export_response(
+    data_iterator: Iterator[Dict[str, Any]],
+    headers: List[str],
+    filename: str,
+    field_formatters: Optional[Dict[str, Callable]] = None,
+    show_progress: bool = True
+) -> FileResponse | StreamingHttpResponse:
+    """
+    创建 CSV 导出响应
+    
+    根据 show_progress 参数选择响应类型：
+    - True: 使用临时文件 + FileResponse，带 Content-Length（浏览器显示下载进度）
+    - False: 使用 StreamingHttpResponse（内存更友好，但无下载进度）
+    
+    Args:
+        data_iterator: 数据迭代器，每个元素是一个字典
+        headers: CSV 表头列表
+        filename: 下载文件名（如 "export_2024.csv"）
+        field_formatters: 字段格式化函数字典
+        show_progress: 是否显示下载进度（默认 True）
+    
+    Returns:
+        FileResponse 或 StreamingHttpResponse
+    
+    Example:
+        >>> data_iter = service.iter_data()
+        >>> headers = ['url', 'host', 'created_at']
+        >>> formatters = {'created_at': format_datetime}
+        >>> response = create_csv_export_response(
+        ...     data_iter, headers, 'websites.csv', formatters
+        ... )
+        >>> return response
+    """
+    if show_progress:
+        return _create_file_response(data_iterator, headers, filename, field_formatters)
+    else:
+        return _create_streaming_response(data_iterator, headers, filename, field_formatters)
+
+
+def _create_file_response(
+    data_iterator: Iterator[Dict[str, Any]],
+    headers: List[str],
+    filename: str,
+    field_formatters: Optional[Dict[str, Callable]] = None
+) -> FileResponse:
+    """
+    创建带 Content-Length 的文件响应（支持浏览器下载进度）
+    
+    实现方式：先写入临时文件，再返回 FileResponse
+    """
+    # 创建临时文件
+    temp_file = tempfile.NamedTemporaryFile(
+        mode='w', 
+        suffix='.csv', 
+        delete=False, 
+        encoding='utf-8'
+    )
+    temp_path = temp_file.name
+    
+    try:
+        # 流式写入 CSV 数据到临时文件
+        for row in generate_csv_rows(data_iterator, headers, field_formatters):
+            temp_file.write(row)
+        temp_file.close()
+        
+        # 获取文件大小
+        file_size = os.path.getsize(temp_path)
+        
+        # 创建文件响应
+        response = FileResponse(
+            open(temp_path, 'rb'),
+            content_type='text/csv; charset=utf-8',
+            as_attachment=True,
+            filename=filename
+        )
+        response['Content-Length'] = file_size
+        
+        # 设置清理回调：响应完成后删除临时文件
+        original_close = response.file_to_stream.close
+        def close_and_cleanup():
+            original_close()
+            try:
+                os.unlink(temp_path)
+            except OSError:
+                pass
+        response.file_to_stream.close = close_and_cleanup
+        
+        return response
+        
+    except Exception as e:
+        # 清理临时文件
+        try:
+            temp_file.close()
+        except:
+            pass
+        try:
+            os.unlink(temp_path)
+        except OSError:
+            pass
+        logger.error(f"创建 CSV 导出响应失败: {e}")
+        raise
+
+
+def _create_streaming_response(
+    data_iterator: Iterator[Dict[str, Any]],
+    headers: List[str],
+    filename: str,
+    field_formatters: Optional[Dict[str, Callable]] = None
+) -> StreamingHttpResponse:
+    """
+    创建流式响应（无 Content-Length，内存更友好）
+    """
+    response = StreamingHttpResponse(
+        generate_csv_rows(data_iterator, headers, field_formatters),
+        content_type='text/csv; charset=utf-8'
+    )
+    response['Content-Disposition'] = f'attachment; filename="{filename}"'
+    return response

backend/apps/common/views/__init__.py

@@ -5,14 +5,17 @@
 - 健康检查视图：Docker 健康检查
 - 认证相关视图：登录、登出、用户信息、修改密码
 - 系统日志视图：实时日志查看
+- 黑名单视图：全局黑名单规则管理
 """
 
 from .health_views import HealthCheckView
 from .auth_views import LoginView, LogoutView, MeView, ChangePasswordView
 from .system_log_views import SystemLogsView, SystemLogFilesView
+from .blacklist_views import GlobalBlacklistView
 
 __all__ = [
     'HealthCheckView',
     'LoginView', 'LogoutView', 'MeView', 'ChangePasswordView',
     'SystemLogsView', 'SystemLogFilesView',
+    'GlobalBlacklistView',
 ]

backend/apps/common/views/blacklist_views.py

@@ -0,0 +1,80 @@
+"""全局黑名单 API 视图"""
+import logging
+
+from rest_framework import status
+from rest_framework.views import APIView
+from rest_framework.permissions import IsAuthenticated
+
+from apps.common.response_helpers import success_response, error_response
+from apps.common.services import BlacklistService
+
+logger = logging.getLogger(__name__)
+
+
+class GlobalBlacklistView(APIView):
+    """
+    全局黑名单规则 API
+    
+    Endpoints:
+    - GET /api/blacklist/rules/ - 获取全局黑名单列表
+    - PUT /api/blacklist/rules/ - 全量替换规则（文本框保存场景）
+    
+    设计说明：
+    - 使用 PUT 全量替换模式，适合"文本框每行一个规则"的前端场景
+    - 用户编辑文本框 -> 点击保存 -> 后端全量替换
+    
+    架构：MVS 模式
+    - View: 参数验证、响应格式化
+    - Service: 业务逻辑（BlacklistService）
+    - Model: 数据持久化（BlacklistRule）
+    """
+    
+    permission_classes = [IsAuthenticated]
+    
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.blacklist_service = BlacklistService()
+    
+    def get(self, request):
+        """
+        获取全局黑名单规则列表
+        
+        返回格式：
+        {
+            "patterns": ["*.gov", "*.edu", "10.0.0.0/8"]
+        }
+        """
+        rules = self.blacklist_service.get_global_rules()
+        patterns = list(rules.values_list('pattern', flat=True))
+        return success_response(data={'patterns': patterns})
+    
+    def put(self, request):
+        """
+        全量替换全局黑名单规则
+        
+        请求格式：
+        {
+            "patterns": ["*.gov", "*.edu", "10.0.0.0/8"]
+        }
+        
+        或者空数组清空所有规则：
+        {
+            "patterns": []
+        }
+        """
+        patterns = request.data.get('patterns', [])
+        
+        # 兼容字符串输入（换行分隔）
+        if isinstance(patterns, str):
+            patterns = [p for p in patterns.split('\n') if p.strip()]
+        
+        if not isinstance(patterns, list):
+            return error_response(
+                code='VALIDATION_ERROR',
+                message='patterns 必须是数组'
+            )
+        
+        # 调用 Service 层全量替换
+        result = self.blacklist_service.replace_global_rules(patterns)
+        
+        return success_response(data=result)

backend/apps/engine/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.2.7 on 2026-01-02 04:45
+# Generated by Django 5.2.7 on 2026-01-06 00:55
 
 from django.db import migrations, models
 

backend/apps/engine/services/fingerprints/goby.py

@@ -16,10 +16,9 @@ class GobyFingerprintService(BaseFingerprintService):
         """
         校验单条 Goby 指纹
         
-        校验规则：
-        - name 字段必须存在且非空
-        - logic 字段必须存在
-        - rule 字段必须是数组
+        支持两种格式：
+        1. 标准格式: {"name": "...", "logic": "...", "rule": [...]}
+        2. JSONL 格式: {"product": "...", "rule": "..."}
         
         Args:
             item: 单条指纹数据
@@ -27,25 +26,43 @@ class GobyFingerprintService(BaseFingerprintService):
         Returns:
             bool: 是否有效
         """
+        # 标准格式：name + logic + rule(数组)
         name = item.get('name', '')
-        logic = item.get('logic', '')
-        rule = item.get('rule')
-        return bool(name and str(name).strip()) and bool(logic) and isinstance(rule, list)
+        if name and item.get('logic') is not None and isinstance(item.get('rule'), list):
+            return bool(str(name).strip())
+        
+        # JSONL 格式：product + rule(字符串)
+        product = item.get('product', '')
+        rule = item.get('rule', '')
+        return bool(product and str(product).strip() and rule and str(rule).strip())
     
     def to_model_data(self, item: dict) -> dict:
         """
         转换 Goby JSON 格式为 Model 字段
         
+        支持两种输入格式：
+        1. 标准格式: {"name": "...", "logic": "...", "rule": [...]}
+        2. JSONL 格式: {"product": "...", "rule": "..."}
+        
         Args:
             item: 原始 Goby JSON 数据
             
         Returns:
             dict: Model 字段数据
         """
+        # 标准格式
+        if 'name' in item and isinstance(item.get('rule'), list):
+            return {
+                'name': str(item.get('name', '')).strip(),
+                'logic': item.get('logic', ''),
+                'rule': item.get('rule', []),
+            }
+        
+        # JSONL 格式：将 rule 字符串转为单元素数组
         return {
-            'name': str(item.get('name', '')).strip(),
-            'logic': item.get('logic', ''),
-            'rule': item.get('rule', []),
+            'name': str(item.get('product', '')).strip(),
+            'logic': 'or',  # JSONL 格式默认 or 逻辑
+            'rule': [item.get('rule', '')] if item.get('rule') else [],
         }
     
     def get_export_data(self) -> list:

backend/apps/engine/views/fingerprints/base.py

@@ -139,7 +139,7 @@ class BaseFingerprintViewSet(viewsets.ModelViewSet):
         POST /api/engine/fingerprints/{type}/import_file/
         
         请求格式：multipart/form-data
-        - file: JSON 文件
+        - file: JSON 文件（支持标准 JSON 和 JSONL 格式）
         
         返回：同 batch_create
         """
@@ -148,9 +148,12 @@ class BaseFingerprintViewSet(viewsets.ModelViewSet):
             raise ValidationError('缺少文件')
         
         try:
-            json_data = json.load(file)
+            content = file.read().decode('utf-8')
+            json_data = self._parse_json_content(content)
         except json.JSONDecodeError as e:
             raise ValidationError(f'无效的 JSON 格式: {e}')
+        except UnicodeDecodeError as e:
+            raise ValidationError(f'文件编码错误: {e}')
         
         fingerprints = self.parse_import_data(json_data)
         if not fingerprints:
@@ -159,6 +162,41 @@ class BaseFingerprintViewSet(viewsets.ModelViewSet):
         result = self.get_service().batch_create_fingerprints(fingerprints)
         return success_response(data=result, status_code=status.HTTP_201_CREATED)
     
+    def _parse_json_content(self, content: str):
+        """
+        解析 JSON 内容，支持标准 JSON 和 JSONL 格式
+        
+        Args:
+            content: 文件内容字符串
+            
+        Returns:
+            解析后的数据（list 或 dict）
+        """
+        content = content.strip()
+        
+        # 尝试标准 JSON 解析
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError:
+            pass
+        
+        # 尝试 JSONL 格式（每行一个 JSON 对象）
+        lines = content.split('\n')
+        result = []
+        for i, line in enumerate(lines):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                result.append(json.loads(line))
+            except json.JSONDecodeError as e:
+                raise json.JSONDecodeError(f'第 {i + 1} 行解析失败: {e.msg}', e.doc, e.pos)
+        
+        if not result:
+            raise json.JSONDecodeError('文件为空或格式无效', content, 0)
+        
+        return result
+    
     @action(detail=False, methods=['post'], url_path='bulk-delete')
     def bulk_delete(self, request):
         """

backend/apps/scan/configs/command_templates.py

@@ -13,12 +13,14 @@ SCAN_TOOLS_BASE_PATH = getattr(settings, 'SCAN_TOOLS_BASE_PATH', '/usr/local/bin
 
 SUBDOMAIN_DISCOVERY_COMMANDS = {
     'subfinder': {
-        # 默认使用所有数据源（更全面，略慢），并始终开启递归
-        # -all       使用所有数据源
-        # -recursive 对支持递归的源启用递归枚举（默认开启）
-        'base': "subfinder -d {domain} -all -recursive -o '{output_file}' -silent",
+        # 使用所有数据源（包括付费源，只要配置了 API key）
+        # -all       使用所有数据源（slow 但全面）
+        # -v         显示详细输出，包括使用的数据源（调试用）
+        # 注意：不要加 -recursive，它会排除不支持递归的源（如 fofa）
+        'base': "subfinder -d {domain} -all -o '{output_file}' -v",
         'optional': {
             'threads': '-t {threads}',              # 控制并发 goroutine 数
+            'provider_config': "-pc '{provider_config}'",  # Provider 配置文件路径
         }
     },
     

backend/apps/scan/flows/directory_scan_flow.py

@@ -33,7 +33,7 @@ from apps.scan.handlers.scan_flow_handlers import (
     on_scan_flow_completed,
     on_scan_flow_failed,
 )
-from apps.scan.utils import config_parser, build_scan_command, ensure_wordlist_local
+from apps.scan.utils import config_parser, build_scan_command, ensure_wordlist_local, user_log
 
 logger = logging.getLogger(__name__)
 
@@ -413,6 +413,7 @@ def _run_scans_concurrently(
         logger.info("="*60)
         logger.info("使用工具: %s (并发模式, max_workers=%d)", tool_name, max_workers)
         logger.info("="*60)
+        user_log(scan_id, "directory_scan", f"Running {tool_name}")
 
         # 如果配置了 wordlist_name，则先确保本地存在对应的字典文件（含 hash 校验）
         wordlist_name = tool_config.get('wordlist_name')
@@ -467,6 +468,11 @@ def _run_scans_concurrently(
         total_tasks = len(scan_params_list)
         logger.info("开始分批执行 %d 个扫描任务（每批 %d 个）...", total_tasks, max_workers)
         
+        # 进度里程碑跟踪
+        last_progress_percent = 0
+        tool_directories = 0
+        tool_processed = 0
+        
         batch_num = 0
         for batch_start in range(0, total_tasks, max_workers):
             batch_end = min(batch_start + max_workers, total_tasks)
@@ -498,7 +504,9 @@ def _run_scans_concurrently(
                     result = future.result()  # 阻塞等待单个任务完成
                     directories_found = result.get('created_directories', 0)
                     total_directories += directories_found
+                    tool_directories += directories_found
                     processed_sites_count += 1
+                    tool_processed += 1
                     
                     logger.info(
                         "✓ [%d/%d] 站点扫描完成: %s - 发现 %d 个目录",
@@ -517,6 +525,19 @@ def _run_scans_concurrently(
                             "✗ [%d/%d] 站点扫描失败: %s - 错误: %s",
                             idx, len(sites), site_url, exc
                         )
+            
+            # 进度里程碑：每 20% 输出一次
+            current_progress = int((batch_end / total_tasks) * 100)
+            if current_progress >= last_progress_percent + 20:
+                user_log(scan_id, "directory_scan", f"Progress: {batch_end}/{total_tasks} sites scanned")
+                last_progress_percent = (current_progress // 20) * 20
+    
+        # 工具完成日志（开发者日志 + 用户日志）
+        logger.info(
+            "✓ 工具 %s 执行完成 - 已处理站点: %d/%d, 发现目录: %d",
+            tool_name, tool_processed, total_tasks, tool_directories
+        )
+        user_log(scan_id, "directory_scan", f"{tool_name} completed: found {tool_directories} directories")
     
     # 输出汇总信息
     if failed_sites:
@@ -605,6 +626,8 @@ def directory_scan_flow(
             "="*60
         )
         
+        user_log(scan_id, "directory_scan", "Starting directory scan")
+        
         # 参数验证
         if scan_id is None:
             raise ValueError("scan_id 不能为空")
@@ -625,7 +648,8 @@ def directory_scan_flow(
         sites_file, site_count = _export_site_urls(target_id, target_name, directory_scan_dir)
         
         if site_count == 0:
-            logger.warning("目标下没有站点，跳过目录扫描")
+            logger.warning("跳过目录扫描：没有站点可扫描 - Scan ID: %s", scan_id)
+            user_log(scan_id, "directory_scan", "Skipped: no sites to scan", "warning")
             return {
                 'success': True,
                 'scan_id': scan_id,
@@ -664,7 +688,9 @@ def directory_scan_flow(
             logger.warning("所有站点扫描均失败 - 总站点数: %d, 失败数: %d", site_count, len(failed_sites))
             # 不抛出异常，让扫描继续
         
-        logger.info("="*60 + "\n✓ 目录扫描完成\n" + "="*60)
+        # 记录 Flow 完成
+        logger.info("✓ 目录扫描完成 - 发现目录: %d", total_directories)
+        user_log(scan_id, "directory_scan", f"directory_scan completed: found {total_directories} directories")
         
         return {
             'success': True,

backend/apps/scan/flows/fingerprint_detect_flow.py

@@ -29,7 +29,7 @@ from apps.scan.tasks.fingerprint_detect import (
     export_urls_for_fingerprint_task,
     run_xingfinger_and_stream_update_tech_task,
 )
-from apps.scan.utils import build_scan_command
+from apps.scan.utils import build_scan_command, user_log
 from apps.scan.utils.fingerprint_helpers import get_fingerprint_paths
 
 logger = logging.getLogger(__name__)
@@ -168,6 +168,7 @@ def _run_fingerprint_detect(
             "开始执行 %s 指纹识别 - URL数: %d, 超时: %ds, 指纹库: %s",
             tool_name, url_count, timeout, list(fingerprint_paths.keys())
         )
+        user_log(scan_id, "fingerprint_detect", f"Running {tool_name}: {command}")
         
         # 6. 执行扫描任务
         try:
@@ -190,17 +191,21 @@ def _run_fingerprint_detect(
                 'fingerprint_libs': list(fingerprint_paths.keys())
             }
             
+            tool_updated = result.get('updated_count', 0)
             logger.info(
                 "✓ 工具 %s 执行完成 - 处理记录: %d, 更新: %d, 未找到: %d",
                 tool_name,
                 result.get('processed_records', 0),
-                result.get('updated_count', 0),
+                tool_updated,
                 result.get('not_found_count', 0)
             )
+            user_log(scan_id, "fingerprint_detect", f"{tool_name} completed: identified {tool_updated} fingerprints")
             
         except Exception as exc:
-            failed_tools.append({'tool': tool_name, 'reason': str(exc)})
+            reason = str(exc)
+            failed_tools.append({'tool': tool_name, 'reason': reason})
             logger.error("工具 %s 执行失败: %s", tool_name, exc, exc_info=True)
+            user_log(scan_id, "fingerprint_detect", f"{tool_name} failed: {reason}", "error")
     
     if failed_tools:
         logger.warning(
@@ -272,6 +277,8 @@ def fingerprint_detect_flow(
             "="*60
         )
         
+        user_log(scan_id, "fingerprint_detect", "Starting fingerprint detection")
+        
         # 参数验证
         if scan_id is None:
             raise ValueError("scan_id 不能为空")
@@ -293,7 +300,8 @@ def fingerprint_detect_flow(
         urls_file, url_count = _export_urls(target_id, fingerprint_dir, source)
         
         if url_count == 0:
-            logger.warning("目标下没有可用的 URL，跳过指纹识别")
+            logger.warning("跳过指纹识别：没有 URL 可扫描 - Scan ID: %s", scan_id)
+            user_log(scan_id, "fingerprint_detect", "Skipped: no URLs to scan", "warning")
             return {
                 'success': True,
                 'scan_id': scan_id,
@@ -332,8 +340,6 @@ def fingerprint_detect_flow(
             source=source
         )
         
-        logger.info("="*60 + "\n✓ 指纹识别完成\n" + "="*60)
-        
         # 动态生成已执行的任务列表
         executed_tasks = ['export_urls_for_fingerprint']
         executed_tasks.extend([f'run_xingfinger ({tool})' for tool in tool_stats.keys()])
@@ -344,6 +350,10 @@ def fingerprint_detect_flow(
         total_created = sum(stats['result'].get('created_count', 0) for stats in tool_stats.values())
         total_snapshots = sum(stats['result'].get('snapshot_count', 0) for stats in tool_stats.values())
         
+        # 记录 Flow 完成
+        logger.info("✓ 指纹识别完成 - 识别指纹: %d", total_updated)
+        user_log(scan_id, "fingerprint_detect", f"fingerprint_detect completed: identified {total_updated} fingerprints")
+        
         successful_tools = [name for name in enabled_tools.keys() 
                            if name not in [f['tool'] for f in failed_tools]]
         

backend/apps/scan/flows/initiate_scan_flow.py

@@ -115,7 +115,7 @@ def initiate_scan_flow(
         # ==================== Task 2: 获取引擎配置 ====================
         from apps.scan.models import Scan
         scan = Scan.objects.get(id=scan_id)
-        engine_config = scan.merged_configuration
+        engine_config = scan.yaml_configuration
         
         # 使用 engine_names 进行显示
         display_engine_name = ', '.join(scan.engine_names) if scan.engine_names else engine_name

backend/apps/scan/flows/port_scan_flow.py

@@ -20,7 +20,7 @@ from pathlib import Path
 from typing import Callable
 from prefect import flow
 from apps.scan.tasks.port_scan import (
-    export_scan_targets_task,
+    export_hosts_task,
     run_and_stream_save_ports_task
 )
 from apps.scan.handlers.scan_flow_handlers import (
@@ -28,7 +28,7 @@ from apps.scan.handlers.scan_flow_handlers import (
     on_scan_flow_completed,
     on_scan_flow_failed,
 )
-from apps.scan.utils import config_parser, build_scan_command
+from apps.scan.utils import config_parser, build_scan_command, user_log
 
 logger = logging.getLogger(__name__)
 
@@ -157,9 +157,9 @@ def _parse_port_count(tool_config: dict) -> int:
 
 
 
-def _export_scan_targets(target_id: int, port_scan_dir: Path) -> tuple[str, int, str]:
+def _export_hosts(target_id: int, port_scan_dir: Path) -> tuple[str, int, str]:
     """
-    导出扫描目标到文件
+    导出主机列表到文件
     
     根据 Target 类型自动决定导出内容：
     - DOMAIN: 从 Subdomain 表导出子域名
@@ -171,31 +171,31 @@ def _export_scan_targets(target_id: int, port_scan_dir: Path) -> tuple[str, int,
         port_scan_dir: 端口扫描目录
         
     Returns:
-        tuple: (targets_file, target_count, target_type)
+        tuple: (hosts_file, host_count, target_type)
     """
-    logger.info("Step 1: 导出扫描目标列表")
+    logger.info("Step 1: 导出主机列表")
     
-    targets_file = str(port_scan_dir / 'targets.txt')
-    export_result = export_scan_targets_task(
+    hosts_file = str(port_scan_dir / 'hosts.txt')
+    export_result = export_hosts_task(
         target_id=target_id,
-        output_file=targets_file,
+        output_file=hosts_file,
         batch_size=1000  # 每次读取 1000 条，优化内存占用
     )
     
-    target_count = export_result['total_count']
+    host_count = export_result['total_count']
     target_type = export_result.get('target_type', 'unknown')
     
     logger.info(
-        "✓ 扫描目标导出完成 - 类型: %s, 文件: %s, 数量: %d",
+        "✓ 主机列表导出完成 - 类型: %s, 文件: %s, 数量: %d",
         target_type,
         export_result['output_file'],
-        target_count
+        host_count
     )
     
-    if target_count == 0:
-        logger.warning("目标下没有可扫描的地址，无法执行端口扫描")
+    if host_count == 0:
+        logger.warning("目标下没有可扫描的主机，无法执行端口扫描")
     
-    return export_result['output_file'], target_count, target_type
+    return export_result['output_file'], host_count, target_type
 
 
 def _run_scans_sequentially(
@@ -265,6 +265,7 @@ def _run_scans_sequentially(
         
         # 3. 执行扫描任务
         logger.info("开始执行 %s 扫描（超时: %d秒）...", tool_name, config_timeout)
+        user_log(scan_id, "port_scan", f"Running {tool_name}: {command}")
         
         try:
             # 直接调用 task（串行执行）
@@ -286,26 +287,31 @@ def _run_scans_sequentially(
                 'result': result,
                 'timeout': config_timeout
             }
-            processed_records += result.get('processed_records', 0)
+            tool_records = result.get('processed_records', 0)
+            processed_records += tool_records
             logger.info(
                 "✓ 工具 %s 流式处理完成 - 记录数: %d",
-                tool_name, result.get('processed_records', 0)
+                tool_name, tool_records
             )
+            user_log(scan_id, "port_scan", f"{tool_name} completed: found {tool_records} ports")
             
         except subprocess.TimeoutExpired as exc:
             # 超时异常单独处理
             # 注意：流式处理任务超时时，已解析的数据已保存到数据库
-            reason = f"执行超时（配置: {config_timeout}秒）"
+            reason = f"timeout after {config_timeout}s"
             failed_tools.append({'tool': tool_name, 'reason': reason})
             logger.warning(
                 "⚠️ 工具 %s 执行超时 - 超时配置: %d秒\n"
                 "注意：超时前已解析的端口数据已保存到数据库，但扫描未完全完成。",
                 tool_name, config_timeout
             )
+            user_log(scan_id, "port_scan", f"{tool_name} failed: {reason}", "error")
         except Exception as exc:
             # 其他异常
-            failed_tools.append({'tool': tool_name, 'reason': str(exc)})
+            reason = str(exc)
+            failed_tools.append({'tool': tool_name, 'reason': reason})
             logger.error("工具 %s 执行失败: %s", tool_name, exc, exc_info=True)
+            user_log(scan_id, "port_scan", f"{tool_name} failed: {reason}", "error")
     
     if failed_tools:
         logger.warning(
@@ -376,8 +382,8 @@ def port_scan_flow(
             'scan_id': int,
             'target': str,
             'scan_workspace_dir': str,
-            'domains_file': str,
-            'domain_count': int,
+            'hosts_file': str,
+            'host_count': int,
             'processed_records': int,
             'executed_tasks': list,
             'tool_stats': {
@@ -420,25 +426,28 @@ def port_scan_flow(
             "="*60
         )
         
+        user_log(scan_id, "port_scan", "Starting port scan")
+        
         # Step 0: 创建工作目录
         from apps.scan.utils import setup_scan_directory
         port_scan_dir = setup_scan_directory(scan_workspace_dir, 'port_scan')
         
-        # Step 1: 导出扫描目标列表到文件（根据 Target 类型自动决定内容）
-        targets_file, target_count, target_type = _export_scan_targets(target_id, port_scan_dir)
+        # Step 1: 导出主机列表到文件（根据 Target 类型自动决定内容）
+        hosts_file, host_count, target_type = _export_hosts(target_id, port_scan_dir)
         
-        if target_count == 0:
-            logger.warning("目标下没有可扫描的地址，跳过端口扫描")
+        if host_count == 0:
+            logger.warning("跳过端口扫描：没有主机可扫描 - Scan ID: %s", scan_id)
+            user_log(scan_id, "port_scan", "Skipped: no hosts to scan", "warning")
             return {
                 'success': True,
                 'scan_id': scan_id,
                 'target': target_name,
                 'scan_workspace_dir': scan_workspace_dir,
-                'targets_file': targets_file,
-                'target_count': 0,
+                'hosts_file': hosts_file,
+                'host_count': 0,
                 'target_type': target_type,
                 'processed_records': 0,
-                'executed_tasks': ['export_scan_targets'],
+                'executed_tasks': ['export_hosts'],
                 'tool_stats': {
                     'total': 0,
                     'successful': 0,
@@ -460,17 +469,19 @@ def port_scan_flow(
         logger.info("Step 3: 串行执行扫描工具")
         tool_stats, processed_records, successful_tool_names, failed_tools = _run_scans_sequentially(
             enabled_tools=enabled_tools,
-            domains_file=targets_file,  # 现在是 targets_file，兼容原参数名
+            domains_file=hosts_file,
             port_scan_dir=port_scan_dir,
             scan_id=scan_id,
             target_id=target_id,
             target_name=target_name
         )
         
-        logger.info("="*60 + "\n✓ 端口扫描完成\n" + "="*60)
+        # 记录 Flow 完成
+        logger.info("✓ 端口扫描完成 - 发现端口: %d", processed_records)
+        user_log(scan_id, "port_scan", f"port_scan completed: found {processed_records} ports")
         
         # 动态生成已执行的任务列表
-        executed_tasks = ['export_scan_targets', 'parse_config']
+        executed_tasks = ['export_hosts', 'parse_config']
         executed_tasks.extend([f'run_and_stream_save_ports ({tool})' for tool in tool_stats.keys()])
         
         return {
@@ -478,8 +489,8 @@ def port_scan_flow(
             'scan_id': scan_id,
             'target': target_name,
             'scan_workspace_dir': scan_workspace_dir,
-            'targets_file': targets_file,
-            'target_count': target_count,
+            'hosts_file': hosts_file,
+            'host_count': host_count,
             'target_type': target_type,
             'processed_records': processed_records,
             'executed_tasks': executed_tasks,
@@ -488,8 +499,8 @@ def port_scan_flow(
                 'successful': len(successful_tool_names),
                 'failed': len(failed_tools),
                 'successful_tools': successful_tool_names,
-                'failed_tools': failed_tools,  # [{'tool': 'naabu_active', 'reason': '超时'}]
-                'details': tool_stats  # 详细结果（保留向后兼容）
+                'failed_tools': failed_tools,
+                'details': tool_stats
             }
         }
 

backend/apps/scan/flows/site_scan_flow.py

@@ -17,6 +17,7 @@ from apps.common.prefect_django_setup import setup_django_for_prefect
 import logging
 import os
 import subprocess
+import time
 from pathlib import Path
 from typing import Callable
 from prefect import flow
@@ -26,7 +27,7 @@ from apps.scan.handlers.scan_flow_handlers import (
     on_scan_flow_completed,
     on_scan_flow_failed,
 )
-from apps.scan.utils import config_parser, build_scan_command
+from apps.scan.utils import config_parser, build_scan_command, user_log
 
 logger = logging.getLogger(__name__)
 
@@ -198,6 +199,7 @@ def _run_scans_sequentially(
             "开始执行 %s 站点扫描 - URL数: %d, 最终超时: %ds",
             tool_name, total_urls, timeout
         )
+        user_log(scan_id, "site_scan", f"Running {tool_name}: {command}")
         
         # 3. 执行扫描任务
         try:
@@ -218,29 +220,35 @@ def _run_scans_sequentially(
                 'result': result,
                 'timeout': timeout
             }
-            processed_records += result.get('processed_records', 0)
+            tool_records = result.get('processed_records', 0)
+            tool_created = result.get('created_websites', 0)
+            processed_records += tool_records
             
             logger.info(
                 "✓ 工具 %s 流式处理完成 - 处理记录: %d, 创建站点: %d, 跳过: %d",
                 tool_name,
-                result.get('processed_records', 0),
-                result.get('created_websites', 0),
+                tool_records,
+                tool_created,
                 result.get('skipped_no_subdomain', 0) + result.get('skipped_failed', 0)
             )
+            user_log(scan_id, "site_scan", f"{tool_name} completed: found {tool_created} websites")
             
         except subprocess.TimeoutExpired as exc:
             # 超时异常单独处理
-            reason = f"执行超时（配置: {timeout}秒）"
+            reason = f"timeout after {timeout}s"
             failed_tools.append({'tool': tool_name, 'reason': reason})
             logger.warning(
                 "⚠️ 工具 %s 执行超时 - 超时配置: %d秒\n"
                 "注意：超时前已解析的站点数据已保存到数据库，但扫描未完全完成。",
                 tool_name, timeout
             )
+            user_log(scan_id, "site_scan", f"{tool_name} failed: {reason}", "error")
         except Exception as exc:
             # 其他异常
-            failed_tools.append({'tool': tool_name, 'reason': str(exc)})
+            reason = str(exc)
+            failed_tools.append({'tool': tool_name, 'reason': reason})
             logger.error("工具 %s 执行失败: %s", tool_name, exc, exc_info=True)
+            user_log(scan_id, "site_scan", f"{tool_name} failed: {reason}", "error")
     
     if failed_tools:
         logger.warning(
@@ -379,6 +387,8 @@ def site_scan_flow(
         if not scan_workspace_dir:
             raise ValueError("scan_workspace_dir 不能为空")
         
+        user_log(scan_id, "site_scan", "Starting site scan")
+        
         # Step 0: 创建工作目录
         from apps.scan.utils import setup_scan_directory
         site_scan_dir = setup_scan_directory(scan_workspace_dir, 'site_scan')
@@ -389,7 +399,8 @@ def site_scan_flow(
         )
         
         if total_urls == 0:
-            logger.warning("目标下没有可用的站点URL，跳过站点扫描")
+            logger.warning("跳过站点扫描：没有站点 URL 可扫描 - Scan ID: %s", scan_id)
+            user_log(scan_id, "site_scan", "Skipped: no site URLs to scan", "warning")
             return {
                 'success': True,
                 'scan_id': scan_id,
@@ -432,8 +443,6 @@ def site_scan_flow(
             target_name=target_name
         )
         
-        logger.info("="*60 + "\n✓ 站点扫描完成\n" + "="*60)
-        
         # 动态生成已执行的任务列表
         executed_tasks = ['export_site_urls', 'parse_config']
         executed_tasks.extend([f'run_and_stream_save_websites ({tool})' for tool in tool_stats.keys()])
@@ -443,6 +452,10 @@ def site_scan_flow(
         total_skipped_no_subdomain = sum(stats['result'].get('skipped_no_subdomain', 0) for stats in tool_stats.values())
         total_skipped_failed = sum(stats['result'].get('skipped_failed', 0) for stats in tool_stats.values())
         
+        # 记录 Flow 完成
+        logger.info("✓ 站点扫描完成 - 创建站点: %d", total_created)
+        user_log(scan_id, "site_scan", f"site_scan completed: found {total_created} websites")
+        
         return {
             'success': True,
             'scan_id': scan_id,

backend/apps/scan/flows/subdomain_discovery_flow.py

@@ -30,7 +30,7 @@ from apps.scan.handlers.scan_flow_handlers import (
     on_scan_flow_completed,
     on_scan_flow_failed,
 )
-from apps.scan.utils import build_scan_command, ensure_wordlist_local
+from apps.scan.utils import build_scan_command, ensure_wordlist_local, user_log
 from apps.engine.services.wordlist_service import WordlistService
 from apps.common.normalizer import normalize_domain
 from apps.common.validators import validate_domain
@@ -77,7 +77,9 @@ def _validate_and_normalize_target(target_name: str) -> str:
 def _run_scans_parallel(
     enabled_tools: dict,
     domain_name: str,
-    result_dir: Path
+    result_dir: Path,
+    scan_id: int,
+    provider_config_path: str = None
 ) -> tuple[list, list, list]:
     """
     并行运行所有启用的子域名扫描工具
@@ -86,6 +88,8 @@ def _run_scans_parallel(
         enabled_tools: 启用的工具配置字典 {'tool_name': {'timeout': 600, ...}}
         domain_name: 目标域名
         result_dir: 结果输出目录
+        scan_id: 扫描任务 ID（用于记录日志）
+        provider_config_path: Provider 配置文件路径（可选，用于 subfinder）
         
     Returns:
         tuple: (result_files, failed_tools, successful_tool_names)
@@ -110,13 +114,19 @@ def _run_scans_parallel(
         
         # 1.2 构建完整命令（变量替换）
         try:
+            command_params = {
+                'domain': domain_name,      # 对应 {domain}
+                'output_file': output_file  # 对应 {output_file}
+            }
+            
+            # 如果是 subfinder 且有 provider_config，添加到参数
+            if tool_name == 'subfinder' and provider_config_path:
+                command_params['provider_config'] = provider_config_path
+            
             command = build_scan_command(
                 tool_name=tool_name,
                 scan_type='subdomain_discovery',
-                command_params={
-                    'domain': domain_name,      # 对应 {domain}
-                    'output_file': output_file  # 对应 {output_file}
-                },
+                command_params=command_params,
                 tool_config=tool_config
             )
         except Exception as e:
@@ -137,6 +147,9 @@ def _run_scans_parallel(
             f"提交任务 - 工具: {tool_name}, 超时: {timeout}s, 输出: {output_file}"
         )
         
+        # 记录工具开始执行日志
+        user_log(scan_id, "subdomain_discovery", f"Running {tool_name}: {command}")
+        
         future = run_subdomain_discovery_task.submit(
             tool=tool_name,
             command=command,
@@ -164,16 +177,19 @@ def _run_scans_parallel(
             if result:
                 result_files.append(result)
                 logger.info("✓ 扫描工具 %s 执行成功: %s", tool_name, result)
+                user_log(scan_id, "subdomain_discovery", f"{tool_name} completed")
             else:
                 failure_msg = f"{tool_name}: 未生成结果文件"
                 failures.append(failure_msg)
                 failed_tools.append({'tool': tool_name, 'reason': '未生成结果文件'})
                 logger.warning("⚠️ 扫描工具 %s 未生成结果文件", tool_name)
+                user_log(scan_id, "subdomain_discovery", f"{tool_name} failed: no output file", "error")
         except Exception as e:
             failure_msg = f"{tool_name}: {str(e)}"
             failures.append(failure_msg)
             failed_tools.append({'tool': tool_name, 'reason': str(e)})
             logger.warning("⚠️ 扫描工具 %s 执行失败: %s", tool_name, str(e))
+            user_log(scan_id, "subdomain_discovery", f"{tool_name} failed: {str(e)}", "error")
     
     # 4. 检查是否有成功的工具
     if not result_files:
@@ -203,7 +219,8 @@ def _run_single_tool(
     tool_config: dict,
     command_params: dict,
     result_dir: Path,
-    scan_type: str = 'subdomain_discovery'
+    scan_type: str = 'subdomain_discovery',
+    scan_id: int = None
 ) -> str:
     """
     运行单个扫描工具
@@ -214,6 +231,7 @@ def _run_single_tool(
         command_params: 命令参数
         result_dir: 结果目录
         scan_type: 扫描类型
+        scan_id: 扫描 ID（用于记录用户日志）
         
     Returns:
         str: 输出文件路径，失败返回空字符串
@@ -242,7 +260,9 @@ def _run_single_tool(
     if timeout == 'auto':
         timeout = 3600
     
-    logger.info(f"执行 {tool_name}: timeout={timeout}s")
+    logger.info(f"执行 {tool_name}: {command}")
+    if scan_id:
+        user_log(scan_id, scan_type, f"Running {tool_name}: {command}")
     
     try:
         result = run_subdomain_discovery_task(
@@ -401,7 +421,6 @@ def subdomain_discovery_flow(
             logger.warning("目标域名无效，跳过子域名发现扫描: %s", e)
             return _empty_result(scan_id, target_name, scan_workspace_dir)
         
-        # 验证成功后打印日志
         logger.info(
             "="*60 + "\n" +
             "开始子域名发现扫描\n" +
@@ -410,6 +429,7 @@ def subdomain_discovery_flow(
             f"  Workspace: {scan_workspace_dir}\n" +
             "="*60
         )
+        user_log(scan_id, "subdomain_discovery", f"Starting subdomain discovery for {domain_name}")
         
         # 解析配置
         passive_tools = scan_config.get('passive_tools', {})
@@ -428,24 +448,37 @@ def subdomain_discovery_flow(
         failed_tools = []
         successful_tool_names = []
         
-        # ==================== Stage 1: 被动收集（并行）====================
-        logger.info("=" * 40)
-        logger.info("Stage 1: 被动收集（并行）")
-        logger.info("=" * 40)
+        # ==================== 生成 Provider 配置文件 ====================
+        # 为 subfinder 生成第三方数据源配置
+        provider_config_path = None
+        try:
+            from apps.scan.services.subfinder_provider_config_service import SubfinderProviderConfigService
+            provider_config_service = SubfinderProviderConfigService()
+            provider_config_path = provider_config_service.generate(str(result_dir))
+            if provider_config_path:
+                logger.info(f"Provider 配置文件已生成: {provider_config_path}")
+                user_log(scan_id, "subdomain_discovery", "Provider config generated for subfinder")
+        except Exception as e:
+            logger.warning(f"生成 Provider 配置文件失败: {e}")
         
+        # ==================== Stage 1: 被动收集（并行）====================
         if enabled_passive_tools:
+            logger.info("=" * 40)
+            logger.info("Stage 1: 被动收集（并行）")
+            logger.info("=" * 40)
             logger.info("启用工具: %s", ', '.join(enabled_passive_tools.keys()))
+            user_log(scan_id, "subdomain_discovery", f"Stage 1: passive collection ({', '.join(enabled_passive_tools.keys())})")
             result_files, stage1_failed, stage1_success = _run_scans_parallel(
                 enabled_tools=enabled_passive_tools,
                 domain_name=domain_name,
-                result_dir=result_dir
+                result_dir=result_dir,
+                scan_id=scan_id,
+                provider_config_path=provider_config_path
             )
             all_result_files.extend(result_files)
             failed_tools.extend(stage1_failed)
             successful_tool_names.extend(stage1_success)
             executed_tasks.extend([f'passive ({tool})' for tool in stage1_success])
-        else:
-            logger.warning("未启用任何被动收集工具")
         
         # 合并 Stage 1 结果
         timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
@@ -456,7 +489,6 @@ def subdomain_discovery_flow(
         else:
             # 创建空文件
             Path(current_result).touch()
-            logger.warning("Stage 1 无结果，创建空文件")
         
         # ==================== Stage 2: 字典爆破（可选）====================
         bruteforce_enabled = bruteforce_config.get('enabled', False)
@@ -464,6 +496,7 @@ def subdomain_discovery_flow(
             logger.info("=" * 40)
             logger.info("Stage 2: 字典爆破")
             logger.info("=" * 40)
+            user_log(scan_id, "subdomain_discovery", "Stage 2: bruteforce")
             
             bruteforce_tool_config = bruteforce_config.get('subdomain_bruteforce', {})
             wordlist_name = bruteforce_tool_config.get('wordlist_name', 'dns_wordlist.txt')
@@ -496,22 +529,16 @@ def subdomain_discovery_flow(
                         **bruteforce_tool_config,
                         'timeout': timeout_value,
                     }
-                    logger.info(
-                        "subdomain_bruteforce 使用自动 timeout: %s 秒 (字典行数=%s, 3秒/行)",
-                        timeout_value,
-                        line_count_int,
-                    )
 
-                brute_output = str(result_dir / f"subs_brute_{timestamp}.txt")
                 brute_result = _run_single_tool(
                     tool_name='subdomain_bruteforce',
                     tool_config=bruteforce_tool_config,
                     command_params={
                         'domain': domain_name,
                         'wordlist': local_wordlist_path,
-                        'output_file': brute_output
                     },
-                    result_dir=result_dir
+                    result_dir=result_dir,
+                    scan_id=scan_id
                 )
                 
                 if brute_result:
@@ -522,11 +549,16 @@ def subdomain_discovery_flow(
                     )
                     successful_tool_names.append('subdomain_bruteforce')
                     executed_tasks.append('bruteforce')
+                    logger.info("✓ subdomain_bruteforce 执行完成")
+                    user_log(scan_id, "subdomain_discovery", "subdomain_bruteforce completed")
                 else:
                     failed_tools.append({'tool': 'subdomain_bruteforce', 'reason': '执行失败'})
+                    logger.warning("⚠️ subdomain_bruteforce 执行失败")
+                    user_log(scan_id, "subdomain_discovery", "subdomain_bruteforce failed: execution failed", "error")
             except Exception as exc:
-                logger.warning("字典准备失败，跳过字典爆破: %s", exc)
                 failed_tools.append({'tool': 'subdomain_bruteforce', 'reason': str(exc)})
+                logger.warning("字典准备失败，跳过字典爆破: %s", exc)
+                user_log(scan_id, "subdomain_discovery", f"subdomain_bruteforce failed: {str(exc)}", "error")
         
         # ==================== Stage 3: 变异生成 + 验证（可选）====================
         permutation_enabled = permutation_config.get('enabled', False)
@@ -534,6 +566,7 @@ def subdomain_discovery_flow(
             logger.info("=" * 40)
             logger.info("Stage 3: 变异生成 + 存活验证（流式管道）")
             logger.info("=" * 40)
+            user_log(scan_id, "subdomain_discovery", "Stage 3: permutation + resolve")
             
             permutation_tool_config = permutation_config.get('subdomain_permutation_resolve', {})
             
@@ -587,20 +620,19 @@ def subdomain_discovery_flow(
                         'tool': 'subdomain_permutation_resolve',
                         'reason': f"采样检测到泛解析 (膨胀率 {ratio:.1f}x)"
                     })
+                    user_log(scan_id, "subdomain_discovery", f"subdomain_permutation_resolve skipped: wildcard detected (ratio {ratio:.1f}x)", "warning")
                 else:
                     # === Step 3.2: 采样通过，执行完整变异 ===
                     logger.info("采样检测通过，执行完整变异...")
                     
-                    permuted_output = str(result_dir / f"subs_permuted_{timestamp}.txt")
-                    
                     permuted_result = _run_single_tool(
                         tool_name='subdomain_permutation_resolve',
                         tool_config=permutation_tool_config,
                         command_params={
                             'input_file': current_result,
-                            'output_file': permuted_output,
                         },
-                        result_dir=result_dir
+                        result_dir=result_dir,
+                        scan_id=scan_id
                     )
                     
                     if permuted_result:
@@ -611,15 +643,21 @@ def subdomain_discovery_flow(
                         )
                         successful_tool_names.append('subdomain_permutation_resolve')
                         executed_tasks.append('permutation')
+                        logger.info("✓ subdomain_permutation_resolve 执行完成")
+                        user_log(scan_id, "subdomain_discovery", "subdomain_permutation_resolve completed")
                     else:
                         failed_tools.append({'tool': 'subdomain_permutation_resolve', 'reason': '执行失败'})
+                        logger.warning("⚠️ subdomain_permutation_resolve 执行失败")
+                        user_log(scan_id, "subdomain_discovery", "subdomain_permutation_resolve failed: execution failed", "error")
                         
             except subprocess.TimeoutExpired:
-                logger.warning(f"采样检测超时 ({SAMPLE_TIMEOUT}秒)，跳过变异")
                 failed_tools.append({'tool': 'subdomain_permutation_resolve', 'reason': '采样检测超时'})
+                logger.warning(f"采样检测超时 ({SAMPLE_TIMEOUT}秒)，跳过变异")
+                user_log(scan_id, "subdomain_discovery", "subdomain_permutation_resolve failed: sample detection timeout", "error")
             except Exception as e:
-                logger.warning(f"采样检测失败: {e}，跳过变异")
                 failed_tools.append({'tool': 'subdomain_permutation_resolve', 'reason': f'采样检测失败: {e}'})
+                logger.warning(f"采样检测失败: {e}，跳过变异")
+                user_log(scan_id, "subdomain_discovery", f"subdomain_permutation_resolve failed: {str(e)}", "error")
         
         # ==================== Stage 4: DNS 存活验证（可选）====================
         # 无论是否启用 Stage 3，只要 resolve.enabled 为 true 就会执行，对当前所有候选子域做统一 DNS 验证
@@ -628,6 +666,7 @@ def subdomain_discovery_flow(
             logger.info("=" * 40)
             logger.info("Stage 4: DNS 存活验证")
             logger.info("=" * 40)
+            user_log(scan_id, "subdomain_discovery", "Stage 4: DNS resolve")
             
             resolve_tool_config = resolve_config.get('subdomain_resolve', {})
 
@@ -651,30 +690,27 @@ def subdomain_discovery_flow(
                     **resolve_tool_config,
                     'timeout': timeout_value,
                 }
-                logger.info(
-                    "subdomain_resolve 使用自动 timeout: %s 秒 (候选子域数=%s, 3秒/域名)",
-                    timeout_value,
-                    line_count_int,
-                )
 
-            alive_output = str(result_dir / f"subs_alive_{timestamp}.txt")
-            
             alive_result = _run_single_tool(
                 tool_name='subdomain_resolve',
                 tool_config=resolve_tool_config,
                 command_params={
                     'input_file': current_result,
-                    'output_file': alive_output,
                 },
-                result_dir=result_dir
+                result_dir=result_dir,
+                scan_id=scan_id
             )
             
             if alive_result:
                 current_result = alive_result
                 successful_tool_names.append('subdomain_resolve')
                 executed_tasks.append('resolve')
+                logger.info("✓ subdomain_resolve 执行完成")
+                user_log(scan_id, "subdomain_discovery", "subdomain_resolve completed")
             else:
                 failed_tools.append({'tool': 'subdomain_resolve', 'reason': '执行失败'})
+                logger.warning("⚠️ subdomain_resolve 执行失败")
+                user_log(scan_id, "subdomain_discovery", "subdomain_resolve failed: execution failed", "error")
         
         # ==================== Final: 保存到数据库 ====================
         logger.info("=" * 40)
@@ -695,7 +731,9 @@ def subdomain_discovery_flow(
         processed_domains = save_result.get('processed_records', 0)
         executed_tasks.append('save_domains')
         
+        # 记录 Flow 完成
         logger.info("="*60 + "\n✓ 子域名发现扫描完成\n" + "="*60)
+        user_log(scan_id, "subdomain_discovery", f"subdomain_discovery completed: found {processed_domains} subdomains")
         
         return {
             'success': True,

backend/apps/scan/flows/url_fetch/domain_name_url_fetch_flow.py

@@ -59,6 +59,8 @@ def domain_name_url_fetch_flow(
     - IP 和 CIDR 类型会自动跳过（waymore 等工具不支持）
     - 工具会自动收集 *.target_name 的所有历史 URL，无需遍历子域名
     """
+    from apps.scan.utils import user_log
+    
     try:
         output_path = Path(output_dir)
         output_path.mkdir(parents=True, exist_ok=True)
@@ -145,6 +147,9 @@ def domain_name_url_fetch_flow(
                 timeout,
             )
 
+            # 记录工具开始执行日志
+            user_log(scan_id, "url_fetch", f"Running {tool_name}: {command}")
+
             future = run_url_fetcher_task.submit(
                 tool_name=tool_name,
                 command=command,
@@ -163,22 +168,28 @@ def domain_name_url_fetch_flow(
                 if result and result.get("success"):
                     result_files.append(result["output_file"])
                     successful_tools.append(tool_name)
+                    url_count = result.get("url_count", 0)
                     logger.info(
                         "✓ 工具 %s 执行成功 - 发现 URL: %d",
                         tool_name,
-                        result.get("url_count", 0),
+                        url_count,
                     )
+                    user_log(scan_id, "url_fetch", f"{tool_name} completed: found {url_count} urls")
                 else:
+                    reason = "未生成结果或无有效 URL"
                     failed_tools.append(
                         {
                             "tool": tool_name,
-                            "reason": "未生成结果或无有效 URL",
+                            "reason": reason,
                         }
                     )
                     logger.warning("⚠️ 工具 %s 未生成有效结果", tool_name)
+                    user_log(scan_id, "url_fetch", f"{tool_name} failed: {reason}", "error")
             except Exception as e:
-                failed_tools.append({"tool": tool_name, "reason": str(e)})
+                reason = str(e)
+                failed_tools.append({"tool": tool_name, "reason": reason})
                 logger.warning("⚠️ 工具 %s 执行失败: %s", tool_name, e)
+                user_log(scan_id, "url_fetch", f"{tool_name} failed: {reason}", "error")
 
         logger.info(
             "基于 domain_name 的 URL 获取完成 - 成功工具: %s, 失败工具: %s",

backend/apps/scan/flows/url_fetch/main_flow.py

@@ -25,6 +25,7 @@ from apps.scan.handlers.scan_flow_handlers import (
     on_scan_flow_completed,
     on_scan_flow_failed,
 )
+from apps.scan.utils import user_log
 
 from .domain_name_url_fetch_flow import domain_name_url_fetch_flow
 from .sites_url_fetch_flow import sites_url_fetch_flow
@@ -291,6 +292,8 @@ def url_fetch_flow(
             "="*60
         )
         
+        user_log(scan_id, "url_fetch", "Starting URL fetch")
+        
         # Step 1: 准备工作目录
         logger.info("Step 1: 准备工作目录")
         from apps.scan.utils import setup_scan_directory
@@ -403,7 +406,9 @@ def url_fetch_flow(
                 target_id=target_id
             )
         
-        logger.info("="*60 + "\n✓ URL 获取扫描完成\n" + "="*60)
+        # 记录 Flow 完成
+        logger.info("✓ URL 获取完成 - 保存 endpoints: %d", saved_count)
+        user_log(scan_id, "url_fetch", f"url_fetch completed: found {saved_count} endpoints")
         
         # 构建已执行的任务列表
         executed_tasks = ['setup_directory', 'classify_tools']

backend/apps/scan/flows/url_fetch/sites_url_fetch_flow.py

@@ -116,7 +116,8 @@ def sites_url_fetch_flow(
             tools=enabled_tools,
             input_file=sites_file,
             input_type="sites_file",
-            output_dir=output_path
+            output_dir=output_path,
+            scan_id=scan_id
         )
         
         logger.info(

backend/apps/scan/flows/url_fetch/utils.py

@@ -152,7 +152,8 @@ def run_tools_parallel(
     tools: dict,
     input_file: str,
     input_type: str,
-    output_dir: Path
+    output_dir: Path,
+    scan_id: int
 ) -> tuple[list, list, list]:
     """
     并行执行工具列表
@@ -162,11 +163,13 @@ def run_tools_parallel(
         input_file: 输入文件路径
         input_type: 输入类型
         output_dir: 输出目录
+        scan_id: 扫描任务 ID（用于记录日志）
         
     Returns:
         tuple: (result_files, failed_tools, successful_tool_names)
     """
     from apps.scan.tasks.url_fetch import run_url_fetcher_task
+    from apps.scan.utils import user_log
 
     futures: dict[str, object] = {}
     failed_tools: list[dict] = []
@@ -192,6 +195,9 @@ def run_tools_parallel(
             exec_params["timeout"],
         )
 
+        # 记录工具开始执行日志
+        user_log(scan_id, "url_fetch", f"Running {tool_name}: {exec_params['command']}")
+
         # 提交并行任务
         future = run_url_fetcher_task.submit(
             tool_name=tool_name,
@@ -208,22 +214,28 @@ def run_tools_parallel(
             result = future.result()
             if result and result['success']:
                 result_files.append(result['output_file'])
+                url_count = result['url_count']
                 logger.info(
                     "✓ 工具 %s 执行成功 - 发现 URL: %d",
-                    tool_name, result['url_count']
+                    tool_name, url_count
                 )
+                user_log(scan_id, "url_fetch", f"{tool_name} completed: found {url_count} urls")
             else:
+                reason = '未生成结果或无有效URL'
                 failed_tools.append({
                     'tool': tool_name,
-                    'reason': '未生成结果或无有效URL'
+                    'reason': reason
                 })
                 logger.warning("⚠️ 工具 %s 未生成有效结果", tool_name)
+                user_log(scan_id, "url_fetch", f"{tool_name} failed: {reason}", "error")
         except Exception as e:
+            reason = str(e)
             failed_tools.append({
                 'tool': tool_name,
-                'reason': str(e)
+                'reason': reason
             })
             logger.warning("⚠️ 工具 %s 执行失败: %s", tool_name, e)
+            user_log(scan_id, "url_fetch", f"{tool_name} failed: {reason}", "error")
 
     # 计算成功的工具列表
     failed_tool_names = [f['tool'] for f in failed_tools]

backend/apps/scan/flows/vuln_scan/endpoints_vuln_scan_flow.py

@@ -12,7 +12,7 @@ from apps.scan.handlers.scan_flow_handlers import (
     on_scan_flow_completed,
     on_scan_flow_failed,
 )
-from apps.scan.utils import build_scan_command, ensure_nuclei_templates_local
+from apps.scan.utils import build_scan_command, ensure_nuclei_templates_local, user_log
 from apps.scan.tasks.vuln_scan import (
     export_endpoints_task,
     run_vuln_tool_task,
@@ -141,6 +141,7 @@ def endpoints_vuln_scan_flow(
             # Dalfox XSS 使用流式任务，一边解析一边保存漏洞结果
             if tool_name == "dalfox_xss":
                 logger.info("开始执行漏洞扫描工具 %s（流式保存漏洞结果，已提交任务）", tool_name)
+                user_log(scan_id, "vuln_scan", f"Running {tool_name}: {command}")
                 future = run_and_stream_save_dalfox_vulns_task.submit(
                     cmd=command,
                     tool_name=tool_name,
@@ -163,6 +164,7 @@ def endpoints_vuln_scan_flow(
             elif tool_name == "nuclei":
                 # Nuclei 使用流式任务
                 logger.info("开始执行漏洞扫描工具 %s（流式保存漏洞结果，已提交任务）", tool_name)
+                user_log(scan_id, "vuln_scan", f"Running {tool_name}: {command}")
                 future = run_and_stream_save_nuclei_vulns_task.submit(
                     cmd=command,
                     tool_name=tool_name,
@@ -185,6 +187,7 @@ def endpoints_vuln_scan_flow(
             else:
                 # 其他工具仍使用非流式执行逻辑
                 logger.info("开始执行漏洞扫描工具 %s（已提交任务）", tool_name)
+                user_log(scan_id, "vuln_scan", f"Running {tool_name}: {command}")
                 future = run_vuln_tool_task.submit(
                     tool_name=tool_name,
                     command=command,
@@ -203,24 +206,34 @@ def endpoints_vuln_scan_flow(
         # 统一收集所有工具的执行结果
         for tool_name, meta in tool_futures.items():
             future = meta["future"]
-            result = future.result()
+            try:
+                result = future.result()
 
-            if meta["mode"] == "streaming":
-                tool_results[tool_name] = {
-                    "command": meta["command"],
-                    "timeout": meta["timeout"],
-                    "processed_records": result.get("processed_records"),
-                    "created_vulns": result.get("created_vulns"),
-                    "command_log_file": meta["log_file"],
-                }
-            else:
-                tool_results[tool_name] = {
-                    "command": meta["command"],
-                    "timeout": meta["timeout"],
-                    "duration": result.get("duration"),
-                    "returncode": result.get("returncode"),
-                    "command_log_file": result.get("command_log_file"),
-                }
+                if meta["mode"] == "streaming":
+                    created_vulns = result.get("created_vulns", 0)
+                    tool_results[tool_name] = {
+                        "command": meta["command"],
+                        "timeout": meta["timeout"],
+                        "processed_records": result.get("processed_records"),
+                        "created_vulns": created_vulns,
+                        "command_log_file": meta["log_file"],
+                    }
+                    logger.info("✓ 工具 %s 执行完成 - 漏洞: %d", tool_name, created_vulns)
+                    user_log(scan_id, "vuln_scan", f"{tool_name} completed: found {created_vulns} vulnerabilities")
+                else:
+                    tool_results[tool_name] = {
+                        "command": meta["command"],
+                        "timeout": meta["timeout"],
+                        "duration": result.get("duration"),
+                        "returncode": result.get("returncode"),
+                        "command_log_file": result.get("command_log_file"),
+                    }
+                    logger.info("✓ 工具 %s 执行完成 - returncode=%s", tool_name, result.get("returncode"))
+                    user_log(scan_id, "vuln_scan", f"{tool_name} completed")
+            except Exception as e:
+                reason = str(e)
+                logger.error("工具 %s 执行失败: %s", tool_name, e, exc_info=True)
+                user_log(scan_id, "vuln_scan", f"{tool_name} failed: {reason}", "error")
 
         return {
             "success": True,

backend/apps/scan/flows/vuln_scan/main_flow.py

@@ -11,6 +11,7 @@ from apps.scan.handlers.scan_flow_handlers import (
     on_scan_flow_failed,
 )
 from apps.scan.configs.command_templates import get_command_template
+from apps.scan.utils import user_log
 from .endpoints_vuln_scan_flow import endpoints_vuln_scan_flow
 
 
@@ -72,6 +73,9 @@ def vuln_scan_flow(
         if not enabled_tools:
             raise ValueError("enabled_tools 不能为空")
 
+        logger.info("开始漏洞扫描 - Scan ID: %s, Target: %s", scan_id, target_name)
+        user_log(scan_id, "vuln_scan", "Starting vulnerability scan")
+
         # Step 1: 分类工具
         endpoints_tools, other_tools = _classify_vuln_tools(enabled_tools)
 
@@ -99,6 +103,14 @@ def vuln_scan_flow(
             enabled_tools=endpoints_tools,
         )
 
+        # 记录 Flow 完成
+        total_vulns = sum(
+            r.get("created_vulns", 0) 
+            for r in endpoint_result.get("tool_results", {}).values()
+        )
+        logger.info("✓ 漏洞扫描完成 - 新增漏洞: %d", total_vulns)
+        user_log(scan_id, "vuln_scan", f"vuln_scan completed: found {total_vulns} vulnerabilities")
+
         # 目前只有一个子 Flow，直接返回其结果
         return endpoint_result
 

backend/apps/scan/handlers/scan_flow_handlers.py

@@ -14,6 +14,7 @@ from prefect import Flow
 from prefect.client.schemas import FlowRun, State
 
 from apps.scan.utils.performance import FlowPerformanceTracker
+from apps.scan.utils import user_log
 
 logger = logging.getLogger(__name__)
 
@@ -136,6 +137,7 @@ def on_scan_flow_failed(flow: Flow, flow_run: FlowRun, state: State) -> None:
     - 更新阶段进度为 failed
     - 发送扫描失败通知
     - 记录性能指标（含错误信息）
+    - 写入 ScanLog 供前端显示
     
     Args:
         flow: Prefect Flow 对象
@@ -152,6 +154,11 @@ def on_scan_flow_failed(flow: Flow, flow_run: FlowRun, state: State) -> None:
     # 提取错误信息
     error_message = str(state.message) if state.message else "未知错误"
     
+    # 写入 ScanLog 供前端显示
+    stage = _get_stage_from_flow_name(flow.name)
+    if scan_id and stage:
+        user_log(scan_id, stage, f"Failed: {error_message}", "error")
+    
     # 记录性能指标（失败情况）
     tracker = _flow_trackers.pop(str(flow_run.id), None)
     if tracker:

backend/apps/scan/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.2.7 on 2026-01-02 04:45
+# Generated by Django 5.2.7 on 2026-01-06 00:55
 
 import django.contrib.postgres.fields
 import django.db.models.deletion
@@ -31,6 +31,20 @@ class Migration(migrations.Migration):
                 'db_table': 'notification_settings',
             },
         ),
+        migrations.CreateModel(
+            name='SubfinderProviderSettings',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('providers', models.JSONField(default=dict, help_text='各 Provider 的 API Key 配置')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+            ],
+            options={
+                'verbose_name': 'Subfinder Provider 配置',
+                'verbose_name_plural': 'Subfinder Provider 配置',
+                'db_table': 'subfinder_provider_settings',
+            },
+        ),
         migrations.CreateModel(
             name='Notification',
             fields=[
@@ -57,7 +71,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(primary_key=True, serialize=False)),
                 ('engine_ids', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), default=list, help_text='引擎 ID 列表', size=None)),
                 ('engine_names', models.JSONField(default=list, help_text='引擎名称列表，如 ["引擎A", "引擎B"]')),
-                ('merged_configuration', models.TextField(default='', help_text='合并后的 YAML 配置')),
+                ('yaml_configuration', models.TextField(default='', help_text='YAML 格式的扫描配置')),
                 ('created_at', models.DateTimeField(auto_now_add=True, help_text='任务创建时间')),
                 ('stopped_at', models.DateTimeField(blank=True, help_text='扫描结束时间', null=True)),
                 ('status', models.CharField(choices=[('cancelled', '已取消'), ('completed', '已完成'), ('failed', '失败'), ('initiated', '初始化'), ('running', '运行中')], db_index=True, default='initiated', help_text='任务状态', max_length=20)),
@@ -87,7 +101,22 @@ class Migration(migrations.Migration):
                 'verbose_name_plural': '扫描任务',
                 'db_table': 'scan',
                 'ordering': ['-created_at'],
-                'indexes': [models.Index(fields=['-created_at'], name='scan_created_0bb6c7_idx'), models.Index(fields=['target'], name='scan_target__718b9d_idx'), models.Index(fields=['deleted_at', '-created_at'], name='scan_deleted_eb17e8_idx')],
+            },
+        ),
+        migrations.CreateModel(
+            name='ScanLog',
+            fields=[
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
+                ('level', models.CharField(choices=[('info', 'Info'), ('warning', 'Warning'), ('error', 'Error')], default='info', help_text='日志级别', max_length=10)),
+                ('content', models.TextField(help_text='日志内容')),
+                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True, help_text='创建时间')),
+                ('scan', models.ForeignKey(help_text='关联的扫描任务', on_delete=django.db.models.deletion.CASCADE, related_name='logs', to='scan.scan')),
+            ],
+            options={
+                'verbose_name': '扫描日志',
+                'verbose_name_plural': '扫描日志',
+                'db_table': 'scan_log',
+                'ordering': ['created_at'],
             },
         ),
         migrations.CreateModel(
@@ -97,7 +126,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(help_text='任务名称', max_length=200)),
                 ('engine_ids', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), default=list, help_text='引擎 ID 列表', size=None)),
                 ('engine_names', models.JSONField(default=list, help_text='引擎名称列表，如 ["引擎A", "引擎B"]')),
-                ('merged_configuration', models.TextField(default='', help_text='合并后的 YAML 配置')),
+                ('yaml_configuration', models.TextField(default='', help_text='YAML 格式的扫描配置')),
                 ('cron_expression', models.CharField(default='0 2 * * *', help_text='Cron 表达式，格式：分 时 日 月 周', max_length=100)),
                 ('is_enabled', models.BooleanField(db_index=True, default=True, help_text='是否启用')),
                 ('run_count', models.IntegerField(default=0, help_text='已执行次数')),
@@ -113,7 +142,34 @@ class Migration(migrations.Migration):
                 'verbose_name_plural': '定时扫描任务',
                 'db_table': 'scheduled_scan',
                 'ordering': ['-created_at'],
-                'indexes': [models.Index(fields=['-created_at'], name='scheduled_s_created_9b9c2e_idx'), models.Index(fields=['is_enabled', '-created_at'], name='scheduled_s_is_enab_23d660_idx'), models.Index(fields=['name'], name='scheduled_s_name_bf332d_idx')],
             },
         ),
+        migrations.AddIndex(
+            model_name='scan',
+            index=models.Index(fields=['-created_at'], name='scan_created_0bb6c7_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scan',
+            index=models.Index(fields=['target'], name='scan_target__718b9d_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scan',
+            index=models.Index(fields=['deleted_at', '-created_at'], name='scan_deleted_eb17e8_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scanlog',
+            index=models.Index(fields=['scan', 'created_at'], name='scan_log_scan_id_c4814a_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scheduledscan',
+            index=models.Index(fields=['-created_at'], name='scheduled_s_created_9b9c2e_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scheduledscan',
+            index=models.Index(fields=['is_enabled', '-created_at'], name='scheduled_s_is_enab_23d660_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scheduledscan',
+            index=models.Index(fields=['name'], name='scheduled_s_name_bf332d_idx'),
+        ),
     ]

backend/apps/scan/models/__init__.py

@@ -0,0 +1,18 @@
+"""Scan Models - 统一导出"""
+
+from .scan_models import Scan, SoftDeleteManager
+from .scan_log_model import ScanLog
+from .scheduled_scan_model import ScheduledScan
+from .subfinder_provider_settings_model import SubfinderProviderSettings
+
+# 兼容旧名称（已废弃，请使用 SubfinderProviderSettings）
+ProviderSettings = SubfinderProviderSettings
+
+__all__ = [
+    'Scan',
+    'ScanLog',
+    'ScheduledScan',
+    'SoftDeleteManager',
+    'SubfinderProviderSettings',
+    'ProviderSettings',  # 兼容旧名称
+]

backend/apps/scan/models/scan_log_model.py

@@ -0,0 +1,41 @@
+"""扫描日志模型"""
+
+from django.db import models
+
+
+class ScanLog(models.Model):
+    """扫描日志模型"""
+    
+    class Level(models.TextChoices):
+        INFO = 'info', 'Info'
+        WARNING = 'warning', 'Warning'
+        ERROR = 'error', 'Error'
+    
+    id = models.BigAutoField(primary_key=True)
+    scan = models.ForeignKey(
+        'Scan',
+        on_delete=models.CASCADE,
+        related_name='logs',
+        db_index=True,
+        help_text='关联的扫描任务'
+    )
+    level = models.CharField(
+        max_length=10,
+        choices=Level.choices,
+        default=Level.INFO,
+        help_text='日志级别'
+    )
+    content = models.TextField(help_text='日志内容')
+    created_at = models.DateTimeField(auto_now_add=True, db_index=True, help_text='创建时间')
+    
+    class Meta:
+        db_table = 'scan_log'
+        verbose_name = '扫描日志'
+        verbose_name_plural = '扫描日志'
+        ordering = ['created_at']
+        indexes = [
+            models.Index(fields=['scan', 'created_at']),
+        ]
+    
+    def __str__(self):
+        return f"[{self.level}] {self.content[:50]}"

backend/apps/scan/models/scan_models.py

@@ -1,9 +1,9 @@
+"""扫描相关模型"""
+
 from django.db import models
 from django.contrib.postgres.fields import ArrayField
 
-from ..common.definitions import ScanStatus
-
-
+from apps.common.definitions import ScanStatus
 
 
 class SoftDeleteManager(models.Manager):
@@ -30,9 +30,9 @@ class Scan(models.Model):
         default=list,
         help_text='引擎名称列表，如 ["引擎A", "引擎B"]'
     )
-    merged_configuration = models.TextField(
+    yaml_configuration = models.TextField(
         default='',
-        help_text='合并后的 YAML 配置'
+        help_text='YAML 格式的扫描配置'
     )
 
     created_at = models.DateTimeField(auto_now_add=True, help_text='任务创建时间')
@@ -97,99 +97,10 @@ class Scan(models.Model):
         verbose_name_plural = '扫描任务'
         ordering = ['-created_at']
         indexes = [
-            models.Index(fields=['-created_at']),  # 优化按创建时间降序排序（list 查询的默认排序）
-            models.Index(fields=['target']),  # 优化按目标查询扫描任务
-            models.Index(fields=['deleted_at', '-created_at']),  # 软删除 + 时间索引
+            models.Index(fields=['-created_at']),
+            models.Index(fields=['target']),
+            models.Index(fields=['deleted_at', '-created_at']),
         ]
 
     def __str__(self):
         return f"Scan #{self.id} - {self.target.name}"
-
-
-class ScheduledScan(models.Model):
-    """
-    定时扫描任务模型
-    
-    调度机制：
-    - APScheduler 每分钟检查 next_run_time
-    - 到期任务通过 task_distributor 分发到 Worker 执行
-    - 支持 cron 表达式进行灵活调度
-    
-    扫描模式（二选一）：
-    - 组织扫描：设置 organization，执行时动态获取组织下所有目标
-    - 目标扫描：设置 target，扫描单个目标
-    - organization 优先级高于 target
-    """
-    
-    id = models.AutoField(primary_key=True)
-    
-    # 基本信息
-    name = models.CharField(max_length=200, help_text='任务名称')
-    
-    # 多引擎支持字段
-    engine_ids = ArrayField(
-        models.IntegerField(),
-        default=list,
-        help_text='引擎 ID 列表'
-    )
-    engine_names = models.JSONField(
-        default=list,
-        help_text='引擎名称列表，如 ["引擎A", "引擎B"]'
-    )
-    merged_configuration = models.TextField(
-        default='',
-        help_text='合并后的 YAML 配置'
-    )
-    
-    # 关联的组织（组织扫描模式：执行时动态获取组织下所有目标）
-    organization = models.ForeignKey(
-        'targets.Organization',
-        on_delete=models.CASCADE,
-        related_name='scheduled_scans',
-        null=True,
-        blank=True,
-        help_text='扫描组织（设置后执行时动态获取组织下所有目标）'
-    )
-    
-    # 关联的目标（目标扫描模式：扫描单个目标）
-    target = models.ForeignKey(
-        'targets.Target',
-        on_delete=models.CASCADE,
-        related_name='scheduled_scans',
-        null=True,
-        blank=True,
-        help_text='扫描单个目标（与 organization 二选一）'
-    )
-    
-    # 调度配置 - 直接使用 Cron 表达式
-    cron_expression = models.CharField(
-        max_length=100,
-        default='0 2 * * *',
-        help_text='Cron 表达式，格式：分 时 日 月 周'
-    )
-    
-    # 状态
-    is_enabled = models.BooleanField(default=True, db_index=True, help_text='是否启用')
-    
-    # 执行统计
-    run_count = models.IntegerField(default=0, help_text='已执行次数')
-    last_run_time = models.DateTimeField(null=True, blank=True, help_text='上次执行时间')
-    next_run_time = models.DateTimeField(null=True, blank=True, help_text='下次执行时间')
-    
-    # 时间戳
-    created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
-    updated_at = models.DateTimeField(auto_now=True, help_text='更新时间')
-    
-    class Meta:
-        db_table = 'scheduled_scan'
-        verbose_name = '定时扫描任务'
-        verbose_name_plural = '定时扫描任务'
-        ordering = ['-created_at']
-        indexes = [
-            models.Index(fields=['-created_at']),
-            models.Index(fields=['is_enabled', '-created_at']),
-            models.Index(fields=['name']),  # 优化 name 搜索
-        ]
-    
-    def __str__(self):
-        return f"ScheduledScan #{self.id} - {self.name}"

backend/apps/scan/models/scheduled_scan_model.py

@@ -0,0 +1,73 @@
+"""定时扫描任务模型"""
+
+from django.db import models
+from django.contrib.postgres.fields import ArrayField
+
+
+class ScheduledScan(models.Model):
+    """定时扫描任务模型"""
+    
+    id = models.AutoField(primary_key=True)
+    
+    name = models.CharField(max_length=200, help_text='任务名称')
+    
+    engine_ids = ArrayField(
+        models.IntegerField(),
+        default=list,
+        help_text='引擎 ID 列表'
+    )
+    engine_names = models.JSONField(
+        default=list,
+        help_text='引擎名称列表，如 ["引擎A", "引擎B"]'
+    )
+    yaml_configuration = models.TextField(
+        default='',
+        help_text='YAML 格式的扫描配置'
+    )
+    
+    organization = models.ForeignKey(
+        'targets.Organization',
+        on_delete=models.CASCADE,
+        related_name='scheduled_scans',
+        null=True,
+        blank=True,
+        help_text='扫描组织（设置后执行时动态获取组织下所有目标）'
+    )
+    
+    target = models.ForeignKey(
+        'targets.Target',
+        on_delete=models.CASCADE,
+        related_name='scheduled_scans',
+        null=True,
+        blank=True,
+        help_text='扫描单个目标（与 organization 二选一）'
+    )
+    
+    cron_expression = models.CharField(
+        max_length=100,
+        default='0 2 * * *',
+        help_text='Cron 表达式，格式：分 时 日 月 周'
+    )
+    
+    is_enabled = models.BooleanField(default=True, db_index=True, help_text='是否启用')
+    
+    run_count = models.IntegerField(default=0, help_text='已执行次数')
+    last_run_time = models.DateTimeField(null=True, blank=True, help_text='上次执行时间')
+    next_run_time = models.DateTimeField(null=True, blank=True, help_text='下次执行时间')
+    
+    created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
+    updated_at = models.DateTimeField(auto_now=True, help_text='更新时间')
+    
+    class Meta:
+        db_table = 'scheduled_scan'
+        verbose_name = '定时扫描任务'
+        verbose_name_plural = '定时扫描任务'
+        ordering = ['-created_at']
+        indexes = [
+            models.Index(fields=['-created_at']),
+            models.Index(fields=['is_enabled', '-created_at']),
+            models.Index(fields=['name']),
+        ]
+    
+    def __str__(self):
+        return f"ScheduledScan #{self.id} - {self.name}"

backend/apps/scan/models/subfinder_provider_settings_model.py

@@ -0,0 +1,64 @@
+"""Subfinder Provider 配置模型（单例模式）
+
+用于存储 subfinder 第三方数据源的 API Key 配置
+"""
+
+from django.db import models
+
+
+class SubfinderProviderSettings(models.Model):
+    """
+    Subfinder Provider 配置（单例模式）
+    存储第三方数据源的 API Key 配置，用于 subfinder 子域名发现
+    
+    支持的 Provider:
+    - fofa: email + api_key (composite)
+    - censys: api_id + api_secret (composite)
+    - hunter, shodan, zoomeye, securitytrails, threatbook, quake: api_key (single)
+    """
+    
+    providers = models.JSONField(
+        default=dict,
+        help_text='各 Provider 的 API Key 配置'
+    )
+    
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+    
+    class Meta:
+        db_table = 'subfinder_provider_settings'
+        verbose_name = 'Subfinder Provider 配置'
+        verbose_name_plural = 'Subfinder Provider 配置'
+    
+    DEFAULT_PROVIDERS = {
+        'fofa': {'enabled': False, 'email': '', 'api_key': ''},
+        'hunter': {'enabled': False, 'api_key': ''},
+        'shodan': {'enabled': False, 'api_key': ''},
+        'censys': {'enabled': False, 'api_id': '', 'api_secret': ''},
+        'zoomeye': {'enabled': False, 'api_key': ''},
+        'securitytrails': {'enabled': False, 'api_key': ''},
+        'threatbook': {'enabled': False, 'api_key': ''},
+        'quake': {'enabled': False, 'api_key': ''},
+    }
+    
+    def save(self, *args, **kwargs):
+        self.pk = 1
+        super().save(*args, **kwargs)
+    
+    @classmethod
+    def get_instance(cls) -> 'SubfinderProviderSettings':
+        """获取或创建单例实例"""
+        obj, _ = cls.objects.get_or_create(
+            pk=1,
+            defaults={'providers': cls.DEFAULT_PROVIDERS.copy()}
+        )
+        return obj
+    
+    def get_provider_config(self, provider: str) -> dict:
+        """获取指定 Provider 的配置"""
+        return self.providers.get(provider, self.DEFAULT_PROVIDERS.get(provider, {}))
+    
+    def is_provider_enabled(self, provider: str) -> bool:
+        """检查指定 Provider 是否启用"""
+        config = self.get_provider_config(provider)
+        return config.get('enabled', False)

backend/apps/scan/notifications/urls.py

@@ -21,9 +21,6 @@ urlpatterns = [
 
     # 标记全部已读
     path('mark-all-as-read/', NotificationMarkAllAsReadView.as_view(), name='mark-all-as-read'),
-    
-    # 测试通知
-    path('test/', views.notifications_test, name='test'),
 ]
 
 # WebSocket 实时通知路由在 routing.py 中定义：ws://host/ws/notifications/

backend/apps/scan/notifications/views.py

@@ -23,45 +23,7 @@ from .services import NotificationService, NotificationSettingsService
 logger = logging.getLogger(__name__)
 
 
-def notifications_test(request):
-    """
-    测试通知推送
-    """
-    try:
-        from .services import create_notification
-        from django.http import JsonResponse
 
-        level_param = request.GET.get('level', NotificationLevel.LOW)
-        try:
-            level_choice = NotificationLevel(level_param)
-        except ValueError:
-            level_choice = NotificationLevel.LOW
-
-        title = request.GET.get('title') or "测试通知"
-        message = request.GET.get('message') or "这是一条测试通知消息"
-
-        # 创建测试通知
-        notification = create_notification(
-            title=title,
-            message=message,
-            level=level_choice
-        )
-        
-        return JsonResponse({
-            'success': True,
-            'message': '测试通知已发送',
-            'notification_id': notification.id
-        })
-        
-    except Exception as e:
-        logger.error(f"发送测试通知失败: {e}")
-        return JsonResponse({
-            'success': False,
-            'error': str(e)
-        }, status=500)
-
-
-# build_api_response 已废弃，请使用 success_response/error_response
 
 
 def _parse_bool(value: str | None) -> bool | None:

backend/apps/scan/repositories/django_scan_repository.py

@@ -104,7 +104,7 @@ class DjangoScanRepository:
         target: Target,
         engine_ids: List[int],
         engine_names: List[str],
-        merged_configuration: str,
+        yaml_configuration: str,
         results_dir: str,
         status: ScanStatus = ScanStatus.INITIATED
     ) -> Scan:
@@ -115,7 +115,7 @@ class DjangoScanRepository:
             target: 扫描目标
             engine_ids: 引擎 ID 列表
             engine_names: 引擎名称列表
-            merged_configuration: 合并后的 YAML 配置
+            yaml_configuration: YAML 格式的扫描配置
             results_dir: 结果目录
             status: 初始状态
         
@@ -126,7 +126,7 @@ class DjangoScanRepository:
             target=target,
             engine_ids=engine_ids,
             engine_names=engine_names,
-            merged_configuration=merged_configuration,
+            yaml_configuration=yaml_configuration,
             results_dir=results_dir,
             status=status,
             container_ids=[]

backend/apps/scan/repositories/scheduled_scan_repository.py

@@ -31,7 +31,7 @@ class ScheduledScanDTO:
     name: str = ''
     engine_ids: List[int] = None  # 多引擎支持
     engine_names: List[str] = None  # 引擎名称列表
-    merged_configuration: str = ''  # 合并后的配置
+    yaml_configuration: str = ''  # YAML 格式的扫描配置
     organization_id: Optional[int] = None  # 组织扫描模式
     target_id: Optional[int] = None  # 目标扫描模式
     cron_expression: Optional[str] = None
@@ -114,7 +114,7 @@ class DjangoScheduledScanRepository:
                 name=dto.name,
                 engine_ids=dto.engine_ids,
                 engine_names=dto.engine_names,
-                merged_configuration=dto.merged_configuration,
+                yaml_configuration=dto.yaml_configuration,
                 organization_id=dto.organization_id,  # 组织扫描模式
                 target_id=dto.target_id if not dto.organization_id else None,  # 目标扫描模式
                 cron_expression=dto.cron_expression,
@@ -147,8 +147,8 @@ class DjangoScheduledScanRepository:
                     scheduled_scan.engine_ids = dto.engine_ids
                 if dto.engine_names is not None:
                     scheduled_scan.engine_names = dto.engine_names
-                if dto.merged_configuration is not None:
-                    scheduled_scan.merged_configuration = dto.merged_configuration
+                if dto.yaml_configuration is not None:
+                    scheduled_scan.yaml_configuration = dto.yaml_configuration
                 if dto.cron_expression is not None:
                     scheduled_scan.cron_expression = dto.cron_expression
                 if dto.is_enabled is not None:

backend/apps/scan/serializers.py

@@ -1,266 +0,0 @@
-from rest_framework import serializers
-from django.db.models import Count
-
-from .models import Scan, ScheduledScan
-
-
-class ScanSerializer(serializers.ModelSerializer):
-    """扫描任务序列化器"""
-    target_name = serializers.SerializerMethodField()
-    
-    class Meta:
-        model = Scan
-        fields = [
-            'id', 'target', 'target_name', 'engine_ids', 'engine_names',
-            'created_at', 'stopped_at', 'status', 'results_dir',
-            'container_ids', 'error_message'
-        ]
-        read_only_fields = [
-            'id', 'created_at', 'stopped_at', 'results_dir',
-            'container_ids', 'error_message', 'status'
-        ]
-    
-    def get_target_name(self, obj):
-        """获取目标名称"""
-        return obj.target.name if obj.target else None
-
-
-class ScanHistorySerializer(serializers.ModelSerializer):
-    """扫描历史列表专用序列化器
-    
-    为前端扫描历史页面提供优化的数据格式，包括：
-    - 扫描汇总统计（子域名、端点、漏洞数量）
-    - 进度百分比和当前阶段
-    - 执行节点信息
-    """
-    
-    # 字段映射
-    target_name = serializers.CharField(source='target.name', read_only=True)
-    worker_name = serializers.CharField(source='worker.name', read_only=True, allow_null=True)
-    
-    # 计算字段
-    summary = serializers.SerializerMethodField()
-    
-    # 进度跟踪字段（直接从模型读取）
-    progress = serializers.IntegerField(read_only=True)
-    current_stage = serializers.CharField(read_only=True)
-    stage_progress = serializers.JSONField(read_only=True)
-    
-    class Meta:
-        model = Scan
-        fields = [
-            'id', 'target', 'target_name', 'engine_ids', 'engine_names', 
-            'worker_name', 'created_at', 'status', 'error_message', 'summary', 
-            'progress', 'current_stage', 'stage_progress'
-        ]
-    
-    def get_summary(self, obj):
-        """获取扫描汇总数据。
-
-        设计原则：
-        - 子域名/网站/端点/IP/目录使用缓存字段（避免实时 COUNT）
-        - 漏洞统计使用 Scan 上的缓存字段，在扫描结束时统一聚合
-        """
-        # 1. 使用缓存字段构建基础统计（子域名、网站、端点、IP、目录）
-        summary = {
-            'subdomains': obj.cached_subdomains_count or 0,
-            'websites': obj.cached_websites_count or 0,
-            'endpoints': obj.cached_endpoints_count or 0,
-            'ips': obj.cached_ips_count or 0,
-            'directories': obj.cached_directories_count or 0,
-        }
-
-        # 2. 使用 Scan 模型上的缓存漏洞统计（按严重性聚合）
-        summary['vulnerabilities'] = {
-            'total': obj.cached_vulns_total or 0,
-            'critical': obj.cached_vulns_critical or 0,
-            'high': obj.cached_vulns_high or 0,
-            'medium': obj.cached_vulns_medium or 0,
-            'low': obj.cached_vulns_low or 0,
-        }
-
-        return summary
-
-
-class QuickScanSerializer(serializers.Serializer):
-    """
-    快速扫描序列化器
-    
-    功能：
-    - 接收目标列表和引擎配置
-    - 自动创建/获取目标
-    - 立即发起扫描
-    """
-    
-    # 批量创建的最大数量限制
-    MAX_BATCH_SIZE = 1000
-    
-    # 目标列表
-    targets = serializers.ListField(
-        child=serializers.DictField(),
-        help_text='目标列表，每个目标包含 name 字段'
-    )
-    
-    # 扫描引擎 ID 列表
-    engine_ids = serializers.ListField(
-        child=serializers.IntegerField(),
-        required=True,
-        help_text='使用的扫描引擎 ID 列表 (必填)'
-    )
-    
-    def validate_targets(self, value):
-        """验证目标列表"""
-        if not value:
-            raise serializers.ValidationError("目标列表不能为空")
-        
-        # 检查数量限制，防止服务器过载
-        if len(value) > self.MAX_BATCH_SIZE:
-            raise serializers.ValidationError(
-                f"快速扫描最多支持 {self.MAX_BATCH_SIZE} 个目标，当前提交了 {len(value)} 个"
-            )
-        
-        # 验证每个目标的必填字段
-        for idx, target in enumerate(value):
-            if 'name' not in target:
-                raise serializers.ValidationError(f"第 {idx + 1} 个目标缺少 name 字段")
-            if not target['name']:
-                raise serializers.ValidationError(f"第 {idx + 1} 个目标的 name 不能为空")
-        
-        return value
-    
-    def validate_engine_ids(self, value):
-        """验证引擎 ID 列表"""
-        if not value:
-            raise serializers.ValidationError("engine_ids 不能为空")
-        return value
-
-
-# ==================== 定时扫描序列化器 ====================
-
-class ScheduledScanSerializer(serializers.ModelSerializer):
-    """定时扫描任务序列化器（用于列表和详情）"""
-    
-    # 关联字段
-    organization_id = serializers.IntegerField(source='organization.id', read_only=True, allow_null=True)
-    organization_name = serializers.CharField(source='organization.name', read_only=True, allow_null=True)
-    target_id = serializers.IntegerField(source='target.id', read_only=True, allow_null=True)
-    target_name = serializers.CharField(source='target.name', read_only=True, allow_null=True)
-    scan_mode = serializers.SerializerMethodField()
-    
-    class Meta:
-        model = ScheduledScan
-        fields = [
-            'id', 'name',
-            'engine_ids', 'engine_names',
-            'organization_id', 'organization_name',
-            'target_id', 'target_name',
-            'scan_mode',
-            'cron_expression',
-            'is_enabled',
-            'run_count', 'last_run_time', 'next_run_time',
-            'created_at', 'updated_at'
-        ]
-        read_only_fields = [
-            'id', 'run_count',
-            'last_run_time', 'next_run_time',
-            'created_at', 'updated_at'
-        ]
-    
-    def get_scan_mode(self, obj):
-        """获取扫描模式：organization 或 target"""
-        return 'organization' if obj.organization_id else 'target'
-
-
-class CreateScheduledScanSerializer(serializers.Serializer):
-    """创建定时扫描任务序列化器
-    
-    扫描模式（二选一）：
-    - 组织扫描：提供 organization_id，执行时动态获取组织下所有目标
-    - 目标扫描：提供 target_id，扫描单个目标
-    """
-    
-    name = serializers.CharField(max_length=200, help_text='任务名称')
-    engine_ids = serializers.ListField(
-        child=serializers.IntegerField(),
-        help_text='扫描引擎 ID 列表'
-    )
-    
-    # 组织扫描模式
-    organization_id = serializers.IntegerField(
-        required=False,
-        allow_null=True,
-        help_text='组织 ID（组织扫描模式：执行时动态获取组织下所有目标）'
-    )
-    
-    # 目标扫描模式
-    target_id = serializers.IntegerField(
-        required=False,
-        allow_null=True,
-        help_text='目标 ID（目标扫描模式：扫描单个目标）'
-    )
-    
-    cron_expression = serializers.CharField(
-        max_length=100,
-        default='0 2 * * *',
-        help_text='Cron 表达式，格式：分 时 日 月 周'
-    )
-    is_enabled = serializers.BooleanField(default=True, help_text='是否立即启用')
-    
-    def validate_engine_ids(self, value):
-        """验证引擎 ID 列表"""
-        if not value:
-            raise serializers.ValidationError("engine_ids 不能为空")
-        return value
-    
-    def validate(self, data):
-        """验证 organization_id 和 target_id 互斥"""
-        organization_id = data.get('organization_id')
-        target_id = data.get('target_id')
-        
-        if not organization_id and not target_id:
-            raise serializers.ValidationError('必须提供 organization_id 或 target_id 其中之一')
-        
-        if organization_id and target_id:
-            raise serializers.ValidationError('organization_id 和 target_id 只能提供其中之一')
-        
-        return data
-
-
-class UpdateScheduledScanSerializer(serializers.Serializer):
-    """更新定时扫描任务序列化器"""
-    
-    name = serializers.CharField(max_length=200, required=False, help_text='任务名称')
-    engine_ids = serializers.ListField(
-        child=serializers.IntegerField(),
-        required=False,
-        help_text='扫描引擎 ID 列表'
-    )
-    
-    # 组织扫描模式
-    organization_id = serializers.IntegerField(
-        required=False,
-        allow_null=True,
-        help_text='组织 ID（设置后清空 target_id）'
-    )
-    
-    # 目标扫描模式
-    target_id = serializers.IntegerField(
-        required=False,
-        allow_null=True,
-        help_text='目标 ID（设置后清空 organization_id）'
-    )
-    
-    cron_expression = serializers.CharField(max_length=100, required=False, help_text='Cron 表达式')
-    is_enabled = serializers.BooleanField(required=False, help_text='是否启用')
-    
-    def validate_engine_ids(self, value):
-        """验证引擎 ID 列表"""
-        if value is not None and not value:
-            raise serializers.ValidationError("engine_ids 不能为空")
-        return value
-
-
-class ToggleScheduledScanSerializer(serializers.Serializer):
-    """切换定时扫描启用状态序列化器"""
-    
-    is_enabled = serializers.BooleanField(help_text='是否启用')

backend/apps/scan/serializers/__init__.py

@@ -0,0 +1,40 @@
+"""Scan Serializers - 统一导出"""
+
+from .mixins import ScanConfigValidationMixin
+from .scan_serializers import (
+    ScanSerializer,
+    ScanHistorySerializer,
+    QuickScanSerializer,
+    InitiateScanSerializer,
+)
+from .scan_log_serializers import ScanLogSerializer
+from .scheduled_scan_serializers import (
+    ScheduledScanSerializer,
+    CreateScheduledScanSerializer,
+    UpdateScheduledScanSerializer,
+    ToggleScheduledScanSerializer,
+)
+from .subfinder_provider_settings_serializers import SubfinderProviderSettingsSerializer
+
+# 兼容旧名称
+ProviderSettingsSerializer = SubfinderProviderSettingsSerializer
+
+__all__ = [
+    # Mixins
+    'ScanConfigValidationMixin',
+    # Scan
+    'ScanSerializer',
+    'ScanHistorySerializer',
+    'QuickScanSerializer',
+    'InitiateScanSerializer',
+    # ScanLog
+    'ScanLogSerializer',
+    # Scheduled Scan
+    'ScheduledScanSerializer',
+    'CreateScheduledScanSerializer',
+    'UpdateScheduledScanSerializer',
+    'ToggleScheduledScanSerializer',
+    # Subfinder Provider Settings
+    'SubfinderProviderSettingsSerializer',
+    'ProviderSettingsSerializer',  # 兼容旧名称
+]

backend/apps/scan/serializers/mixins.py

@@ -0,0 +1,57 @@
+"""序列化器通用 Mixin 和工具类"""
+
+from rest_framework import serializers
+import yaml
+
+
+class DuplicateKeyLoader(yaml.SafeLoader):
+    """自定义 YAML Loader，检测重复 key"""
+    pass
+
+
+def _check_duplicate_keys(loader, node, deep=False):
+    """检测 YAML mapping 中的重复 key"""
+    mapping = {}
+    for key_node, value_node in node.value:
+        key = loader.construct_object(key_node, deep=deep)
+        if key in mapping:
+            raise yaml.constructor.ConstructorError(
+                "while constructing a mapping", node.start_mark,
+                f"发现重复的配置项 '{key}'，后面的配置会覆盖前面的配置，请删除重复项", key_node.start_mark
+            )
+        mapping[key] = loader.construct_object(value_node, deep=deep)
+    return mapping
+
+
+DuplicateKeyLoader.add_constructor(
+    yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+    _check_duplicate_keys
+)
+
+
+class ScanConfigValidationMixin:
+    """扫描配置验证 Mixin"""
+    
+    def validate_configuration(self, value):
+        """验证 YAML 配置格式"""
+        if not value or not value.strip():
+            raise serializers.ValidationError("configuration 不能为空")
+        
+        try:
+            yaml.load(value, Loader=DuplicateKeyLoader)
+        except yaml.YAMLError as e:
+            raise serializers.ValidationError(f"无效的 YAML 格式: {str(e)}")
+        
+        return value
+    
+    def validate_engine_ids(self, value):
+        """验证引擎 ID 列表"""
+        if not value:
+            raise serializers.ValidationError("engine_ids 不能为空，请至少选择一个扫描引擎")
+        return value
+    
+    def validate_engine_names(self, value):
+        """验证引擎名称列表"""
+        if not value:
+            raise serializers.ValidationError("engine_names 不能为空")
+        return value

backend/apps/scan/serializers/scan_log_serializers.py

@@ -0,0 +1,13 @@
+"""扫描日志序列化器"""
+
+from rest_framework import serializers
+
+from ..models import ScanLog
+
+
+class ScanLogSerializer(serializers.ModelSerializer):
+    """扫描日志序列化器"""
+    
+    class Meta:
+        model = ScanLog
+        fields = ['id', 'level', 'content', 'created_at']

backend/apps/scan/serializers/scan_serializers.py

@@ -0,0 +1,111 @@
+"""扫描任务序列化器"""
+
+from rest_framework import serializers
+
+from ..models import Scan
+from .mixins import ScanConfigValidationMixin
+
+
+class ScanSerializer(serializers.ModelSerializer):
+    """扫描任务序列化器"""
+    target_name = serializers.SerializerMethodField()
+    
+    class Meta:
+        model = Scan
+        fields = [
+            'id', 'target', 'target_name', 'engine_ids', 'engine_names',
+            'created_at', 'stopped_at', 'status', 'results_dir',
+            'container_ids', 'error_message'
+        ]
+        read_only_fields = [
+            'id', 'created_at', 'stopped_at', 'results_dir',
+            'container_ids', 'error_message', 'status'
+        ]
+    
+    def get_target_name(self, obj):
+        return obj.target.name if obj.target else None
+
+
+class ScanHistorySerializer(serializers.ModelSerializer):
+    """扫描历史列表序列化器"""
+    
+    target_name = serializers.CharField(source='target.name', read_only=True)
+    worker_name = serializers.CharField(source='worker.name', read_only=True, allow_null=True)
+    summary = serializers.SerializerMethodField()
+    progress = serializers.IntegerField(read_only=True)
+    current_stage = serializers.CharField(read_only=True)
+    stage_progress = serializers.JSONField(read_only=True)
+    
+    class Meta:
+        model = Scan
+        fields = [
+            'id', 'target', 'target_name', 'engine_ids', 'engine_names', 
+            'worker_name', 'created_at', 'status', 'error_message', 'summary', 
+            'progress', 'current_stage', 'stage_progress'
+        ]
+    
+    def get_summary(self, obj):
+        summary = {
+            'subdomains': obj.cached_subdomains_count or 0,
+            'websites': obj.cached_websites_count or 0,
+            'endpoints': obj.cached_endpoints_count or 0,
+            'ips': obj.cached_ips_count or 0,
+            'directories': obj.cached_directories_count or 0,
+        }
+        summary['vulnerabilities'] = {
+            'total': obj.cached_vulns_total or 0,
+            'critical': obj.cached_vulns_critical or 0,
+            'high': obj.cached_vulns_high or 0,
+            'medium': obj.cached_vulns_medium or 0,
+            'low': obj.cached_vulns_low or 0,
+        }
+        return summary
+
+
+class QuickScanSerializer(ScanConfigValidationMixin, serializers.Serializer):
+    """快速扫描序列化器"""
+    
+    MAX_BATCH_SIZE = 5000
+    
+    targets = serializers.ListField(
+        child=serializers.DictField(),
+        help_text='目标列表，每个目标包含 name 字段'
+    )
+    configuration = serializers.CharField(required=True, help_text='YAML 格式的扫描配置')
+    engine_ids = serializers.ListField(child=serializers.IntegerField(), required=True)
+    engine_names = serializers.ListField(child=serializers.CharField(), required=True)
+    
+    def validate_targets(self, value):
+        if not value:
+            raise serializers.ValidationError("目标列表不能为空")
+        if len(value) > self.MAX_BATCH_SIZE:
+            raise serializers.ValidationError(
+                f"快速扫描最多支持 {self.MAX_BATCH_SIZE} 个目标，当前提交了 {len(value)} 个"
+            )
+        for idx, target in enumerate(value):
+            if 'name' not in target:
+                raise serializers.ValidationError(f"第 {idx + 1} 个目标缺少 name 字段")
+            if not target['name']:
+                raise serializers.ValidationError(f"第 {idx + 1} 个目标的 name 不能为空")
+        return value
+
+
+class InitiateScanSerializer(ScanConfigValidationMixin, serializers.Serializer):
+    """发起扫描任务序列化器"""
+    
+    configuration = serializers.CharField(required=True, help_text='YAML 格式的扫描配置')
+    engine_ids = serializers.ListField(child=serializers.IntegerField(), required=True)
+    engine_names = serializers.ListField(child=serializers.CharField(), required=True)
+    organization_id = serializers.IntegerField(required=False, allow_null=True)
+    target_id = serializers.IntegerField(required=False, allow_null=True)
+    
+    def validate(self, data):
+        organization_id = data.get('organization_id')
+        target_id = data.get('target_id')
+        
+        if not organization_id and not target_id:
+            raise serializers.ValidationError('必须提供 organization_id 或 target_id 其中之一')
+        if organization_id and target_id:
+            raise serializers.ValidationError('organization_id 和 target_id 只能提供其中之一')
+        
+        return data

backend/apps/scan/serializers/scheduled_scan_serializers.py

@@ -0,0 +1,84 @@
+"""定时扫描序列化器"""
+
+from rest_framework import serializers
+
+from ..models import ScheduledScan
+from .mixins import ScanConfigValidationMixin
+
+
+class ScheduledScanSerializer(serializers.ModelSerializer):
+    """定时扫描任务序列化器（用于列表和详情）"""
+    
+    organization_id = serializers.IntegerField(source='organization.id', read_only=True, allow_null=True)
+    organization_name = serializers.CharField(source='organization.name', read_only=True, allow_null=True)
+    target_id = serializers.IntegerField(source='target.id', read_only=True, allow_null=True)
+    target_name = serializers.CharField(source='target.name', read_only=True, allow_null=True)
+    scan_mode = serializers.SerializerMethodField()
+    
+    class Meta:
+        model = ScheduledScan
+        fields = [
+            'id', 'name',
+            'engine_ids', 'engine_names',
+            'organization_id', 'organization_name',
+            'target_id', 'target_name',
+            'scan_mode',
+            'cron_expression',
+            'is_enabled',
+            'run_count', 'last_run_time', 'next_run_time',
+            'created_at', 'updated_at'
+        ]
+        read_only_fields = [
+            'id', 'run_count',
+            'last_run_time', 'next_run_time',
+            'created_at', 'updated_at'
+        ]
+    
+    def get_scan_mode(self, obj):
+        return 'organization' if obj.organization_id else 'target'
+
+
+class CreateScheduledScanSerializer(ScanConfigValidationMixin, serializers.Serializer):
+    """创建定时扫描任务序列化器"""
+    
+    name = serializers.CharField(max_length=200, help_text='任务名称')
+    configuration = serializers.CharField(required=True, help_text='YAML 格式的扫描配置')
+    engine_ids = serializers.ListField(child=serializers.IntegerField(), required=True)
+    engine_names = serializers.ListField(child=serializers.CharField(), required=True)
+    organization_id = serializers.IntegerField(required=False, allow_null=True)
+    target_id = serializers.IntegerField(required=False, allow_null=True)
+    cron_expression = serializers.CharField(max_length=100, default='0 2 * * *')
+    is_enabled = serializers.BooleanField(default=True)
+    
+    def validate(self, data):
+        organization_id = data.get('organization_id')
+        target_id = data.get('target_id')
+        
+        if not organization_id and not target_id:
+            raise serializers.ValidationError('必须提供 organization_id 或 target_id 其中之一')
+        if organization_id and target_id:
+            raise serializers.ValidationError('organization_id 和 target_id 只能提供其中之一')
+        
+        return data
+
+
+class UpdateScheduledScanSerializer(serializers.Serializer):
+    """更新定时扫描任务序列化器"""
+    
+    name = serializers.CharField(max_length=200, required=False)
+    engine_ids = serializers.ListField(child=serializers.IntegerField(), required=False)
+    organization_id = serializers.IntegerField(required=False, allow_null=True)
+    target_id = serializers.IntegerField(required=False, allow_null=True)
+    cron_expression = serializers.CharField(max_length=100, required=False)
+    is_enabled = serializers.BooleanField(required=False)
+    
+    def validate_engine_ids(self, value):
+        if value is not None and not value:
+            raise serializers.ValidationError("engine_ids 不能为空")
+        return value
+
+
+class ToggleScheduledScanSerializer(serializers.Serializer):
+    """切换定时扫描启用状态序列化器"""
+    
+    is_enabled = serializers.BooleanField(help_text='是否启用')

backend/apps/scan/serializers/subfinder_provider_settings_serializers.py

@@ -0,0 +1,55 @@
+"""Subfinder Provider 配置序列化器"""
+
+from rest_framework import serializers
+
+
+class SubfinderProviderSettingsSerializer(serializers.Serializer):
+    """Subfinder Provider 配置序列化器
+    
+    支持的 Provider:
+    - fofa: email + api_key (composite)
+    - censys: api_id + api_secret (composite)
+    - hunter, shodan, zoomeye, securitytrails, threatbook, quake: api_key (single)
+    
+    注意：djangorestframework-camel-case 会自动处理 camelCase <-> snake_case 转换
+    所以这里统一使用 snake_case
+    """
+    
+    VALID_PROVIDERS = {
+        'fofa', 'hunter', 'shodan', 'censys', 
+        'zoomeye', 'securitytrails', 'threatbook', 'quake'
+    }
+    
+    def to_internal_value(self, data):
+        """验证并转换输入数据"""
+        if not isinstance(data, dict):
+            raise serializers.ValidationError('Expected a dictionary')
+        
+        result = {}
+        for provider, config in data.items():
+            if provider not in self.VALID_PROVIDERS:
+                continue
+            
+            if not isinstance(config, dict):
+                continue
+            
+            db_config = {'enabled': bool(config.get('enabled', False))}
+            
+            if provider == 'fofa':
+                db_config['email'] = str(config.get('email', ''))
+                db_config['api_key'] = str(config.get('api_key', ''))
+            elif provider == 'censys':
+                db_config['api_id'] = str(config.get('api_id', ''))
+                db_config['api_secret'] = str(config.get('api_secret', ''))
+            else:
+                db_config['api_key'] = str(config.get('api_key', ''))
+            
+            result[provider] = db_config
+        
+        return result
+    
+    def to_representation(self, instance):
+        """输出数据（数据库格式，camel-case 中间件会自动转换）"""
+        if isinstance(instance, dict):
+            return instance
+        return instance.providers if hasattr(instance, 'providers') else {}

backend/apps/scan/services/__init__.py

@@ -17,7 +17,6 @@ from .scan_state_service import ScanStateService
 from .scan_control_service import ScanControlService
 from .scan_stats_service import ScanStatsService
 from .scheduled_scan_service import ScheduledScanService
-from .blacklist_service import BlacklistService
 from .target_export_service import TargetExportService
 
 __all__ = [
@@ -27,7 +26,6 @@ __all__ = [
     'ScanControlService',
     'ScanStatsService',
     'ScheduledScanService',
-    'BlacklistService',      # 黑名单过滤服务
     'TargetExportService',   # 目标导出服务
 ]
 

backend/apps/scan/services/blacklist_service.py

@@ -1,82 +0,0 @@
-"""
-黑名单过滤服务
-
-过滤敏感域名（如 .gov、.edu、.mil 等）
-
-当前版本使用默认规则，后续将支持从前端配置加载。
-"""
-
-from typing import List, Optional
-from django.db.models import QuerySet
-import re
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-class BlacklistService:
-    """
-    黑名单过滤服务 - 过滤敏感域名
-    
-    TODO: 后续版本支持从前端配置加载黑名单规则
-    - 用户在开始扫描时配置黑名单 URL、域名、IP
-    - 黑名单规则存储在数据库中，与 Scan 或 Engine 关联
-    """
-    
-    # 默认黑名单正则规则
-    DEFAULT_PATTERNS = [
-        r'\.gov$',           # .gov 结尾
-        r'\.gov\.[a-z]{2}$', # .gov.cn, .gov.uk 等
-    ]
-    
-    def __init__(self, patterns: Optional[List[str]] = None):
-        """
-        初始化黑名单服务
-        
-        Args:
-            patterns: 正则表达式列表，None 使用默认规则
-        """
-        self.patterns = patterns or self.DEFAULT_PATTERNS
-        self._compiled_patterns = [re.compile(p) for p in self.patterns]
-    
-    def filter_queryset(
-        self,
-        queryset: QuerySet,
-        url_field: str = 'url'
-    ) -> QuerySet:
-        """
-        数据库层面过滤 queryset
-        
-        使用 PostgreSQL 正则表达式排除黑名单 URL
-        
-        Args:
-            queryset: 原始 queryset
-            url_field: URL 字段名
-            
-        Returns:
-            QuerySet: 过滤后的 queryset
-        """
-        for pattern in self.patterns:
-            queryset = queryset.exclude(**{f'{url_field}__regex': pattern})
-        return queryset
-    
-    def filter_url(self, url: str) -> bool:
-        """
-        检查单个 URL 是否通过黑名单过滤
-        
-        Args:
-            url: 要检查的 URL
-            
-        Returns:
-            bool: True 表示通过（不在黑名单），False 表示被过滤
-        """
-        for pattern in self._compiled_patterns:
-            if pattern.search(url):
-                return False
-        return True
-    
-    # TODO: 后续版本实现
-    # @classmethod
-    # def from_scan(cls, scan_id: int) -> 'BlacklistService':
-    #     """从数据库加载扫描配置的黑名单规则"""
-    #     pass

backend/apps/scan/services/scan_creation_service.py

@@ -282,7 +282,7 @@ class ScanCreationService:
         targets: List[Target],
         engine_ids: List[int],
         engine_names: List[str],
-        merged_configuration: str,
+        yaml_configuration: str,
         scheduled_scan_name: str | None = None
     ) -> List[Scan]:
         """
@@ -292,7 +292,7 @@ class ScanCreationService:
             targets: 目标列表
             engine_ids: 引擎 ID 列表
             engine_names: 引擎名称列表
-            merged_configuration: 合并后的 YAML 配置
+            yaml_configuration: YAML 格式的扫描配置
             scheduled_scan_name: 定时扫描任务名称（可选，用于通知显示）
         
         Returns:
@@ -312,7 +312,7 @@ class ScanCreationService:
                     target=target,
                     engine_ids=engine_ids,
                     engine_names=engine_names,
-                    merged_configuration=merged_configuration,
+                    yaml_configuration=yaml_configuration,
                     results_dir=scan_workspace_dir,
                     status=ScanStatus.INITIATED,
                     container_ids=[],

backend/apps/scan/services/scan_service.py

@@ -117,12 +117,12 @@ class ScanService:
         targets: List[Target],
         engine_ids: List[int],
         engine_names: List[str],
-        merged_configuration: str,
+        yaml_configuration: str,
         scheduled_scan_name: str | None = None
     ) -> List[Scan]:
         """批量创建扫描任务（委托给 ScanCreationService）"""
         return self.creation_service.create_scans(
-            targets, engine_ids, engine_names, merged_configuration, scheduled_scan_name
+            targets, engine_ids, engine_names, yaml_configuration, scheduled_scan_name
         )
     
     # ==================== 状态管理方法（委托给 ScanStateService） ====================

backend/apps/scan/services/scheduled_scan_service.py

@@ -54,7 +54,7 @@ class ScheduledScanService:
     
     def create(self, dto: ScheduledScanDTO) -> ScheduledScan:
         """
-        创建定时扫描任务
+        创建定时扫描任务（使用引擎 ID 合并配置）
         
         流程：
         1. 验证参数
@@ -88,7 +88,7 @@ class ScheduledScanService:
         
         # 设置 DTO 的合并配置和引擎名称
         dto.engine_names = engine_names
-        dto.merged_configuration = merged_configuration
+        dto.yaml_configuration = merged_configuration
         
         # 3. 创建数据库记录
         scheduled_scan = self.repo.create(dto)
@@ -107,12 +107,49 @@ class ScheduledScanService:
         
         return scheduled_scan
     
-    def _validate_create_dto(self, dto: ScheduledScanDTO) -> None:
-        """验证创建 DTO"""
-        from apps.targets.repositories import DjangoOrganizationRepository
+    def create_with_configuration(self, dto: ScheduledScanDTO) -> ScheduledScan:
+        """
+        创建定时扫描任务（直接使用前端传递的配置）
         
-        if not dto.name:
-            raise ValidationError('任务名称不能为空')
+        流程：
+        1. 验证参数
+        2. 直接使用 dto.yaml_configuration
+        3. 创建数据库记录
+        4. 计算并设置 next_run_time
+        
+        Args:
+            dto: 定时扫描 DTO（必须包含 yaml_configuration）
+        
+        Returns:
+            创建的 ScheduledScan 对象
+        
+        Raises:
+            ValidationError: 参数验证失败
+        """
+        # 1. 验证参数
+        self._validate_create_dto_with_configuration(dto)
+        
+        # 2. 创建数据库记录（直接使用 dto 中的配置）
+        scheduled_scan = self.repo.create(dto)
+        
+        # 3. 如果有 cron 表达式且已启用，计算下次执行时间
+        if scheduled_scan.cron_expression and scheduled_scan.is_enabled:
+            next_run_time = self._calculate_next_run_time(scheduled_scan)
+            if next_run_time:
+                self.repo.update_next_run_time(scheduled_scan.id, next_run_time)
+                scheduled_scan.next_run_time = next_run_time
+        
+        logger.info(
+            "创建定时扫描任务 - ID: %s, 名称: %s, 下次执行: %s",
+            scheduled_scan.id, scheduled_scan.name, scheduled_scan.next_run_time
+        )
+        
+        return scheduled_scan
+    
+    def _validate_create_dto(self, dto: ScheduledScanDTO) -> None:
+        """验证创建 DTO（使用引擎 ID）"""
+        # 基础验证
+        self._validate_base_dto(dto)
         
         if not dto.engine_ids:
             raise ValidationError('必须选择扫描引擎')
@@ -121,6 +158,21 @@ class ScheduledScanService:
         for engine_id in dto.engine_ids:
             if not self.engine_repo.get_by_id(engine_id):
                 raise ValidationError(f'扫描引擎 ID {engine_id} 不存在')
+    
+    def _validate_create_dto_with_configuration(self, dto: ScheduledScanDTO) -> None:
+        """验证创建 DTO（使用前端传递的配置）"""
+        # 基础验证
+        self._validate_base_dto(dto)
+        
+        if not dto.yaml_configuration:
+            raise ValidationError('配置不能为空')
+    
+    def _validate_base_dto(self, dto: ScheduledScanDTO) -> None:
+        """验证 DTO 的基础字段（公共逻辑）"""
+        from apps.targets.repositories import DjangoOrganizationRepository
+        
+        if not dto.name:
+            raise ValidationError('任务名称不能为空')
         
         # 验证扫描模式（organization_id 和 target_id 互斥）
         if not dto.organization_id and not dto.target_id:
@@ -178,7 +230,7 @@ class ScheduledScanService:
             
             merged_configuration = merge_engine_configs(engines)
             dto.engine_names = engine_names
-            dto.merged_configuration = merged_configuration
+            dto.yaml_configuration = merged_configuration
         
         # 更新数据库记录
         scheduled_scan = self.repo.update(scheduled_scan_id, dto)
@@ -329,7 +381,7 @@ class ScheduledScanService:
         立即触发扫描（支持组织扫描和目标扫描两种模式）
         
         复用 ScanService 的逻辑，与 API 调用保持一致。
-        使用存储的 merged_configuration 而不是重新合并。
+        使用存储的 yaml_configuration 而不是重新合并。
         """
         from apps.scan.services.scan_service import ScanService
         
@@ -347,7 +399,7 @@ class ScheduledScanService:
             targets=targets,
             engine_ids=scheduled_scan.engine_ids,
             engine_names=scheduled_scan.engine_names,
-            merged_configuration=scheduled_scan.merged_configuration,
+            yaml_configuration=scheduled_scan.yaml_configuration,
             scheduled_scan_name=scheduled_scan.name
         )
         

backend/apps/scan/services/subfinder_provider_config_service.py

@@ -0,0 +1,138 @@
+"""Subfinder Provider 配置文件生成服务
+
+负责生成 subfinder 的 provider-config.yaml 配置文件
+"""
+
+import logging
+import os
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+from ..models import SubfinderProviderSettings
+
+logger = logging.getLogger(__name__)
+
+
+class SubfinderProviderConfigService:
+    """Subfinder Provider 配置文件生成服务"""
+    
+    # Provider 格式定义
+    PROVIDER_FORMATS = {
+        'fofa': {'type': 'composite', 'format': '{email}:{api_key}'},
+        'censys': {'type': 'composite', 'format': '{api_id}:{api_secret}'},
+        'hunter': {'type': 'single', 'field': 'api_key'},
+        'shodan': {'type': 'single', 'field': 'api_key'},
+        'zoomeye': {'type': 'single', 'field': 'api_key'},
+        'securitytrails': {'type': 'single', 'field': 'api_key'},
+        'threatbook': {'type': 'single', 'field': 'api_key'},
+        'quake': {'type': 'single', 'field': 'api_key'},
+    }
+    
+    def generate(self, output_dir: str) -> Optional[str]:
+        """
+        生成 provider-config.yaml 文件
+        
+        Args:
+            output_dir: 输出目录路径
+            
+        Returns:
+            生成的配置文件路径，如果没有启用的 provider 则返回 None
+        """
+        settings = SubfinderProviderSettings.get_instance()
+        
+        config = {}
+        has_enabled = False
+        
+        for provider, format_info in self.PROVIDER_FORMATS.items():
+            provider_config = settings.providers.get(provider, {})
+            
+            if not provider_config.get('enabled'):
+                config[provider] = []
+                continue
+            
+            value = self._build_provider_value(provider, provider_config)
+            if value:
+                config[provider] = [value]  # 单个 key 放入数组
+                has_enabled = True
+                logger.debug(f"Provider {provider} 已启用")
+            else:
+                config[provider] = []
+        
+        # 检查是否有任何启用的 provider
+        if not has_enabled:
+            logger.info("没有启用的 Provider，跳过配置文件生成")
+            return None
+        
+        # 确保输出目录存在
+        output_path = Path(output_dir) / 'provider-config.yaml'
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        # 写入 YAML 文件（使用默认列表格式，和 subfinder 一致）
+        with open(output_path, 'w', encoding='utf-8') as f:
+            yaml.dump(config, f, default_flow_style=False, allow_unicode=True)
+        
+        # 设置文件权限为 600（仅所有者可读写）
+        os.chmod(output_path, 0o600)
+        
+        logger.info(f"Provider 配置文件已生成: {output_path}")
+        return str(output_path)
+    
+    def _build_provider_value(self, provider: str, config: dict) -> Optional[str]:
+        """根据 provider 格式规则构建配置值
+        
+        Args:
+            provider: provider 名称
+            config: provider 配置字典
+            
+        Returns:
+            构建的配置值字符串，如果配置不完整则返回 None
+        """
+        format_info = self.PROVIDER_FORMATS.get(provider)
+        if not format_info:
+            return None
+        
+        if format_info['type'] == 'composite':
+            # 复合格式：需要多个字段
+            format_str = format_info['format']
+            try:
+                # 提取格式字符串中的字段名
+                # 例如 '{email}:{api_key}' -> ['email', 'api_key']
+                import re
+                fields = re.findall(r'\{(\w+)\}', format_str)
+                
+                # 检查所有字段是否都有值
+                values = {}
+                for field in fields:
+                    value = config.get(field, '').strip()
+                    if not value:
+                        logger.debug(f"Provider {provider} 缺少字段 {field}")
+                        return None
+                    values[field] = value
+                
+                return format_str.format(**values)
+            except (KeyError, ValueError) as e:
+                logger.warning(f"构建 {provider} 配置值失败: {e}")
+                return None
+        else:
+            # 单字段格式
+            field = format_info['field']
+            value = config.get(field, '').strip()
+            if not value:
+                logger.debug(f"Provider {provider} 缺少字段 {field}")
+                return None
+            return value
+    
+    def cleanup(self, config_path: str) -> None:
+        """清理配置文件
+        
+        Args:
+            config_path: 配置文件路径
+        """
+        try:
+            if config_path and Path(config_path).exists():
+                Path(config_path).unlink()
+                logger.debug(f"已清理配置文件: {config_path}")
+        except Exception as e:
+            logger.warning(f"清理配置文件失败: {config_path} - {e}")

backend/apps/scan/services/target_export_service.py

@@ -10,37 +10,58 @@
 import ipaddress
 import logging
 from pathlib import Path
-from typing import Dict, Any, Optional, Iterator
+from typing import Dict, Any, Optional, List
 
 from django.db.models import QuerySet
 
-from .blacklist_service import BlacklistService
+from apps.common.utils import BlacklistFilter
 
 logger = logging.getLogger(__name__)
 
 
+def create_export_service(target_id: int) -> 'TargetExportService':
+    """
+    工厂函数：创建带黑名单过滤的导出服务
+    
+    Args:
+        target_id: 目标 ID，用于加载黑名单规则
+        
+    Returns:
+        TargetExportService: 配置好黑名单过滤器的导出服务实例
+    """
+    from apps.common.services import BlacklistService
+    
+    rules = BlacklistService().get_rules(target_id)
+    blacklist_filter = BlacklistFilter(rules)
+    return TargetExportService(blacklist_filter=blacklist_filter)
+
+
 class TargetExportService:
     """
     目标导出服务 - 提供统一的目标提取和文件导出功能
     
     使用方式：
-        # Task 层决定数据源
-        queryset = WebSite.objects.filter(target_id=target_id).values_list('url', flat=True)
+        from apps.common.services import BlacklistService
+        from apps.common.utils import BlacklistFilter
+        
+        # 获取规则并创建过滤器
+        blacklist_service = BlacklistService()
+        rules = blacklist_service.get_rules(target_id)
+        blacklist_filter = BlacklistFilter(rules)
         
         # 使用导出服务
-        blacklist_service = BlacklistService()
-        export_service = TargetExportService(blacklist_service=blacklist_service)
+        export_service = TargetExportService(blacklist_filter=blacklist_filter)
         result = export_service.export_urls(target_id, output_path, queryset)
     """
     
-    def __init__(self, blacklist_service: Optional[BlacklistService] = None):
+    def __init__(self, blacklist_filter: Optional[BlacklistFilter] = None):
         """
         初始化导出服务
         
         Args:
-            blacklist_service: 黑名单过滤服务，None 表示禁用过滤
+            blacklist_filter: 黑名单过滤器，None 表示禁用过滤
         """
-        self.blacklist_service = blacklist_service
+        self.blacklist_filter = blacklist_filter
     
     def export_urls(
         self,
@@ -79,19 +100,15 @@ class TargetExportService:
         
         logger.info("开始导出 URL - target_id=%s, output=%s", target_id, output_path)
         
-        # 应用黑名单过滤（数据库层面）
-        if self.blacklist_service:
-            # 注意：queryset 应该是原始 queryset，不是 values_list
-            # 这里假设 Task 层传入的是 values_list，需要在 Task 层处理过滤
-            pass
-        
         total_count = 0
+        filtered_count = 0
         try:
             with open(output_file, 'w', encoding='utf-8', buffering=8192) as f:
                 for url in queryset.iterator(chunk_size=batch_size):
                     if url:
-                        # Python 层面黑名单过滤
-                        if self.blacklist_service and not self.blacklist_service.filter_url(url):
+                        # 黑名单过滤
+                        if self.blacklist_filter and not self.blacklist_filter.is_allowed(url):
+                            filtered_count += 1
                             continue
                         f.write(f"{url}\n")
                         total_count += 1
@@ -102,6 +119,9 @@ class TargetExportService:
             logger.error("文件写入失败: %s - %s", output_path, e)
             raise
         
+        if filtered_count > 0:
+            logger.info("黑名单过滤: 过滤 %d 个 URL", filtered_count)
+        
         # 默认值回退模式
         if total_count == 0:
             total_count = self._generate_default_urls(target_id, output_file)
@@ -206,18 +226,18 @@ class TargetExportService:
     
     def _should_write_url(self, url: str) -> bool:
         """检查 URL 是否应该写入（通过黑名单过滤）"""
-        if self.blacklist_service:
-            return self.blacklist_service.filter_url(url)
+        if self.blacklist_filter:
+            return self.blacklist_filter.is_allowed(url)
         return True
 
-    def export_targets(
+    def export_hosts(
         self,
         target_id: int,
         output_path: str,
         batch_size: int = 1000
     ) -> Dict[str, Any]:
         """
-        域名/IP 导出函数（用于端口扫描）
+        主机列表导出函数（用于端口扫描）
         
         根据 Target 类型选择导出逻辑：
         - DOMAIN: 从 Subdomain 表流式导出子域名
@@ -255,7 +275,7 @@ class TargetExportService:
         target_name = target.name
         
         logger.info(
-            "开始导出扫描目标 - Target ID: %d, Name: %s, Type: %s, 输出文件: %s",
+            "开始导出主机列表 - Target ID: %d, Name: %s, Type: %s, 输出文件: %s",
             target_id, target_name, target_type, output_path
         )
         
@@ -277,7 +297,7 @@ class TargetExportService:
             raise ValueError(f"不支持的目标类型: {target_type}")
         
         logger.info(
-            "✓ 扫描目标导出完成 - 类型: %s, 总数: %d, 文件: %s",
+            "✓ 主机列表导出完成 - 类型: %s, 总数: %d, 文件: %s",
             type_desc, total_count, output_path
         )
         
@@ -295,7 +315,7 @@ class TargetExportService:
         output_path: Path,
         batch_size: int
     ) -> int:
-        """导出域名类型目标的子域名"""
+        """导出域名类型目标的根域名 + 子域名"""
         from apps.asset.services.asset.subdomain_service import SubdomainService
         
         subdomain_service = SubdomainService()
@@ -305,23 +325,27 @@ class TargetExportService:
         )
         
         total_count = 0
+        written_domains = set()  # 去重（子域名表可能已包含根域名）
+        
         with open(output_path, 'w', encoding='utf-8', buffering=8192) as f:
+            # 1. 先写入根域名
+            if self._should_write_target(target_name):
+                f.write(f"{target_name}\n")
+                written_domains.add(target_name)
+                total_count += 1
+            
+            # 2. 再写入子域名（跳过已写入的根域名）
             for domain_name in domain_iterator:
+                if domain_name in written_domains:
+                    continue
                 if self._should_write_target(domain_name):
                     f.write(f"{domain_name}\n")
+                    written_domains.add(domain_name)
                     total_count += 1
                     
                     if total_count % 10000 == 0:
                         logger.info("已导出 %d 个域名...", total_count)
         
-        # 默认值模式：如果没有子域名，使用根域名
-        if total_count == 0:
-            logger.info("采用默认域名：%s (target_id=%d)", target_name, target_id)
-            if self._should_write_target(target_name):
-                with open(output_path, 'w', encoding='utf-8') as f:
-                    f.write(f"{target_name}\n")
-                total_count = 1
-        
         return total_count
     
     def _export_ip(self, target_name: str, output_path: Path) -> int:
@@ -359,6 +383,6 @@ class TargetExportService:
     
     def _should_write_target(self, target: str) -> bool:
         """检查目标是否应该写入（通过黑名单过滤）"""
-        if self.blacklist_service:
-            return self.blacklist_service.filter_url(target)
+        if self.blacklist_filter:
+            return self.blacklist_filter.is_allowed(target)
         return True

backend/apps/scan/tasks/directory_scan/export_sites_task.py

@@ -8,7 +8,8 @@ import logging
 from prefect import task
 
 from apps.asset.models import WebSite
-from apps.scan.services import TargetExportService, BlacklistService
+from apps.scan.services import TargetExportService
+from apps.scan.services.target_export_service import create_export_service
 
 logger = logging.getLogger(__name__)
 
@@ -49,9 +50,8 @@ def export_sites_task(
     # 构建数据源 queryset（Task 层决定数据源）
     queryset = WebSite.objects.filter(target_id=target_id).values_list('url', flat=True)
     
-    # 使用 TargetExportService 处理导出
-    blacklist_service = BlacklistService()
-    export_service = TargetExportService(blacklist_service=blacklist_service)
+    # 使用工厂函数创建导出服务
+    export_service = create_export_service(target_id)
     
     result = export_service.export_urls(
         target_id=target_id,

backend/apps/scan/tasks/fingerprint_detect/export_urls_task.py

@@ -10,7 +10,7 @@ import logging
 from prefect import task
 
 from apps.asset.models import WebSite
-from apps.scan.services import TargetExportService, BlacklistService
+from apps.scan.services.target_export_service import create_export_service
 
 logger = logging.getLogger(__name__)
 
@@ -46,9 +46,8 @@ def export_urls_for_fingerprint_task(
     # 构建数据源 queryset（Task 层决定数据源）
     queryset = WebSite.objects.filter(target_id=target_id).values_list('url', flat=True)
     
-    # 使用 TargetExportService 处理导出
-    blacklist_service = BlacklistService()
-    export_service = TargetExportService(blacklist_service=blacklist_service)
+    # 使用工厂函数创建导出服务
+    export_service = create_export_service(target_id)
     
     result = export_service.export_urls(
         target_id=target_id,

backend/apps/scan/tasks/port_scan/__init__.py

@@ -4,12 +4,12 @@
 提供端口扫描流程所需的原子化任务
 """
 
-from .export_scan_targets_task import export_scan_targets_task
+from .export_hosts_task import export_hosts_task
 from .run_and_stream_save_ports_task import run_and_stream_save_ports_task
 from .types import PortScanRecord
 
 __all__ = [
-    'export_scan_targets_task',
+    'export_hosts_task',
     'run_and_stream_save_ports_task',
     'PortScanRecord',
 ]

backend/apps/scan/tasks/port_scan/export_hosts_task.py

@@ -1,7 +1,7 @@
 """
-导出扫描目标到 TXT 文件的 Task
+导出主机列表到 TXT 文件的 Task
 
-使用 TargetExportService.export_targets() 统一处理导出逻辑
+使用 TargetExportService.export_hosts() 统一处理导出逻辑
 
 根据 Target 类型决定导出内容：
 - DOMAIN: 从 Subdomain 表导出子域名
@@ -11,19 +11,19 @@
 import logging
 from prefect import task
 
-from apps.scan.services import TargetExportService, BlacklistService
+from apps.scan.services.target_export_service import create_export_service
 
 logger = logging.getLogger(__name__)
 
 
-@task(name="export_scan_targets")
-def export_scan_targets_task(
+@task(name="export_hosts")
+def export_hosts_task(
     target_id: int,
     output_file: str,
     batch_size: int = 1000
 ) -> dict:
     """
-    导出扫描目标到 TXT 文件
+    导出主机列表到 TXT 文件
     
     根据 Target 类型自动决定导出内容：
     - DOMAIN: 从 Subdomain 表导出子域名（流式处理，支持 10万+ 域名）
@@ -47,11 +47,10 @@ def export_scan_targets_task(
         ValueError: Target 不存在
         IOError: 文件写入失败
     """
-    # 使用 TargetExportService 处理导出
-    blacklist_service = BlacklistService()
-    export_service = TargetExportService(blacklist_service=blacklist_service)
+    # 使用工厂函数创建导出服务
+    export_service = create_export_service(target_id)
     
-    result = export_service.export_targets(
+    result = export_service.export_hosts(
         target_id=target_id,
         output_path=output_file,
         batch_size=batch_size

backend/apps/scan/tasks/site_scan/export_site_urls_task.py

@@ -14,7 +14,9 @@ from pathlib import Path
 from prefect import task
 
 from apps.asset.services import HostPortMappingService
-from apps.scan.services import TargetExportService, BlacklistService
+from apps.scan.services.target_export_service import create_export_service
+from apps.common.services import BlacklistService
+from apps.common.utils import BlacklistFilter
 
 logger = logging.getLogger(__name__)
 
@@ -80,8 +82,8 @@ def export_site_urls_task(
     output_path = Path(output_file)
     output_path.parent.mkdir(parents=True, exist_ok=True)
     
-    # 初始化黑名单服务
-    blacklist_service = BlacklistService()
+    # 获取规则并创建过滤器
+    blacklist_filter = BlacklistFilter(BlacklistService().get_rules(target_id))
     
     # 直接查询 HostPortMapping 表，按 host 排序
     service = HostPortMappingService()
@@ -100,11 +102,14 @@ def export_site_urls_task(
             host = assoc['host']
             port = assoc['port']
             
+            # 先校验 host，通过了再生成 URL
+            if not blacklist_filter.is_allowed(host):
+                continue
+            
             # 根据端口号生成URL
             for url in _generate_urls_from_port(host, port):
-                if blacklist_service.filter_url(url):
-                    f.write(f"{url}\n")
-                    total_urls += 1
+                f.write(f"{url}\n")
+                total_urls += 1
             
             if association_count % 1000 == 0:
                 logger.info("已处理 %d 条关联，生成 %d 个URL...", association_count, total_urls)
@@ -114,9 +119,9 @@ def export_site_urls_task(
         association_count, total_urls, str(output_path)
     )
     
-    # 默认值回退模式：使用 TargetExportService
+    # 默认值回退模式：使用工厂函数创建导出服务
     if total_urls == 0:
-        export_service = TargetExportService(blacklist_service=blacklist_service)
+        export_service = create_export_service(target_id)
         total_urls = export_service._generate_default_urls(target_id, output_path)
     
     return {

backend/apps/scan/tasks/subdomain_discovery/save_domains_task.py

@@ -111,6 +111,7 @@ def save_domains_task(
                     continue
                 
                 # 只有通过验证的域名才添加到批次和计数
+                # 注意：不在此处过滤黑名单，最大化资产发现
                 batch.append(domain)
                 total_domains += 1
                 

backend/apps/scan/tasks/url_fetch/export_sites_task.py

@@ -10,7 +10,7 @@ from prefect import task
 from typing import Optional
 
 from apps.asset.models import WebSite
-from apps.scan.services import TargetExportService, BlacklistService
+from apps.scan.services.target_export_service import create_export_service
 
 logger = logging.getLogger(__name__)
 
@@ -56,9 +56,8 @@ def export_sites_task(
     # 构建数据源 queryset（Task 层决定数据源）
     queryset = WebSite.objects.filter(target_id=target_id).values_list('url', flat=True)
     
-    # 使用 TargetExportService 处理导出
-    blacklist_service = BlacklistService()
-    export_service = TargetExportService(blacklist_service=blacklist_service)
+    # 使用工厂函数创建导出服务
+    export_service = create_export_service(target_id)
     
     result = export_service.export_urls(
         target_id=target_id,

backend/apps/scan/tasks/vuln_scan/export_endpoints_task.py

@@ -1,16 +1,21 @@
 """导出 Endpoint URL 到文件的 Task
 
 使用 TargetExportService 统一处理导出逻辑和默认值回退
-数据源: Endpoint.url
+
+数据源优先级（回退链）：
+1. Endpoint.url - 最精细的 URL（含路径、参数等）
+2. WebSite.url - 站点级别 URL
+3. 默认生成 - 根据 Target 类型生成 http(s)://target_name
 """
 
 import logging
-from typing import Dict, Optional
+from pathlib import Path
+from typing import Dict
 
 from prefect import task
 
-from apps.asset.models import Endpoint
-from apps.scan.services import TargetExportService, BlacklistService
+from apps.asset.models import Endpoint, WebSite
+from apps.scan.services.target_export_service import create_export_service
 
 logger = logging.getLogger(__name__)
 
@@ -23,13 +28,10 @@ def export_endpoints_task(
 ) -> Dict[str, object]:
     """导出目标下的所有 Endpoint URL 到文本文件。
 
-    数据源: Endpoint.url
-    
-    懒加载模式：
-    - 如果数据库为空，根据 Target 类型生成默认 URL
-    - DOMAIN: http(s)://domain
-    - IP: http(s)://ip
-    - CIDR: 展开为所有 IP 的 URL
+    数据源优先级（回退链）：
+    1. Endpoint 表 - 最精细的 URL（含路径、参数等）
+    2. WebSite 表 - 站点级别 URL
+    3. 默认生成 - 根据 Target 类型生成 http(s)://target_name
 
     Args:
         target_id: 目标 ID
@@ -41,25 +43,55 @@ def export_endpoints_task(
             "success": bool,
             "output_file": str,
             "total_count": int,
+            "source": str,  # 数据来源: "endpoint" | "website" | "default"
         }
     """
-    # 构建数据源 queryset（Task 层决定数据源）
-    queryset = Endpoint.objects.filter(target_id=target_id).values_list('url', flat=True)
-    
-    # 使用 TargetExportService 处理导出
-    blacklist_service = BlacklistService()
-    export_service = TargetExportService(blacklist_service=blacklist_service)
+    export_service = create_export_service(target_id)
+    output_path = Path(output_file)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
     
+    # 1. 优先从 Endpoint 表导出
+    endpoint_queryset = Endpoint.objects.filter(target_id=target_id).values_list('url', flat=True)
     result = export_service.export_urls(
         target_id=target_id,
         output_path=output_file,
-        queryset=queryset,
+        queryset=endpoint_queryset,
         batch_size=batch_size
     )
     
-    # 保持返回值格式不变（向后兼容）
+    if result['total_count'] > 0:
+        logger.info("从 Endpoint 表导出 %d 条 URL", result['total_count'])
+        return {
+            "success": True,
+            "output_file": result['output_file'],
+            "total_count": result['total_count'],
+            "source": "endpoint",
+        }
+    
+    # 2. Endpoint 为空，回退到 WebSite 表
+    logger.info("Endpoint 表为空，回退到 WebSite 表")
+    website_queryset = WebSite.objects.filter(target_id=target_id).values_list('url', flat=True)
+    result = export_service.export_urls(
+        target_id=target_id,
+        output_path=output_file,
+        queryset=website_queryset,
+        batch_size=batch_size
+    )
+    
+    if result['total_count'] > 0:
+        logger.info("从 WebSite 表导出 %d 条 URL", result['total_count'])
+        return {
+            "success": True,
+            "output_file": result['output_file'],
+            "total_count": result['total_count'],
+            "source": "website",
+        }
+    
+    # 3. WebSite 也为空，生成默认 URL（export_urls 内部已处理）
+    logger.info("WebSite 表也为空，使用默认 URL 生成")
     return {
-        "success": result['success'],
+        "success": True,
         "output_file": result['output_file'],
         "total_count": result['total_count'],
+        "source": "default",
     }

backend/apps/scan/urls.py

@@ -1,6 +1,6 @@
 from django.urls import path, include
 from rest_framework.routers import DefaultRouter
-from .views import ScanViewSet, ScheduledScanViewSet
+from .views import ScanViewSet, ScheduledScanViewSet, ScanLogListView, SubfinderProviderSettingsView
 from .notifications.views import notification_callback
 from apps.asset.views import (
     SubdomainSnapshotViewSet, WebsiteSnapshotViewSet, DirectorySnapshotViewSet,
@@ -31,6 +31,10 @@ urlpatterns = [
     path('', include(router.urls)),
     # Worker 回调 API
     path('callbacks/notification/', notification_callback, name='notification-callback'),
+    # API Key 配置
+    path('settings/api-keys/', SubfinderProviderSettingsView.as_view(), name='subfinder-provider-settings'),
+    # 扫描日志 API
+    path('scans/<int:scan_id>/logs/', ScanLogListView.as_view(), name='scan-logs-list'),
     # 嵌套路由：/api/scans/{scan_pk}/xxx/
     path('scans/<int:scan_pk>/subdomains/', scan_subdomains_list, name='scan-subdomains-list'),
     path('scans/<int:scan_pk>/subdomains/export/', scan_subdomains_export, name='scan-subdomains-export'),

backend/apps/scan/utils/__init__.py

@@ -11,6 +11,7 @@ from .wordlist_helpers import ensure_wordlist_local
 from .nuclei_helpers import ensure_nuclei_templates_local
 from .performance import FlowPerformanceTracker, CommandPerformanceTracker
 from .workspace_utils import setup_scan_workspace, setup_scan_directory
+from .user_logger import user_log
 from . import config_parser
 
 __all__ = [
@@ -31,6 +32,8 @@ __all__ = [
     # 性能监控
     'FlowPerformanceTracker',      # Flow 性能追踪器（含系统资源采样）
     'CommandPerformanceTracker',   # 命令性能追踪器
+    # 扫描日志
+    'user_log',              # 用户可见扫描日志记录
     # 配置解析
     'config_parser',
 ]

backend/apps/scan/utils/command_executor.py

@@ -48,7 +48,7 @@ ENABLE_COMMAND_LOGGING = getattr(settings, 'ENABLE_COMMAND_LOGGING', True)
 # 动态并发控制阈值（可在 Django settings 中覆盖）
 SCAN_CPU_HIGH = getattr(settings, 'SCAN_CPU_HIGH', 90.0)   # CPU 高水位（百分比）
 SCAN_MEM_HIGH = getattr(settings, 'SCAN_MEM_HIGH', 80.0)   # 内存高水位（百分比）
-SCAN_LOAD_CHECK_INTERVAL = getattr(settings, 'SCAN_LOAD_CHECK_INTERVAL', 30)  # 负载检查间隔（秒）
+SCAN_LOAD_CHECK_INTERVAL = getattr(settings, 'SCAN_LOAD_CHECK_INTERVAL', 180)  # 负载检查间隔（秒）
 SCAN_COMMAND_STARTUP_DELAY = getattr(settings, 'SCAN_COMMAND_STARTUP_DELAY', 5)  # 命令启动前等待（秒）
 
 _ACTIVE_COMMANDS = 0
@@ -74,7 +74,7 @@ def _wait_for_system_load() -> None:
             return
 
         logger.info(
-            "系统负载较高，暂缓启动: cpu=%.1f%% (阈值 %.1f%%), mem=%.1f%% (阈值 %.1f%%)",
+            "系统负载较高，任务将排队执行，防止oom: cpu=%.1f%% (阈值 %.1f%%), mem=%.1f%% (阈值 %.1f%%)",
             cpu,
             SCAN_CPU_HIGH,
             mem,

backend/apps/scan/utils/user_logger.py

@@ -0,0 +1,56 @@
+"""
+扫描日志记录器
+
+提供统一的日志记录接口，用于在 Flow 中记录用户可见的扫描进度日志。
+
+特性：
+- 简单的函数式 API
+- 只写入数据库（ScanLog 表），不写 Python logging
+- 错误容忍（数据库失败不影响扫描执行）
+
+职责分离：
+- user_log: 用户可见日志（写数据库，前端展示）
+- logger: 开发者日志（写日志文件/控制台，调试用）
+
+使用示例：
+    from apps.scan.utils import user_log
+    
+    # 用户日志（写数据库）
+    user_log(scan_id, "port_scan", "Starting port scan")
+    user_log(scan_id, "port_scan", "naabu completed: found 120 ports")
+    
+    # 开发者日志（写日志文件）
+    logger.info("✓ 工具 %s 执行完成 - 记录数: %d", tool_name, count)
+"""
+
+import logging
+from django.db import DatabaseError
+
+logger = logging.getLogger(__name__)
+
+
+def user_log(scan_id: int, stage: str, message: str, level: str = "info"):
+    """
+    记录用户可见的扫描日志（只写数据库）
+    
+    Args:
+        scan_id: 扫描任务 ID
+        stage: 阶段名称，如 "port_scan", "site_scan"
+        message: 日志消息
+        level: 日志级别，默认 "info"，可选 "warning", "error"
+        
+    数据库 content 格式: "[{stage}] {message}"
+    """
+    formatted = f"[{stage}] {message}"
+    
+    try:
+        from apps.scan.models import ScanLog
+        ScanLog.objects.create(
+            scan_id=scan_id,
+            level=level,
+            content=formatted
+        )
+    except DatabaseError as e:
+        logger.error("ScanLog write failed - scan_id=%s, error=%s", scan_id, e)
+    except Exception as e:
+        logger.error("ScanLog write unexpected error - scan_id=%s, error=%s", scan_id, e)

backend/apps/scan/views/__init__.py

@@ -2,8 +2,12 @@
 
 from .scan_views import ScanViewSet
 from .scheduled_scan_views import ScheduledScanViewSet
+from .scan_log_views import ScanLogListView
+from .subfinder_provider_settings_views import SubfinderProviderSettingsView
 
 __all__ = [
     'ScanViewSet',
     'ScheduledScanViewSet',
+    'ScanLogListView',
+    'SubfinderProviderSettingsView',
 ]

backend/apps/scan/views/scan_log_views.py

@@ -0,0 +1,56 @@
+"""
+扫描日志 API
+
+提供扫描日志查询接口，支持游标分页用于增量轮询。
+"""
+
+from rest_framework.views import APIView
+from rest_framework.response import Response
+
+from apps.scan.models import ScanLog
+from apps.scan.serializers import ScanLogSerializer
+
+
+class ScanLogListView(APIView):
+    """
+    GET /scans/{scan_id}/logs/
+    
+    游标分页 API，用于增量查询日志
+    
+    查询参数：
+    - afterId: 只返回此 ID 之后的日志（用于增量轮询，避免时间戳重复导致的重复日志）
+    - limit: 返回数量限制（默认 200，最大 1000）
+    
+    返回：
+    - results: 日志列表
+    - hasMore: 是否还有更多日志
+    """
+    
+    def get(self, request, scan_id: int):
+        # 参数解析
+        after_id = request.query_params.get('afterId')
+        try:
+            limit = min(int(request.query_params.get('limit', 200)), 1000)
+        except (ValueError, TypeError):
+            limit = 200
+        
+        # 查询日志（按 ID 排序，ID 是自增的，保证顺序一致）
+        queryset = ScanLog.objects.filter(scan_id=scan_id).order_by('id')
+        
+        # 游标过滤（使用 ID 而非时间戳，避免同一时间戳多条日志导致重复）
+        if after_id:
+            try:
+                queryset = queryset.filter(id__gt=int(after_id))
+            except (ValueError, TypeError):
+                pass
+        
+        # 限制返回数量（多取一条用于判断 hasMore）
+        logs = list(queryset[:limit + 1])
+        has_more = len(logs) > limit
+        if has_more:
+            logs = logs[:limit]
+        
+        return Response({
+            'results': ScanLogSerializer(logs, many=True).data,
+            'hasMore': has_more,
+        })

backend/apps/scan/views/scan_views.py

@@ -3,6 +3,7 @@ from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework.exceptions import NotFound, APIException
 from rest_framework.filters import SearchFilter
+from django_filters.rest_framework import DjangoFilterBackend
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db.utils import DatabaseError, IntegrityError, OperationalError
 import logging
@@ -16,7 +17,7 @@ logger = logging.getLogger(__name__)
 from ..models import Scan, ScheduledScan
 from ..serializers import (
     ScanSerializer, ScanHistorySerializer, QuickScanSerializer,
-    ScheduledScanSerializer, CreateScheduledScanSerializer,
+    InitiateScanSerializer, ScheduledScanSerializer, CreateScheduledScanSerializer,
     UpdateScheduledScanSerializer, ToggleScheduledScanSerializer
 )
 from ..services.scan_service import ScanService
@@ -33,7 +34,8 @@ class ScanViewSet(viewsets.ModelViewSet):
     """扫描任务视图集"""
     serializer_class = ScanSerializer
     pagination_class = BasePagination
-    filter_backends = [SearchFilter]
+    filter_backends = [DjangoFilterBackend, SearchFilter]
+    filterset_fields = ['target']  # 支持 ?target=123 过滤
     search_fields = ['target__name']  # 按目标名称搜索
     
     def get_queryset(self):
@@ -111,7 +113,7 @@ class ScanViewSet(viewsets.ModelViewSet):
         快速扫描接口
         
         功能：
-        1. 接收目标列表和引擎配置
+        1. 接收目标列表和 YAML 配置
         2. 自动解析输入（支持 URL、域名、IP、CIDR）
         3. 批量创建 Target、Website、Endpoint 资产
         4. 立即发起批量扫描
@@ -119,7 +121,9 @@ class ScanViewSet(viewsets.ModelViewSet):
         请求参数：
         {
             "targets": [{"name": "example.com"}, {"name": "https://example.com/api"}],
-            "engine_ids": [1, 2]
+            "configuration": "subdomain_discovery:\n  enabled: true\n  ...",
+            "engine_ids": [1, 2],  // 可选，用于记录
+            "engine_names": ["引擎A", "引擎B"]  // 可选，用于记录
         }
         
         支持的输入格式：
@@ -134,7 +138,9 @@ class ScanViewSet(viewsets.ModelViewSet):
         serializer.is_valid(raise_exception=True)
         
         targets_data = serializer.validated_data['targets']
-        engine_ids = serializer.validated_data.get('engine_ids')
+        configuration = serializer.validated_data['configuration']
+        engine_ids = serializer.validated_data.get('engine_ids', [])
+        engine_names = serializer.validated_data.get('engine_names', [])
         
         try:
             # 提取输入字符串列表
@@ -154,19 +160,13 @@ class ScanViewSet(viewsets.ModelViewSet):
                     status_code=status.HTTP_400_BAD_REQUEST
                 )
             
-            # 2. 准备多引擎扫描
+            # 2. 直接使用前端传递的配置创建扫描
             scan_service = ScanService()
-            _, merged_configuration, engine_names, engine_ids = scan_service.prepare_initiate_scan_multi_engine(
-                target_id=targets[0].id,  # 使用第一个目标来验证引擎
-                engine_ids=engine_ids
-            )
-            
-            # 3. 批量发起扫描
             created_scans = scan_service.create_scans(
                 targets=targets,
                 engine_ids=engine_ids,
                 engine_names=engine_names,
-                merged_configuration=merged_configuration
+                yaml_configuration=configuration
             )
             
             # 检查是否成功创建扫描任务
@@ -195,17 +195,6 @@ class ScanViewSet(viewsets.ModelViewSet):
                 },
                 status_code=status.HTTP_201_CREATED
             )
-        
-        except ConfigConflictError as e:
-            return error_response(
-                code='CONFIG_CONFLICT',
-                message=str(e),
-                details=[
-                    {'key': k, 'engines': [e1, e2]} 
-                    for k, e1, e2 in e.conflicts
-                ],
-                status_code=status.HTTP_400_BAD_REQUEST
-            )
             
         except ValidationError as e:
             return error_response(
@@ -228,48 +217,53 @@ class ScanViewSet(viewsets.ModelViewSet):
         请求参数:
         - organization_id: 组织ID (int, 可选)
         - target_id: 目标ID (int, 可选)
+        - configuration: YAML 配置字符串 (str, 必填)
         - engine_ids: 扫描引擎ID列表 (list[int], 必填)
+        - engine_names: 引擎名称列表 (list[str], 必填)
         
         注意: organization_id 和 target_id 二选一
         
         返回:
         - 扫描任务详情（单个或多个）
         """
-        # 获取请求数据
-        organization_id = request.data.get('organization_id')
-        target_id = request.data.get('target_id')
-        engine_ids = request.data.get('engine_ids')
+        # 使用 serializer 验证请求数据
+        serializer = InitiateScanSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
         
-        # 验证 engine_ids
-        if not engine_ids:
-            return error_response(
-                code=ErrorCodes.VALIDATION_ERROR,
-                message='缺少必填参数: engine_ids',
-                status_code=status.HTTP_400_BAD_REQUEST
-            )
-        
-        if not isinstance(engine_ids, list):
-            return error_response(
-                code=ErrorCodes.VALIDATION_ERROR,
-                message='engine_ids 必须是数组',
-                status_code=status.HTTP_400_BAD_REQUEST
-            )
+        # 获取验证后的数据
+        organization_id = serializer.validated_data.get('organization_id')
+        target_id = serializer.validated_data.get('target_id')
+        configuration = serializer.validated_data['configuration']
+        engine_ids = serializer.validated_data['engine_ids']
+        engine_names = serializer.validated_data['engine_names']
         
         try:
-            # 步骤1：准备多引擎扫描所需的数据
+            # 获取目标列表
             scan_service = ScanService()
-            targets, merged_configuration, engine_names, engine_ids = scan_service.prepare_initiate_scan_multi_engine(
-                organization_id=organization_id,
-                target_id=target_id,
-                engine_ids=engine_ids
-            )
             
-            # 步骤2：批量创建扫描记录并分发扫描任务
+            if organization_id:
+                from apps.targets.repositories import DjangoOrganizationRepository
+                org_repo = DjangoOrganizationRepository()
+                organization = org_repo.get_by_id(organization_id)
+                if not organization:
+                    raise ObjectDoesNotExist(f'Organization ID {organization_id} 不存在')
+                targets = org_repo.get_targets(organization_id)
+                if not targets:
+                    raise ValidationError(f'组织 ID {organization_id} 下没有目标')
+            else:
+                from apps.targets.repositories import DjangoTargetRepository
+                target_repo = DjangoTargetRepository()
+                target = target_repo.get_by_id(target_id)
+                if not target:
+                    raise ObjectDoesNotExist(f'Target ID {target_id} 不存在')
+                targets = [target]
+            
+            # 直接使用前端传递的配置创建扫描
             created_scans = scan_service.create_scans(
                 targets=targets,
                 engine_ids=engine_ids,
                 engine_names=engine_names,
-                merged_configuration=merged_configuration
+                yaml_configuration=configuration
             )
             
             # 检查是否成功创建扫描任务
@@ -290,17 +284,6 @@ class ScanViewSet(viewsets.ModelViewSet):
                 },
                 status_code=status.HTTP_201_CREATED
             )
-        
-        except ConfigConflictError as e:
-            return error_response(
-                code='CONFIG_CONFLICT',
-                message=str(e),
-                details=[
-                    {'key': k, 'engines': [e1, e2]} 
-                    for k, e1, e2 in e.conflicts
-                ],
-                status_code=status.HTTP_400_BAD_REQUEST
-            )
             
         except ObjectDoesNotExist as e:
             # 资源不存在错误（由 service 层抛出）

backend/apps/scan/views/scheduled_scan_views.py

@@ -37,6 +37,11 @@ class ScheduledScanViewSet(viewsets.ModelViewSet):
     - PUT    /scheduled-scans/{id}/      更新定时扫描
     - DELETE /scheduled-scans/{id}/      删除定时扫描
     - POST   /scheduled-scans/{id}/toggle/   切换启用状态
+    
+    查询参数：
+    - target_id: 按目标 ID 过滤
+    - organization_id: 按组织 ID 过滤
+    - search: 按名称搜索
     """
     
     queryset = ScheduledScan.objects.all().order_by('-created_at')
@@ -49,6 +54,19 @@ class ScheduledScanViewSet(viewsets.ModelViewSet):
         super().__init__(*args, **kwargs)
         self.service = ScheduledScanService()
     
+    def get_queryset(self):
+        """支持按 target_id 和 organization_id 过滤"""
+        queryset = super().get_queryset()
+        target_id = self.request.query_params.get('target_id')
+        organization_id = self.request.query_params.get('organization_id')
+        
+        if target_id:
+            queryset = queryset.filter(target_id=target_id)
+        if organization_id:
+            queryset = queryset.filter(organization_id=organization_id)
+        
+        return queryset
+    
     def get_serializer_class(self):
         """根据 action 返回不同的序列化器"""
         if self.action == 'create':
@@ -68,30 +86,22 @@ class ScheduledScanViewSet(viewsets.ModelViewSet):
             data = serializer.validated_data
             dto = ScheduledScanDTO(
                 name=data['name'],
-                engine_ids=data['engine_ids'],
+                engine_ids=data.get('engine_ids', []),
+                engine_names=data.get('engine_names', []),
+                yaml_configuration=data['configuration'],
                 organization_id=data.get('organization_id'),
                 target_id=data.get('target_id'),
                 cron_expression=data.get('cron_expression', '0 2 * * *'),
                 is_enabled=data.get('is_enabled', True),
             )
             
-            scheduled_scan = self.service.create(dto)
+            scheduled_scan = self.service.create_with_configuration(dto)
             response_serializer = ScheduledScanSerializer(scheduled_scan)
             
             return success_response(
                 data=response_serializer.data,
                 status_code=status.HTTP_201_CREATED
             )
-        except ConfigConflictError as e:
-            return error_response(
-                code='CONFIG_CONFLICT',
-                message=str(e),
-                details=[
-                    {'key': k, 'engines': [e1, e2]} 
-                    for k, e1, e2 in e.conflicts
-                ],
-                status_code=status.HTTP_400_BAD_REQUEST
-            )
         except ValidationError as e:
             return error_response(
                 code=ErrorCodes.VALIDATION_ERROR,

backend/apps/scan/views/subfinder_provider_settings_views.py

@@ -0,0 +1,38 @@
+"""Subfinder Provider 配置视图"""
+
+import logging
+from rest_framework import status
+from rest_framework.views import APIView
+from rest_framework.response import Response
+
+from ..models import SubfinderProviderSettings
+from ..serializers import SubfinderProviderSettingsSerializer
+
+logger = logging.getLogger(__name__)
+
+
+class SubfinderProviderSettingsView(APIView):
+    """Subfinder Provider 配置视图
+    
+    GET /api/settings/api-keys/ - 获取配置
+    PUT /api/settings/api-keys/ - 更新配置
+    """
+    
+    def get(self, request):
+        """获取 Subfinder Provider 配置"""
+        settings = SubfinderProviderSettings.get_instance()
+        serializer = SubfinderProviderSettingsSerializer(settings.providers)
+        return Response(serializer.data)
+    
+    def put(self, request):
+        """更新 Subfinder Provider 配置"""
+        serializer = SubfinderProviderSettingsSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        
+        settings = SubfinderProviderSettings.get_instance()
+        settings.providers.update(serializer.validated_data)
+        settings.save()
+        
+        logger.info("Subfinder Provider 配置已更新")
+        
+        return Response(SubfinderProviderSettingsSerializer(settings.providers).data)

backend/apps/targets/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.2.7 on 2026-01-02 04:45
+# Generated by Django 5.2.7 on 2026-01-06 00:55
 
 from django.db import migrations, models
 

backend/apps/targets/services/target_service.py

@@ -99,31 +99,6 @@ class TargetService:
     
     # ==================== 创建操作 ====================
     
-    def create_or_get_target(
-        self, 
-        name: str, 
-        target_type: str
-    ) -> Tuple[Target, bool]:
-        """
-        创建或获取目标
-        
-        Args:
-            name: 目标名称
-            target_type: 目标类型
-        
-        Returns:
-            (Target对象, 是否新创建)
-        """
-        logger.debug("创建或获取目标 - Name: %s, Type: %s", name, target_type)
-        target, created = self.repo.get_or_create(name, target_type)
-        
-        if created:
-            logger.info("创建新目标 - ID: %s, Name: %s", target.id, name)
-        else:
-            logger.debug("目标已存在 - ID: %s, Name: %s", target.id, name)
-        
-        return target, created
-    
     def batch_create_targets(
         self,
         targets_data: List[Dict[str, Any]],

backend/apps/targets/views.py

@@ -11,6 +11,8 @@ from .services.target_service import TargetService
 from .services.organization_service import OrganizationService
 from apps.common.pagination import BasePagination
 from apps.common.response_helpers import success_response
+from apps.common.models import BlacklistRule
+from apps.common.serializers import TargetBlacklistRuleSerializer
 
 logger = logging.getLogger(__name__)
 
@@ -405,3 +407,48 @@ class TargetViewSet(viewsets.ModelViewSet):
     # GET /api/targets/{id}/ip-addresses/ -> HostPortMappingViewSet
     # GET /api/targets/{id}/ip-addresses/export/ -> HostPortMappingViewSet.export
     # GET /api/targets/{id}/vulnerabilities/ -> VulnerabilityViewSet
+
+    # ==================== 黑名单管理 ====================
+    
+    @action(detail=True, methods=['get', 'put'], url_path='blacklist')
+    def blacklist(self, request, pk=None):
+        """
+        Target 黑名单规则管理
+        
+        GET /api/targets/{id}/blacklist/ - 获取 Target 黑名单列表
+        PUT /api/targets/{id}/blacklist/ - 全量替换规则（文本框保存场景）
+        
+        设计说明：
+        - 使用 PUT 全量替换模式，适合"文本框每行一个规则"的前端场景
+        - 用户编辑文本框 -> 点击保存 -> 后端全量替换
+        
+        架构：MVS 模式
+        - View: 参数验证、响应格式化
+        - Service: 业务逻辑（BlacklistService）
+        - Model: 数据持久化（BlacklistRule）
+        """
+        from apps.common.services import BlacklistService
+        
+        target = self.get_object()
+        blacklist_service = BlacklistService()
+        
+        if request.method == 'GET':
+            # 获取 Target 的黑名单规则
+            rules = blacklist_service.get_target_rules(target.id)
+            patterns = list(rules.values_list('pattern', flat=True))
+            return success_response(data={'patterns': patterns})
+        
+        elif request.method == 'PUT':
+            # 全量替换
+            patterns = request.data.get('patterns', [])
+            
+            if not isinstance(patterns, list):
+                return Response(
+                    {'error': {'code': 'VALIDATION_ERROR', 'message': 'patterns 必须是数组'}},
+                    status=status.HTTP_400_BAD_REQUEST
+                )
+            
+            # 调用 Service 层全量替换
+            result = blacklist_service.replace_target_rules(target, patterns)
+            
+            return success_response(data=result)

backend/config/settings.py

@@ -51,6 +51,7 @@ INSTALLED_APPS = [
     'django.contrib.staticfiles',
     # 第三方应用
     'rest_framework',
+    'django_filters',  # DRF 过滤器支持
     'drf_yasg',
     'corsheaders',
     'channels',  # WebSocket 支持
@@ -219,6 +220,8 @@ REST_FRAMEWORK = {
 # 允许所有来源（前后端分离项目，安全性由认证系统保障）
 CORS_ALLOW_ALL_ORIGINS = os.getenv('CORS_ALLOW_ALL_ORIGINS', 'True').lower() == 'true'
 CORS_ALLOW_CREDENTIALS = True
+# 暴露额外的响应头给前端（Content-Disposition 用于文件下载获取文件名）
+CORS_EXPOSE_HEADERS = ['Content-Disposition']
 
 # ==================== CSRF 配置 ====================
 CSRF_TRUSTED_ORIGINS = os.getenv('CSRF_TRUSTED_ORIGINS', 'http://localhost:3000,http://127.0.0.1:3000').split(',')

backend/requirements.txt

@@ -11,6 +11,9 @@ setuptools==75.6.0
 # CORS 支持
 django-cors-headers==4.3.1
 
+# 过滤器支持
+django-filter==24.3
+
 # 环境变量管理
 python-dotenv==1.0.1
 

backend/scripts/generate_test_data_sql.py

@@ -636,7 +636,7 @@ class TestDataGenerator:
                 
                 cur.execute("""
                     INSERT INTO scan (
-                        target_id, engine_ids, engine_names, merged_configuration, status, worker_id, progress, current_stage,
+                        target_id, engine_ids, engine_names, yaml_configuration, status, worker_id, progress, current_stage,
                         results_dir, error_message, container_ids, stage_progress,
                         cached_subdomains_count, cached_websites_count, cached_endpoints_count,
                         cached_ips_count, cached_directories_count, cached_vulns_total,
@@ -749,7 +749,7 @@ class TestDataGenerator:
             
             cur.execute("""
                 INSERT INTO scheduled_scan (
-                    name, engine_ids, engine_names, merged_configuration, organization_id, target_id, cron_expression, is_enabled,
+                    name, engine_ids, engine_names, yaml_configuration, organization_id, target_id, cron_expression, is_enabled,
                     run_count, last_run_time, next_run_time, created_at, updated_at
                 ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW() - INTERVAL '%s days', NOW())
                 ON CONFLICT DO NOTHING

docker/nginx/nginx.conf

@@ -38,6 +38,8 @@ http {
       proxy_set_header Host $host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_read_timeout 300s;  # 5分钟，支持大数据量导出
+      proxy_send_timeout 300s;
       proxy_pass http://backend;
     }
 

docker/scripts/init-data.sh

@@ -63,11 +63,7 @@ wait_for_server() {
 run_migrations() {
     log_step "执行数据库迁移..."
     
-    # 开发环境：先 makemigrations
-    if [ "$DEV_MODE" = "true" ]; then
-        docker compose exec -T server python backend/manage.py makemigrations --noinput 2>/dev/null || true
-    fi
-    
+    # 迁移文件应手动生成并提交到仓库，这里只执行 migrate
     docker compose exec -T server python backend/manage.py migrate --noinput
     log_info "数据库迁移完成"
 }

docker/server/start.sh

@@ -3,26 +3,21 @@ set -e
 
 echo "[START] 启动 XingRin Server..."
 
-# 1. 生成和迁移数据库
-echo "  [1/3] 生成数据库迁移文件..."
+# 1. 执行数据库迁移（迁移文件应提交到仓库，这里只执行 migrate）
+echo "  [1/3] 执行数据库迁移..."
 cd /app/backend
-python manage.py makemigrations
-echo "  ✓ 迁移文件生成完成"
-
-echo "  [1.1/3] 执行数据库迁移..."
 python manage.py migrate --noinput
 echo "  ✓ 数据库迁移完成"
 
-echo "  [1.2/3] 初始化默认扫描引擎..."
+echo "  [1.1/3] 初始化默认扫描引擎..."
 python manage.py init_default_engine
 echo "  ✓ 默认扫描引擎已就绪"
 
-echo "  [1.3/3] 初始化默认目录字典..."
+echo "  [1.2/3] 初始化默认目录字典..."
 python manage.py init_wordlists
 echo "  ✓ 默认目录字典已就绪"
 
-
-echo "  [1.4/3] 初始化默认指纹库..."
+echo "  [1.3/3] 初始化默认指纹库..."
 python manage.py init_fingerprints
 echo "  ✓ 默认指纹库已就绪"
 

docker/start.sh

@@ -155,7 +155,11 @@ fi
 echo -e "${GREEN}[OK]${NC} 服务已启动"
 
 # 数据初始化
-./scripts/init-data.sh
+if [ "$DEV_MODE" = true ]; then
+    ./scripts/init-data.sh --dev
+else
+    ./scripts/init-data.sh
+fi
 
 # 静默模式下不显示结果（由调用方显示）
 if [ "$QUIET_MODE" = true ]; then

docker/worker/Dockerfile

@@ -1,6 +1,5 @@
 # 第一阶段：使用 Go 官方镜像编译工具
-# 锁定 digest 避免上游更新导致缓存失效
-FROM golang:1.24@sha256:7e050c14ae9ca5ae56408a288336545b18632f51402ab0ec8e7be0e649a1fc42 AS go-builder
+FROM golang:1.24 AS go-builder
 
 ENV GOPROXY=https://goproxy.cn,direct
 # Naabu 需要 CGO 和 libpcap
@@ -37,8 +36,7 @@ RUN CGO_ENABLED=0 go install -v github.com/owasp-amass/amass/v5/cmd/amass@main
 RUN go install github.com/hahwul/dalfox/v2@latest
 
 # 第二阶段：运行时镜像
-# 锁定 digest 避免上游更新导致缓存失效
-FROM ubuntu:24.04@sha256:4fdf0125919d24aec972544669dcd7d6a26a8ad7e6561c73d5549bd6db258ac2
+FROM ubuntu:24.04
 
 # 避免交互式提示
 ENV DEBIAN_FRONTEND=noninteractive
@@ -104,7 +102,11 @@ RUN pip install uv --break-system-packages && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-# 6. 复制后端代码
+# 6. 设置 Prefect 配置目录（避免 home 目录不存在的警告）
+ENV PREFECT_HOME=/app/.prefect
+RUN mkdir -p /app/.prefect
+
+# 7. 复制后端代码
 COPY backend /app/backend
 ENV PYTHONPATH=/app/backend
 

frontend/app/[locale]/settings/api-keys/page.tsx

@@ -0,0 +1,306 @@
+"use client"
+
+import React, { useState, useEffect } from 'react'
+import { IconEye, IconEyeOff, IconWorldSearch, IconRadar2 } from '@tabler/icons-react'
+
+import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/components/ui/card'
+import { Button } from '@/components/ui/button'
+import { Input } from '@/components/ui/input'
+import { Switch } from '@/components/ui/switch'
+import { Separator } from '@/components/ui/separator'
+import { Badge } from '@/components/ui/badge'
+import { Skeleton } from '@/components/ui/skeleton'
+import { useApiKeySettings, useUpdateApiKeySettings } from '@/hooks/use-api-key-settings'
+import type { ApiKeySettings } from '@/types/api-key-settings.types'
+
+// 密码输入框组件（带显示/隐藏切换）
+function PasswordInput({ value, onChange, placeholder, disabled }: {
+  value: string
+  onChange: (value: string) => void
+  placeholder?: string
+  disabled?: boolean
+}) {
+  const [show, setShow] = useState(false)
+  return (
+    <div className="relative">
+      <Input
+        type={show ? 'text' : 'password'}
+        value={value}
+        onChange={(e) => onChange(e.target.value)}
+        placeholder={placeholder}
+        disabled={disabled}
+        className="pr-10"
+      />
+      <button
+        type="button"
+        onClick={() => setShow(!show)}
+        className="absolute right-3 top-1/2 -translate-y-1/2 text-muted-foreground hover:text-foreground"
+      >
+        {show ? <IconEyeOff className="h-4 w-4" /> : <IconEye className="h-4 w-4" />}
+      </button>
+    </div>
+  )
+}
+
+// Provider 配置定义
+const PROVIDERS = [
+  {
+    key: 'fofa',
+    name: 'FOFA',
+    description: '网络空间测绘平台，提供全球互联网资产搜索',
+    icon: IconWorldSearch,
+    color: 'text-blue-500',
+    bgColor: 'bg-blue-500/10',
+    fields: [
+      { name: 'email', label: '邮箱', type: 'text', placeholder: 'your@email.com' },
+      { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 FOFA API Key' },
+    ],
+    docUrl: 'https://fofa.info/api',
+  },
+  {
+    key: 'hunter',
+    name: 'Hunter (鹰图)',
+    description: '奇安信威胁情报平台，提供网络空间资产测绘',
+    icon: IconRadar2,
+    color: 'text-orange-500',
+    bgColor: 'bg-orange-500/10',
+    fields: [
+      { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 Hunter API Key' },
+    ],
+    docUrl: 'https://hunter.qianxin.com/',
+  },
+  {
+    key: 'shodan',
+    name: 'Shodan',
+    description: '全球最大的互联网设备搜索引擎',
+    icon: IconWorldSearch,
+    color: 'text-red-500',
+    bgColor: 'bg-red-500/10',
+    fields: [
+      { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 Shodan API Key' },
+    ],
+    docUrl: 'https://developer.shodan.io/',
+  },
+  {
+    key: 'censys',
+    name: 'Censys',
+    description: '互联网资产搜索和监控平台',
+    icon: IconWorldSearch,
+    color: 'text-purple-500',
+    bgColor: 'bg-purple-500/10',
+    fields: [
+      { name: 'apiId', label: 'API ID', type: 'text', placeholder: '输入 Censys API ID' },
+      { name: 'apiSecret', label: 'API Secret', type: 'password', placeholder: '输入 Censys API Secret' },
+    ],
+    docUrl: 'https://search.censys.io/api',
+  },
+  {
+    key: 'zoomeye',
+    name: 'ZoomEye (钟馗之眼)',
+    description: '知道创宇网络空间搜索引擎',
+    icon: IconWorldSearch,
+    color: 'text-green-500',
+    bgColor: 'bg-green-500/10',
+    fields: [
+      { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 ZoomEye API Key' },
+    ],
+    docUrl: 'https://www.zoomeye.org/doc',
+  },
+  {
+    key: 'securitytrails',
+    name: 'SecurityTrails',
+    description: 'DNS 历史记录和子域名数据平台',
+    icon: IconWorldSearch,
+    color: 'text-cyan-500',
+    bgColor: 'bg-cyan-500/10',
+    fields: [
+      { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 SecurityTrails API Key' },
+    ],
+    docUrl: 'https://securitytrails.com/corp/api',
+  },
+  {
+    key: 'threatbook',
+    name: 'ThreatBook (微步在线)',
+    description: '威胁情报平台，提供域名和 IP 情报查询',
+    icon: IconWorldSearch,
+    color: 'text-indigo-500',
+    bgColor: 'bg-indigo-500/10',
+    fields: [
+      { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 ThreatBook API Key' },
+    ],
+    docUrl: 'https://x.threatbook.com/api',
+  },
+  {
+    key: 'quake',
+    name: 'Quake (360)',
+    description: '360 网络空间测绘系统',
+    icon: IconWorldSearch,
+    color: 'text-teal-500',
+    bgColor: 'bg-teal-500/10',
+    fields: [
+      { name: 'apiKey', label: 'API Key', type: 'password', placeholder: '输入 Quake API Key' },
+    ],
+    docUrl: 'https://quake.360.net/quake/#/help',
+  },
+]
+
+// 默认配置
+const DEFAULT_SETTINGS: ApiKeySettings = {
+  fofa: { enabled: false, email: '', apiKey: '' },
+  hunter: { enabled: false, apiKey: '' },
+  shodan: { enabled: false, apiKey: '' },
+  censys: { enabled: false, apiId: '', apiSecret: '' },
+  zoomeye: { enabled: false, apiKey: '' },
+  securitytrails: { enabled: false, apiKey: '' },
+  threatbook: { enabled: false, apiKey: '' },
+  quake: { enabled: false, apiKey: '' },
+}
+
+export default function ApiKeysSettingsPage() {
+  const { data: settings, isLoading } = useApiKeySettings()
+  const updateMutation = useUpdateApiKeySettings()
+  
+  const [formData, setFormData] = useState<ApiKeySettings>(DEFAULT_SETTINGS)
+  const [hasChanges, setHasChanges] = useState(false)
+
+  // 当数据加载完成后，更新表单数据
+  useEffect(() => {
+    if (settings) {
+      setFormData({ ...DEFAULT_SETTINGS, ...settings })
+      setHasChanges(false)
+    }
+  }, [settings])
+
+  const updateProvider = (providerKey: string, field: string, value: any) => {
+    setFormData(prev => ({
+      ...prev,
+      [providerKey]: {
+        ...prev[providerKey as keyof ApiKeySettings],
+        [field]: value,
+      }
+    }))
+    setHasChanges(true)
+  }
+
+  const handleSave = async () => {
+    updateMutation.mutate(formData)
+    setHasChanges(false)
+  }
+
+  const enabledCount = Object.values(formData).filter((p: any) => p?.enabled).length
+
+  if (isLoading) {
+    return (
+      <div className="p-4 md:p-6 space-y-6">
+        <div>
+          <Skeleton className="h-8 w-48" />
+          <Skeleton className="h-4 w-96 mt-2" />
+        </div>
+        <div className="grid gap-4">
+          {[1, 2, 3].map((i) => (
+            <Skeleton key={i} className="h-24 w-full" />
+          ))}
+        </div>
+      </div>
+    )
+  }
+
+  return (
+    <div className="p-4 md:p-6 space-y-6">
+      {/* 页面标题 */}
+      <div>
+        <div className="flex items-center gap-2">
+          <h1 className="text-2xl font-semibold">API 密钥配置</h1>
+          {enabledCount > 0 && (
+            <Badge variant="secondary">{enabledCount} 个已启用</Badge>
+          )}
+        </div>
+        <p className="text-muted-foreground mt-1">
+          配置第三方数据源的 API 密钥，用于增强子域名发现能力。启用后将在 subfinder 扫描时自动使用。
+        </p>
+      </div>
+
+      {/* Provider 卡片列表 */}
+      <div className="grid gap-4">
+        {PROVIDERS.map((provider) => {
+          const data = formData[provider.key as keyof ApiKeySettings] || {}
+          const isEnabled = (data as any)?.enabled || false
+
+          return (
+            <Card key={provider.key}>
+              <CardHeader className="pb-4">
+                <div className="flex items-center justify-between">
+                  <div className="flex items-center gap-3">
+                    <div className={`flex h-10 w-10 items-center justify-center rounded-lg ${provider.bgColor}`}>
+                      <provider.icon className={`h-5 w-5 ${provider.color}`} />
+                    </div>
+                    <div>
+                      <div className="flex items-center gap-2">
+                        <CardTitle className="text-base">{provider.name}</CardTitle>
+                        {isEnabled && <Badge variant="outline" className="text-xs text-green-600">已启用</Badge>}
+                      </div>
+                      <CardDescription>{provider.description}</CardDescription>
+                    </div>
+                  </div>
+                  <Switch
+                    checked={isEnabled}
+                    onCheckedChange={(checked) => updateProvider(provider.key, 'enabled', checked)}
+                  />
+                </div>
+              </CardHeader>
+
+              {/* 展开的配置表单 */}
+              {isEnabled && (
+                <CardContent className="pt-0">
+                  <Separator className="mb-4" />
+                  <div className="space-y-4">
+                    {provider.fields.map((field) => (
+                      <div key={field.name} className="space-y-2">
+                        <label className="text-sm font-medium">{field.label}</label>
+                        {field.type === 'password' ? (
+                          <PasswordInput
+                            value={(data as any)[field.name] || ''}
+                            onChange={(value) => updateProvider(provider.key, field.name, value)}
+                            placeholder={field.placeholder}
+                          />
+                        ) : (
+                          <Input
+                            type="text"
+                            value={(data as any)[field.name] || ''}
+                            onChange={(e) => updateProvider(provider.key, field.name, e.target.value)}
+                            placeholder={field.placeholder}
+                          />
+                        )}
+                      </div>
+                    ))}
+                    <p className="text-xs text-muted-foreground">
+                      获取 API Key：
+                      <a 
+                        href={provider.docUrl} 
+                        target="_blank" 
+                        rel="noopener noreferrer"
+                        className="text-primary hover:underline ml-1"
+                      >
+                        {provider.docUrl}
+                      </a>
+                    </p>
+                  </div>
+                </CardContent>
+              )}
+            </Card>
+          )
+        })}
+      </div>
+
+      {/* 保存按钮 */}
+      <div className="flex justify-end">
+        <Button 
+          onClick={handleSave} 
+          disabled={updateMutation.isPending || !hasChanges}
+        >
+          {updateMutation.isPending ? '保存中...' : '保存配置'}
+        </Button>
+      </div>
+    </div>
+  )
+}

frontend/app/[locale]/settings/blacklist/page.tsx

@@ -0,0 +1,132 @@
+"use client"
+
+import React, { useState, useEffect } from "react"
+import { useTranslations } from "next-intl"
+import { AlertTriangle, Loader2, Ban } from "lucide-react"
+import { Button } from "@/components/ui/button"
+import { Textarea } from "@/components/ui/textarea"
+import { Skeleton } from "@/components/ui/skeleton"
+import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"
+import { useGlobalBlacklist, useUpdateGlobalBlacklist } from "@/hooks/use-global-blacklist"
+
+/**
+ * Global blacklist settings page
+ */
+export default function GlobalBlacklistPage() {
+  const t = useTranslations("pages.settings.blacklist")
+  
+  const [blacklistText, setBlacklistText] = useState("")
+  const [hasChanges, setHasChanges] = useState(false)
+
+  const { data, isLoading, error } = useGlobalBlacklist()
+  const updateBlacklist = useUpdateGlobalBlacklist()
+
+  // Initialize text when data loads
+  useEffect(() => {
+    if (data?.patterns) {
+      setBlacklistText(data.patterns.join("\n"))
+      setHasChanges(false)
+    }
+  }, [data])
+
+  // Handle text change
+  const handleTextChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
+    setBlacklistText(e.target.value)
+    setHasChanges(true)
+  }
+
+  // Handle save
+  const handleSave = () => {
+    const patterns = blacklistText
+      .split("\n")
+      .map((line) => line.trim())
+      .filter((line) => line.length > 0)
+
+    updateBlacklist.mutate(
+      { patterns },
+      {
+        onSuccess: () => {
+          setHasChanges(false)
+        },
+      }
+    )
+  }
+
+  if (isLoading) {
+    return (
+      <div className="flex flex-1 flex-col gap-4 p-4">
+        <div className="space-y-2">
+          <Skeleton className="h-8 w-48" />
+          <Skeleton className="h-4 w-96" />
+        </div>
+        <Skeleton className="h-[400px] w-full" />
+      </div>
+    )
+  }
+
+  if (error) {
+    return (
+      <div className="flex flex-1 flex-col items-center justify-center py-12">
+        <AlertTriangle className="h-10 w-10 text-destructive mb-4" />
+        <p className="text-muted-foreground">{t("loadError")}</p>
+      </div>
+    )
+  }
+
+  return (
+    <div className="flex flex-1 flex-col gap-4 p-4">
+      {/* Page header */}
+      <div>
+        <h1 className="text-2xl font-bold">{t("title")}</h1>
+        <p className="text-muted-foreground">{t("description")}</p>
+      </div>
+
+      {/* Blacklist card */}
+      <Card>
+        <CardHeader>
+          <div className="flex items-center gap-2">
+            <Ban className="h-5 w-5 text-muted-foreground" />
+            <CardTitle>{t("card.title")}</CardTitle>
+          </div>
+          <CardDescription>{t("card.description")}</CardDescription>
+        </CardHeader>
+        <CardContent className="space-y-4">
+          {/* Rules hint */}
+          <div className="flex flex-wrap items-center gap-x-4 gap-y-2 text-sm text-muted-foreground">
+            <span className="font-medium text-foreground">{t("rules.title")}:</span>
+            <span><code className="bg-muted px-1.5 py-0.5 rounded text-xs">*.gov</code> {t("rules.domain")}</span>
+            <span><code className="bg-muted px-1.5 py-0.5 rounded text-xs">*cdn*</code> {t("rules.keyword")}</span>
+            <span><code className="bg-muted px-1.5 py-0.5 rounded text-xs">192.168.1.1</code> {t("rules.ip")}</span>
+            <span><code className="bg-muted px-1.5 py-0.5 rounded text-xs">10.0.0.0/8</code> {t("rules.cidr")}</span>
+          </div>
+
+          {/* Scope hint */}
+          <div className="rounded-lg border bg-muted/50 p-3 text-sm">
+            <p className="text-muted-foreground">{t("scopeHint")}</p>
+          </div>
+
+          {/* Input */}
+          <Textarea
+            value={blacklistText}
+            onChange={handleTextChange}
+            placeholder={t("placeholder")}
+            className="min-h-[320px] font-mono text-sm"
+          />
+
+          {/* Save button */}
+          <div className="flex justify-end">
+            <Button
+              onClick={handleSave}
+              disabled={!hasChanges || updateBlacklist.isPending}
+            >
+              {updateBlacklist.isPending && (
+                <Loader2 className="mr-2 h-4 w-4 animate-spin" />
+              )}
+              {t("save")}
+            </Button>
+          </div>
+        </CardContent>
+      </Card>
+    </div>
+  )
+}

frontend/app/[locale]/target/[id]/details/page.tsx

@@ -5,15 +5,15 @@ import { useEffect } from "react"
 
 /**
  * Target detail page (compatible with old routes)
- * Automatically redirects to websites page
+ * Automatically redirects to overview page
  */
 export default function TargetDetailsPage() {
   const { id } = useParams<{ id: string }>()
   const router = useRouter()
 
   useEffect(() => {
-    // Redirect to websites page
-    router.replace(`/target/${id}/websites/`)
+    // Redirect to overview page
+    router.replace(`/target/${id}/overview/`)
   }, [id, router])
 
   return null

frontend/app/[locale]/target/[id]/layout.tsx

@@ -12,7 +12,8 @@ import { useTranslations } from "next-intl"
 
 /**
  * Target detail layout
- * Provides shared target information and navigation for all sub-pages
+ * Two-level navigation: Overview / Assets / Vulnerabilities
+ * Assets has secondary navigation for different asset types
  */
 export default function TargetLayout({
   children,
@@ -30,26 +31,52 @@ export default function TargetLayout({
     error
   } = useTarget(Number(id))
 
-  // Get currently active tab
-  const getActiveTab = () => {
-    if (pathname.includes("/subdomain")) return "subdomain"
-    if (pathname.includes("/endpoints")) return "endpoints"
-    if (pathname.includes("/websites")) return "websites"
-    if (pathname.includes("/directories")) return "directories"
+  // Get primary navigation active tab
+  const getPrimaryTab = () => {
+    if (pathname.includes("/overview")) return "overview"
     if (pathname.includes("/vulnerabilities")) return "vulnerabilities"
-    if (pathname.includes("/ip-addresses")) return "ip-addresses"
-    return ""
+    if (pathname.includes("/settings")) return "settings"
+    // All asset pages fall under "assets"
+    if (
+      pathname.includes("/websites") ||
+      pathname.includes("/subdomain") ||
+      pathname.includes("/ip-addresses") ||
+      pathname.includes("/endpoints") ||
+      pathname.includes("/directories")
+    ) {
+      return "assets"
+    }
+    return "overview"
   }
 
+  // Get secondary navigation active tab (for assets)
+  const getSecondaryTab = () => {
+    if (pathname.includes("/websites")) return "websites"
+    if (pathname.includes("/subdomain")) return "subdomain"
+    if (pathname.includes("/ip-addresses")) return "ip-addresses"
+    if (pathname.includes("/endpoints")) return "endpoints"
+    if (pathname.includes("/directories")) return "directories"
+    return "websites"
+  }
+
+  // Check if we should show secondary navigation
+  const showSecondaryNav = getPrimaryTab() === "assets"
+
   // Tab path mapping
   const basePath = `/target/${id}`
-  const tabPaths = {
-    subdomain: `${basePath}/subdomain/`,
-    endpoints: `${basePath}/endpoints/`,
-    websites: `${basePath}/websites/`,
-    directories: `${basePath}/directories/`,
+  const primaryPaths = {
+    overview: `${basePath}/overview/`,
+    assets: `${basePath}/websites/`, // Default to websites when clicking assets
     vulnerabilities: `${basePath}/vulnerabilities/`,
+    settings: `${basePath}/settings/`,
+  }
+
+  const secondaryPaths = {
+    websites: `${basePath}/websites/`,
+    subdomain: `${basePath}/subdomain/`,
     "ip-addresses": `${basePath}/ip-addresses/`,
+    endpoints: `${basePath}/endpoints/`,
+    directories: `${basePath}/directories/`,
   }
 
   // Get counts for each tab from target data
@@ -62,27 +89,24 @@ export default function TargetLayout({
     "ip-addresses": (target as any)?.summary?.ips || 0,
   }
 
+  // Calculate total assets count
+  const totalAssets = counts.websites + counts.subdomain + counts["ip-addresses"] + counts.endpoints + counts.directories
+
   // Loading state
   if (isLoading) {
     return (
       <div className="flex flex-col gap-4 py-4 md:gap-6 md:py-6">
-        {/* Page header skeleton */}
-        <div className="flex items-center justify-between px-4 lg:px-6">
-          <div className="w-full max-w-xl space-y-2">
-            <div className="flex items-center gap-2">
-              <Skeleton className="h-6 w-6 rounded-md" />
-              <Skeleton className="h-7 w-48" />
-            </div>
-            <Skeleton className="h-4 w-72" />
-          </div>
+        {/* Header skeleton */}
+        <div className="flex items-center gap-2 px-4 lg:px-6">
+          <Skeleton className="h-4 w-16" />
+          <span className="text-muted-foreground">/</span>
+          <Skeleton className="h-4 w-32" />
         </div>
-
-        {/* Tabs navigation skeleton */}
-        <div className="flex items-center justify-between px-4 lg:px-6">
-          <div className="flex gap-2">
-            <Skeleton className="h-9 w-20" />
-            <Skeleton className="h-9 w-24" />
-          </div>
+        {/* Tabs skeleton */}
+        <div className="flex gap-1 px-4 lg:px-6">
+          <Skeleton className="h-9 w-20" />
+          <Skeleton className="h-9 w-20" />
+          <Skeleton className="h-9 w-24" />
         </div>
       </div>
     )
@@ -123,74 +147,38 @@ export default function TargetLayout({
 
   return (
     <div className="flex flex-col gap-4 py-4 md:gap-6 md:py-6">
-      {/* Page header */}
-      <div className="flex items-center justify-between px-4 lg:px-6">
-        <div>
-          <h2 className="text-2xl font-bold tracking-tight flex items-center gap-2">
-            <Target />
-            {target.name}
-          </h2>
-          <p className="text-muted-foreground">{target.description || t("noDescription")}</p>
-        </div>
+      {/* Header: Page label + Target name */}
+      <div className="flex items-center gap-2 text-sm px-4 lg:px-6">
+        <span className="text-muted-foreground">{t("breadcrumb.targetDetail")}</span>
+        <span className="text-muted-foreground">/</span>
+        <span className="font-medium flex items-center gap-1.5">
+          <Target className="h-4 w-4" />
+          {target.name}
+        </span>
       </div>
 
-      {/* Tabs navigation - Use Link to ensure progress bar is triggered */}
-      <div className="flex items-center justify-between px-4 lg:px-6">
-        <Tabs value={getActiveTab()} className="w-full">
+      {/* Primary navigation */}
+      <div className="px-4 lg:px-6">
+        <Tabs value={getPrimaryTab()}>
           <TabsList>
-            <TabsTrigger value="websites" asChild>
-              <Link href={tabPaths.websites} className="flex items-center gap-0.5">
-                Websites
-                {counts.websites > 0 && (
-                  <Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
-                    {counts.websites}
-                  </Badge>
-                )}
+            <TabsTrigger value="overview" asChild>
+              <Link href={primaryPaths.overview} className="flex items-center gap-0.5">
+                {t("tabs.overview")}
               </Link>
             </TabsTrigger>
-            <TabsTrigger value="subdomain" asChild>
-              <Link href={tabPaths.subdomain} className="flex items-center gap-0.5">
-                Subdomains
-                {counts.subdomain > 0 && (
+            <TabsTrigger value="assets" asChild>
+              <Link href={primaryPaths.assets} className="flex items-center gap-0.5">
+                {t("tabs.assets")}
+                {totalAssets > 0 && (
                   <Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
-                    {counts.subdomain}
-                  </Badge>
-                )}
-              </Link>
-            </TabsTrigger>
-            <TabsTrigger value="ip-addresses" asChild>
-              <Link href={tabPaths["ip-addresses"]} className="flex items-center gap-0.5">
-                IP Addresses
-                {counts["ip-addresses"] > 0 && (
-                  <Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
-                    {counts["ip-addresses"]}
-                  </Badge>
-                )}
-              </Link>
-            </TabsTrigger>
-            <TabsTrigger value="endpoints" asChild>
-              <Link href={tabPaths.endpoints} className="flex items-center gap-0.5">
-                URLs
-                {counts.endpoints > 0 && (
-                  <Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
-                    {counts.endpoints}
-                  </Badge>
-                )}
-              </Link>
-            </TabsTrigger>
-            <TabsTrigger value="directories" asChild>
-              <Link href={tabPaths.directories} className="flex items-center gap-0.5">
-                Directories
-                {counts.directories > 0 && (
-                  <Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
-                    {counts.directories}
+                    {totalAssets}
                   </Badge>
                 )}
               </Link>
             </TabsTrigger>
             <TabsTrigger value="vulnerabilities" asChild>
-              <Link href={tabPaths.vulnerabilities} className="flex items-center gap-0.5">
-                Vulnerabilities
+              <Link href={primaryPaths.vulnerabilities} className="flex items-center gap-0.5">
+                {t("tabs.vulnerabilities")}
                 {counts.vulnerabilities > 0 && (
                   <Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
                     {counts.vulnerabilities}
@@ -198,10 +186,75 @@ export default function TargetLayout({
                 )}
               </Link>
             </TabsTrigger>
+            <TabsTrigger value="settings" asChild>
+              <Link href={primaryPaths.settings} className="flex items-center gap-0.5">
+                {t("tabs.settings")}
+              </Link>
+            </TabsTrigger>
           </TabsList>
         </Tabs>
       </div>
 
+      {/* Secondary navigation (only for assets) */}
+      {showSecondaryNav && (
+        <div className="flex items-center px-4 lg:px-6">
+          <Tabs value={getSecondaryTab()} className="w-full">
+            <TabsList variant="underline">
+              <TabsTrigger value="websites" variant="underline" asChild>
+                <Link href={secondaryPaths.websites} className="flex items-center gap-0.5">
+                  Websites
+                  {counts.websites > 0 && (
+                    <Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
+                      {counts.websites}
+                    </Badge>
+                  )}
+                </Link>
+              </TabsTrigger>
+              <TabsTrigger value="subdomain" variant="underline" asChild>
+                <Link href={secondaryPaths.subdomain} className="flex items-center gap-0.5">
+                  Subdomains
+                  {counts.subdomain > 0 && (
+                    <Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
+                      {counts.subdomain}
+                    </Badge>
+                  )}
+                </Link>
+              </TabsTrigger>
+              <TabsTrigger value="ip-addresses" variant="underline" asChild>
+                <Link href={secondaryPaths["ip-addresses"]} className="flex items-center gap-0.5">
+                  IPs
+                  {counts["ip-addresses"] > 0 && (
+                    <Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
+                      {counts["ip-addresses"]}
+                    </Badge>
+                  )}
+                </Link>
+              </TabsTrigger>
+              <TabsTrigger value="endpoints" variant="underline" asChild>
+                <Link href={secondaryPaths.endpoints} className="flex items-center gap-0.5">
+                  URLs
+                  {counts.endpoints > 0 && (
+                    <Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
+                      {counts.endpoints}
+                    </Badge>
+                  )}
+                </Link>
+              </TabsTrigger>
+              <TabsTrigger value="directories" variant="underline" asChild>
+                <Link href={secondaryPaths.directories} className="flex items-center gap-0.5">
+                  Directories
+                  {counts.directories > 0 && (
+                    <Badge variant="secondary" className="ml-1.5 h-5 min-w-5 rounded-full px-1.5 text-xs">
+                      {counts.directories}
+                    </Badge>
+                  )}
+                </Link>
+              </TabsTrigger>
+            </TabsList>
+          </Tabs>
+        </div>
+      )}
+
       {/* Sub-page content */}
       {children}
     </div>

frontend/app/[locale]/target/[id]/overview/page.tsx

@@ -0,0 +1,19 @@
+"use client"
+
+import { useParams } from "next/navigation"
+import { TargetOverview } from "@/components/target/target-overview"
+
+/**
+ * Target overview page
+ * Displays target statistics and summary information
+ */
+export default function TargetOverviewPage() {
+  const { id } = useParams<{ id: string }>()
+  const targetId = Number(id)
+
+  return (
+    <div className="px-4 lg:px-6">
+      <TargetOverview targetId={targetId} />
+    </div>
+  )
+}

frontend/app/[locale]/target/[id]/page.tsx

@@ -5,15 +5,15 @@ import { useEffect } from "react"
 
 /**
  * Target detail default page
- * Automatically redirects to websites page
+ * Automatically redirects to overview page
  */
 export default function TargetDetailPage() {
   const { id } = useParams<{ id: string }>()
   const router = useRouter()
 
   useEffect(() => {
-    // Redirect to websites page
-    router.replace(`/target/${id}/websites/`)
+    // Redirect to overview page
+    router.replace(`/target/${id}/overview/`)
   }, [id, router])
 
   return null