feat(notifications): add WeChat Work (WeChat Enterprise) notification support

- Add wecom notification channel configuration to mock notification settings
- Initialize wecom with disabled state and empty webhook URL by default
- Update notification settings response to include wecom configuration
- Enable WeChat Work as an alternative notification channel alongside Discord

.github/workflows/docker-build.yml

@@ -19,7 +19,8 @@ permissions:
   contents: write
 
 jobs:
-  build:
+  # AMD64 构建（原生 x64 runner）
+  build-amd64:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -27,39 +28,30 @@ jobs:
           - image: xingrin-server
             dockerfile: docker/server/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
           - image: xingrin-frontend
             dockerfile: docker/frontend/Dockerfile
             context: .
-            platforms: linux/amd64  # ARM64 构建时 Next.js 在 QEMU 下会崩溃
           - image: xingrin-worker
             dockerfile: docker/worker/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
           - image: xingrin-nginx
             dockerfile: docker/nginx/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
           - image: xingrin-agent
             dockerfile: docker/agent/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
+          - image: xingrin-postgres
+            dockerfile: docker/postgres/Dockerfile
+            context: docker/postgres
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Free disk space (for large builds like worker)
+      - name: Free disk space
         run: |
-          echo "=== Before cleanup ==="
-          df -h
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /usr/local/lib/android
-          sudo rm -rf /opt/ghc
-          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
           sudo docker image prune -af
-          echo "=== After cleanup ==="
-          df -h
 
       - name: Generate SSL certificates for nginx build
         if: matrix.image == 'xingrin-nginx'
@@ -69,10 +61,6 @@ jobs:
             -keyout docker/nginx/ssl/privkey.pem \
             -out docker/nginx/ssl/fullchain.pem \
             -subj "/CN=localhost"
-          echo "SSL certificates generated for CI build"
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -83,7 +71,120 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Get version from git tag
+      - name: Get version
+        id: version
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+          else
+            echo "VERSION=dev-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build and push AMD64
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
+          platforms: linux/amd64
+          push: true
+          tags: ${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}-amd64
+          build-args: IMAGE_TAG=${{ steps.version.outputs.VERSION }}
+          cache-from: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-amd64
+          cache-to: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-amd64,mode=max
+          provenance: false
+          sbom: false
+
+  # ARM64 构建（原生 ARM64 runner）
+  build-arm64:
+    runs-on: ubuntu-22.04-arm
+    strategy:
+      matrix:
+        include:
+          - image: xingrin-server
+            dockerfile: docker/server/Dockerfile
+            context: .
+          - image: xingrin-frontend
+            dockerfile: docker/frontend/Dockerfile
+            context: .
+          - image: xingrin-worker
+            dockerfile: docker/worker/Dockerfile
+            context: .
+          - image: xingrin-nginx
+            dockerfile: docker/nginx/Dockerfile
+            context: .
+          - image: xingrin-agent
+            dockerfile: docker/agent/Dockerfile
+            context: .
+          - image: xingrin-postgres
+            dockerfile: docker/postgres/Dockerfile
+            context: docker/postgres
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Generate SSL certificates for nginx build
+        if: matrix.image == 'xingrin-nginx'
+        run: |
+          mkdir -p docker/nginx/ssl
+          openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
+            -keyout docker/nginx/ssl/privkey.pem \
+            -out docker/nginx/ssl/fullchain.pem \
+            -subj "/CN=localhost"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get version
+        id: version
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+          else
+            echo "VERSION=dev-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build and push ARM64
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
+          platforms: linux/arm64
+          push: true
+          tags: ${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}-arm64
+          build-args: IMAGE_TAG=${{ steps.version.outputs.VERSION }}
+          cache-from: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-arm64
+          cache-to: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-arm64,mode=max
+          provenance: false
+          sbom: false
+
+  # 合并多架构 manifest
+  merge-manifests:
+    runs-on: ubuntu-latest
+    needs: [build-amd64, build-arm64]
+    strategy:
+      matrix:
+        image:
+          - xingrin-server
+          - xingrin-frontend
+          - xingrin-worker
+          - xingrin-nginx
+          - xingrin-agent
+          - xingrin-postgres
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get version
         id: version
         run: |
           if [[ $GITHUB_REF == refs/tags/* ]]; then
@@ -94,28 +195,27 @@ jobs:
             echo "IS_RELEASE=false" >> $GITHUB_OUTPUT
           fi
 
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: ${{ matrix.context }}
-          file: ${{ matrix.dockerfile }}
-          platforms: ${{ matrix.platforms }}
-          push: true
-          tags: |
-            ${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}
-            ${{ steps.version.outputs.IS_RELEASE == 'true' && format('{0}/{1}:latest', env.IMAGE_PREFIX, matrix.image) || '' }}
-          build-args: |
-            IMAGE_TAG=${{ steps.version.outputs.VERSION }}
-          cache-from: type=gha,scope=${{ matrix.image }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.image }}
-          provenance: false
-          sbom: false
+      - name: Create and push multi-arch manifest
+        run: |
+          VERSION=${{ steps.version.outputs.VERSION }}
+          IMAGE=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}
+          
+          docker manifest create ${IMAGE}:${VERSION} \
+            ${IMAGE}:${VERSION}-amd64 \
+            ${IMAGE}:${VERSION}-arm64
+          docker manifest push ${IMAGE}:${VERSION}
+          
+          if [[ "${{ steps.version.outputs.IS_RELEASE }}" == "true" ]]; then
+            docker manifest create ${IMAGE}:latest \
+              ${IMAGE}:${VERSION}-amd64 \
+              ${IMAGE}:${VERSION}-arm64
+            docker manifest push ${IMAGE}:latest
+          fi
 
-  # 所有镜像构建成功后，更新 VERSION 文件
-  # 根据 tag 所在的分支更新对应分支的 VERSION 文件
+  # 更新 VERSION 文件
   update-version:
     runs-on: ubuntu-latest
-    needs: build
+    needs: merge-manifests
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
       - name: Checkout repository

.gitignore

@@ -64,6 +64,7 @@ backend/.env.local
 .coverage
 htmlcov/
 *.cover
+.hypothesis/
 
 # ============================
 # 后端 (Go) 相关

README.md

@@ -25,6 +25,13 @@
 
 ---
 
+## 🌐 在线 Demo
+
+ **[https://xingrin.vercel.app/](https://xingrin.vercel.app/)**
+
+> ⚠️ 仅用于 UI 展示，未接入后端数据库
+
+---
 
 <p align="center">
   <b>🎨 现代化 UI </b>
@@ -51,23 +58,33 @@
 
 ## ✨ 功能特性
 
-### 🎯 目标与资产管理
-- **组织管理** - 多层级目标组织，灵活分组
-- **目标管理** - 支持域名、IP目标类型
-- **资产发现** - 子域名、网站、端点、目录自动发现
-- **资产快照** - 扫描结果快照对比，追踪资产变化
+### 扫描能力
 
-### 🔍 漏洞扫描
-- **多引擎支持** - 集成 Nuclei 等主流扫描引擎
-- **自定义流程** - YAML 配置扫描流程，灵活编排
-- **定时扫描** - Cron 表达式配置，自动化周期扫描
+| 功能 | 状态 | 工具 | 说明 |
+|------|------|------|------|
+| 子域名扫描 | ✅ | Subfinder, Amass, PureDNS | 被动收集 + 主动爆破，聚合 50+ 数据源 |
+| 端口扫描 | ✅ | Naabu | 自定义端口范围 |
+| 站点发现 | ✅ | HTTPX | HTTP 探测，自动获取标题、状态码、技术栈 |
+| 指纹识别 | ✅ | XingFinger | 2.7W+ 指纹规则，多源指纹库 |
+| URL 收集 | ✅ | Waymore, Katana | 历史数据 + 主动爬取 |
+| 目录扫描 | ✅ | FFUF | 高速爆破，智能字典 |
+| 漏洞扫描 | ✅ | Nuclei, Dalfox | 9000+ POC 模板，XSS 检测 |
+| 站点截图 | ✅ | Playwright | WebP 高压缩存储 |
 
-### 🔖 指纹识别
-- **多源指纹库** - 内置 EHole、Goby、Wappalyzer、Fingers、FingerPrintHub、ARL 等 2.7W+ 指纹规则
-- **自动识别** - 扫描流程自动执行，识别 Web 应用技术栈
-- **指纹管理** - 支持查询、导入、导出指纹规则
+### 平台能力
 
-#### 扫描流程架构
+| 功能 | 状态 | 说明 |
+|------|------|------|
+| 目标管理 | ✅ | 多层级组织，支持域名/IP 目标 |
+| 资产快照 | ✅ | 扫描结果对比，追踪资产变化 |
+| 黑名单过滤 | ✅ | 全局 + Target 级，支持通配符/CIDR |
+| 定时任务 | ✅ | Cron 表达式，自动化周期扫描 |
+| 分布式扫描 | ✅ | 多 Worker 节点，负载感知调度 |
+| 全局搜索 | ✅ | 表达式语法，多字段组合查询 |
+| 通知推送 | ✅ | 企业微信、Telegram、Discord |
+| API 密钥管理 | ✅ | 可视化配置各数据源 API Key |
+
+### 扫描流程架构
 
 完整的扫描流程包括：子域名发现、端口扫描、站点发现、指纹识别、URL 收集、目录扫描、漏洞扫描等阶段
 
@@ -88,6 +105,7 @@ flowchart LR
         direction TB
         URL["URL 收集<br/>waymore, katana"]
         DIR["目录扫描<br/>ffuf"]
+        SCREENSHOT["站点截图<br/>playwright"]
     end
     
     subgraph STAGE3["阶段 3: 漏洞检测"]
@@ -112,6 +130,7 @@ flowchart LR
     style FINGER fill:#5dade2,stroke:#3498db,stroke-width:1px,color:#fff
     style URL fill:#bb8fce,stroke:#9b59b6,stroke-width:1px,color:#fff
     style DIR fill:#bb8fce,stroke:#9b59b6,stroke-width:1px,color:#fff
+    style SCREENSHOT fill:#bb8fce,stroke:#9b59b6,stroke-width:1px,color:#fff
     style VULN fill:#f0b27a,stroke:#e67e22,stroke-width:1px,color:#fff
 ```
 
@@ -189,6 +208,7 @@ url="/api/v1" && status!="404"
 ### 📊 可视化界面
 - **数据统计** - 资产/漏洞统计仪表盘
 - **实时通知** - WebSocket 消息推送
+- **通知推送** - 实时企业微信，tg，discard消息推送服务
 
 ---
 
@@ -197,6 +217,7 @@ url="/api/v1" && status!="404"
 ### 环境要求
 
 - **操作系统**: Ubuntu 20.04+ / Debian 11+ 
+- **系统架构**: AMD64 (x86_64) / ARM64 (aarch64)
 - **硬件**: 2核 4G 内存起步，20GB+ 磁盘空间
 
 ### 一键安装
@@ -216,7 +237,6 @@ sudo ./install.sh --mirror
 > **💡 --mirror 参数说明**
 > - 自动配置 Docker 镜像加速（国内镜像源）
 > - 加速 Git 仓库克隆（Nuclei 模板等）
-> - 大幅提升安装速度，避免网络超时
 
 ### 访问服务
 
@@ -241,17 +261,40 @@ sudo ./uninstall.sh
 
 ## 🤝 反馈与贡献
 
-- 🐛 **如果发现 Bug** 可以点击右边链接进行提交 [Issue](https://github.com/yyhuni/xingrin/issues)
-- 💡 **有新想法，比如UI设计，功能设计等** 欢迎点击右边链接进行提交建议 [Issue](https://github.com/yyhuni/xingrin/issues)
+- 💡 **发现 Bug，有新想法，比如UI设计，功能设计等** 欢迎点击右边链接进行提交建议 [Issue](https://github.com/yyhuni/xingrin/issues) 或者公众号私信
 
 ## 📧 联系
-- 目前版本就我个人使用，可能会有很多边界问题
-- 如有问题，建议，其他，优先提交[Issue](https://github.com/yyhuni/xingrin/issues)，也可以直接给我的公众号发消息，我都会回复的
-
 - 微信公众号: **塔罗安全学苑**
+- 微信群去公众号底下的菜单，有个交流群，点击就可以看到了，链接过期可以私信我拉你
 
 <img src="docs/wechat-qrcode.png" alt="微信公众号" width="200">
 
+### 🎁 关注公众号免费领取指纹库
+
+| 指纹库 | 数量 |
+|--------|------|
+| ehole.json | 21,977 |
+| ARL.yaml | 9,264 |
+| goby.json | 7,086 |
+| FingerprintHub.json | 3,147 |
+
+> 💡 关注公众号回复「指纹」即可获取
+
+## ☕ 赞助支持
+
+如果这个项目对你有帮助，谢谢请我能喝杯蜜雪冰城，你的star和赞助是我免费更新的动力
+
+<p>
+  <img src="docs/wx_pay.jpg" alt="微信支付" width="200">
+  <img src="docs/zfb_pay.jpg" alt="支付宝" width="200">
+</p>
+
+### 🙏 感谢以下赞助
+
+| 昵称 | 金额 |
+|------|------|
+| X（闭关中） | ¥88 |
+
 
 ## ⚠️ 免责声明
 

VERSION

@@ -1 +1 @@
-v1.3.2-dev
+v1.5.4-dev

backend/.gitignore

@@ -7,6 +7,7 @@ __pycache__/
 *.egg-info/
 dist/
 build/
+.hypothesis/  # Hypothesis 属性测试缓存
 
 # 虚拟环境
 venv/

backend/apps/asset/apps.py

@@ -1,106 +1,6 @@
-import logging
-import sys
-
 from django.apps import AppConfig
 
-logger = logging.getLogger(__name__)
-
 
 class AssetConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'apps.asset'
-    
-    def ready(self):
-        # 导入所有模型以确保Django发现并注册
-        from . import models
-        
-        # 启用 pg_trgm 扩展（用于文本模糊搜索索引）
-        # 用于已有数据库升级场景
-        self._ensure_pg_trgm_extension()
-        
-        # 验证 pg_ivm 扩展是否可用（用于 IMMV 增量维护）
-        self._verify_pg_ivm_extension()
-    
-    def _ensure_pg_trgm_extension(self):
-        """
-        确保 pg_trgm 扩展已启用。
-        该扩展用于 response_body 和 response_headers 字段的 GIN 索引，
-        支持高效的文本模糊搜索。
-        """
-        from django.db import connection
-        
-        # 检查是否为 PostgreSQL 数据库
-        if connection.vendor != 'postgresql':
-            logger.debug("跳过 pg_trgm 扩展：当前数据库不是 PostgreSQL")
-            return
-        
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
-                logger.debug("pg_trgm 扩展已启用")
-        except Exception as e:
-            # 记录错误但不阻止应用启动
-            # 常见原因：权限不足（需要超级用户权限）
-            logger.warning(
-                "无法创建 pg_trgm 扩展: %s。"
-                "这可能导致 response_body 和 response_headers 字段的 GIN 索引无法正常工作。"
-                "请手动执行: CREATE EXTENSION IF NOT EXISTS pg_trgm;",
-                str(e)
-            )
-    
-    def _verify_pg_ivm_extension(self):
-        """
-        验证 pg_ivm 扩展是否可用。
-        pg_ivm 用于 IMMV（增量维护物化视图），是系统必需的扩展。
-        如果不可用，将记录错误并退出。
-        """
-        from django.db import connection
-        
-        # 检查是否为 PostgreSQL 数据库
-        if connection.vendor != 'postgresql':
-            logger.debug("跳过 pg_ivm 验证：当前数据库不是 PostgreSQL")
-            return
-        
-        # 跳过某些管理命令（如 migrate、makemigrations）
-        import sys
-        if len(sys.argv) > 1 and sys.argv[1] in ('migrate', 'makemigrations', 'collectstatic', 'check'):
-            logger.debug("跳过 pg_ivm 验证：当前为管理命令")
-            return
-        
-        try:
-            with connection.cursor() as cursor:
-                # 检查 pg_ivm 扩展是否已安装
-                cursor.execute("""
-                    SELECT COUNT(*) FROM pg_extension WHERE extname = 'pg_ivm'
-                """)
-                count = cursor.fetchone()[0]
-                
-                if count > 0:
-                    logger.info("✓ pg_ivm 扩展已启用")
-                else:
-                    # 尝试创建扩展
-                    try:
-                        cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_ivm;")
-                        logger.info("✓ pg_ivm 扩展已创建并启用")
-                    except Exception as create_error:
-                        logger.error(
-                            "=" * 60 + "\n"
-                            "错误: pg_ivm 扩展未安装\n"
-                            "=" * 60 + "\n"
-                            "pg_ivm 是系统必需的扩展，用于增量维护物化视图。\n\n"
-                            "请在 PostgreSQL 服务器上安装 pg_ivm：\n"
-                            "  curl -sSL https://raw.githubusercontent.com/yyhuni/xingrin/main/docker/scripts/install-pg-ivm.sh | sudo bash\n\n"
-                            "或手动安装：\n"
-                            "  1. apt install build-essential postgresql-server-dev-15 git\n"
-                            "  2. git clone https://github.com/sraoss/pg_ivm.git && cd pg_ivm && make && make install\n"
-                            "  3. 在 postgresql.conf 中添加: shared_preload_libraries = 'pg_ivm'\n"
-                            "  4. 重启 PostgreSQL\n"
-                            "=" * 60
-                        )
-                        # 在生产环境中退出，开发环境中仅警告
-                        from django.conf import settings
-                        if not settings.DEBUG:
-                            sys.exit(1)
-                        
-        except Exception as e:
-            logger.error(f"pg_ivm 扩展验证失败: {e}")

backend/apps/asset/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.2.7 on 2026-01-02 04:45
+# Generated by Django 5.2.7 on 2026-01-06 00:55
 
 import django.contrib.postgres.fields
 import django.contrib.postgres.indexes

backend/apps/asset/migrations/0002_create_search_materialized_view.py

@@ -1,187 +0,0 @@
-"""
-创建资产搜索 IMMV（增量维护物化视图）
-
-使用 pg_ivm 扩展创建 IMMV，数据变更时自动增量更新，无需手动刷新。
-
-包含：
-1. asset_search_view - Website 搜索视图
-2. endpoint_search_view - Endpoint 搜索视图
-"""
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('asset', '0001_initial'),
-    ]
-
-    operations = [
-        # 1. 确保 pg_ivm 扩展已启用
-        migrations.RunSQL(
-            sql="CREATE EXTENSION IF NOT EXISTS pg_ivm;",
-            reverse_sql="-- pg_ivm extension kept for other uses"
-        ),
-        
-        # ==================== Website IMMV ====================
-        
-        # 2. 创建 asset_search_view IMMV
-        migrations.RunSQL(
-            sql="""
-                SELECT pgivm.create_immv('asset_search_view', $$
-                    SELECT 
-                        w.id,
-                        w.url,
-                        w.host,
-                        w.title,
-                        w.tech,
-                        w.status_code,
-                        w.response_headers,
-                        w.response_body,
-                        w.content_type,
-                        w.content_length,
-                        w.webserver,
-                        w.location,
-                        w.vhost,
-                        w.created_at,
-                        w.target_id
-                    FROM website w
-                $$);
-            """,
-            reverse_sql="SELECT pgivm.drop_immv('asset_search_view');"
-        ),
-        
-        # 3. 创建 asset_search_view 索引
-        migrations.RunSQL(
-            sql="""
-                -- 唯一索引
-                CREATE UNIQUE INDEX IF NOT EXISTS asset_search_view_id_idx 
-                ON asset_search_view (id);
-                
-                -- host 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_host_trgm_idx 
-                ON asset_search_view USING gin (host gin_trgm_ops);
-                
-                -- title 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_title_trgm_idx 
-                ON asset_search_view USING gin (title gin_trgm_ops);
-                
-                -- url 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_url_trgm_idx 
-                ON asset_search_view USING gin (url gin_trgm_ops);
-                
-                -- response_headers 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_headers_trgm_idx 
-                ON asset_search_view USING gin (response_headers gin_trgm_ops);
-                
-                -- response_body 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_body_trgm_idx 
-                ON asset_search_view USING gin (response_body gin_trgm_ops);
-                
-                -- tech 数组索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_tech_idx 
-                ON asset_search_view USING gin (tech);
-                
-                -- status_code 索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_status_idx 
-                ON asset_search_view (status_code);
-                
-                -- created_at 排序索引
-                CREATE INDEX IF NOT EXISTS asset_search_view_created_idx 
-                ON asset_search_view (created_at DESC);
-            """,
-            reverse_sql="""
-                DROP INDEX IF EXISTS asset_search_view_id_idx;
-                DROP INDEX IF EXISTS asset_search_view_host_trgm_idx;
-                DROP INDEX IF EXISTS asset_search_view_title_trgm_idx;
-                DROP INDEX IF EXISTS asset_search_view_url_trgm_idx;
-                DROP INDEX IF EXISTS asset_search_view_headers_trgm_idx;
-                DROP INDEX IF EXISTS asset_search_view_body_trgm_idx;
-                DROP INDEX IF EXISTS asset_search_view_tech_idx;
-                DROP INDEX IF EXISTS asset_search_view_status_idx;
-                DROP INDEX IF EXISTS asset_search_view_created_idx;
-            """
-        ),
-
-        # ==================== Endpoint IMMV ====================
-        
-        # 4. 创建 endpoint_search_view IMMV
-        migrations.RunSQL(
-            sql="""
-                SELECT pgivm.create_immv('endpoint_search_view', $$
-                    SELECT 
-                        e.id,
-                        e.url,
-                        e.host,
-                        e.title,
-                        e.tech,
-                        e.status_code,
-                        e.response_headers,
-                        e.response_body,
-                        e.content_type,
-                        e.content_length,
-                        e.webserver,
-                        e.location,
-                        e.vhost,
-                        e.matched_gf_patterns,
-                        e.created_at,
-                        e.target_id
-                    FROM endpoint e
-                $$);
-            """,
-            reverse_sql="SELECT pgivm.drop_immv('endpoint_search_view');"
-        ),
-        
-        # 5. 创建 endpoint_search_view 索引
-        migrations.RunSQL(
-            sql="""
-                -- 唯一索引
-                CREATE UNIQUE INDEX IF NOT EXISTS endpoint_search_view_id_idx 
-                ON endpoint_search_view (id);
-                
-                -- host 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_host_trgm_idx 
-                ON endpoint_search_view USING gin (host gin_trgm_ops);
-                
-                -- title 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_title_trgm_idx 
-                ON endpoint_search_view USING gin (title gin_trgm_ops);
-                
-                -- url 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_url_trgm_idx 
-                ON endpoint_search_view USING gin (url gin_trgm_ops);
-                
-                -- response_headers 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_headers_trgm_idx 
-                ON endpoint_search_view USING gin (response_headers gin_trgm_ops);
-                
-                -- response_body 模糊搜索索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_body_trgm_idx 
-                ON endpoint_search_view USING gin (response_body gin_trgm_ops);
-                
-                -- tech 数组索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_tech_idx 
-                ON endpoint_search_view USING gin (tech);
-                
-                -- status_code 索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_status_idx 
-                ON endpoint_search_view (status_code);
-                
-                -- created_at 排序索引
-                CREATE INDEX IF NOT EXISTS endpoint_search_view_created_idx 
-                ON endpoint_search_view (created_at DESC);
-            """,
-            reverse_sql="""
-                DROP INDEX IF EXISTS endpoint_search_view_id_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_host_trgm_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_title_trgm_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_url_trgm_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_headers_trgm_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_body_trgm_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_tech_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_status_idx;
-                DROP INDEX IF EXISTS endpoint_search_view_created_idx;
-            """
-        ),
-    ]

backend/apps/asset/migrations/0002_create_search_views.py

@@ -0,0 +1,104 @@
+"""
+创建资产搜索物化视图（使用 pg_ivm 增量维护）
+
+这些视图用于资产搜索功能，提供高性能的全文搜索能力。
+"""
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    """创建资产搜索所需的增量物化视图"""
+
+    dependencies = [
+        ('asset', '0001_initial'),
+    ]
+
+    operations = [
+        # 1. 确保 pg_ivm 扩展已安装
+        migrations.RunSQL(
+            sql="CREATE EXTENSION IF NOT EXISTS pg_ivm;",
+            reverse_sql="DROP EXTENSION IF EXISTS pg_ivm;",
+        ),
+        
+        # 2. 创建 Website 搜索视图
+        # 注意：pg_ivm 不支持 ArrayField，所以 tech 字段需要从原表 JOIN 获取
+        migrations.RunSQL(
+            sql="""
+                SELECT pgivm.create_immv('asset_search_view', $$
+                    SELECT 
+                        w.id,
+                        w.url,
+                        w.host,
+                        w.title,
+                        w.status_code,
+                        w.response_headers,
+                        w.response_body,
+                        w.content_type,
+                        w.content_length,
+                        w.webserver,
+                        w.location,
+                        w.vhost,
+                        w.created_at,
+                        w.target_id
+                    FROM website w
+                $$);
+            """,
+            reverse_sql="DROP TABLE IF EXISTS asset_search_view CASCADE;",
+        ),
+        
+        # 3. 创建 Endpoint 搜索视图
+        migrations.RunSQL(
+            sql="""
+                SELECT pgivm.create_immv('endpoint_search_view', $$
+                    SELECT 
+                        e.id,
+                        e.url,
+                        e.host,
+                        e.title,
+                        e.status_code,
+                        e.response_headers,
+                        e.response_body,
+                        e.content_type,
+                        e.content_length,
+                        e.webserver,
+                        e.location,
+                        e.vhost,
+                        e.created_at,
+                        e.target_id
+                    FROM endpoint e
+                $$);
+            """,
+            reverse_sql="DROP TABLE IF EXISTS endpoint_search_view CASCADE;",
+        ),
+        
+        # 4. 为搜索视图创建索引（加速查询）
+        migrations.RunSQL(
+            sql=[
+                # Website 搜索视图索引
+                "CREATE INDEX IF NOT EXISTS asset_search_view_host_idx ON asset_search_view (host);",
+                "CREATE INDEX IF NOT EXISTS asset_search_view_url_idx ON asset_search_view (url);",
+                "CREATE INDEX IF NOT EXISTS asset_search_view_title_idx ON asset_search_view (title);",
+                "CREATE INDEX IF NOT EXISTS asset_search_view_status_idx ON asset_search_view (status_code);",
+                "CREATE INDEX IF NOT EXISTS asset_search_view_created_idx ON asset_search_view (created_at DESC);",
+                # Endpoint 搜索视图索引
+                "CREATE INDEX IF NOT EXISTS endpoint_search_view_host_idx ON endpoint_search_view (host);",
+                "CREATE INDEX IF NOT EXISTS endpoint_search_view_url_idx ON endpoint_search_view (url);",
+                "CREATE INDEX IF NOT EXISTS endpoint_search_view_title_idx ON endpoint_search_view (title);",
+                "CREATE INDEX IF NOT EXISTS endpoint_search_view_status_idx ON endpoint_search_view (status_code);",
+                "CREATE INDEX IF NOT EXISTS endpoint_search_view_created_idx ON endpoint_search_view (created_at DESC);",
+            ],
+            reverse_sql=[
+                "DROP INDEX IF EXISTS asset_search_view_host_idx;",
+                "DROP INDEX IF EXISTS asset_search_view_url_idx;",
+                "DROP INDEX IF EXISTS asset_search_view_title_idx;",
+                "DROP INDEX IF EXISTS asset_search_view_status_idx;",
+                "DROP INDEX IF EXISTS asset_search_view_created_idx;",
+                "DROP INDEX IF EXISTS endpoint_search_view_host_idx;",
+                "DROP INDEX IF EXISTS endpoint_search_view_url_idx;",
+                "DROP INDEX IF EXISTS endpoint_search_view_title_idx;",
+                "DROP INDEX IF EXISTS endpoint_search_view_status_idx;",
+                "DROP INDEX IF EXISTS endpoint_search_view_created_idx;",
+            ],
+        ),
+    ]

backend/apps/asset/migrations/0003_add_screenshot_models.py

@@ -0,0 +1,53 @@
+# Generated by Django 5.2.7 on 2026-01-07 02:21
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('asset', '0002_create_search_views'),
+        ('scan', '0001_initial'),
+        ('targets', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Screenshot',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('url', models.TextField(help_text='截图对应的 URL')),
+                ('image', models.BinaryField(help_text='截图 WebP 二进制数据（压缩后）')),
+                ('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
+                ('updated_at', models.DateTimeField(auto_now=True, help_text='更新时间')),
+                ('target', models.ForeignKey(help_text='所属目标', on_delete=django.db.models.deletion.CASCADE, related_name='screenshots', to='targets.target')),
+            ],
+            options={
+                'verbose_name': '截图',
+                'verbose_name_plural': '截图',
+                'db_table': 'screenshot',
+                'ordering': ['-created_at'],
+                'indexes': [models.Index(fields=['target'], name='screenshot_target__2f01f6_idx'), models.Index(fields=['-created_at'], name='screenshot_created_c0ad4b_idx')],
+                'constraints': [models.UniqueConstraint(fields=('target', 'url'), name='unique_screenshot_per_target')],
+            },
+        ),
+        migrations.CreateModel(
+            name='ScreenshotSnapshot',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('url', models.TextField(help_text='截图对应的 URL')),
+                ('image', models.BinaryField(help_text='截图 WebP 二进制数据（压缩后）')),
+                ('created_at', models.DateTimeField(auto_now_add=True, help_text='创建时间')),
+                ('scan', models.ForeignKey(help_text='所属的扫描任务', on_delete=django.db.models.deletion.CASCADE, related_name='screenshot_snapshots', to='scan.scan')),
+            ],
+            options={
+                'verbose_name': '截图快照',
+                'verbose_name_plural': '截图快照',
+                'db_table': 'screenshot_snapshot',
+                'ordering': ['-created_at'],
+                'indexes': [models.Index(fields=['scan'], name='screenshot__scan_id_fb8c4d_idx'), models.Index(fields=['-created_at'], name='screenshot__created_804117_idx')],
+                'constraints': [models.UniqueConstraint(fields=('scan', 'url'), name='unique_screenshot_per_scan_snapshot')],
+            },
+        ),
+    ]

backend/apps/asset/migrations/0004_add_status_code_to_screenshot.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.2.7 on 2026-01-07 13:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('asset', '0003_add_screenshot_models'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='screenshot',
+            name='status_code',
+            field=models.SmallIntegerField(blank=True, help_text='HTTP 响应状态码', null=True),
+        ),
+        migrations.AddField(
+            model_name='screenshotsnapshot',
+            name='status_code',
+            field=models.SmallIntegerField(blank=True, help_text='HTTP 响应状态码', null=True),
+        ),
+    ]

backend/apps/asset/models/__init__.py

@@ -20,6 +20,12 @@ from .snapshot_models import (
     VulnerabilitySnapshot,
 )
 
+# 截图模型
+from .screenshot_models import (
+    Screenshot,
+    ScreenshotSnapshot,
+)
+
 # 统计模型
 from .statistics_models import AssetStatistics, StatisticsHistory
 
@@ -39,6 +45,9 @@ __all__ = [
     'HostPortMappingSnapshot',
     'EndpointSnapshot',
     'VulnerabilitySnapshot',
+    # 截图模型
+    'Screenshot',
+    'ScreenshotSnapshot',
     # 统计模型
     'AssetStatistics',
     'StatisticsHistory',

backend/apps/asset/models/screenshot_models.py

@@ -0,0 +1,80 @@
+from django.db import models
+
+
+class ScreenshotSnapshot(models.Model):
+    """
+    截图快照
+    
+    记录：某次扫描中捕获的网站截图
+    """
+
+    id = models.AutoField(primary_key=True)
+    scan = models.ForeignKey(
+        'scan.Scan',
+        on_delete=models.CASCADE,
+        related_name='screenshot_snapshots',
+        help_text='所属的扫描任务'
+    )
+    url = models.TextField(help_text='截图对应的 URL')
+    status_code = models.SmallIntegerField(null=True, blank=True, help_text='HTTP 响应状态码')
+    image = models.BinaryField(help_text='截图 WebP 二进制数据（压缩后）')
+    created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
+
+    class Meta:
+        db_table = 'screenshot_snapshot'
+        verbose_name = '截图快照'
+        verbose_name_plural = '截图快照'
+        ordering = ['-created_at']
+        indexes = [
+            models.Index(fields=['scan']),
+            models.Index(fields=['-created_at']),
+        ]
+        constraints = [
+            models.UniqueConstraint(
+                fields=['scan', 'url'],
+                name='unique_screenshot_per_scan_snapshot'
+            ),
+        ]
+
+    def __str__(self):
+        return f'{self.url} (Scan #{self.scan_id})'
+
+
+class Screenshot(models.Model):
+    """
+    截图资产
+    
+    存储：目标的最新截图（从快照同步）
+    """
+
+    id = models.AutoField(primary_key=True)
+    target = models.ForeignKey(
+        'targets.Target',
+        on_delete=models.CASCADE,
+        related_name='screenshots',
+        help_text='所属目标'
+    )
+    url = models.TextField(help_text='截图对应的 URL')
+    status_code = models.SmallIntegerField(null=True, blank=True, help_text='HTTP 响应状态码')
+    image = models.BinaryField(help_text='截图 WebP 二进制数据（压缩后）')
+    created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
+    updated_at = models.DateTimeField(auto_now=True, help_text='更新时间')
+
+    class Meta:
+        db_table = 'screenshot'
+        verbose_name = '截图'
+        verbose_name_plural = '截图'
+        ordering = ['-created_at']
+        indexes = [
+            models.Index(fields=['target']),
+            models.Index(fields=['-created_at']),
+        ]
+        constraints = [
+            models.UniqueConstraint(
+                fields=['target', 'url'],
+                name='unique_screenshot_per_target'
+            ),
+        ]
+
+    def __str__(self):
+        return f'{self.url} (Target #{self.target_id})'

backend/apps/asset/serializers.py

@@ -7,6 +7,7 @@ from .models.snapshot_models import (
     EndpointSnapshot,
     VulnerabilitySnapshot,
 )
+from .models.screenshot_models import Screenshot, ScreenshotSnapshot
 
 
 # 注意：IPAddress 和 Port 模型已被重构为 HostPortMapping
@@ -290,3 +291,23 @@ class EndpointSnapshotSerializer(serializers.ModelSerializer):
             'created_at',
         ]
         read_only_fields = fields
+
+
+# ==================== 截图序列化器 ====================
+
+class ScreenshotListSerializer(serializers.ModelSerializer):
+    """截图资产列表序列化器（不包含 image 字段）"""
+    
+    class Meta:
+        model = Screenshot
+        fields = ['id', 'url', 'status_code', 'created_at', 'updated_at']
+        read_only_fields = fields
+
+
+class ScreenshotSnapshotListSerializer(serializers.ModelSerializer):
+    """截图快照列表序列化器（不包含 image 字段）"""
+    
+    class Meta:
+        model = ScreenshotSnapshot
+        fields = ['id', 'url', 'status_code', 'created_at']
+        read_only_fields = fields

backend/apps/asset/services/playwright_screenshot_service.py

@@ -0,0 +1,186 @@
+"""
+Playwright 截图服务
+
+使用 Playwright 异步批量捕获网站截图
+"""
+import asyncio
+import logging
+from typing import Optional, AsyncGenerator
+
+logger = logging.getLogger(__name__)
+
+
+class PlaywrightScreenshotService:
+    """Playwright 截图服务 - 异步多 Page 并发截图"""
+    
+    # 内置默认值（不暴露给用户）
+    DEFAULT_VIEWPORT_WIDTH = 1920
+    DEFAULT_VIEWPORT_HEIGHT = 1080
+    DEFAULT_TIMEOUT = 30000  # 毫秒
+    DEFAULT_JPEG_QUALITY = 85
+    
+    def __init__(
+        self,
+        viewport_width: int = DEFAULT_VIEWPORT_WIDTH,
+        viewport_height: int = DEFAULT_VIEWPORT_HEIGHT,
+        timeout: int = DEFAULT_TIMEOUT,
+        concurrency: int = 5
+    ):
+        """
+        初始化 Playwright 截图服务
+        
+        Args:
+            viewport_width: 视口宽度（像素）
+            viewport_height: 视口高度（像素）
+            timeout: 页面加载超时时间（毫秒）
+            concurrency: 并发截图数
+        """
+        self.viewport_width = viewport_width
+        self.viewport_height = viewport_height
+        self.timeout = timeout
+        self.concurrency = concurrency
+    
+    async def capture_screenshot(self, url: str, page) -> tuple[Optional[bytes], Optional[int]]:
+        """
+        捕获单个 URL 的截图
+        
+        Args:
+            url: 目标 URL
+            page: Playwright Page 对象
+        
+        Returns:
+            (screenshot_bytes, status_code) 元组
+            - screenshot_bytes: JPEG 格式的截图字节数据，失败返回 None
+            - status_code: HTTP 响应状态码，失败返回 None
+        """
+        status_code = None
+        try:
+            # 尝试加载页面，即使返回错误状态码也继续截图
+            try:
+                response = await page.goto(url, timeout=self.timeout, wait_until='networkidle')
+                if response:
+                    status_code = response.status
+            except Exception as goto_error:
+                # 页面加载失败（4xx/5xx 或其他错误），但页面可能已部分渲染
+                # 仍然尝试截图以捕获错误页面
+                logger.debug("页面加载异常但尝试截图: %s, 错误: %s", url, str(goto_error)[:50])
+            
+            # 尝试截图（即使 goto 失败）
+            screenshot_bytes = await page.screenshot(
+                type='jpeg',
+                quality=self.DEFAULT_JPEG_QUALITY,
+                full_page=False
+            )
+            return (screenshot_bytes, status_code)
+        except asyncio.TimeoutError:
+            logger.warning("截图超时: %s", url)
+            return (None, None)
+        except Exception as e:
+            logger.warning("截图失败: %s, 错误: %s", url, str(e)[:100])
+            return (None, None)
+    
+    async def _capture_with_semaphore(
+        self,
+        url: str,
+        context,
+        semaphore: asyncio.Semaphore
+    ) -> tuple[str, Optional[bytes], Optional[int]]:
+        """
+        使用信号量控制并发的截图任务
+        
+        Args:
+            url: 目标 URL
+            context: Playwright BrowserContext
+            semaphore: 并发控制信号量
+        
+        Returns:
+            (url, screenshot_bytes, status_code) 元组
+        """
+        async with semaphore:
+            page = await context.new_page()
+            try:
+                screenshot_bytes, status_code = await self.capture_screenshot(url, page)
+                return (url, screenshot_bytes, status_code)
+            finally:
+                await page.close()
+    
+    async def capture_batch(
+        self,
+        urls: list[str]
+    ) -> AsyncGenerator[tuple[str, Optional[bytes], Optional[int]], None]:
+        """
+        批量捕获截图（异步生成器）
+        
+        使用单个 BrowserContext + 多 Page 并发模式
+        通过 Semaphore 控制并发数
+        
+        Args:
+            urls: URL 列表
+        
+        Yields:
+            (url, screenshot_bytes, status_code) 元组
+        """
+        if not urls:
+            return
+        
+        from playwright.async_api import async_playwright
+        
+        async with async_playwright() as p:
+            # 启动浏览器（headless 模式）
+            browser = await p.chromium.launch(
+                headless=True,
+                args=[
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-dev-shm-usage',
+                    '--disable-gpu'
+                ]
+            )
+            
+            try:
+                # 创建单个 context
+                context = await browser.new_context(
+                    viewport={
+                        'width': self.viewport_width,
+                        'height': self.viewport_height
+                    },
+                    ignore_https_errors=True,
+                    user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+                )
+                
+                # 使用 Semaphore 控制并发
+                semaphore = asyncio.Semaphore(self.concurrency)
+                
+                # 创建所有任务
+                tasks = [
+                    self._capture_with_semaphore(url, context, semaphore)
+                    for url in urls
+                ]
+                
+                # 使用 as_completed 实现流式返回
+                for coro in asyncio.as_completed(tasks):
+                    result = await coro
+                    yield result
+                
+                await context.close()
+                
+            finally:
+                await browser.close()
+    
+    async def capture_batch_collect(
+        self,
+        urls: list[str]
+    ) -> list[tuple[str, Optional[bytes], Optional[int]]]:
+        """
+        批量捕获截图（收集所有结果）
+        
+        Args:
+            urls: URL 列表
+        
+        Returns:
+            [(url, screenshot_bytes, status_code), ...] 列表
+        """
+        results = []
+        async for result in self.capture_batch(urls):
+            results.append(result)
+        return results

backend/apps/asset/services/screenshot_service.py

@@ -0,0 +1,185 @@
+"""
+截图服务
+
+负责截图的压缩、保存和同步
+"""
+import io
+import logging
+import os
+from typing import Optional
+
+from PIL import Image
+
+logger = logging.getLogger(__name__)
+
+
+class ScreenshotService:
+    """截图服务 - 负责压缩、保存和同步"""
+    
+    def __init__(self, max_width: int = 800, target_kb: int = 100):
+        """
+        初始化截图服务
+        
+        Args:
+            max_width: 最大宽度（像素）
+            target_kb: 目标文件大小（KB）
+        """
+        self.max_width = max_width
+        self.target_kb = target_kb
+    
+    def compress_screenshot(self, image_path: str) -> Optional[bytes]:
+        """
+        压缩截图为 WebP 格式
+        
+        Args:
+            image_path: PNG 截图文件路径
+        
+        Returns:
+            压缩后的 WebP 二进制数据，失败返回 None
+        """
+        if not os.path.exists(image_path):
+            logger.warning(f"截图文件不存在: {image_path}")
+            return None
+        
+        try:
+            with Image.open(image_path) as img:
+                return self._compress_image(img)
+        except Exception as e:
+            logger.error(f"压缩截图失败: {image_path}, 错误: {e}")
+            return None
+    
+    def compress_from_bytes(self, image_bytes: bytes) -> Optional[bytes]:
+        """
+        从字节数据压缩截图为 WebP 格式
+        
+        Args:
+            image_bytes: JPEG/PNG 图片字节数据
+        
+        Returns:
+            压缩后的 WebP 二进制数据，失败返回 None
+        """
+        if not image_bytes:
+            return None
+        
+        try:
+            img = Image.open(io.BytesIO(image_bytes))
+            return self._compress_image(img)
+        except Exception as e:
+            logger.error(f"从字节压缩截图失败: {e}")
+            return None
+    
+    def _compress_image(self, img: Image.Image) -> Optional[bytes]:
+        """
+        压缩 PIL Image 对象为 WebP 格式
+        
+        Args:
+            img: PIL Image 对象
+        
+        Returns:
+            压缩后的 WebP 二进制数据
+        """
+        try:
+            if img.mode in ('RGBA', 'P'):
+                img = img.convert('RGB')
+            
+            width, height = img.size
+            if width > self.max_width:
+                ratio = self.max_width / width
+                new_size = (self.max_width, int(height * ratio))
+                img = img.resize(new_size, Image.Resampling.LANCZOS)
+            
+            quality = 80
+            while quality >= 10:
+                buffer = io.BytesIO()
+                img.save(buffer, format='WEBP', quality=quality, method=6)
+                if len(buffer.getvalue()) <= self.target_kb * 1024:
+                    return buffer.getvalue()
+                quality -= 10
+            
+            return buffer.getvalue()
+        except Exception as e:
+            logger.error(f"压缩图片失败: {e}")
+            return None
+    
+    def save_screenshot_snapshot(
+        self,
+        scan_id: int,
+        url: str,
+        image_data: bytes,
+        status_code: int | None = None
+    ) -> bool:
+        """
+        保存截图快照到 ScreenshotSnapshot 表
+        
+        Args:
+            scan_id: 扫描 ID
+            url: 截图对应的 URL
+            image_data: 压缩后的图片二进制数据
+            status_code: HTTP 响应状态码
+        
+        Returns:
+            是否保存成功
+        """
+        from apps.asset.models import ScreenshotSnapshot
+        
+        try:
+            ScreenshotSnapshot.objects.update_or_create(
+                scan_id=scan_id,
+                url=url,
+                defaults={'image': image_data, 'status_code': status_code}
+            )
+            return True
+        except Exception as e:
+            logger.error(f"保存截图快照失败: scan_id={scan_id}, url={url}, 错误: {e}")
+            return False
+    
+    def sync_screenshots_to_asset(self, scan_id: int, target_id: int) -> int:
+        """
+        将扫描的截图快照同步到资产表
+        
+        Args:
+            scan_id: 扫描 ID
+            target_id: 目标 ID
+        
+        Returns:
+            同步的截图数量
+        """
+        from apps.asset.models import Screenshot, ScreenshotSnapshot
+        
+        snapshots = ScreenshotSnapshot.objects.filter(scan_id=scan_id)
+        count = 0
+        
+        for snapshot in snapshots:
+            try:
+                Screenshot.objects.update_or_create(
+                    target_id=target_id,
+                    url=snapshot.url,
+                    defaults={
+                        'image': snapshot.image,
+                        'status_code': snapshot.status_code
+                    }
+                )
+                count += 1
+            except Exception as e:
+                logger.error(f"同步截图到资产表失败: url={snapshot.url}, 错误: {e}")
+        
+        logger.info(f"同步截图完成: scan_id={scan_id}, target_id={target_id}, 数量={count}")
+        return count
+    
+    def process_and_save_screenshot(self, scan_id: int, url: str, image_path: str) -> bool:
+        """
+        处理并保存截图（压缩 + 保存快照）
+        
+        Args:
+            scan_id: 扫描 ID
+            url: 截图对应的 URL
+            image_path: PNG 截图文件路径
+        
+        Returns:
+            是否处理成功
+        """
+        image_data = self.compress_screenshot(image_path)
+        if image_data is None:
+            return False
+        
+        return self.save_screenshot_snapshot(scan_id, url, image_data)

backend/apps/asset/services/search_service.py

@@ -11,7 +11,7 @@
 
 import logging
 import re
-from typing import Optional, List, Dict, Any, Tuple, Literal
+from typing import Optional, List, Dict, Any, Tuple, Literal, Iterator
 
 from django.db import connection
 
@@ -37,46 +37,55 @@ VIEW_MAPPING = {
     'endpoint': 'endpoint_search_view',
 }
 
+# 资产类型到原表名的映射（用于 JOIN 获取数组字段）
+# ⚠️ 重要：pg_ivm 不支持 ArrayField，所有数组字段必须从原表 JOIN 获取
+TABLE_MAPPING = {
+    'website': 'website',
+    'endpoint': 'endpoint',
+}
+
 # 有效的资产类型
 VALID_ASSET_TYPES = {'website', 'endpoint'}
 
-# Website 查询字段
+# Website 查询字段（v=视图，t=原表）
+# ⚠️ 注意：t.tech 从原表获取，因为 pg_ivm 不支持 ArrayField
 WEBSITE_SELECT_FIELDS = """
-    id,
-    url,
-    host,
-    title,
-    tech,
-    status_code,
-    response_headers,
-    response_body,
-    content_type,
-    content_length,
-    webserver,
-    location,
-    vhost,
-    created_at,
-    target_id
+    v.id,
+    v.url,
+    v.host,
+    v.title,
+    t.tech,  -- ArrayField，从 website 表 JOIN 获取
+    v.status_code,
+    v.response_headers,
+    v.response_body,
+    v.content_type,
+    v.content_length,
+    v.webserver,
+    v.location,
+    v.vhost,
+    v.created_at,
+    v.target_id
 """
 
-# Endpoint 查询字段（包含 matched_gf_patterns）
+# Endpoint 查询字段
+# ⚠️ 注意：t.tech 和 t.matched_gf_patterns 从原表获取，因为 pg_ivm 不支持 ArrayField
 ENDPOINT_SELECT_FIELDS = """
-    id,
-    url,
-    host,
-    title,
-    tech,
-    status_code,
-    response_headers,
-    response_body,
-    content_type,
-    content_length,
-    webserver,
-    location,
-    vhost,
-    matched_gf_patterns,
-    created_at,
-    target_id
+    v.id,
+    v.url,
+    v.host,
+    v.title,
+    t.tech,  -- ArrayField，从 endpoint 表 JOIN 获取
+    v.status_code,
+    v.response_headers,
+    v.response_body,
+    v.content_type,
+    v.content_length,
+    v.webserver,
+    v.location,
+    v.vhost,
+    t.matched_gf_patterns,  -- ArrayField，从 endpoint 表 JOIN 获取
+    v.created_at,
+    v.target_id
 """
 
 
@@ -119,8 +128,8 @@ class SearchQueryParser:
         
         # 检查是否包含操作符语法，如果不包含则作为 host 模糊搜索
         if not cls.CONDITION_PATTERN.search(query):
-            # 裸文本，默认作为 host 模糊搜索
-            return "host ILIKE %s", [f"%{query}%"]
+            # 裸文本，默认作为 host 模糊搜索（v 是视图别名）
+            return "v.host ILIKE %s", [f"%{query}%"]
         
         # 按 || 分割为 OR 组
         or_groups = cls._split_by_or(query)
@@ -273,45 +282,45 @@ class SearchQueryParser:
     def _build_like_condition(cls, field: str, value: str, is_array: bool) -> Tuple[str, List[Any]]:
         """构建模糊匹配条件"""
         if is_array:
-            # 数组字段：检查数组中是否有元素包含该值
-            return f"EXISTS (SELECT 1 FROM unnest({field}) AS t WHERE t ILIKE %s)", [f"%{value}%"]
+            # 数组字段：检查数组中是否有元素包含该值（从原表 t 获取）
+            return f"EXISTS (SELECT 1 FROM unnest(t.{field}) AS elem WHERE elem ILIKE %s)", [f"%{value}%"]
         elif field == 'status_code':
             # 状态码是整数，模糊匹配转为精确匹配
             try:
-                return f"{field} = %s", [int(value)]
+                return f"v.{field} = %s", [int(value)]
             except ValueError:
-                return f"{field}::text ILIKE %s", [f"%{value}%"]
+                return f"v.{field}::text ILIKE %s", [f"%{value}%"]
         else:
-            return f"{field} ILIKE %s", [f"%{value}%"]
+            return f"v.{field} ILIKE %s", [f"%{value}%"]
     
     @classmethod
     def _build_exact_condition(cls, field: str, value: str, is_array: bool) -> Tuple[str, List[Any]]:
         """构建精确匹配条件"""
         if is_array:
-            # 数组字段：检查数组中是否包含该精确值
-            return f"%s = ANY({field})", [value]
+            # 数组字段：检查数组中是否包含该精确值（从原表 t 获取）
+            return f"%s = ANY(t.{field})", [value]
         elif field == 'status_code':
             # 状态码是整数
             try:
-                return f"{field} = %s", [int(value)]
+                return f"v.{field} = %s", [int(value)]
             except ValueError:
-                return f"{field}::text = %s", [value]
+                return f"v.{field}::text = %s", [value]
         else:
-            return f"{field} = %s", [value]
+            return f"v.{field} = %s", [value]
     
     @classmethod
     def _build_not_equal_condition(cls, field: str, value: str, is_array: bool) -> Tuple[str, List[Any]]:
         """构建不等于条件"""
         if is_array:
-            # 数组字段：检查数组中不包含该值
-            return f"NOT (%s = ANY({field}))", [value]
+            # 数组字段：检查数组中不包含该值（从原表 t 获取）
+            return f"NOT (%s = ANY(t.{field}))", [value]
         elif field == 'status_code':
             try:
-                return f"({field} IS NULL OR {field} != %s)", [int(value)]
+                return f"(v.{field} IS NULL OR v.{field} != %s)", [int(value)]
             except ValueError:
-                return f"({field} IS NULL OR {field}::text != %s)", [value]
+                return f"(v.{field} IS NULL OR v.{field}::text != %s)", [value]
         else:
-            return f"({field} IS NULL OR {field} != %s)", [value]
+            return f"(v.{field} IS NULL OR v.{field} != %s)", [value]
 
 
 AssetType = Literal['website', 'endpoint']
@@ -339,15 +348,18 @@ class AssetSearchService:
         """
         where_clause, params = SearchQueryParser.parse(query)
         
-        # 根据资产类型选择视图和字段
+        # 根据资产类型选择视图、原表和字段
         view_name = VIEW_MAPPING.get(asset_type, 'asset_search_view')
+        table_name = TABLE_MAPPING.get(asset_type, 'website')
         select_fields = ENDPOINT_SELECT_FIELDS if asset_type == 'endpoint' else WEBSITE_SELECT_FIELDS
         
+        # JOIN 原表获取数组字段（tech, matched_gf_patterns）
         sql = f"""
             SELECT {select_fields}
-            FROM {view_name}
+            FROM {view_name} v
+            JOIN {table_name} t ON v.id = t.id
             WHERE {where_clause}
-            ORDER BY created_at DESC
+            ORDER BY v.created_at DESC
         """
         
         # 添加 LIMIT
@@ -369,26 +381,31 @@ class AssetSearchService:
             logger.error(f"搜索查询失败: {e}, SQL: {sql}, params: {params}")
             raise
     
-    def count(self, query: str, asset_type: AssetType = 'website') -> int:
+    def count(self, query: str, asset_type: AssetType = 'website', statement_timeout_ms: int = 300000) -> int:
         """
         统计搜索结果数量
         
         Args:
             query: 搜索查询字符串
             asset_type: 资产类型 ('website' 或 'endpoint')
+            statement_timeout_ms: SQL 语句超时时间（毫秒），默认 5 分钟
         
         Returns:
             int: 结果总数
         """
         where_clause, params = SearchQueryParser.parse(query)
         
-        # 根据资产类型选择视图
+        # 根据资产类型选择视图和原表
         view_name = VIEW_MAPPING.get(asset_type, 'asset_search_view')
+        table_name = TABLE_MAPPING.get(asset_type, 'website')
         
-        sql = f"SELECT COUNT(*) FROM {view_name} WHERE {where_clause}"
+        # JOIN 原表以支持数组字段查询
+        sql = f"SELECT COUNT(*) FROM {view_name} v JOIN {table_name} t ON v.id = t.id WHERE {where_clause}"
         
         try:
             with connection.cursor() as cursor:
+                # 为导出设置更长的超时时间（仅影响当前会话）
+                cursor.execute(f"SET LOCAL statement_timeout = {statement_timeout_ms}")
                 cursor.execute(sql, params)
                 return cursor.fetchone()[0]
         except Exception as e:
@@ -399,41 +416,62 @@ class AssetSearchService:
         self, 
         query: str, 
         asset_type: AssetType = 'website',
-        batch_size: int = 1000
-    ):
+        batch_size: int = 1000,
+        statement_timeout_ms: int = 300000
+    ) -> Iterator[Dict[str, Any]]:
         """
-        流式搜索资产（使用服务端游标，内存友好）
+        流式搜索资产（使用分批查询，内存友好）
         
         Args:
             query: 搜索查询字符串
             asset_type: 资产类型 ('website' 或 'endpoint')
             batch_size: 每批获取的数量
+            statement_timeout_ms: SQL 语句超时时间（毫秒），默认 5 分钟
         
         Yields:
             Dict: 单条搜索结果
         """
         where_clause, params = SearchQueryParser.parse(query)
         
-        # 根据资产类型选择视图和字段
+        # 根据资产类型选择视图、原表和字段
         view_name = VIEW_MAPPING.get(asset_type, 'asset_search_view')
+        table_name = TABLE_MAPPING.get(asset_type, 'website')
         select_fields = ENDPOINT_SELECT_FIELDS if asset_type == 'endpoint' else WEBSITE_SELECT_FIELDS
         
-        sql = f"""
-            SELECT {select_fields}
-            FROM {view_name}
-            WHERE {where_clause}
-            ORDER BY created_at DESC
-        """
+        # 使用 OFFSET/LIMIT 分批查询（Django 不支持命名游标）
+        offset = 0
         
         try:
-            # 使用服务端游标，避免一次性加载所有数据到内存
-            with connection.cursor(name='export_cursor') as cursor:
-                cursor.itersize = batch_size
-                cursor.execute(sql, params)
-                columns = [col[0] for col in cursor.description]
+            while True:
+                # JOIN 原表获取数组字段
+                sql = f"""
+                    SELECT {select_fields}
+                    FROM {view_name} v
+                    JOIN {table_name} t ON v.id = t.id
+                    WHERE {where_clause}
+                    ORDER BY v.created_at DESC
+                    LIMIT {batch_size} OFFSET {offset}
+                """
                 
-                for row in cursor:
+                with connection.cursor() as cursor:
+                    # 为导出设置更长的超时时间（仅影响当前会话）
+                    cursor.execute(f"SET LOCAL statement_timeout = {statement_timeout_ms}")
+                    cursor.execute(sql, params)
+                    columns = [col[0] for col in cursor.description]
+                    rows = cursor.fetchall()
+                
+                if not rows:
+                    break
+                
+                for row in rows:
                     yield dict(zip(columns, row))
+                
+                # 如果返回的行数少于 batch_size，说明已经是最后一批
+                if len(rows) < batch_size:
+                    break
+                
+                offset += batch_size
+                
         except Exception as e:
             logger.error(f"流式搜索查询失败: {e}, SQL: {sql}, params: {params}")
             raise

backend/apps/asset/tasks/__init__.py

@@ -1,7 +0,0 @@
-"""
-Asset 应用的任务模块
-
-注意：物化视图刷新已移至 APScheduler 定时任务（apps.engine.scheduler）
-"""
-
-__all__ = []

backend/apps/asset/urls.py

@@ -12,17 +12,22 @@ from .views import (
     AssetStatisticsViewSet,
     AssetSearchView,
     AssetSearchExportView,
+    EndpointViewSet,
+    HostPortMappingViewSet,
+    ScreenshotViewSet,
 )
 
 # 创建 DRF 路由器
 router = DefaultRouter()
 
 # 注册 ViewSet
-# 注意：IPAddress 模型已被重构为 HostPortMapping，相关路由已移除
 router.register(r'subdomains', SubdomainViewSet, basename='subdomain')
 router.register(r'websites', WebSiteViewSet, basename='website')
 router.register(r'directories', DirectoryViewSet, basename='directory')
+router.register(r'endpoints', EndpointViewSet, basename='endpoint')
+router.register(r'ip-addresses', HostPortMappingViewSet, basename='ip-address')
 router.register(r'vulnerabilities', VulnerabilityViewSet, basename='vulnerability')
+router.register(r'screenshots', ScreenshotViewSet, basename='screenshot')
 router.register(r'statistics', AssetStatisticsViewSet, basename='asset-statistics')
 
 urlpatterns = [

backend/apps/asset/views/__init__.py

@@ -18,6 +18,8 @@ from .asset_views import (
     EndpointSnapshotViewSet,
     HostPortMappingSnapshotViewSet,
     VulnerabilitySnapshotViewSet,
+    ScreenshotViewSet,
+    ScreenshotSnapshotViewSet,
 )
 from .search_views import AssetSearchView, AssetSearchExportView
 
@@ -35,6 +37,8 @@ __all__ = [
     'EndpointSnapshotViewSet',
     'HostPortMappingSnapshotViewSet',
     'VulnerabilitySnapshotViewSet',
+    'ScreenshotViewSet',
+    'ScreenshotSnapshotViewSet',
     'AssetSearchView',
     'AssetSearchExportView',
 ]

backend/apps/asset/views/asset_views.py

@@ -8,7 +8,6 @@ from rest_framework.request import Request
 from rest_framework.exceptions import NotFound, ValidationError as DRFValidationError
 from django.core.exceptions import ValidationError, ObjectDoesNotExist
 from django.db import DatabaseError, IntegrityError, OperationalError
-from django.http import StreamingHttpResponse
 
 from ..serializers import (
     SubdomainListSerializer, WebSiteSerializer, DirectorySerializer, 
@@ -243,7 +242,7 @@ class SubdomainViewSet(viewsets.ModelViewSet):
         
         CSV 列：name, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         target_pk = self.kwargs.get('target_pk')
         if not target_pk:
@@ -254,12 +253,41 @@ class SubdomainViewSet(viewsets.ModelViewSet):
         headers = ['name', 'created_at']
         formatters = {'created_at': format_datetime}
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"target-{target_pk}-subdomains.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-subdomains.csv"'
-        return response
+
+    @action(detail=False, methods=['post'], url_path='bulk-delete')
+    def bulk_delete(self, request, **kwargs):
+        """批量删除子域名
+        
+        POST /api/assets/subdomains/bulk-delete/
+        
+        请求体: {"ids": [1, 2, 3]}
+        响应: {"deletedCount": 3}
+        """
+        ids = request.data.get('ids', [])
+        if not ids or not isinstance(ids, list):
+            return error_response(
+                code=ErrorCodes.VALIDATION_ERROR,
+                message='ids is required and must be a list',
+                status_code=status.HTTP_400_BAD_REQUEST
+            )
+        
+        try:
+            from ..models import Subdomain
+            deleted_count, _ = Subdomain.objects.filter(id__in=ids).delete()
+            return success_response(data={'deletedCount': deleted_count})
+        except Exception as e:
+            logger.exception("批量删除子域名失败")
+            return error_response(
+                code=ErrorCodes.SERVER_ERROR,
+                message='Failed to delete subdomains',
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
+            )
 
 
 class WebSiteViewSet(viewsets.ModelViewSet):
@@ -369,7 +397,7 @@ class WebSiteViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
+        from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
         
         target_pk = self.kwargs.get('target_pk')
         if not target_pk:
@@ -387,12 +415,41 @@ class WebSiteViewSet(viewsets.ModelViewSet):
             'tech': lambda x: format_list_field(x, separator=','),
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"target-{target_pk}-websites.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-websites.csv"'
-        return response
+
+    @action(detail=False, methods=['post'], url_path='bulk-delete')
+    def bulk_delete(self, request, **kwargs):
+        """批量删除网站
+        
+        POST /api/assets/websites/bulk-delete/
+        
+        请求体: {"ids": [1, 2, 3]}
+        响应: {"deletedCount": 3}
+        """
+        ids = request.data.get('ids', [])
+        if not ids or not isinstance(ids, list):
+            return error_response(
+                code=ErrorCodes.VALIDATION_ERROR,
+                message='ids is required and must be a list',
+                status_code=status.HTTP_400_BAD_REQUEST
+            )
+        
+        try:
+            from ..models import WebSite
+            deleted_count, _ = WebSite.objects.filter(id__in=ids).delete()
+            return success_response(data={'deletedCount': deleted_count})
+        except Exception as e:
+            logger.exception("批量删除网站失败")
+            return error_response(
+                code=ErrorCodes.SERVER_ERROR,
+                message='Failed to delete websites',
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
+            )
 
 
 class DirectoryViewSet(viewsets.ModelViewSet):
@@ -499,7 +556,7 @@ class DirectoryViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, status, content_length, words, lines, content_type, duration, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         target_pk = self.kwargs.get('target_pk')
         if not target_pk:
@@ -515,12 +572,41 @@ class DirectoryViewSet(viewsets.ModelViewSet):
             'created_at': format_datetime,
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"target-{target_pk}-directories.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-directories.csv"'
-        return response
+
+    @action(detail=False, methods=['post'], url_path='bulk-delete')
+    def bulk_delete(self, request, **kwargs):
+        """批量删除目录
+        
+        POST /api/assets/directories/bulk-delete/
+        
+        请求体: {"ids": [1, 2, 3]}
+        响应: {"deletedCount": 3}
+        """
+        ids = request.data.get('ids', [])
+        if not ids or not isinstance(ids, list):
+            return error_response(
+                code=ErrorCodes.VALIDATION_ERROR,
+                message='ids is required and must be a list',
+                status_code=status.HTTP_400_BAD_REQUEST
+            )
+        
+        try:
+            from ..models import Directory
+            deleted_count, _ = Directory.objects.filter(id__in=ids).delete()
+            return success_response(data={'deletedCount': deleted_count})
+        except Exception as e:
+            logger.exception("批量删除目录失败")
+            return error_response(
+                code=ErrorCodes.SERVER_ERROR,
+                message='Failed to delete directories',
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
+            )
 
 
 class EndpointViewSet(viewsets.ModelViewSet):
@@ -630,7 +716,7 @@ class EndpointViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, matched_gf_patterns, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
+        from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
         
         target_pk = self.kwargs.get('target_pk')
         if not target_pk:
@@ -649,12 +735,41 @@ class EndpointViewSet(viewsets.ModelViewSet):
             'matched_gf_patterns': lambda x: format_list_field(x, separator=','),
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"target-{target_pk}-endpoints.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-endpoints.csv"'
-        return response
+
+    @action(detail=False, methods=['post'], url_path='bulk-delete')
+    def bulk_delete(self, request, **kwargs):
+        """批量删除端点
+        
+        POST /api/assets/endpoints/bulk-delete/
+        
+        请求体: {"ids": [1, 2, 3]}
+        响应: {"deletedCount": 3}
+        """
+        ids = request.data.get('ids', [])
+        if not ids or not isinstance(ids, list):
+            return error_response(
+                code=ErrorCodes.VALIDATION_ERROR,
+                message='ids is required and must be a list',
+                status_code=status.HTTP_400_BAD_REQUEST
+            )
+        
+        try:
+            from ..models import Endpoint
+            deleted_count, _ = Endpoint.objects.filter(id__in=ids).delete()
+            return success_response(data={'deletedCount': deleted_count})
+        except Exception as e:
+            logger.exception("批量删除端点失败")
+            return error_response(
+                code=ErrorCodes.SERVER_ERROR,
+                message='Failed to delete endpoints',
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
+            )
 
 
 class HostPortMappingViewSet(viewsets.ModelViewSet):
@@ -707,7 +822,7 @@ class HostPortMappingViewSet(viewsets.ModelViewSet):
         
         CSV 列：ip, host, port, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         target_pk = self.kwargs.get('target_pk')
         if not target_pk:
@@ -722,14 +837,44 @@ class HostPortMappingViewSet(viewsets.ModelViewSet):
             'created_at': format_datetime
         }
         
-        # 生成流式响应
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"target-{target_pk}-ip-addresses.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="target-{target_pk}-ip-addresses.csv"'
+
+    @action(detail=False, methods=['post'], url_path='bulk-delete')
+    def bulk_delete(self, request, **kwargs):
+        """批量删除 IP 地址映射
         
-        return response
+        POST /api/assets/ip-addresses/bulk-delete/
+        
+        请求体: {"ips": ["192.168.1.1", "10.0.0.1"]}
+        响应: {"deletedCount": 3}
+        
+        注意：由于 IP 地址是聚合显示的，删除时传入 IP 列表，
+        会删除该 IP 下的所有 host:port 映射记录
+        """
+        ips = request.data.get('ips', [])
+        if not ips or not isinstance(ips, list):
+            return error_response(
+                code=ErrorCodes.VALIDATION_ERROR,
+                message='ips is required and must be a list',
+                status_code=status.HTTP_400_BAD_REQUEST
+            )
+        
+        try:
+            from ..models import HostPortMapping
+            deleted_count, _ = HostPortMapping.objects.filter(ip__in=ips).delete()
+            return success_response(data={'deletedCount': deleted_count})
+        except Exception as e:
+            logger.exception("批量删除 IP 地址映射失败")
+            return error_response(
+                code=ErrorCodes.SERVER_ERROR,
+                message='Failed to delete ip addresses',
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
+            )
 
 
 class VulnerabilityViewSet(viewsets.ModelViewSet):
@@ -801,7 +946,7 @@ class SubdomainSnapshotViewSet(viewsets.ModelViewSet):
         
         CSV 列：name, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         scan_pk = self.kwargs.get('scan_pk')
         if not scan_pk:
@@ -812,12 +957,12 @@ class SubdomainSnapshotViewSet(viewsets.ModelViewSet):
         headers = ['name', 'created_at']
         formatters = {'created_at': format_datetime}
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"scan-{scan_pk}-subdomains.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-subdomains.csv"'
-        return response
 
 
 class WebsiteSnapshotViewSet(viewsets.ModelViewSet):
@@ -855,7 +1000,7 @@ class WebsiteSnapshotViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
+        from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
         
         scan_pk = self.kwargs.get('scan_pk')
         if not scan_pk:
@@ -873,12 +1018,12 @@ class WebsiteSnapshotViewSet(viewsets.ModelViewSet):
             'tech': lambda x: format_list_field(x, separator=','),
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"scan-{scan_pk}-websites.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-websites.csv"'
-        return response
 
 
 class DirectorySnapshotViewSet(viewsets.ModelViewSet):
@@ -913,7 +1058,7 @@ class DirectorySnapshotViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, status, content_length, words, lines, content_type, duration, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         scan_pk = self.kwargs.get('scan_pk')
         if not scan_pk:
@@ -929,12 +1074,12 @@ class DirectorySnapshotViewSet(viewsets.ModelViewSet):
             'created_at': format_datetime,
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"scan-{scan_pk}-directories.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-directories.csv"'
-        return response
 
 
 class EndpointSnapshotViewSet(viewsets.ModelViewSet):
@@ -972,7 +1117,7 @@ class EndpointSnapshotViewSet(viewsets.ModelViewSet):
         
         CSV 列：url, host, location, title, status_code, content_length, content_type, webserver, tech, response_body, response_headers, vhost, matched_gf_patterns, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime, format_list_field
+        from apps.common.utils import create_csv_export_response, format_datetime, format_list_field
         
         scan_pk = self.kwargs.get('scan_pk')
         if not scan_pk:
@@ -991,12 +1136,12 @@ class EndpointSnapshotViewSet(viewsets.ModelViewSet):
             'matched_gf_patterns': lambda x: format_list_field(x, separator=','),
         }
         
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"scan-{scan_pk}-endpoints.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-endpoints.csv"'
-        return response
 
 
 class HostPortMappingSnapshotViewSet(viewsets.ModelViewSet):
@@ -1031,7 +1176,7 @@ class HostPortMappingSnapshotViewSet(viewsets.ModelViewSet):
         
         CSV 列：ip, host, port, created_at
         """
-        from apps.common.utils import generate_csv_rows, format_datetime
+        from apps.common.utils import create_csv_export_response, format_datetime
         
         scan_pk = self.kwargs.get('scan_pk')
         if not scan_pk:
@@ -1046,14 +1191,12 @@ class HostPortMappingSnapshotViewSet(viewsets.ModelViewSet):
             'created_at': format_datetime
         }
         
-        # 生成流式响应
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=f"scan-{scan_pk}-ip-addresses.csv",
+            field_formatters=formatters
         )
-        response['Content-Disposition'] = f'attachment; filename="scan-{scan_pk}-ip-addresses.csv"'
-        
-        return response
 
 
 class VulnerabilitySnapshotViewSet(viewsets.ModelViewSet):
@@ -1082,3 +1225,162 @@ class VulnerabilitySnapshotViewSet(viewsets.ModelViewSet):
         if scan_pk:
             return self.service.get_by_scan(scan_pk, filter_query=filter_query)
         return self.service.get_all(filter_query=filter_query)
+
+
+# ==================== 截图 ViewSet ====================
+
+class ScreenshotViewSet(viewsets.ModelViewSet):
+    """截图资产 ViewSet
+    
+    支持两种访问方式：
+    1. 嵌套路由：GET /api/targets/{target_pk}/screenshots/
+    2. 独立路由：GET /api/screenshots/（全局查询）
+    
+    支持智能过滤语法（filter 参数）：
+    - url="example"      URL 模糊匹配
+    """
+    
+    from ..serializers import ScreenshotListSerializer
+    
+    serializer_class = ScreenshotListSerializer
+    pagination_class = BasePagination
+    filter_backends = [filters.OrderingFilter]
+    ordering = ['-created_at']
+    
+    def get_queryset(self):
+        """根据是否有 target_pk 参数决定查询范围"""
+        from ..models import Screenshot
+        
+        target_pk = self.kwargs.get('target_pk')
+        filter_query = self.request.query_params.get('filter', None)
+        
+        queryset = Screenshot.objects.all()
+        if target_pk:
+            queryset = queryset.filter(target_id=target_pk)
+        
+        if filter_query:
+            # 简单的 URL 模糊匹配
+            queryset = queryset.filter(url__icontains=filter_query)
+        
+        return queryset.order_by('-created_at')
+    
+    @action(detail=True, methods=['get'], url_path='image')
+    def image(self, request, pk=None, **kwargs):
+        """获取截图图片
+        
+        GET /api/assets/screenshots/{id}/image/
+        
+        返回 WebP 格式的图片二进制数据
+        """
+        from django.http import HttpResponse
+        from ..models import Screenshot
+        
+        try:
+            screenshot = Screenshot.objects.get(pk=pk)
+            if not screenshot.image:
+                return error_response(
+                    code=ErrorCodes.NOT_FOUND,
+                    message='Screenshot image not found',
+                    status_code=status.HTTP_404_NOT_FOUND
+                )
+            
+            response = HttpResponse(screenshot.image, content_type='image/webp')
+            response['Content-Disposition'] = f'inline; filename="screenshot_{pk}.webp"'
+            return response
+        except Screenshot.DoesNotExist:
+            return error_response(
+                code=ErrorCodes.NOT_FOUND,
+                message='Screenshot not found',
+                status_code=status.HTTP_404_NOT_FOUND
+            )
+    
+    @action(detail=False, methods=['post'], url_path='bulk-delete')
+    def bulk_delete(self, request, **kwargs):
+        """批量删除截图
+        
+        POST /api/assets/screenshots/bulk-delete/
+        
+        请求体: {"ids": [1, 2, 3]}
+        响应: {"deletedCount": 3}
+        """
+        ids = request.data.get('ids', [])
+        if not ids or not isinstance(ids, list):
+            return error_response(
+                code=ErrorCodes.VALIDATION_ERROR,
+                message='ids is required and must be a list',
+                status_code=status.HTTP_400_BAD_REQUEST
+            )
+        
+        try:
+            from ..models import Screenshot
+            deleted_count, _ = Screenshot.objects.filter(id__in=ids).delete()
+            return success_response(data={'deletedCount': deleted_count})
+        except Exception as e:
+            logger.exception("批量删除截图失败")
+            return error_response(
+                code=ErrorCodes.SERVER_ERROR,
+                message='Failed to delete screenshots',
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR
+            )
+
+
+class ScreenshotSnapshotViewSet(viewsets.ModelViewSet):
+    """截图快照 ViewSet - 嵌套路由：GET /api/scans/{scan_pk}/screenshots/
+    
+    支持智能过滤语法（filter 参数）：
+    - url="example"      URL 模糊匹配
+    """
+    
+    from ..serializers import ScreenshotSnapshotListSerializer
+    
+    serializer_class = ScreenshotSnapshotListSerializer
+    pagination_class = BasePagination
+    filter_backends = [filters.OrderingFilter]
+    ordering = ['-created_at']
+    
+    def get_queryset(self):
+        """根据 scan_pk 参数查询"""
+        from ..models import ScreenshotSnapshot
+        
+        scan_pk = self.kwargs.get('scan_pk')
+        filter_query = self.request.query_params.get('filter', None)
+        
+        queryset = ScreenshotSnapshot.objects.all()
+        if scan_pk:
+            queryset = queryset.filter(scan_id=scan_pk)
+        
+        if filter_query:
+            # 简单的 URL 模糊匹配
+            queryset = queryset.filter(url__icontains=filter_query)
+        
+        return queryset.order_by('-created_at')
+    
+    @action(detail=True, methods=['get'], url_path='image')
+    def image(self, request, pk=None, **kwargs):
+        """获取截图快照图片
+        
+        GET /api/scans/{scan_pk}/screenshots/{id}/image/
+        
+        返回 WebP 格式的图片二进制数据
+        """
+        from django.http import HttpResponse
+        from ..models import ScreenshotSnapshot
+        
+        try:
+            screenshot = ScreenshotSnapshot.objects.get(pk=pk)
+            if not screenshot.image:
+                return error_response(
+                    code=ErrorCodes.NOT_FOUND,
+                    message='Screenshot image not found',
+                    status_code=status.HTTP_404_NOT_FOUND
+                )
+            
+            response = HttpResponse(screenshot.image, content_type='image/webp')
+            response['Content-Disposition'] = f'inline; filename="screenshot_snapshot_{pk}.webp"'
+            return response
+        except ScreenshotSnapshot.DoesNotExist:
+            return error_response(
+                code=ErrorCodes.NOT_FOUND,
+                message='Screenshot snapshot not found',
+                status_code=status.HTTP_404_NOT_FOUND
+            )

backend/apps/asset/views/search_views.py

@@ -33,7 +33,6 @@ from urllib.parse import urlparse, urlunparse
 from rest_framework import status
 from rest_framework.views import APIView
 from rest_framework.request import Request
-from django.http import StreamingHttpResponse
 from django.db import connection
 
 from apps.common.response_helpers import success_response, error_response
@@ -285,7 +284,7 @@ class AssetSearchExportView(APIView):
         asset_type: 资产类型 ('website' 或 'endpoint'，默认 'website')
     
     Response:
-        CSV 文件流（使用服务端游标，支持大数据量导出）
+        CSV 文件（带 Content-Length，支持浏览器显示下载进度）
     """
     
     def __init__(self, **kwargs):
@@ -313,8 +312,8 @@ class AssetSearchExportView(APIView):
         return headers, formatters
     
     def get(self, request: Request):
-        """导出搜索结果为 CSV（流式导出，无数量限制）"""
-        from apps.common.utils import generate_csv_rows
+        """导出搜索结果为 CSV（带 Content-Length，支持下载进度显示）"""
+        from apps.common.utils import create_csv_export_response
         
         # 获取搜索查询
         query = request.query_params.get('q', '').strip()
@@ -347,18 +346,16 @@ class AssetSearchExportView(APIView):
         # 获取表头和格式化器
         headers, formatters = self._get_headers_and_formatters(asset_type)
         
-        # 获取流式数据迭代器
-        data_iterator = self.service.search_iter(query, asset_type)
-        
         # 生成文件名
         timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
         filename = f'search_{asset_type}_{timestamp}.csv'
         
-        # 返回流式响应
-        response = StreamingHttpResponse(
-            generate_csv_rows(data_iterator, headers, formatters),
-            content_type='text/csv; charset=utf-8'
+        # 使用通用导出工具
+        data_iterator = self.service.search_iter(query, asset_type)
+        return create_csv_export_response(
+            data_iterator=data_iterator,
+            headers=headers,
+            filename=filename,
+            field_formatters=formatters,
+            show_progress=True  # 显示下载进度
         )
-        response['Content-Disposition'] = f'attachment; filename="{filename}"'
-        
-        return response

backend/apps/common/migrations/0001_initial.py

@@ -0,0 +1,34 @@
+# Generated by Django 5.2.7 on 2026-01-06 00:55
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('targets', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='BlacklistRule',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('pattern', models.CharField(help_text='规则模式，如 *.gov, 10.0.0.0/8, 192.168.1.1', max_length=255)),
+                ('rule_type', models.CharField(choices=[('domain', '域名'), ('ip', 'IP地址'), ('cidr', 'CIDR范围'), ('keyword', '关键词')], help_text='规则类型：domain, ip, cidr', max_length=20)),
+                ('scope', models.CharField(choices=[('global', '全局规则'), ('target', 'Target规则')], db_index=True, help_text='作用域：global 或 target', max_length=20)),
+                ('description', models.CharField(blank=True, default='', help_text='规则描述', max_length=500)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('target', models.ForeignKey(blank=True, help_text='关联的 Target（仅 scope=target 时有值）', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='blacklist_rules', to='targets.target')),
+            ],
+            options={
+                'db_table': 'blacklist_rule',
+                'ordering': ['-created_at'],
+                'indexes': [models.Index(fields=['scope', 'rule_type'], name='blacklist_r_scope_6ff77f_idx'), models.Index(fields=['target', 'scope'], name='blacklist_r_target__191441_idx')],
+                'constraints': [models.UniqueConstraint(fields=('pattern', 'scope', 'target'), name='unique_blacklist_rule')],
+            },
+        ),
+    ]

backend/apps/common/models/__init__.py

@@ -0,0 +1,4 @@
+"""Common models"""
+from apps.common.models.blacklist import BlacklistRule
+
+__all__ = ['BlacklistRule']

backend/apps/common/models/blacklist.py

@@ -0,0 +1,71 @@
+"""黑名单规则模型"""
+from django.db import models
+
+
+class BlacklistRule(models.Model):
+    """黑名单规则模型
+    
+    用于存储黑名单过滤规则，支持域名、IP、CIDR 三种类型。
+    支持两层作用域：全局规则和 Target 级规则。
+    """
+    
+    class RuleType(models.TextChoices):
+        DOMAIN = 'domain', '域名'
+        IP = 'ip', 'IP地址'
+        CIDR = 'cidr', 'CIDR范围'
+        KEYWORD = 'keyword', '关键词'
+    
+    class Scope(models.TextChoices):
+        GLOBAL = 'global', '全局规则'
+        TARGET = 'target', 'Target规则'
+    
+    id = models.AutoField(primary_key=True)
+    pattern = models.CharField(
+        max_length=255, 
+        help_text='规则模式，如 *.gov, 10.0.0.0/8, 192.168.1.1'
+    )
+    rule_type = models.CharField(
+        max_length=20, 
+        choices=RuleType.choices,
+        help_text='规则类型：domain, ip, cidr'
+    )
+    scope = models.CharField(
+        max_length=20, 
+        choices=Scope.choices, 
+        db_index=True,
+        help_text='作用域：global 或 target'
+    )
+    target = models.ForeignKey(
+        'targets.Target',
+        on_delete=models.CASCADE,
+        null=True, 
+        blank=True,
+        related_name='blacklist_rules',
+        help_text='关联的 Target（仅 scope=target 时有值）'
+    )
+    description = models.CharField(
+        max_length=500, 
+        blank=True, 
+        default='', 
+        help_text='规则描述'
+    )
+    created_at = models.DateTimeField(auto_now_add=True)
+    
+    class Meta:
+        db_table = 'blacklist_rule'
+        indexes = [
+            models.Index(fields=['scope', 'rule_type']),
+            models.Index(fields=['target', 'scope']),
+        ]
+        constraints = [
+            models.UniqueConstraint(
+                fields=['pattern', 'scope', 'target'],
+                name='unique_blacklist_rule'
+            ),
+        ]
+        ordering = ['-created_at']
+    
+    def __str__(self):
+        if self.scope == self.Scope.TARGET and self.target:
+            return f"[{self.scope}:{self.target_id}] {self.pattern}"
+        return f"[{self.scope}] {self.pattern}"

backend/apps/common/serializers/__init__.py

@@ -0,0 +1,12 @@
+"""Common serializers"""
+from .blacklist_serializers import (
+    BlacklistRuleSerializer,
+    GlobalBlacklistRuleSerializer,
+    TargetBlacklistRuleSerializer,
+)
+
+__all__ = [
+    'BlacklistRuleSerializer',
+    'GlobalBlacklistRuleSerializer',
+    'TargetBlacklistRuleSerializer',
+]

backend/apps/common/serializers/blacklist_serializers.py

@@ -0,0 +1,68 @@
+"""黑名单规则序列化器"""
+from rest_framework import serializers
+
+from apps.common.models import BlacklistRule
+from apps.common.utils import detect_rule_type
+
+
+class BlacklistRuleSerializer(serializers.ModelSerializer):
+    """黑名单规则序列化器"""
+    
+    class Meta:
+        model = BlacklistRule
+        fields = [
+            'id',
+            'pattern',
+            'rule_type',
+            'scope',
+            'target',
+            'description',
+            'created_at',
+        ]
+        read_only_fields = ['id', 'rule_type', 'created_at']
+    
+    def validate_pattern(self, value):
+        """验证规则模式"""
+        if not value or not value.strip():
+            raise serializers.ValidationError("规则模式不能为空")
+        return value.strip()
+    
+    def create(self, validated_data):
+        """创建规则时自动识别规则类型"""
+        pattern = validated_data.get('pattern', '')
+        validated_data['rule_type'] = detect_rule_type(pattern)
+        return super().create(validated_data)
+    
+    def update(self, instance, validated_data):
+        """更新规则时重新识别规则类型"""
+        if 'pattern' in validated_data:
+            pattern = validated_data['pattern']
+            validated_data['rule_type'] = detect_rule_type(pattern)
+        return super().update(instance, validated_data)
+
+
+class GlobalBlacklistRuleSerializer(BlacklistRuleSerializer):
+    """全局黑名单规则序列化器"""
+    
+    class Meta(BlacklistRuleSerializer.Meta):
+        fields = ['id', 'pattern', 'rule_type', 'description', 'created_at']
+        read_only_fields = ['id', 'rule_type', 'created_at']
+    
+    def create(self, validated_data):
+        """创建全局规则"""
+        validated_data['scope'] = BlacklistRule.Scope.GLOBAL
+        validated_data['target'] = None
+        return super().create(validated_data)
+
+
+class TargetBlacklistRuleSerializer(BlacklistRuleSerializer):
+    """Target 黑名单规则序列化器"""
+    
+    class Meta(BlacklistRuleSerializer.Meta):
+        fields = ['id', 'pattern', 'rule_type', 'description', 'created_at']
+        read_only_fields = ['id', 'rule_type', 'created_at']
+    
+    def create(self, validated_data):
+        """创建 Target 规则（target_id 由 view 设置）"""
+        validated_data['scope'] = BlacklistRule.Scope.TARGET
+        return super().create(validated_data)

backend/apps/common/services/__init__.py

@@ -3,13 +3,16 @@
 
 提供系统级别的公共服务，包括：
 - SystemLogService: 系统日志读取服务
+- BlacklistService: 黑名单过滤服务
 
 注意：FilterService 已移至 apps.common.utils.filter_utils
 推荐使用: from apps.common.utils.filter_utils import apply_filters
 """
 
 from .system_log_service import SystemLogService
+from .blacklist_service import BlacklistService
 
 __all__ = [
     'SystemLogService',
+    'BlacklistService',
 ]

backend/apps/common/services/blacklist_service.py

@@ -0,0 +1,176 @@
+"""
+黑名单规则管理服务
+
+负责黑名单规则的 CRUD 操作（数据库层面）。
+过滤逻辑请使用 apps.common.utils.BlacklistFilter。
+
+架构说明：
+- Model: BlacklistRule (apps.common.models.blacklist)
+- Service: BlacklistService (本文件) - 规则 CRUD
+- Utils: BlacklistFilter (apps.common.utils.blacklist_filter) - 过滤逻辑
+- View: GlobalBlacklistView, TargetViewSet.blacklist
+"""
+
+import logging
+from typing import List, Dict, Any, Optional
+
+from django.db.models import QuerySet
+
+from apps.common.utils import detect_rule_type
+
+logger = logging.getLogger(__name__)
+
+
+def _normalize_patterns(patterns: List[str]) -> List[str]:
+    """
+    规范化规则列表：去重 + 过滤空行
+    
+    Args:
+        patterns: 原始规则列表
+        
+    Returns:
+        List[str]: 去重后的规则列表（保持顺序）
+    """
+    return list(dict.fromkeys(filter(None, (p.strip() for p in patterns))))
+
+
+class BlacklistService:
+    """
+    黑名单规则管理服务
+    
+    只负责规则的 CRUD 操作，不包含过滤逻辑。
+    过滤逻辑请使用 BlacklistFilter 工具类。
+    """
+    
+    def get_global_rules(self) -> QuerySet:
+        """
+        获取全局黑名单规则列表
+        
+        Returns:
+            QuerySet: 全局规则查询集
+        """
+        from apps.common.models import BlacklistRule
+        return BlacklistRule.objects.filter(scope=BlacklistRule.Scope.GLOBAL)
+    
+    def get_target_rules(self, target_id: int) -> QuerySet:
+        """
+        获取 Target 级黑名单规则列表
+        
+        Args:
+            target_id: Target ID
+            
+        Returns:
+            QuerySet: Target 级规则查询集
+        """
+        from apps.common.models import BlacklistRule
+        return BlacklistRule.objects.filter(
+            scope=BlacklistRule.Scope.TARGET,
+            target_id=target_id
+        )
+    
+    def get_rules(self, target_id: Optional[int] = None) -> List:
+        """
+        获取黑名单规则（全局 + Target 级）
+        
+        Args:
+            target_id: Target ID，用于加载 Target 级规则
+            
+        Returns:
+            List[BlacklistRule]: 规则列表
+        """
+        from apps.common.models import BlacklistRule
+        
+        # 加载全局规则
+        rules = list(BlacklistRule.objects.filter(scope=BlacklistRule.Scope.GLOBAL))
+        
+        # 加载 Target 级规则
+        if target_id:
+            target_rules = BlacklistRule.objects.filter(
+                scope=BlacklistRule.Scope.TARGET,
+                target_id=target_id
+            )
+            rules.extend(target_rules)
+        
+        return rules
+    
+    def replace_global_rules(self, patterns: List[str]) -> Dict[str, Any]:
+        """
+        全量替换全局黑名单规则（PUT 语义）
+        
+        Args:
+            patterns: 新的规则模式列表
+            
+        Returns:
+            Dict: {'count': int} 最终规则数量
+        """
+        from apps.common.models import BlacklistRule
+        
+        count = self._replace_rules(
+            patterns=patterns,
+            scope=BlacklistRule.Scope.GLOBAL,
+            target=None
+        )
+        
+        logger.info("全量替换全局黑名单规则: %d 条", count)
+        return {'count': count}
+    
+    def replace_target_rules(self, target, patterns: List[str]) -> Dict[str, Any]:
+        """
+        全量替换 Target 级黑名单规则（PUT 语义）
+        
+        Args:
+            target: Target 对象
+            patterns: 新的规则模式列表
+            
+        Returns:
+            Dict: {'count': int} 最终规则数量
+        """
+        from apps.common.models import BlacklistRule
+        
+        count = self._replace_rules(
+            patterns=patterns,
+            scope=BlacklistRule.Scope.TARGET,
+            target=target
+        )
+        
+        logger.info("全量替换 Target 黑名单规则: %d 条 (Target: %s)", count, target.name)
+        return {'count': count}
+    
+    def _replace_rules(self, patterns: List[str], scope: str, target=None) -> int:
+        """
+        内部方法：全量替换规则
+        
+        Args:
+            patterns: 规则模式列表
+            scope: 规则作用域 (GLOBAL/TARGET)
+            target: Target 对象（仅 TARGET 作用域需要）
+            
+        Returns:
+            int: 最终规则数量
+        """
+        from apps.common.models import BlacklistRule
+        from django.db import transaction
+        
+        patterns = _normalize_patterns(patterns)
+        
+        with transaction.atomic():
+            # 1. 删除旧规则
+            delete_filter = {'scope': scope}
+            if target:
+                delete_filter['target'] = target
+            BlacklistRule.objects.filter(**delete_filter).delete()
+            
+            # 2. 创建新规则
+            if patterns:
+                rules = [
+                    BlacklistRule(
+                        pattern=pattern,
+                        rule_type=detect_rule_type(pattern),
+                        scope=scope,
+                        target=target
+                    )
+                    for pattern in patterns
+                ]
+                BlacklistRule.objects.bulk_create(rules)
+        
+        return len(patterns)

backend/apps/common/urls.py

@@ -2,13 +2,20 @@
 通用模块 URL 配置
 
 路由说明：
-- /api/health/    健康检查接口（无需认证）
-- /api/auth/*     认证相关接口（登录、登出、用户信息）
-- /api/system/*   系统管理接口（日志查看等）
+- /api/health/       健康检查接口（无需认证）
+- /api/auth/*        认证相关接口（登录、登出、用户信息）
+- /api/system/*      系统管理接口（日志查看等）
+- /api/blacklist/*   黑名单管理接口
 """
 
 from django.urls import path
-from .views import LoginView, LogoutView, MeView, ChangePasswordView, SystemLogsView, SystemLogFilesView, HealthCheckView
+
+from .views import (
+    LoginView, LogoutView, MeView, ChangePasswordView,
+    SystemLogsView, SystemLogFilesView, HealthCheckView,
+    GlobalBlacklistView,
+    VersionView, CheckUpdateView,
+)
 
 urlpatterns = [
     # 健康检查（无需认证）
@@ -23,4 +30,9 @@ urlpatterns = [
     # 系统管理
     path('system/logs/', SystemLogsView.as_view(), name='system-logs'),
     path('system/logs/files/', SystemLogFilesView.as_view(), name='system-log-files'),
+    path('system/version/', VersionView.as_view(), name='system-version'),
+    path('system/check-update/', CheckUpdateView.as_view(), name='system-check-update'),
+    
+    # 黑名单管理（PUT 全量替换模式）
+    path('blacklist/rules/', GlobalBlacklistView.as_view(), name='blacklist-rules'),
 ]

backend/apps/common/utils/__init__.py

@@ -11,8 +11,14 @@ from .csv_utils import (
     generate_csv_rows,
     format_list_field,
     format_datetime,
+    create_csv_export_response,
     UTF8_BOM,
 )
+from .blacklist_filter import (
+    BlacklistFilter,
+    detect_rule_type,
+    extract_host,
+)
 
 __all__ = [
     'deduplicate_for_bulk',
@@ -24,5 +30,9 @@ __all__ = [
     'generate_csv_rows',
     'format_list_field',
     'format_datetime',
+    'create_csv_export_response',
     'UTF8_BOM',
+    'BlacklistFilter',
+    'detect_rule_type',
+    'extract_host',
 ]

backend/apps/common/utils/blacklist_filter.py

@@ -0,0 +1,246 @@
+"""
+黑名单过滤工具
+
+提供域名、IP、CIDR、关键词的黑名单匹配功能。
+纯工具类，不涉及数据库操作。
+
+支持的规则类型：
+    1. 域名精确匹配: example.com
+       - 规则: example.com
+       - 匹配: example.com
+       - 不匹配: sub.example.com, other.com
+    
+    2. 域名后缀匹配: *.example.com
+       - 规则: *.example.com
+       - 匹配: sub.example.com, a.b.example.com, example.com
+       - 不匹配: other.com, example.com.cn
+    
+    3. 关键词匹配: *cdn*
+       - 规则: *cdn*
+       - 匹配: cdn.example.com, a.cdn.b.com, mycdn123.com
+       - 不匹配: example.com (不包含 cdn)
+    
+    4. IP 精确匹配: 192.168.1.1
+       - 规则: 192.168.1.1
+       - 匹配: 192.168.1.1
+       - 不匹配: 192.168.1.2
+    
+    5. CIDR 范围匹配: 192.168.0.0/24
+       - 规则: 192.168.0.0/24
+       - 匹配: 192.168.0.1, 192.168.0.255
+       - 不匹配: 192.168.1.1
+
+使用方式：
+    from apps.common.utils import BlacklistFilter
+    
+    # 创建过滤器（传入规则列表）
+    rules = BlacklistRule.objects.filter(...)
+    filter = BlacklistFilter(rules)
+    
+    # 检查单个目标
+    if filter.is_allowed('http://example.com'):
+        process(url)
+    
+    # 流式处理
+    for url in urls:
+        if filter.is_allowed(url):
+            process(url)
+"""
+
+import ipaddress
+import logging
+from typing import List, Optional
+from urllib.parse import urlparse
+
+from apps.common.validators import is_valid_ip, validate_cidr
+
+logger = logging.getLogger(__name__)
+
+
+def detect_rule_type(pattern: str) -> str:
+    """
+    自动识别规则类型
+    
+    支持的模式：
+    - 域名精确匹配: example.com
+    - 域名后缀匹配: *.example.com
+    - 关键词匹配: *cdn* (匹配包含 cdn 的域名)
+    - IP 精确匹配: 192.168.1.1
+    - CIDR 范围: 192.168.0.0/24
+    
+    Args:
+        pattern: 规则模式字符串
+        
+    Returns:
+        str: 规则类型 ('domain', 'ip', 'cidr', 'keyword')
+    """
+    if not pattern:
+        return 'domain'
+    
+    pattern = pattern.strip()
+    
+    # 检查关键词模式: *keyword* (前后都有星号，中间无点)
+    if pattern.startswith('*') and pattern.endswith('*') and len(pattern) > 2:
+        keyword = pattern[1:-1]
+        # 关键词中不能有点（否则可能是域名模式）
+        if '.' not in keyword:
+            return 'keyword'
+    
+    # 检查 CIDR（包含 /）
+    if '/' in pattern:
+        try:
+            validate_cidr(pattern)
+            return 'cidr'
+        except ValueError:
+            pass
+    
+    # 检查 IP（去掉通配符前缀后验证）
+    clean_pattern = pattern.lstrip('*').lstrip('.')
+    if is_valid_ip(clean_pattern):
+        return 'ip'
+    
+    # 默认为域名
+    return 'domain'
+
+
+def extract_host(target: str) -> str:
+    """
+    从目标字符串中提取主机名
+    
+    支持：
+    - 纯域名：example.com
+    - 纯 IP：192.168.1.1
+    - URL：http://example.com/path
+    
+    Args:
+        target: 目标字符串
+        
+    Returns:
+        str: 提取的主机名
+    """
+    if not target:
+        return ''
+    
+    target = target.strip()
+    
+    # 如果是 URL，提取 hostname
+    if '://' in target:
+        try:
+            parsed = urlparse(target)
+            return parsed.hostname or target
+        except Exception:
+            return target
+    
+    return target
+
+
+class BlacklistFilter:
+    """
+    黑名单过滤器
+    
+    预编译规则，提供高效的匹配功能。
+    """
+    
+    def __init__(self, rules: List):
+        """
+        初始化过滤器
+        
+        Args:
+            rules: BlacklistRule 对象列表
+        """
+        from apps.common.models import BlacklistRule
+        
+        # 预解析：按类型分类 + CIDR 预编译
+        self._domain_rules = []  # (pattern, is_wildcard, suffix)
+        self._ip_rules = set()   # 精确 IP 用 set，O(1) 查找
+        self._cidr_rules = []    # (pattern, network_obj)
+        self._keyword_rules = [] # 关键词列表（小写）
+        
+        # 去重：跨 scope 可能有重复规则
+        seen_patterns = set()
+        
+        for rule in rules:
+            if rule.pattern in seen_patterns:
+                continue
+            seen_patterns.add(rule.pattern)
+            if rule.rule_type == BlacklistRule.RuleType.DOMAIN:
+                pattern = rule.pattern.lower()
+                if pattern.startswith('*.'):
+                    self._domain_rules.append((pattern, True, pattern[1:]))
+                else:
+                    self._domain_rules.append((pattern, False, None))
+            elif rule.rule_type == BlacklistRule.RuleType.IP:
+                self._ip_rules.add(rule.pattern)
+            elif rule.rule_type == BlacklistRule.RuleType.CIDR:
+                try:
+                    network = ipaddress.ip_network(rule.pattern, strict=False)
+                    self._cidr_rules.append((rule.pattern, network))
+                except ValueError:
+                    pass
+            elif rule.rule_type == BlacklistRule.RuleType.KEYWORD:
+                # *cdn* -> cdn
+                keyword = rule.pattern[1:-1].lower()
+                self._keyword_rules.append(keyword)
+    
+    def is_allowed(self, target: str) -> bool:
+        """
+        检查目标是否通过过滤
+        
+        Args:
+            target: 要检查的目标（域名/IP/URL）
+            
+        Returns:
+            bool: True 表示通过（不在黑名单），False 表示被过滤
+        """
+        if not target:
+            return True
+        
+        host = extract_host(target)
+        if not host:
+            return True
+        
+        # 先判断输入类型，再走对应分支
+        if is_valid_ip(host):
+            return self._check_ip_rules(host)
+        else:
+            return self._check_domain_rules(host)
+    
+    def _check_domain_rules(self, host: str) -> bool:
+        """检查域名规则（精确匹配 + 后缀匹配 + 关键词匹配）"""
+        host_lower = host.lower()
+        
+        # 1. 域名规则（精确 + 后缀）
+        for pattern, is_wildcard, suffix in self._domain_rules:
+            if is_wildcard:
+                if host_lower.endswith(suffix) or host_lower == pattern[2:]:
+                    return False
+            else:
+                if host_lower == pattern:
+                    return False
+        
+        # 2. 关键词匹配（字符串 in 操作，O(n*m)）
+        for keyword in self._keyword_rules:
+            if keyword in host_lower:
+                return False
+        
+        return True
+    
+    def _check_ip_rules(self, host: str) -> bool:
+        """检查 IP 规则（精确匹配 + CIDR）"""
+        # 1. IP 精确匹配（O(1)）
+        if host in self._ip_rules:
+            return False
+        
+        # 2. CIDR 匹配
+        if self._cidr_rules:
+            try:
+                ip_obj = ipaddress.ip_address(host)
+                for _, network in self._cidr_rules:
+                    if ip_obj in network:
+                        return False
+            except ValueError:
+                pass
+        
+        return True
+    
+

backend/apps/common/utils/csv_utils.py

@@ -4,13 +4,21 @@
 - UTF-8 BOM（Excel 兼容）
 - RFC 4180 规范转义
 - 流式生成（内存友好）
+- 带 Content-Length 的文件响应（支持浏览器下载进度显示）
 """
 
 import csv
 import io
+import os
+import tempfile
+import logging
 from datetime import datetime
 from typing import Iterator, Dict, Any, List, Callable, Optional
 
+from django.http import FileResponse, StreamingHttpResponse
+
+logger = logging.getLogger(__name__)
+
 # UTF-8 BOM，确保 Excel 正确识别编码
 UTF8_BOM = '\ufeff'
 
@@ -114,3 +122,123 @@ def format_datetime(dt: Optional[datetime]) -> str:
         dt = timezone.localtime(dt)
     
     return dt.strftime('%Y-%m-%d %H:%M:%S')
+
+
+def create_csv_export_response(
+    data_iterator: Iterator[Dict[str, Any]],
+    headers: List[str],
+    filename: str,
+    field_formatters: Optional[Dict[str, Callable]] = None,
+    show_progress: bool = True
+) -> FileResponse | StreamingHttpResponse:
+    """
+    创建 CSV 导出响应
+    
+    根据 show_progress 参数选择响应类型：
+    - True: 使用临时文件 + FileResponse，带 Content-Length（浏览器显示下载进度）
+    - False: 使用 StreamingHttpResponse（内存更友好，但无下载进度）
+    
+    Args:
+        data_iterator: 数据迭代器，每个元素是一个字典
+        headers: CSV 表头列表
+        filename: 下载文件名（如 "export_2024.csv"）
+        field_formatters: 字段格式化函数字典
+        show_progress: 是否显示下载进度（默认 True）
+    
+    Returns:
+        FileResponse 或 StreamingHttpResponse
+    
+    Example:
+        >>> data_iter = service.iter_data()
+        >>> headers = ['url', 'host', 'created_at']
+        >>> formatters = {'created_at': format_datetime}
+        >>> response = create_csv_export_response(
+        ...     data_iter, headers, 'websites.csv', formatters
+        ... )
+        >>> return response
+    """
+    if show_progress:
+        return _create_file_response(data_iterator, headers, filename, field_formatters)
+    else:
+        return _create_streaming_response(data_iterator, headers, filename, field_formatters)
+
+
+def _create_file_response(
+    data_iterator: Iterator[Dict[str, Any]],
+    headers: List[str],
+    filename: str,
+    field_formatters: Optional[Dict[str, Callable]] = None
+) -> FileResponse:
+    """
+    创建带 Content-Length 的文件响应（支持浏览器下载进度）
+    
+    实现方式：先写入临时文件，再返回 FileResponse
+    """
+    # 创建临时文件
+    temp_file = tempfile.NamedTemporaryFile(
+        mode='w', 
+        suffix='.csv', 
+        delete=False, 
+        encoding='utf-8'
+    )
+    temp_path = temp_file.name
+    
+    try:
+        # 流式写入 CSV 数据到临时文件
+        for row in generate_csv_rows(data_iterator, headers, field_formatters):
+            temp_file.write(row)
+        temp_file.close()
+        
+        # 获取文件大小
+        file_size = os.path.getsize(temp_path)
+        
+        # 创建文件响应
+        response = FileResponse(
+            open(temp_path, 'rb'),
+            content_type='text/csv; charset=utf-8',
+            as_attachment=True,
+            filename=filename
+        )
+        response['Content-Length'] = file_size
+        
+        # 设置清理回调：响应完成后删除临时文件
+        original_close = response.file_to_stream.close
+        def close_and_cleanup():
+            original_close()
+            try:
+                os.unlink(temp_path)
+            except OSError:
+                pass
+        response.file_to_stream.close = close_and_cleanup
+        
+        return response
+        
+    except Exception as e:
+        # 清理临时文件
+        try:
+            temp_file.close()
+        except:
+            pass
+        try:
+            os.unlink(temp_path)
+        except OSError:
+            pass
+        logger.error(f"创建 CSV 导出响应失败: {e}")
+        raise
+
+
+def _create_streaming_response(
+    data_iterator: Iterator[Dict[str, Any]],
+    headers: List[str],
+    filename: str,
+    field_formatters: Optional[Dict[str, Callable]] = None
+) -> StreamingHttpResponse:
+    """
+    创建流式响应（无 Content-Length，内存更友好）
+    """
+    response = StreamingHttpResponse(
+        generate_csv_rows(data_iterator, headers, field_formatters),
+        content_type='text/csv; charset=utf-8'
+    )
+    response['Content-Disposition'] = f'attachment; filename="{filename}"'
+    return response

backend/apps/common/views/__init__.py

@@ -5,14 +5,20 @@
 - 健康检查视图：Docker 健康检查
 - 认证相关视图：登录、登出、用户信息、修改密码
 - 系统日志视图：实时日志查看
+- 黑名单视图：全局黑名单规则管理
+- 版本视图：系统版本和更新检查
 """
 
 from .health_views import HealthCheckView
 from .auth_views import LoginView, LogoutView, MeView, ChangePasswordView
 from .system_log_views import SystemLogsView, SystemLogFilesView
+from .blacklist_views import GlobalBlacklistView
+from .version_views import VersionView, CheckUpdateView
 
 __all__ = [
     'HealthCheckView',
     'LoginView', 'LogoutView', 'MeView', 'ChangePasswordView',
     'SystemLogsView', 'SystemLogFilesView',
+    'GlobalBlacklistView',
+    'VersionView', 'CheckUpdateView',
 ]

backend/apps/common/views/blacklist_views.py

@@ -0,0 +1,80 @@
+"""全局黑名单 API 视图"""
+import logging
+
+from rest_framework import status
+from rest_framework.views import APIView
+from rest_framework.permissions import IsAuthenticated
+
+from apps.common.response_helpers import success_response, error_response
+from apps.common.services import BlacklistService
+
+logger = logging.getLogger(__name__)
+
+
+class GlobalBlacklistView(APIView):
+    """
+    全局黑名单规则 API
+    
+    Endpoints:
+    - GET /api/blacklist/rules/ - 获取全局黑名单列表
+    - PUT /api/blacklist/rules/ - 全量替换规则（文本框保存场景）
+    
+    设计说明：
+    - 使用 PUT 全量替换模式，适合"文本框每行一个规则"的前端场景
+    - 用户编辑文本框 -> 点击保存 -> 后端全量替换
+    
+    架构：MVS 模式
+    - View: 参数验证、响应格式化
+    - Service: 业务逻辑（BlacklistService）
+    - Model: 数据持久化（BlacklistRule）
+    """
+    
+    permission_classes = [IsAuthenticated]
+    
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.blacklist_service = BlacklistService()
+    
+    def get(self, request):
+        """
+        获取全局黑名单规则列表
+        
+        返回格式：
+        {
+            "patterns": ["*.gov", "*.edu", "10.0.0.0/8"]
+        }
+        """
+        rules = self.blacklist_service.get_global_rules()
+        patterns = list(rules.values_list('pattern', flat=True))
+        return success_response(data={'patterns': patterns})
+    
+    def put(self, request):
+        """
+        全量替换全局黑名单规则
+        
+        请求格式：
+        {
+            "patterns": ["*.gov", "*.edu", "10.0.0.0/8"]
+        }
+        
+        或者空数组清空所有规则：
+        {
+            "patterns": []
+        }
+        """
+        patterns = request.data.get('patterns', [])
+        
+        # 兼容字符串输入（换行分隔）
+        if isinstance(patterns, str):
+            patterns = [p for p in patterns.split('\n') if p.strip()]
+        
+        if not isinstance(patterns, list):
+            return error_response(
+                code='VALIDATION_ERROR',
+                message='patterns 必须是数组'
+            )
+        
+        # 调用 Service 层全量替换
+        result = self.blacklist_service.replace_global_rules(patterns)
+        
+        return success_response(data=result)

backend/apps/common/views/version_views.py

@@ -0,0 +1,121 @@
+"""
+系统版本相关视图
+"""
+
+import logging
+from pathlib import Path
+
+import requests
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from apps.common.error_codes import ErrorCodes
+from apps.common.response_helpers import error_response, success_response
+
+logger = logging.getLogger(__name__)
+
+# GitHub 仓库信息
+GITHUB_REPO = "yyhuni/xingrin"
+GITHUB_API_URL = f"https://api.github.com/repos/{GITHUB_REPO}/releases/latest"
+GITHUB_RELEASES_URL = f"https://github.com/{GITHUB_REPO}/releases"
+
+
+def get_current_version() -> str:
+    """读取当前版本号"""
+    version_file = Path(__file__).parent.parent.parent.parent.parent / 'VERSION'
+    try:
+        return version_file.read_text(encoding='utf-8').strip()
+    except FileNotFoundError:
+        return "unknown"
+
+
+def compare_versions(current: str, latest: str) -> bool:
+    """
+    比较版本号，判断是否有更新
+
+    Returns:
+        True 表示有更新可用
+    """
+    def parse_version(v: str) -> tuple:
+        v = v.lstrip('v')
+        parts = v.split('.')
+        result = []
+        for part in parts:
+            if '-' in part:
+                num, _ = part.split('-', 1)
+                result.append(int(num))
+            else:
+                result.append(int(part))
+        return tuple(result)
+
+    try:
+        return parse_version(latest) > parse_version(current)
+    except (ValueError, AttributeError):
+        return False
+
+
+class VersionView(APIView):
+    """获取当前系统版本"""
+
+    def get(self, _request: Request) -> Response:
+        """获取当前版本信息"""
+        return success_response(data={
+            'version': get_current_version(),
+            'github_repo': GITHUB_REPO,
+        })
+
+
+class CheckUpdateView(APIView):
+    """检查系统更新"""
+
+    def get(self, _request: Request) -> Response:
+        """
+        检查是否有新版本
+
+        Returns:
+            - current_version: 当前版本
+            - latest_version: 最新版本
+            - has_update: 是否有更新
+            - release_url: 发布页面 URL
+            - release_notes: 更新说明（如果有）
+        """
+        current_version = get_current_version()
+
+        try:
+            response = requests.get(
+                GITHUB_API_URL,
+                headers={'Accept': 'application/vnd.github.v3+json'},
+                timeout=10
+            )
+
+            if response.status_code == 404:
+                return success_response(data={
+                    'current_version': current_version,
+                    'latest_version': current_version,
+                    'has_update': False,
+                    'release_url': GITHUB_RELEASES_URL,
+                    'release_notes': None,
+                })
+
+            response.raise_for_status()
+            release_data = response.json()
+
+            latest_version = release_data.get('tag_name', current_version)
+            has_update = compare_versions(current_version, latest_version)
+
+            return success_response(data={
+                'current_version': current_version,
+                'latest_version': latest_version,
+                'has_update': has_update,
+                'release_url': release_data.get('html_url', GITHUB_RELEASES_URL),
+                'release_notes': release_data.get('body'),
+                'published_at': release_data.get('published_at'),
+            })
+
+        except requests.RequestException as e:
+            logger.warning("检查更新失败: %s", e)
+            return error_response(
+                code=ErrorCodes.SERVER_ERROR,
+                message="无法连接到 GitHub，请稍后重试",
+            )

backend/apps/engine/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.2.7 on 2026-01-02 04:45
+# Generated by Django 5.2.7 on 2026-01-06 00:55
 
 from django.db import migrations, models
 

backend/apps/engine/services/fingerprints/goby.py

@@ -16,10 +16,9 @@ class GobyFingerprintService(BaseFingerprintService):
         """
         校验单条 Goby 指纹
         
-        校验规则：
-        - name 字段必须存在且非空
-        - logic 字段必须存在
-        - rule 字段必须是数组
+        支持两种格式：
+        1. 标准格式: {"name": "...", "logic": "...", "rule": [...]}
+        2. JSONL 格式: {"product": "...", "rule": "..."}
         
         Args:
             item: 单条指纹数据
@@ -27,25 +26,43 @@ class GobyFingerprintService(BaseFingerprintService):
         Returns:
             bool: 是否有效
         """
+        # 标准格式：name + logic + rule(数组)
         name = item.get('name', '')
-        logic = item.get('logic', '')
-        rule = item.get('rule')
-        return bool(name and str(name).strip()) and bool(logic) and isinstance(rule, list)
+        if name and item.get('logic') is not None and isinstance(item.get('rule'), list):
+            return bool(str(name).strip())
+        
+        # JSONL 格式：product + rule(字符串)
+        product = item.get('product', '')
+        rule = item.get('rule', '')
+        return bool(product and str(product).strip() and rule and str(rule).strip())
     
     def to_model_data(self, item: dict) -> dict:
         """
         转换 Goby JSON 格式为 Model 字段
         
+        支持两种输入格式：
+        1. 标准格式: {"name": "...", "logic": "...", "rule": [...]}
+        2. JSONL 格式: {"product": "...", "rule": "..."}
+        
         Args:
             item: 原始 Goby JSON 数据
             
         Returns:
             dict: Model 字段数据
         """
+        # 标准格式
+        if 'name' in item and isinstance(item.get('rule'), list):
+            return {
+                'name': str(item.get('name', '')).strip(),
+                'logic': item.get('logic', ''),
+                'rule': item.get('rule', []),
+            }
+        
+        # JSONL 格式：将 rule 字符串转为单元素数组
         return {
-            'name': str(item.get('name', '')).strip(),
-            'logic': item.get('logic', ''),
-            'rule': item.get('rule', []),
+            'name': str(item.get('product', '')).strip(),
+            'logic': 'or',  # JSONL 格式默认 or 逻辑
+            'rule': [item.get('rule', '')] if item.get('rule') else [],
         }
     
     def get_export_data(self) -> list:

backend/apps/engine/services/task_distributor.py

@@ -312,7 +312,11 @@ class TaskDistributor:
         # - 本地 Worker：install.sh 已预拉取镜像，直接使用本地版本
         # - 远程 Worker：deploy 时已预拉取镜像，直接使用本地版本
         # - 避免每次任务都检查 Docker Hub，提升性能和稳定性
+        # OOM 优先级：--oom-score-adj=1000 让 Worker 在内存不足时优先被杀
+        # - 范围 -1000 到 1000，值越大越容易被 OOM Killer 选中
+        # - 保护 server/nginx/frontend 等核心服务，确保 Web 界面可用
         cmd = f'''docker run --rm -d --pull=missing {network_arg} \\
+            --oom-score-adj=1000 \\
             {' '.join(env_vars)} \\
             {' '.join(volumes)} \\
             {self.docker_image} \\

backend/apps/engine/views/fingerprints/base.py

@@ -139,7 +139,7 @@ class BaseFingerprintViewSet(viewsets.ModelViewSet):
         POST /api/engine/fingerprints/{type}/import_file/
         
         请求格式：multipart/form-data
-        - file: JSON 文件
+        - file: JSON 文件（支持标准 JSON 和 JSONL 格式）
         
         返回：同 batch_create
         """
@@ -148,9 +148,12 @@ class BaseFingerprintViewSet(viewsets.ModelViewSet):
             raise ValidationError('缺少文件')
         
         try:
-            json_data = json.load(file)
+            content = file.read().decode('utf-8')
+            json_data = self._parse_json_content(content)
         except json.JSONDecodeError as e:
             raise ValidationError(f'无效的 JSON 格式: {e}')
+        except UnicodeDecodeError as e:
+            raise ValidationError(f'文件编码错误: {e}')
         
         fingerprints = self.parse_import_data(json_data)
         if not fingerprints:
@@ -159,6 +162,41 @@ class BaseFingerprintViewSet(viewsets.ModelViewSet):
         result = self.get_service().batch_create_fingerprints(fingerprints)
         return success_response(data=result, status_code=status.HTTP_201_CREATED)
     
+    def _parse_json_content(self, content: str):
+        """
+        解析 JSON 内容，支持标准 JSON 和 JSONL 格式
+        
+        Args:
+            content: 文件内容字符串
+            
+        Returns:
+            解析后的数据（list 或 dict）
+        """
+        content = content.strip()
+        
+        # 尝试标准 JSON 解析
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError:
+            pass
+        
+        # 尝试 JSONL 格式（每行一个 JSON 对象）
+        lines = content.split('\n')
+        result = []
+        for i, line in enumerate(lines):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                result.append(json.loads(line))
+            except json.JSONDecodeError as e:
+                raise json.JSONDecodeError(f'第 {i + 1} 行解析失败: {e.msg}', e.doc, e.pos)
+        
+        if not result:
+            raise json.JSONDecodeError('文件为空或格式无效', content, 0)
+        
+        return result
+    
     @action(detail=False, methods=['post'], url_path='bulk-delete')
     def bulk_delete(self, request):
         """

backend/apps/scan/configs/command_templates.py

@@ -13,27 +13,17 @@ SCAN_TOOLS_BASE_PATH = getattr(settings, 'SCAN_TOOLS_BASE_PATH', '/usr/local/bin
 
 SUBDOMAIN_DISCOVERY_COMMANDS = {
     'subfinder': {
-        # 默认使用所有数据源（更全面，略慢），并始终开启递归
-        # -all       使用所有数据源
-        # -recursive 对支持递归的源启用递归枚举（默认开启）
-        'base': "subfinder -d {domain} -all -recursive -o '{output_file}' -silent",
+        # 使用所有数据源（包括付费源，只要配置了 API key）
+        # -all       使用所有数据源（slow 但全面）
+        # -v         显示详细输出，包括使用的数据源（调试用）
+        # 注意：不要加 -recursive，它会排除不支持递归的源（如 fofa）
+        'base': "subfinder -d {domain} -all -o '{output_file}' -v",
         'optional': {
             'threads': '-t {threads}',              # 控制并发 goroutine 数
+            'provider_config': "-pc '{provider_config}'",  # Provider 配置文件路径
         }
     },
     
-    'amass_passive': {
-        # 先执行被动枚举，将结果写入 amass 内部数据库，然后从数据库中导出纯域名（names）到 output_file
-        # -silent 禁用进度条和其他输出
-        'base': "amass enum -passive -silent -d {domain} && amass subs -names -d {domain} > '{output_file}'"
-    },
-    
-    'amass_active': {
-        # 先执行主动枚举 + 爆破，将结果写入 amass 内部数据库，然后从数据库中导出纯域名（names）到 output_file
-        # -silent 禁用进度条和其他输出
-        'base': "amass enum -active -silent -d {domain} -brute && amass subs -names -d {domain} > '{output_file}'"
-    },
-    
     'sublist3r': {
         'base': "python3 '/usr/local/share/Sublist3r/sublist3r.py' -d {domain} -o '{output_file}'",
         'optional': {
@@ -261,11 +251,16 @@ COMMAND_TEMPLATES = {
     'directory_scan': DIRECTORY_SCAN_COMMANDS,
     'url_fetch': URL_FETCH_COMMANDS,
     'vuln_scan': VULN_SCAN_COMMANDS,
+    'screenshot': {},  # 使用 Python 原生库（Playwright），无命令模板
 }
 
 # ==================== 扫描类型配置 ====================
 
 # 执行阶段定义（按顺序执行）
+# Stage 1: 资产发现 - 子域名 → 端口 → 站点探测 → 指纹识别
+# Stage 2: URL 收集 - URL 获取 + 目录扫描（并行）
+# Stage 3: 截图 - 在 URL 收集完成后执行，捕获更多发现的页面
+# Stage 4: 漏洞扫描 - 最后执行
 EXECUTION_STAGES = [
     {
         'mode': 'sequential',
@@ -275,6 +270,10 @@ EXECUTION_STAGES = [
         'mode': 'parallel',
         'flows': ['url_fetch', 'directory_scan']
     },
+    {
+        'mode': 'sequential',
+        'flows': ['screenshot']
+    },
     {
         'mode': 'sequential',
         'flows': ['vuln_scan']

backend/apps/scan/configs/engine_config_example.yaml

@@ -17,14 +17,6 @@ subdomain_discovery:
       timeout: 3600      # 1小时
       # threads: 10      # 并发 goroutine 数
       
-    amass_passive:
-      enabled: true
-      timeout: 3600
-      
-    amass_active:
-      enabled: true      # 主动枚举 + 爆破
-      timeout: 3600
-      
     sublist3r:
       enabled: true
       timeout: 3600
@@ -62,7 +54,7 @@ port_scan:
       threads: 200       # 并发连接数（默认 5）
       # ports: 1-65535   # 扫描端口范围（默认 1-65535）
       top-ports: 100     # 扫描 nmap top 100 端口
-      rate: 10           # 扫描速率（默认 10）
+      rate: 50           # 扫描速率
       
     naabu_passive:
       enabled: true
@@ -101,6 +93,16 @@ directory_scan:
       match-codes: 200,201,301,302,401,403  # 匹配的 HTTP 状态码
       # rate: 0                           # 每秒请求数（默认 0 不限制）
 
+screenshot:
+  # ==================== 网站截图 ====================
+  # 使用 Playwright 对网站进行截图，保存为 WebP 格式
+  # 在 Stage 2 与 url_fetch、directory_scan 并行执行
+  tools:
+    playwright:
+      enabled: true
+      concurrency: 5                        # 并发截图数（默认 5）
+      url_sources: [websites]               # URL 来源，当前对website截图，还可以用 [websites, endpoints]
+
 url_fetch:
   # ==================== URL 获取 ====================
   tools:

backend/apps/scan/flows/directory_scan_flow.py

@@ -10,30 +10,30 @@
 - 配置由 YAML 解析
 """
 
-# Django 环境初始化（导入即生效）
-from apps.common.prefect_django_setup import setup_django_for_prefect
-
-from prefect import flow
-from prefect.task_runners import ThreadPoolTaskRunner
-
 import hashlib
 import logging
-import os
 import subprocess
 from datetime import datetime
 from pathlib import Path
 from typing import List, Tuple
 
-from apps.scan.tasks.directory_scan import (
-    export_sites_task,
-    run_and_stream_save_directories_task
-)
+from prefect import flow
+
 from apps.scan.handlers.scan_flow_handlers import (
-    on_scan_flow_running,
     on_scan_flow_completed,
     on_scan_flow_failed,
+    on_scan_flow_running,
+)
+from apps.scan.tasks.directory_scan import (
+    export_sites_task,
+    run_and_stream_save_directories_task,
+)
+from apps.scan.utils import (
+    build_scan_command,
+    ensure_wordlist_local,
+    user_log,
+    wait_for_system_load,
 )
-from apps.scan.utils import config_parser, build_scan_command, ensure_wordlist_local
 
 logger = logging.getLogger(__name__)
 
@@ -45,496 +45,343 @@ def calculate_directory_scan_timeout(
     tool_config: dict,
     base_per_word: float = 1.0,
     min_timeout: int = 60,
-    max_timeout: int = 7200
 ) -> int:
     """
     根据字典行数计算目录扫描超时时间
-    
+
     计算公式：超时时间 = 字典行数 × 每个单词基础时间
-    超时范围：60秒 ~ 2小时（7200秒）
-    
+    超时范围：最小 60 秒，无上限
+
     Args:
         tool_config: 工具配置字典，包含 wordlist 路径
         base_per_word: 每个单词的基础时间（秒），默认 1.0秒
         min_timeout: 最小超时时间（秒），默认 60秒
-        max_timeout: 最大超时时间（秒），默认 7200秒（2小时）
-    
+
     Returns:
-        int: 计算出的超时时间（秒），范围：60 ~ 7200
-    
-    Example:
-        # 1000行字典 × 1.0秒 = 1000秒 → 限制为7200秒中的 1000秒
-        # 10000行字典 × 1.0秒 = 10000秒 → 限制为7200秒（最大值）
-        timeout = calculate_directory_scan_timeout(
-            tool_config={'wordlist': '/path/to/wordlist.txt'}
-        )
+        int: 计算出的超时时间（秒）
     """
+    import os
+
+    wordlist_path = tool_config.get('wordlist')
+    if not wordlist_path:
+        logger.warning("工具配置中未指定 wordlist，使用默认超时: %d秒", min_timeout)
+        return min_timeout
+
+    wordlist_path = os.path.expanduser(wordlist_path)
+
+    if not os.path.exists(wordlist_path):
+        logger.warning("字典文件不存在: %s，使用默认超时: %d秒", wordlist_path, min_timeout)
+        return min_timeout
+
     try:
-        # 从 tool_config 中获取 wordlist 路径
-        wordlist_path = tool_config.get('wordlist')
-        if not wordlist_path:
-            logger.warning("工具配置中未指定 wordlist，使用默认超时: %d秒", min_timeout)
-            return min_timeout
-        
-        # 展开用户目录（~）
-        wordlist_path = os.path.expanduser(wordlist_path)
-        
-        # 检查文件是否存在
-        if not os.path.exists(wordlist_path):
-            logger.warning("字典文件不存在: %s，使用默认超时: %d秒", wordlist_path, min_timeout)
-            return min_timeout
-        
-        # 使用 wc -l 快速统计字典行数
         result = subprocess.run(
             ['wc', '-l', wordlist_path],
             capture_output=True,
             text=True,
             check=True
         )
-        # wc -l 输出格式：行数 + 空格 + 文件名
         line_count = int(result.stdout.strip().split()[0])
-        
-        # 计算超时时间
-        timeout = int(line_count * base_per_word)
-        
-        # 设置合理的下限（不再设置上限）
-        timeout = max(min_timeout, timeout)
-        
+        timeout = max(min_timeout, int(line_count * base_per_word))
+
         logger.info(
             "目录扫描超时计算 - 字典: %s, 行数: %d, 基础时间: %.3f秒/词, 计算超时: %d秒",
             wordlist_path, line_count, base_per_word, timeout
         )
-        
         return timeout
-        
-    except subprocess.CalledProcessError as e:
-        logger.error("统计字典行数失败: %s", e)
-        # 失败时返回默认超时
-        return min_timeout
-    except (ValueError, IndexError) as e:
-        logger.error("解析字典行数失败: %s", e)
-        return min_timeout
-    except Exception as e:
-        logger.error("计算超时时间异常: %s", e)
+
+    except (subprocess.CalledProcessError, ValueError, IndexError) as e:
+        logger.error("计算超时时间失败: %s", e)
         return min_timeout
 
 
 def _get_max_workers(tool_config: dict, default: int = DEFAULT_MAX_WORKERS) -> int:
-    """
-    从单个工具配置中获取 max_workers 参数
-    
-    Args:
-        tool_config: 单个工具的配置字典，如 {'max_workers': 10, 'threads': 5, ...}
-        default: 默认值，默认为 5
-        
-    Returns:
-        int: max_workers 值
-    """
+    """从单个工具配置中获取 max_workers 参数"""
     if not isinstance(tool_config, dict):
         return default
-    
-    # 支持 max_workers 和 max-workers（YAML 中划线会被转换）
+
     max_workers = tool_config.get('max_workers') or tool_config.get('max-workers')
-    if max_workers is not None and isinstance(max_workers, int) and max_workers > 0:
+    if isinstance(max_workers, int) and max_workers > 0:
         return max_workers
     return default
 
 
-
-
-
-def _export_site_urls(target_id: int, target_name: str, directory_scan_dir: Path) -> tuple[str, int]:
+def _export_site_urls(
+    target_id: int,
+    directory_scan_dir: Path
+) -> Tuple[str, int]:
     """
-    导出目标下的所有站点 URL 到文件（支持懒加载）
-    
+    导出目标下的所有站点 URL 到文件
+
     Args:
         target_id: 目标 ID
-        target_name: 目标名称（用于懒加载创建默认站点）
         directory_scan_dir: 目录扫描目录
-        
+
     Returns:
         tuple: (sites_file, site_count)
-        
-    Raises:
-        ValueError: 站点数量为 0
     """
     logger.info("Step 1: 导出目标的所有站点 URL")
-    
+
     sites_file = str(directory_scan_dir / 'sites.txt')
     export_result = export_sites_task(
         target_id=target_id,
         output_file=sites_file,
-        batch_size=1000  # 每次读取 1000 条，优化内存占用
+        batch_size=1000
     )
-    
+
     site_count = export_result['total_count']
-    
     logger.info(
         "✓ 站点 URL 导出完成 - 文件: %s, 数量: %d",
         export_result['output_file'],
         site_count
     )
-    
+
     if site_count == 0:
         logger.warning("目标下没有站点，无法执行目录扫描")
-        # 不抛出异常，由上层决定如何处理
-        # raise ValueError("目标下没有站点，无法执行目录扫描")
-    
+
     return export_result['output_file'], site_count
 
 
-def _run_scans_sequentially(
-    enabled_tools: dict,
-    sites_file: str,
-    directory_scan_dir: Path,
-    scan_id: int,
-    target_id: int,
-    site_count: int,
-    target_name: str
-) -> tuple[int, int, list]:
-    """
-    串行执行目录扫描任务（支持多工具）- 已废弃，保留用于兼容
-    
-    Args:
-        enabled_tools: 启用的工具配置字典
-        sites_file: 站点文件路径
-        directory_scan_dir: 目录扫描目录
-        scan_id: 扫描任务 ID
-        target_id: 目标 ID
-        site_count: 站点数量
-        target_name: 目标名称（用于错误日志）
-        
-    Returns:
-        tuple: (total_directories, processed_sites, failed_sites)
-    """
-    # 读取站点列表
-    sites = []
-    with open(sites_file, 'r', encoding='utf-8') as f:
-        for line in f:
-            site_url = line.strip()
-            if site_url:
-                sites.append(site_url)
-    
-    logger.info("准备扫描 %d 个站点，使用工具: %s", len(sites), ', '.join(enabled_tools.keys()))
-    
-    total_directories = 0
-    processed_sites_set = set()  # 使用 set 避免重复计数
-    failed_sites = []
-    
-    # 遍历每个工具
-    for tool_name, tool_config in enabled_tools.items():
-        logger.info("="*60)
-        logger.info("使用工具: %s", tool_name)
-        logger.info("="*60)
-
-        # 如果配置了 wordlist_name，则先确保本地存在对应的字典文件（含 hash 校验）
-        wordlist_name = tool_config.get('wordlist_name')
-        if wordlist_name:
-            try:
-                local_wordlist_path = ensure_wordlist_local(wordlist_name)
-                tool_config['wordlist'] = local_wordlist_path
-            except Exception as exc:
-                logger.error("为工具 %s 准备字典失败: %s", tool_name, exc)
-                # 当前工具无法执行，将所有站点视为失败，继续下一个工具
-                failed_sites.extend(sites)
-                continue
-        
-        # 逐个站点执行扫描
-        for idx, site_url in enumerate(sites, 1):
-            logger.info(
-                "[%d/%d] 开始扫描站点: %s (工具: %s)",
-                idx, len(sites), site_url, tool_name
-            )
-            
-            # 使用统一的命令构建器
-            try:
-                command = build_scan_command(
-                    tool_name=tool_name,
-                    scan_type='directory_scan',
-                    command_params={
-                        'url': site_url
-                    },
-                    tool_config=tool_config
-                )
-            except Exception as e:
-                logger.error(
-                    "✗ [%d/%d] 构建 %s 命令失败: %s - 站点: %s",
-                    idx, len(sites), tool_name, e, site_url
-                )
-                failed_sites.append(site_url)
-                continue
-            
-            # 单个站点超时：从配置中获取（支持 'auto' 动态计算）
-            # ffuf 逐个站点扫描，timeout 就是单个站点的超时时间
-            site_timeout = tool_config.get('timeout', 300)
-            if site_timeout == 'auto':
-                # 动态计算超时时间（基于字典行数）
-                site_timeout = calculate_directory_scan_timeout(tool_config)
-                logger.info(f"✓ 工具 {tool_name} 动态计算 timeout: {site_timeout}秒")
-            
-            # 生成日志文件路径
-            from datetime import datetime
-            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-            log_file = directory_scan_dir / f"{tool_name}_{timestamp}_{idx}.log"
-            
-            try:
-                # 直接调用 task（串行执行）
-                result = run_and_stream_save_directories_task(
-                    cmd=command,
-                    tool_name=tool_name,  # 新增：工具名称
-                    scan_id=scan_id,
-                    target_id=target_id,
-                    site_url=site_url,
-                    cwd=str(directory_scan_dir),
-                    shell=True,
-                    batch_size=1000,
-                    timeout=site_timeout,
-                    log_file=str(log_file)  # 新增：日志文件路径
-                )
-                
-                total_directories += result.get('created_directories', 0)
-                processed_sites_set.add(site_url)  # 使用 set 记录成功的站点
-                
-                logger.info(
-                    "✓ [%d/%d] 站点扫描完成: %s - 发现 %d 个目录",
-                    idx, len(sites), site_url,
-                    result.get('created_directories', 0)
-                )
-                
-            except subprocess.TimeoutExpired as exc:
-                # 超时异常单独处理
-                failed_sites.append(site_url)
-                logger.warning(
-                    "⚠️ [%d/%d] 站点扫描超时: %s - 超时配置: %d秒\n"
-                    "注意：超时前已解析的目录数据已保存到数据库，但扫描未完全完成。",
-                    idx, len(sites), site_url, site_timeout
-                )
-            except Exception as exc:
-                # 其他异常
-                failed_sites.append(site_url)
-                logger.error(
-                    "✗ [%d/%d] 站点扫描失败: %s - 错误: %s",
-                    idx, len(sites), site_url, exc
-                )
-            
-            # 每 10 个站点输出进度
-            if idx % 10 == 0:
-                logger.info(
-                    "进度: %d/%d (%.1f%%) - 已发现 %d 个目录",
-                    idx, len(sites), idx/len(sites)*100, total_directories
-                )
-    
-    # 计算成功和失败的站点数
-    processed_count = len(processed_sites_set)
-    
-    if failed_sites:
-        logger.warning(
-            "部分站点扫描失败: %d/%d",
-            len(failed_sites), len(sites)
-        )
-    
-    logger.info(
-        "✓ 串行目录扫描执行完成 - 成功: %d/%d, 失败: %d, 总目录数: %d",
-        processed_count, len(sites), len(failed_sites), total_directories
-    )
-    
-    return total_directories, processed_count, failed_sites
-
-
-def _generate_log_filename(tool_name: str, site_url: str, directory_scan_dir: Path) -> Path:
-    """
-    生成唯一的日志文件名
-    
-    使用 URL 的 hash 确保并发时不会冲突
-    
-    Args:
-        tool_name: 工具名称
-        site_url: 站点 URL
-        directory_scan_dir: 目录扫描目录
-        
-    Returns:
-        Path: 日志文件路径
-    """
-    url_hash = hashlib.md5(site_url.encode()).hexdigest()[:8]
+def _generate_log_filename(
+    tool_name: str,
+    site_url: str,
+    directory_scan_dir: Path
+) -> Path:
+    """生成唯一的日志文件名（使用 URL 的 hash 确保并发时不会冲突）"""
+    url_hash = hashlib.md5(
+        site_url.encode(),
+        usedforsecurity=False
+    ).hexdigest()[:8]
     timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
     return directory_scan_dir / f"{tool_name}_{url_hash}_{timestamp}.log"
 
 
+def _prepare_tool_wordlist(tool_name: str, tool_config: dict) -> bool:
+    """准备工具的字典文件，返回是否成功"""
+    wordlist_name = tool_config.get('wordlist_name')
+    if not wordlist_name:
+        return True
+
+    try:
+        local_wordlist_path = ensure_wordlist_local(wordlist_name)
+        tool_config['wordlist'] = local_wordlist_path
+        return True
+    except Exception as exc:
+        logger.error("为工具 %s 准备字典失败: %s", tool_name, exc)
+        return False
+
+
+def _build_scan_params(
+    tool_name: str,
+    tool_config: dict,
+    sites: List[str],
+    directory_scan_dir: Path,
+    site_timeout: int
+) -> Tuple[List[dict], List[str]]:
+    """构建所有站点的扫描参数，返回 (scan_params_list, failed_sites)"""
+    scan_params_list = []
+    failed_sites = []
+
+    for idx, site_url in enumerate(sites, 1):
+        try:
+            command = build_scan_command(
+                tool_name=tool_name,
+                scan_type='directory_scan',
+                command_params={'url': site_url},
+                tool_config=tool_config
+            )
+            log_file = _generate_log_filename(tool_name, site_url, directory_scan_dir)
+            scan_params_list.append({
+                'idx': idx,
+                'site_url': site_url,
+                'command': command,
+                'log_file': str(log_file),
+                'timeout': site_timeout
+            })
+        except Exception as e:
+            logger.error(
+                "✗ [%d/%d] 构建 %s 命令失败: %s - 站点: %s",
+                idx, len(sites), tool_name, e, site_url
+            )
+            failed_sites.append(site_url)
+
+    return scan_params_list, failed_sites
+
+
+def _execute_batch(
+    batch_params: List[dict],
+    tool_name: str,
+    scan_id: int,
+    target_id: int,
+    directory_scan_dir: Path,
+    total_sites: int
+) -> Tuple[int, List[str]]:
+    """执行一批扫描任务，返回 (directories_found, failed_sites)"""
+    directories_found = 0
+    failed_sites = []
+
+    # 提交任务
+    futures = []
+    for params in batch_params:
+        future = run_and_stream_save_directories_task.submit(
+            cmd=params['command'],
+            tool_name=tool_name,
+            scan_id=scan_id,
+            target_id=target_id,
+            site_url=params['site_url'],
+            cwd=str(directory_scan_dir),
+            shell=True,
+            batch_size=1000,
+            timeout=params['timeout'],
+            log_file=params['log_file']
+        )
+        futures.append((params['idx'], params['site_url'], future))
+
+    # 等待结果
+    for idx, site_url, future in futures:
+        try:
+            result = future.result()
+            dirs_count = result.get('created_directories', 0)
+            directories_found += dirs_count
+            logger.info(
+                "✓ [%d/%d] 站点扫描完成: %s - 发现 %d 个目录",
+                idx, total_sites, site_url, dirs_count
+            )
+        except Exception as exc:
+            failed_sites.append(site_url)
+            if 'timeout' in str(exc).lower():
+                logger.warning(
+                    "⚠️ [%d/%d] 站点扫描超时: %s - 错误: %s",
+                    idx, total_sites, site_url, exc
+                )
+            else:
+                logger.error(
+                    "✗ [%d/%d] 站点扫描失败: %s - 错误: %s",
+                    idx, total_sites, site_url, exc
+                )
+
+    return directories_found, failed_sites
+
+
 def _run_scans_concurrently(
     enabled_tools: dict,
     sites_file: str,
     directory_scan_dir: Path,
     scan_id: int,
     target_id: int,
-    site_count: int,
-    target_name: str
 ) -> Tuple[int, int, List[str]]:
     """
-    并发执行目录扫描任务（使用 ThreadPoolTaskRunner）
-    
-    Args:
-        enabled_tools: 启用的工具配置字典
-        sites_file: 站点文件路径
-        directory_scan_dir: 目录扫描目录
-        scan_id: 扫描任务 ID
-        target_id: 目标 ID
-        site_count: 站点数量
-        target_name: 目标名称（用于错误日志）
-        
+    并发执行目录扫描任务
+
     Returns:
         tuple: (total_directories, processed_sites, failed_sites)
     """
     # 读取站点列表
     sites: List[str] = []
     with open(sites_file, 'r', encoding='utf-8') as f:
-        for line in f:
-            site_url = line.strip()
-            if site_url:
-                sites.append(site_url)
-    
+        sites = [line.strip() for line in f if line.strip()]
+
     if not sites:
         logger.warning("站点列表为空")
         return 0, 0, []
-    
+
     logger.info(
         "准备并发扫描 %d 个站点，使用工具: %s",
         len(sites), ', '.join(enabled_tools.keys())
     )
-    
+
     total_directories = 0
     processed_sites_count = 0
     failed_sites: List[str] = []
-    
-    # 遍历每个工具
-    for tool_name, tool_config in enabled_tools.items():
-        # 每个工具独立获取 max_workers 配置
-        max_workers = _get_max_workers(tool_config)
-        
-        logger.info("="*60)
-        logger.info("使用工具: %s (并发模式, max_workers=%d)", tool_name, max_workers)
-        logger.info("="*60)
 
-        # 如果配置了 wordlist_name，则先确保本地存在对应的字典文件（含 hash 校验）
-        wordlist_name = tool_config.get('wordlist_name')
-        if wordlist_name:
-            try:
-                local_wordlist_path = ensure_wordlist_local(wordlist_name)
-                tool_config['wordlist'] = local_wordlist_path
-            except Exception as exc:
-                logger.error("为工具 %s 准备字典失败: %s", tool_name, exc)
-                # 当前工具无法执行，将所有站点视为失败，继续下一个工具
-                failed_sites.extend(sites)
-                continue
-        
-        # 计算超时时间（所有站点共用）
+    for tool_name, tool_config in enabled_tools.items():
+        max_workers = _get_max_workers(tool_config)
+
+        logger.info("=" * 60)
+        logger.info("使用工具: %s (并发模式, max_workers=%d)", tool_name, max_workers)
+        logger.info("=" * 60)
+        user_log(scan_id, "directory_scan", f"Running {tool_name}")
+
+        # 准备字典文件
+        if not _prepare_tool_wordlist(tool_name, tool_config):
+            failed_sites.extend(sites)
+            continue
+
+        # 计算超时时间
         site_timeout = tool_config.get('timeout', 300)
         if site_timeout == 'auto':
             site_timeout = calculate_directory_scan_timeout(tool_config)
-            logger.info(f"✓ 工具 {tool_name} 动态计算 timeout: {site_timeout}秒")
-        
-        # 准备所有站点的扫描参数
-        scan_params_list = []
-        for idx, site_url in enumerate(sites, 1):
-            try:
-                command = build_scan_command(
-                    tool_name=tool_name,
-                    scan_type='directory_scan',
-                    command_params={'url': site_url},
-                    tool_config=tool_config
-                )
-                log_file = _generate_log_filename(tool_name, site_url, directory_scan_dir)
-                scan_params_list.append({
-                    'idx': idx,
-                    'site_url': site_url,
-                    'command': command,
-                    'log_file': str(log_file),
-                    'timeout': site_timeout
-                })
-            except Exception as e:
-                logger.error(
-                    "✗ [%d/%d] 构建 %s 命令失败: %s - 站点: %s",
-                    idx, len(sites), tool_name, e, site_url
-                )
-                failed_sites.append(site_url)
-        
+            logger.info("✓ 工具 %s 动态计算 timeout: %d秒", tool_name, site_timeout)
+
+        # 构建扫描参数
+        scan_params_list, build_failed = _build_scan_params(
+            tool_name, tool_config, sites, directory_scan_dir, site_timeout
+        )
+        failed_sites.extend(build_failed)
+
         if not scan_params_list:
             logger.warning("没有有效的扫描任务")
             continue
-        
-        # ============================================================
-        # 分批执行策略：控制实际并发的 ffuf 进程数
-        # ============================================================
+
+        # 分批执行
         total_tasks = len(scan_params_list)
         logger.info("开始分批执行 %d 个扫描任务（每批 %d 个）...", total_tasks, max_workers)
-        
-        batch_num = 0
+
+        last_progress_percent = 0
+        tool_directories = 0
+        tool_processed = 0
+
         for batch_start in range(0, total_tasks, max_workers):
             batch_end = min(batch_start + max_workers, total_tasks)
             batch_params = scan_params_list[batch_start:batch_end]
-            batch_num += 1
-            
-            logger.info("执行第 %d 批任务（%d-%d/%d）...", batch_num, batch_start + 1, batch_end, total_tasks)
-            
-            # 提交当前批次的任务（非阻塞，立即返回 future）
-            futures = []
-            for params in batch_params:
-                future = run_and_stream_save_directories_task.submit(
-                    cmd=params['command'],
-                    tool_name=tool_name,
-                    scan_id=scan_id,
-                    target_id=target_id,
-                    site_url=params['site_url'],
-                    cwd=str(directory_scan_dir),
-                    shell=True,
-                    batch_size=1000,
-                    timeout=params['timeout'],
-                    log_file=params['log_file']
+            batch_num = batch_start // max_workers + 1
+
+            logger.info(
+                "执行第 %d 批任务（%d-%d/%d）...",
+                batch_num, batch_start + 1, batch_end, total_tasks
+            )
+
+            dirs_found, batch_failed = _execute_batch(
+                batch_params, tool_name, scan_id, target_id,
+                directory_scan_dir, len(sites)
+            )
+
+            total_directories += dirs_found
+            tool_directories += dirs_found
+            tool_processed += len(batch_params) - len(batch_failed)
+            processed_sites_count += len(batch_params) - len(batch_failed)
+            failed_sites.extend(batch_failed)
+
+            # 进度里程碑：每 20% 输出一次
+            current_progress = int((batch_end / total_tasks) * 100)
+            if current_progress >= last_progress_percent + 20:
+                user_log(
+                    scan_id, "directory_scan",
+                    f"Progress: {batch_end}/{total_tasks} sites scanned"
                 )
-                futures.append((params['idx'], params['site_url'], future))
-            
-            # 等待当前批次所有任务完成（阻塞，确保本批完成后再启动下一批）
-            for idx, site_url, future in futures:
-                try:
-                    result = future.result()  # 阻塞等待单个任务完成
-                    directories_found = result.get('created_directories', 0)
-                    total_directories += directories_found
-                    processed_sites_count += 1
-                    
-                    logger.info(
-                        "✓ [%d/%d] 站点扫描完成: %s - 发现 %d 个目录",
-                        idx, len(sites), site_url, directories_found
-                    )
-                    
-                except Exception as exc:
-                    failed_sites.append(site_url)
-                    if 'timeout' in str(exc).lower() or isinstance(exc, subprocess.TimeoutExpired):
-                        logger.warning(
-                            "⚠️ [%d/%d] 站点扫描超时: %s - 错误: %s",
-                            idx, len(sites), site_url, exc
-                        )
-                    else:
-                        logger.error(
-                            "✗ [%d/%d] 站点扫描失败: %s - 错误: %s",
-                            idx, len(sites), site_url, exc
-                        )
-    
-    # 输出汇总信息
-    if failed_sites:
-        logger.warning(
-            "部分站点扫描失败: %d/%d",
-            len(failed_sites), len(sites)
+                last_progress_percent = (current_progress // 20) * 20
+
+        logger.info(
+            "✓ 工具 %s 执行完成 - 已处理站点: %d/%d, 发现目录: %d",
+            tool_name, tool_processed, total_tasks, tool_directories
         )
-    
+        user_log(
+            scan_id, "directory_scan",
+            f"{tool_name} completed: found {tool_directories} directories"
+        )
+
+    if failed_sites:
+        logger.warning("部分站点扫描失败: %d/%d", len(failed_sites), len(sites))
+
     logger.info(
         "✓ 并发目录扫描执行完成 - 成功: %d/%d, 失败: %d, 总目录数: %d",
         processed_sites_count, len(sites), len(failed_sites), total_directories
     )
-    
+
     return total_directories, processed_sites_count, failed_sites
 
 
 @flow(
-    name="directory_scan", 
+    name="directory_scan",
     log_prints=True,
     on_running=[on_scan_flow_running],
     on_completion=[on_scan_flow_completed],
@@ -549,62 +396,31 @@ def directory_scan_flow(
 ) -> dict:
     """
     目录扫描 Flow
-    
+
     主要功能：
         1. 从 target 获取所有站点的 URL
         2. 对每个站点 URL 执行目录扫描（支持 ffuf 等工具）
         3. 流式保存扫描结果到数据库 Directory 表
-    
-    工作流程：
-        Step 0: 创建工作目录
-        Step 1: 导出站点 URL 列表到文件（供扫描工具使用）
-        Step 2: 验证工具配置
-        Step 3: 并发执行扫描工具并实时保存结果（使用 ThreadPoolTaskRunner）
-    
-    ffuf 输出字段：
-        - url: 发现的目录/文件 URL
-        - length: 响应内容长度
-        - status: HTTP 状态码
-        - words: 响应内容单词数
-        - lines: 响应内容行数
-        - content_type: 内容类型
-        - duration: 请求耗时
-    
+
     Args:
         scan_id: 扫描任务 ID
         target_name: 目标名称
         target_id: 目标 ID
         scan_workspace_dir: 扫描工作空间目录
         enabled_tools: 启用的工具配置字典
-        
+
     Returns:
-        dict: {
-            'success': bool,
-            'scan_id': int,
-            'target': str,
-            'scan_workspace_dir': str,
-            'sites_file': str,
-            'site_count': int,
-            'total_directories': int,  # 发现的总目录数
-            'processed_sites': int,  # 成功处理的站点数
-            'failed_sites_count': int,  # 失败的站点数
-            'executed_tasks': list
-        }
-    
-    Raises:
-        ValueError: 参数错误
-        RuntimeError: 执行失败
+        dict: 扫描结果
     """
     try:
+        wait_for_system_load(context="directory_scan_flow")
+
         logger.info(
-            "="*60 + "\n" +
-            "开始目录扫描\n" +
-            f"  Scan ID: {scan_id}\n" +
-            f"  Target: {target_name}\n" +
-            f"  Workspace: {scan_workspace_dir}\n" +
-            "="*60
+            "开始目录扫描 - Scan ID: %s, Target: %s, Workspace: %s",
+            scan_id, target_name, scan_workspace_dir
         )
-        
+        user_log(scan_id, "directory_scan", "Starting directory scan")
+
         # 参数验证
         if scan_id is None:
             raise ValueError("scan_id 不能为空")
@@ -616,16 +432,17 @@ def directory_scan_flow(
             raise ValueError("scan_workspace_dir 不能为空")
         if not enabled_tools:
             raise ValueError("enabled_tools 不能为空")
-        
+
         # Step 0: 创建工作目录
         from apps.scan.utils import setup_scan_directory
         directory_scan_dir = setup_scan_directory(scan_workspace_dir, 'directory_scan')
-        
-        # Step 1: 导出站点 URL（支持懒加载）
-        sites_file, site_count = _export_site_urls(target_id, target_name, directory_scan_dir)
-        
+
+        # Step 1: 导出站点 URL
+        sites_file, site_count = _export_site_urls(target_id, directory_scan_dir)
+
         if site_count == 0:
-            logger.warning("目标下没有站点，跳过目录扫描")
+            logger.warning("跳过目录扫描：没有站点可扫描 - Scan ID: %s", scan_id)
+            user_log(scan_id, "directory_scan", "Skipped: no sites to scan", "warning")
             return {
                 'success': True,
                 'scan_id': scan_id,
@@ -638,16 +455,16 @@ def directory_scan_flow(
                 'failed_sites_count': 0,
                 'executed_tasks': ['export_sites']
             }
-        
+
         # Step 2: 工具配置信息
         logger.info("Step 2: 工具配置信息")
-        tool_info = []
-        for tool_name, tool_config in enabled_tools.items():
-            mw = _get_max_workers(tool_config)
-            tool_info.append(f"{tool_name}(max_workers={mw})")
+        tool_info = [
+            f"{name}(max_workers={_get_max_workers(cfg)})"
+            for name, cfg in enabled_tools.items()
+        ]
         logger.info("✓ 启用工具: %s", ', '.join(tool_info))
-        
-        # Step 3: 并发执行扫描工具并实时保存结果
+
+        # Step 3: 并发执行扫描
         logger.info("Step 3: 并发执行扫描工具并实时保存结果")
         total_directories, processed_sites, failed_sites = _run_scans_concurrently(
             enabled_tools=enabled_tools,
@@ -655,17 +472,20 @@ def directory_scan_flow(
             directory_scan_dir=directory_scan_dir,
             scan_id=scan_id,
             target_id=target_id,
-            site_count=site_count,
-            target_name=target_name
         )
-        
-        # 检查是否所有站点都失败
+
         if processed_sites == 0 and site_count > 0:
-            logger.warning("所有站点扫描均失败 - 总站点数: %d, 失败数: %d", site_count, len(failed_sites))
-            # 不抛出异常，让扫描继续
-        
-        logger.info("="*60 + "\n✓ 目录扫描完成\n" + "="*60)
-        
+            logger.warning(
+                "所有站点扫描均失败 - 总站点数: %d, 失败数: %d",
+                site_count, len(failed_sites)
+            )
+
+        logger.info("✓ 目录扫描完成 - 发现目录: %d", total_directories)
+        user_log(
+            scan_id, "directory_scan",
+            f"directory_scan completed: found {total_directories} directories"
+        )
+
         return {
             'success': True,
             'scan_id': scan_id,
@@ -678,7 +498,7 @@ def directory_scan_flow(
             'failed_sites_count': len(failed_sites),
             'executed_tasks': ['export_sites', 'run_and_stream_save_directories']
         }
-        
+
     except Exception as e:
         logger.exception("目录扫描失败: %s", e)
-        raise
+        raise

backend/apps/scan/flows/fingerprint_detect_flow.py

@@ -10,26 +10,22 @@
 - 流式处理输出，批量更新数据库
 """
 
-# Django 环境初始化（导入即生效）
-from apps.common.prefect_django_setup import setup_django_for_prefect
-
 import logging
-import os
 from datetime import datetime
 from pathlib import Path
 
 from prefect import flow
 
 from apps.scan.handlers.scan_flow_handlers import (
-    on_scan_flow_running,
     on_scan_flow_completed,
     on_scan_flow_failed,
+    on_scan_flow_running,
 )
 from apps.scan.tasks.fingerprint_detect import (
     export_urls_for_fingerprint_task,
     run_xingfinger_and_stream_update_tech_task,
 )
-from apps.scan.utils import build_scan_command
+from apps.scan.utils import build_scan_command, user_log, wait_for_system_load
 from apps.scan.utils.fingerprint_helpers import get_fingerprint_paths
 
 logger = logging.getLogger(__name__)
@@ -42,22 +38,19 @@ def calculate_fingerprint_detect_timeout(
 ) -> int:
     """
     根据 URL 数量计算超时时间
-    
+
     公式：超时时间 = URL 数量 × 每 URL 基础时间
-    最小值：300秒
-    无上限
-    
+    最小值：300秒，无上限
+
     Args:
         url_count: URL 数量
         base_per_url: 每 URL 基础时间（秒），默认 10秒
         min_timeout: 最小超时时间（秒），默认 300秒
-        
+
     Returns:
         int: 计算出的超时时间（秒）
-        
     """
-    timeout = int(url_count * base_per_url)
-    return max(min_timeout, timeout)
+    return max(min_timeout, int(url_count * base_per_url))
 
 
 
@@ -70,17 +63,17 @@ def _export_urls(
 ) -> tuple[str, int]:
     """
     导出 URL 到文件
-    
+
     Args:
         target_id: 目标 ID
         fingerprint_dir: 指纹识别目录
         source: 数据源类型
-        
+
     Returns:
         tuple: (urls_file, total_count)
     """
     logger.info("Step 1: 导出 URL 列表 (source=%s)", source)
-    
+
     urls_file = str(fingerprint_dir / 'urls.txt')
     export_result = export_urls_for_fingerprint_task(
         target_id=target_id,
@@ -88,15 +81,14 @@ def _export_urls(
         source=source,
         batch_size=1000
     )
-    
+
     total_count = export_result['total_count']
-    
     logger.info(
         "✓ URL 导出完成 - 文件: %s, 数量: %d",
         export_result['output_file'],
         total_count
     )
-    
+
     return export_result['output_file'], total_count
 
 
@@ -111,7 +103,7 @@ def _run_fingerprint_detect(
 ) -> tuple[dict, list]:
     """
     执行指纹识别任务
-    
+
     Args:
         enabled_tools: 已启用的工具配置字典
         urls_file: URL 文件路径
@@ -120,55 +112,54 @@ def _run_fingerprint_detect(
         scan_id: 扫描任务 ID
         target_id: 目标 ID
         source: 数据源类型
-        
+
     Returns:
         tuple: (tool_stats, failed_tools)
     """
     tool_stats = {}
     failed_tools = []
-    
+
     for tool_name, tool_config in enabled_tools.items():
         # 1. 获取指纹库路径
         lib_names = tool_config.get('fingerprint_libs', ['ehole'])
         fingerprint_paths = get_fingerprint_paths(lib_names)
-        
+
         if not fingerprint_paths:
             reason = f"没有可用的指纹库: {lib_names}"
             logger.warning(reason)
             failed_tools.append({'tool': tool_name, 'reason': reason})
             continue
-        
+
         # 2. 将指纹库路径合并到 tool_config（用于命令构建）
         tool_config_with_paths = {**tool_config, **fingerprint_paths}
-        
+
         # 3. 构建命令
         try:
             command = build_scan_command(
                 tool_name=tool_name,
                 scan_type='fingerprint_detect',
-                command_params={
-                    'urls_file': urls_file
-                },
+                command_params={'urls_file': urls_file},
                 tool_config=tool_config_with_paths
             )
         except Exception as e:
-            reason = f"命令构建失败: {str(e)}"
+            reason = f"命令构建失败: {e}"
             logger.error("构建 %s 命令失败: %s", tool_name, e)
             failed_tools.append({'tool': tool_name, 'reason': reason})
             continue
-        
+
         # 4. 计算超时时间
         timeout = calculate_fingerprint_detect_timeout(url_count)
-        
+
         # 5. 生成日志文件路径
         timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
         log_file = fingerprint_dir / f"{tool_name}_{timestamp}.log"
-        
+
         logger.info(
             "开始执行 %s 指纹识别 - URL数: %d, 超时: %ds, 指纹库: %s",
             tool_name, url_count, timeout, list(fingerprint_paths.keys())
         )
-        
+        user_log(scan_id, "fingerprint_detect", f"Running {tool_name}: {command}")
+
         # 6. 执行扫描任务
         try:
             result = run_xingfinger_and_stream_update_tech_task(
@@ -182,32 +173,39 @@ def _run_fingerprint_detect(
                 log_file=str(log_file),
                 batch_size=100
             )
-            
+
             tool_stats[tool_name] = {
                 'command': command,
                 'result': result,
                 'timeout': timeout,
                 'fingerprint_libs': list(fingerprint_paths.keys())
             }
-            
+
+            tool_updated = result.get('updated_count', 0)
             logger.info(
                 "✓ 工具 %s 执行完成 - 处理记录: %d, 更新: %d, 未找到: %d",
                 tool_name,
                 result.get('processed_records', 0),
-                result.get('updated_count', 0),
+                tool_updated,
                 result.get('not_found_count', 0)
             )
-            
+            user_log(
+                scan_id, "fingerprint_detect",
+                f"{tool_name} completed: identified {tool_updated} fingerprints"
+            )
+
         except Exception as exc:
-            failed_tools.append({'tool': tool_name, 'reason': str(exc)})
+            reason = str(exc)
+            failed_tools.append({'tool': tool_name, 'reason': reason})
             logger.error("工具 %s 执行失败: %s", tool_name, exc, exc_info=True)
-    
+            user_log(scan_id, "fingerprint_detect", f"{tool_name} failed: {reason}", "error")
+
     if failed_tools:
         logger.warning(
             "以下指纹识别工具执行失败: %s",
             ', '.join([f['tool'] for f in failed_tools])
         )
-    
+
     return tool_stats, failed_tools
 
 
@@ -227,51 +225,38 @@ def fingerprint_detect_flow(
 ) -> dict:
     """
     指纹识别 Flow
-    
+
     主要功能：
         1. 从数据库导出目标下所有 WebSite URL 到文件
         2. 使用 xingfinger 进行技术栈识别
         3. 解析结果并更新 WebSite.tech 字段（合并去重）
-    
+
     工作流程：
         Step 0: 创建工作目录
         Step 1: 导出 URL 列表
         Step 2: 解析配置，获取启用的工具
         Step 3: 执行 xingfinger 并解析结果
-    
+
     Args:
         scan_id: 扫描任务 ID
         target_name: 目标名称
         target_id: 目标 ID
         scan_workspace_dir: 扫描工作空间目录
         enabled_tools: 启用的工具配置（xingfinger）
-        
+
     Returns:
-        dict: {
-            'success': bool,
-            'scan_id': int,
-            'target': str,
-            'scan_workspace_dir': str,
-            'urls_file': str,
-            'url_count': int,
-            'processed_records': int,
-            'updated_count': int,
-            'created_count': int,
-            'snapshot_count': int,
-            'executed_tasks': list,
-            'tool_stats': dict
-        }
+        dict: 扫描结果
     """
     try:
+        # 负载检查：等待系统资源充足
+        wait_for_system_load(context="fingerprint_detect_flow")
+
         logger.info(
-            "="*60 + "\n" +
-            "开始指纹识别\n" +
-            f"  Scan ID: {scan_id}\n" +
-            f"  Target: {target_name}\n" +
-            f"  Workspace: {scan_workspace_dir}\n" +
-            "="*60
+            "开始指纹识别 - Scan ID: %s, Target: %s, Workspace: %s",
+            scan_id, target_name, scan_workspace_dir
         )
-        
+        user_log(scan_id, "fingerprint_detect", "Starting fingerprint detection")
+
         # 参数验证
         if scan_id is None:
             raise ValueError("scan_id 不能为空")
@@ -281,45 +266,26 @@ def fingerprint_detect_flow(
             raise ValueError("target_id 不能为空")
         if not scan_workspace_dir:
             raise ValueError("scan_workspace_dir 不能为空")
-        
+
         # 数据源类型（当前只支持 website）
         source = 'website'
-        
+
         # Step 0: 创建工作目录
         from apps.scan.utils import setup_scan_directory
         fingerprint_dir = setup_scan_directory(scan_workspace_dir, 'fingerprint_detect')
-        
+
         # Step 1: 导出 URL（支持懒加载）
         urls_file, url_count = _export_urls(target_id, fingerprint_dir, source)
-        
+
         if url_count == 0:
-            logger.warning("目标下没有可用的 URL，跳过指纹识别")
-            return {
-                'success': True,
-                'scan_id': scan_id,
-                'target': target_name,
-                'scan_workspace_dir': scan_workspace_dir,
-                'urls_file': urls_file,
-                'url_count': 0,
-                'processed_records': 0,
-                'updated_count': 0,
-                'created_count': 0,
-                'snapshot_count': 0,
-                'executed_tasks': ['export_urls_for_fingerprint'],
-                'tool_stats': {
-                    'total': 0,
-                    'successful': 0,
-                    'failed': 0,
-                    'successful_tools': [],
-                    'failed_tools': [],
-                    'details': {}
-                }
-            }
-        
+            logger.warning("跳过指纹识别：没有 URL 可扫描 - Scan ID: %s", scan_id)
+            user_log(scan_id, "fingerprint_detect", "Skipped: no URLs to scan", "warning")
+            return _build_empty_result(scan_id, target_name, scan_workspace_dir, urls_file)
+
         # Step 2: 工具配置信息
         logger.info("Step 2: 工具配置信息")
         logger.info("✓ 启用工具: %s", ', '.join(enabled_tools.keys()))
-        
+
         # Step 3: 执行指纹识别
         logger.info("Step 3: 执行指纹识别")
         tool_stats, failed_tools = _run_fingerprint_detect(
@@ -331,22 +297,37 @@ def fingerprint_detect_flow(
             target_id=target_id,
             source=source
         )
-        
-        logger.info("="*60 + "\n✓ 指纹识别完成\n" + "="*60)
-        
+
         # 动态生成已执行的任务列表
         executed_tasks = ['export_urls_for_fingerprint']
-        executed_tasks.extend([f'run_xingfinger ({tool})' for tool in tool_stats.keys()])
-        
+        executed_tasks.extend([f'run_xingfinger ({tool})' for tool in tool_stats])
+
         # 汇总所有工具的结果
-        total_processed = sum(stats['result'].get('processed_records', 0) for stats in tool_stats.values())
-        total_updated = sum(stats['result'].get('updated_count', 0) for stats in tool_stats.values())
-        total_created = sum(stats['result'].get('created_count', 0) for stats in tool_stats.values())
-        total_snapshots = sum(stats['result'].get('snapshot_count', 0) for stats in tool_stats.values())
-        
-        successful_tools = [name for name in enabled_tools.keys() 
-                           if name not in [f['tool'] for f in failed_tools]]
-        
+        total_processed = sum(
+            stats['result'].get('processed_records', 0) for stats in tool_stats.values()
+        )
+        total_updated = sum(
+            stats['result'].get('updated_count', 0) for stats in tool_stats.values()
+        )
+        total_created = sum(
+            stats['result'].get('created_count', 0) for stats in tool_stats.values()
+        )
+        total_snapshots = sum(
+            stats['result'].get('snapshot_count', 0) for stats in tool_stats.values()
+        )
+
+        # 记录 Flow 完成
+        logger.info("✓ 指纹识别完成 - 识别指纹: %d", total_updated)
+        user_log(
+            scan_id, "fingerprint_detect",
+            f"fingerprint_detect completed: identified {total_updated} fingerprints"
+        )
+
+        successful_tools = [
+            name for name in enabled_tools
+            if name not in [f['tool'] for f in failed_tools]
+        ]
+
         return {
             'success': True,
             'scan_id': scan_id,
@@ -368,7 +349,7 @@ def fingerprint_detect_flow(
                 'details': tool_stats
             }
         }
-        
+
     except ValueError as e:
         logger.error("配置错误: %s", e)
         raise
@@ -378,3 +359,33 @@ def fingerprint_detect_flow(
     except Exception as e:
         logger.exception("指纹识别失败: %s", e)
         raise
+
+
+def _build_empty_result(
+    scan_id: int,
+    target_name: str,
+    scan_workspace_dir: str,
+    urls_file: str
+) -> dict:
+    """构建空结果（无 URL 可扫描时）"""
+    return {
+        'success': True,
+        'scan_id': scan_id,
+        'target': target_name,
+        'scan_workspace_dir': scan_workspace_dir,
+        'urls_file': urls_file,
+        'url_count': 0,
+        'processed_records': 0,
+        'updated_count': 0,
+        'created_count': 0,
+        'snapshot_count': 0,
+        'executed_tasks': ['export_urls_for_fingerprint'],
+        'tool_stats': {
+            'total': 0,
+            'successful': 0,
+            'failed': 0,
+            'successful_tools': [],
+            'failed_tools': [],
+            'details': {}
+        }
+    }

backend/apps/scan/flows/initiate_scan_flow.py

@@ -99,15 +99,13 @@ def initiate_scan_flow(
             raise ValueError("engine_name is required")
         
         
-        logger.info(
-            "="*60 + "\n" +
-            "开始初始化扫描任务\n" +
-            f"  Scan ID: {scan_id}\n" +
-            f"  Target: {target_name}\n" +
-            f"  Engine: {engine_name}\n" +
-            f"  Workspace: {scan_workspace_dir}\n" +
-            "="*60
-        )
+        logger.info("="*60)
+        logger.info("开始初始化扫描任务")
+        logger.info(f"Scan ID: {scan_id}")
+        logger.info(f"Target: {target_name}")
+        logger.info(f"Engine: {engine_name}")
+        logger.info(f"Workspace: {scan_workspace_dir}")
+        logger.info("="*60)
         
         # ==================== Task 1: 创建 Scan 工作空间 ====================
         scan_workspace_path = setup_scan_workspace(scan_workspace_dir)
@@ -115,7 +113,7 @@ def initiate_scan_flow(
         # ==================== Task 2: 获取引擎配置 ====================
         from apps.scan.models import Scan
         scan = Scan.objects.get(id=scan_id)
-        engine_config = scan.merged_configuration
+        engine_config = scan.yaml_configuration
         
         # 使用 engine_names 进行显示
         display_engine_name = ', '.join(scan.engine_names) if scan.engine_names else engine_name
@@ -126,11 +124,9 @@ def initiate_scan_flow(
         # FlowOrchestrator 已经解析了所有工具配置
         enabled_tools_by_type = orchestrator.enabled_tools_by_type
         
-        logger.info(
-            f"执行计划生成成功：\n"
-            f"  扫描类型: {' → '.join(orchestrator.scan_types)}\n"
-            f"  总共 {len(orchestrator.scan_types)} 个 Flow"
-        )
+        logger.info("执行计划生成成功")
+        logger.info(f"扫描类型: {' → '.join(orchestrator.scan_types)}")
+        logger.info(f"总共 {len(orchestrator.scan_types)} 个 Flow")
         
         # ==================== 初始化阶段进度 ====================
         # 在解析完配置后立即初始化，此时已有完整的 scan_types 列表
@@ -209,9 +205,13 @@ def initiate_scan_flow(
         for mode, enabled_flows in orchestrator.get_execution_stages():
             if mode == 'sequential':
                 # 顺序执行
-                logger.info(f"\n{'='*60}\n顺序执行阶段: {', '.join(enabled_flows)}\n{'='*60}")
+                logger.info("="*60)
+                logger.info(f"顺序执行阶段: {', '.join(enabled_flows)}")
+                logger.info("="*60)
                 for scan_type, flow_func, flow_specific_kwargs in get_valid_flows(enabled_flows):
-                    logger.info(f"\n{'='*60}\n执行 Flow: {scan_type}\n{'='*60}")
+                    logger.info("="*60)
+                    logger.info(f"执行 Flow: {scan_type}")
+                    logger.info("="*60)
                     try:
                         result = flow_func(**flow_specific_kwargs)
                         record_flow_result(scan_type, result=result)
@@ -220,12 +220,16 @@ def initiate_scan_flow(
                     
             elif mode == 'parallel':
                 # 并行执行阶段：通过 Task 包装子 Flow，并使用 Prefect TaskRunner 并发运行
-                logger.info(f"\n{'='*60}\n并行执行阶段: {', '.join(enabled_flows)}\n{'='*60}")
+                logger.info("="*60)
+                logger.info(f"并行执行阶段: {', '.join(enabled_flows)}")
+                logger.info("="*60)
                 futures = []
 
                 # 提交所有并行子 Flow 任务
                 for scan_type, flow_func, flow_specific_kwargs in get_valid_flows(enabled_flows):
-                    logger.info(f"\n{'='*60}\n提交并行子 Flow 任务: {scan_type}\n{'='*60}")
+                    logger.info("="*60)
+                    logger.info(f"提交并行子 Flow 任务: {scan_type}")
+                    logger.info("="*60)
                     future = _run_subflow_task.submit(
                         scan_type=scan_type,
                         flow_func=flow_func,
@@ -246,12 +250,10 @@ def initiate_scan_flow(
                             record_flow_result(scan_type, error=e)
 
         # ==================== 完成 ====================
-        logger.info(
-            "="*60 + "\n" +
-            "✓ 扫描任务初始化完成\n" +
-            f"  执行的 Flow: {', '.join(executed_flows)}\n" +
-            "="*60
-        )
+        logger.info("="*60)
+        logger.info("✓ 扫描任务初始化完成")
+        logger.info(f"执行的 Flow: {', '.join(executed_flows)}")
+        logger.info("="*60)
         
         # ==================== 返回结果 ====================
         return {

backend/apps/scan/flows/port_scan_flow.py

@@ -1,4 +1,4 @@
-"""  
+"""
 端口扫描 Flow
 
 负责编排端口扫描的完整流程
@@ -10,25 +10,23 @@
 - 配置由 YAML 解析
 """
 
-# Django 环境初始化（导入即生效）
-from apps.common.prefect_django_setup import setup_django_for_prefect
-
 import logging
-import os
 import subprocess
+from datetime import datetime
 from pathlib import Path
-from typing import Callable
+
 from prefect import flow
-from apps.scan.tasks.port_scan import (
-    export_scan_targets_task,
-    run_and_stream_save_ports_task
-)
+
 from apps.scan.handlers.scan_flow_handlers import (
-    on_scan_flow_running,
     on_scan_flow_completed,
     on_scan_flow_failed,
+    on_scan_flow_running,
 )
-from apps.scan.utils import config_parser, build_scan_command
+from apps.scan.tasks.port_scan import (
+    export_hosts_task,
+    run_and_stream_save_ports_task,
+)
+from apps.scan.utils import build_scan_command, user_log, wait_for_system_load
 
 logger = logging.getLogger(__name__)
 
@@ -40,28 +38,19 @@ def calculate_port_scan_timeout(
 ) -> int:
     """
     根据目标数量和端口数量计算超时时间
-    
+
     计算公式：超时时间 = 目标数 × 端口数 × base_per_pair
-    超时范围：60秒 ~ 2天（172800秒）
-    
+    超时范围：60秒 ~ 无上限
+
     Args:
         tool_config: 工具配置字典，包含端口配置（ports, top-ports等）
         file_path: 目标文件路径（域名/IP列表）
         base_per_pair: 每个"端口-目标对"的基础时间（秒），默认 0.5秒
-    
+
     Returns:
-        int: 计算出的超时时间（秒），范围：60 ~ 172800
-    
-    Example:
-        # 100个目标 × 100个端口 × 0.5秒 = 5000秒
-        # 10个目标 × 1000个端口 × 0.5秒 = 5000秒
-        timeout = calculate_port_scan_timeout(
-            tool_config={'top-ports': 100},
-            file_path='/path/to/domains.txt'
-        )
+        int: 计算出的超时时间（秒），最小 60 秒
     """
     try:
-        # 1. 统计目标数量
         result = subprocess.run(
             ['wc', '-l', file_path],
             capture_output=True,
@@ -69,133 +58,116 @@ def calculate_port_scan_timeout(
             check=True
         )
         target_count = int(result.stdout.strip().split()[0])
-        
-        # 2. 解析端口数量
         port_count = _parse_port_count(tool_config)
-        
-        # 3. 计算超时时间
-        # 总工作量 = 目标数 × 端口数
         total_work = target_count * port_count
-        timeout = int(total_work * base_per_pair)
-        
-        # 4. 设置合理的下限（不再设置上限）
-        min_timeout = 60       # 最小 60 秒
-        timeout = max(min_timeout, timeout)
-        
+        timeout = max(60, int(total_work * base_per_pair))
+
         logger.info(
-            f"计算端口扫描 timeout - "
-            f"目标数: {target_count}, "
-            f"端口数: {port_count}, "
-            f"总工作量: {total_work}, "
-            f"超时: {timeout}秒"
+            "计算端口扫描 timeout - 目标数: %d, 端口数: %d, 总工作量: %d, 超时: %d秒",
+            target_count, port_count, total_work, timeout
         )
         return timeout
-        
+
     except Exception as e:
-        logger.warning(f"计算 timeout 失败: {e}，使用默认值 600秒")
+        logger.warning("计算 timeout 失败: %s，使用默认值 600秒", e)
         return 600
 
 
 def _parse_port_count(tool_config: dict) -> int:
     """
     从工具配置中解析端口数量
-    
+
     优先级：
     1. top-ports: N  → 返回 N
     2. ports: "80,443,8080"  → 返回逗号分隔的数量
     3. ports: "1-1000"  → 返回范围的大小
     4. ports: "1-65535"  → 返回 65535
     5. 默认  → 返回 100（naabu 默认扫描 top 100）
-    
+
     Args:
         tool_config: 工具配置字典
-    
+
     Returns:
         int: 端口数量
     """
-    # 1. 检查 top-ports 配置
+    # 检查 top-ports 配置
     if 'top-ports' in tool_config:
         top_ports = tool_config['top-ports']
         if isinstance(top_ports, int) and top_ports > 0:
             return top_ports
-        logger.warning(f"top-ports 配置无效: {top_ports}，使用默认值")
-    
-    # 2. 检查 ports 配置
+        logger.warning("top-ports 配置无效: %s，使用默认值", top_ports)
+
+    # 检查 ports 配置
     if 'ports' in tool_config:
         ports_str = str(tool_config['ports']).strip()
-        
-        # 2.1 逗号分隔的端口列表：80,443,8080
+
+        # 逗号分隔的端口列表：80,443,8080
         if ',' in ports_str:
-            port_list = [p.strip() for p in ports_str.split(',') if p.strip()]
-            return len(port_list)
-        
-        # 2.2 端口范围：1-1000
+            return len([p.strip() for p in ports_str.split(',') if p.strip()])
+
+        # 端口范围：1-1000
         if '-' in ports_str:
             try:
                 start, end = ports_str.split('-', 1)
                 start_port = int(start.strip())
                 end_port = int(end.strip())
-                
                 if 1 <= start_port <= end_port <= 65535:
                     return end_port - start_port + 1
-                logger.warning(f"端口范围无效: {ports_str}，使用默认值")
+                logger.warning("端口范围无效: %s，使用默认值", ports_str)
             except ValueError:
-                logger.warning(f"端口范围解析失败: {ports_str}，使用默认值")
-        
-        # 2.3 单个端口
+                logger.warning("端口范围解析失败: %s，使用默认值", ports_str)
+
+        # 单个端口
         try:
             port = int(ports_str)
             if 1 <= port <= 65535:
                 return 1
         except ValueError:
-            logger.warning(f"端口配置解析失败: {ports_str}，使用默认值")
-    
-    # 3. 默认值：naabu 默认扫描 top 100 端口
+            logger.warning("端口配置解析失败: %s，使用默认值", ports_str)
+
+    # 默认值：naabu 默认扫描 top 100 端口
     return 100
 
 
 
 
 
-def _export_scan_targets(target_id: int, port_scan_dir: Path) -> tuple[str, int, str]:
+def _export_hosts(target_id: int, port_scan_dir: Path) -> tuple[str, int, str]:
     """
-    导出扫描目标到文件
-    
+    导出主机列表到文件
+
     根据 Target 类型自动决定导出内容：
     - DOMAIN: 从 Subdomain 表导出子域名
     - IP: 直接写入 target.name
     - CIDR: 展开 CIDR 范围内的所有 IP
-    
+
     Args:
         target_id: 目标 ID
         port_scan_dir: 端口扫描目录
-        
+
     Returns:
-        tuple: (targets_file, target_count, target_type)
+        tuple: (hosts_file, host_count, target_type)
     """
-    logger.info("Step 1: 导出扫描目标列表")
-    
-    targets_file = str(port_scan_dir / 'targets.txt')
-    export_result = export_scan_targets_task(
+    logger.info("Step 1: 导出主机列表")
+
+    hosts_file = str(port_scan_dir / 'hosts.txt')
+    export_result = export_hosts_task(
         target_id=target_id,
-        output_file=targets_file,
-        batch_size=1000  # 每次读取 1000 条，优化内存占用
+        output_file=hosts_file,
     )
-    
-    target_count = export_result['total_count']
+
+    host_count = export_result['total_count']
     target_type = export_result.get('target_type', 'unknown')
-    
+
     logger.info(
-        "✓ 扫描目标导出完成 - 类型: %s, 文件: %s, 数量: %d",
-        target_type,
-        export_result['output_file'],
-        target_count
+        "✓ 主机列表导出完成 - 类型: %s, 文件: %s, 数量: %d",
+        target_type, export_result['output_file'], host_count
     )
-    
-    if target_count == 0:
-        logger.warning("目标下没有可扫描的地址，无法执行端口扫描")
-    
-    return export_result['output_file'], target_count, target_type
+
+    if host_count == 0:
+        logger.warning("目标下没有可扫描的主机，无法执行端口扫描")
+
+    return export_result['output_file'], host_count, target_type
 
 
 def _run_scans_sequentially(
@@ -208,7 +180,7 @@ def _run_scans_sequentially(
 ) -> tuple[dict, int, list, list]:
     """
     串行执行端口扫描任务
-    
+
     Args:
         enabled_tools: 已启用的工具配置字典
         domains_file: 域名文件路径
@@ -216,125 +188,109 @@ def _run_scans_sequentially(
         scan_id: 扫描任务 ID
         target_id: 目标 ID
         target_name: 目标名称（用于错误日志）
-        
+
     Returns:
         tuple: (tool_stats, processed_records, successful_tool_names, failed_tools)
-        注意：端口扫描是流式输出，不生成结果文件
-        
-    Raises:
-        RuntimeError: 所有工具均失败
     """
-    # ==================== 构建命令并串行执行 ====================
-    
     tool_stats = {}
     processed_records = 0
-    failed_tools = []      # 记录失败的工具（含原因）
-    
-    # for循环执行工具：按顺序串行运行每个启用的端口扫描工具
+    failed_tools = []
+
     for tool_name, tool_config in enabled_tools.items():
-        # 1. 构建完整命令（变量替换）
+        # 构建命令
         try:
             command = build_scan_command(
                 tool_name=tool_name,
                 scan_type='port_scan',
-                command_params={
-                    'domains_file': domains_file  # 对应 {domains_file}
-                },
-                tool_config=tool_config     #yaml的工具配置
+                command_params={'domains_file': domains_file},
+                tool_config=tool_config
             )
         except Exception as e:
-            reason = f"命令构建失败: {str(e)}"
-            logger.error(f"构建 {tool_name} 命令失败: {e}")
+            reason = f"命令构建失败: {e}"
+            logger.error("构建 %s 命令失败: %s", tool_name, e)
             failed_tools.append({'tool': tool_name, 'reason': reason})
             continue
-        
-        # 2. 获取超时时间（支持 'auto' 动态计算）
+
+        # 获取超时时间
         config_timeout = tool_config['timeout']
         if config_timeout == 'auto':
-            # 动态计算超时时间
-            config_timeout = calculate_port_scan_timeout(
-                tool_config=tool_config,
-                file_path=str(domains_file)
-            )
-            logger.info(f"✓ 工具 {tool_name} 动态计算 timeout: {config_timeout}秒")
-        
-        # 2.1 生成日志文件路径
-        from datetime import datetime
+            config_timeout = calculate_port_scan_timeout(tool_config, str(domains_file))
+            logger.info("✓ 工具 %s 动态计算 timeout: %d秒", tool_name, config_timeout)
+
+        # 生成日志文件路径
         timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
         log_file = port_scan_dir / f"{tool_name}_{timestamp}.log"
-        
-        # 3. 执行扫描任务
+
         logger.info("开始执行 %s 扫描（超时: %d秒）...", tool_name, config_timeout)
-        
+        user_log(scan_id, "port_scan", f"Running {tool_name}: {command}")
+
+        # 执行扫描任务
         try:
-            # 直接调用 task（串行执行）
-            # 注意：端口扫描是流式输出到 stdout，不使用 output_file
             result = run_and_stream_save_ports_task(
                 cmd=command,
-                tool_name=tool_name,  # 工具名称
+                tool_name=tool_name,
                 scan_id=scan_id,
                 target_id=target_id,
                 cwd=str(port_scan_dir),
                 shell=True,
                 batch_size=1000,
                 timeout=config_timeout,
-                log_file=str(log_file)  # 新增：日志文件路径
+                log_file=str(log_file)
             )
-            
+
             tool_stats[tool_name] = {
                 'command': command,
                 'result': result,
                 'timeout': config_timeout
             }
-            processed_records += result.get('processed_records', 0)
-            logger.info(
-                "✓ 工具 %s 流式处理完成 - 记录数: %d",
-                tool_name, result.get('processed_records', 0)
-            )
-            
-        except subprocess.TimeoutExpired as exc:
-            # 超时异常单独处理
-            # 注意：流式处理任务超时时，已解析的数据已保存到数据库
-            reason = f"执行超时（配置: {config_timeout}秒）"
+            tool_records = result.get('processed_records', 0)
+            processed_records += tool_records
+            logger.info("✓ 工具 %s 流式处理完成 - 记录数: %d", tool_name, tool_records)
+            user_log(scan_id, "port_scan", f"{tool_name} completed: found {tool_records} ports")
+
+        except subprocess.TimeoutExpired:
+            reason = f"timeout after {config_timeout}s"
             failed_tools.append({'tool': tool_name, 'reason': reason})
             logger.warning(
                 "⚠️ 工具 %s 执行超时 - 超时配置: %d秒\n"
                 "注意：超时前已解析的端口数据已保存到数据库，但扫描未完全完成。",
                 tool_name, config_timeout
             )
+            user_log(scan_id, "port_scan", f"{tool_name} failed: {reason}", "error")
         except Exception as exc:
-            # 其他异常
-            failed_tools.append({'tool': tool_name, 'reason': str(exc)})
+            reason = str(exc)
+            failed_tools.append({'tool': tool_name, 'reason': reason})
             logger.error("工具 %s 执行失败: %s", tool_name, exc, exc_info=True)
-    
+            user_log(scan_id, "port_scan", f"{tool_name} failed: {reason}", "error")
+
     if failed_tools:
         logger.warning(
             "以下扫描工具执行失败: %s",
             ', '.join([f['tool'] for f in failed_tools])
         )
-    
+
     if not tool_stats:
         error_details = "; ".join([f"{f['tool']}: {f['reason']}" for f in failed_tools])
         logger.warning("所有端口扫描工具均失败 - 目标: %s, 失败工具: %s", target_name, error_details)
-        # 返回空结果，不抛出异常，让扫描继续
         return {}, 0, [], failed_tools
-    
-    # 动态计算成功的工具列表
-    successful_tool_names = [name for name in enabled_tools.keys() 
-                              if name not in [f['tool'] for f in failed_tools]]
-    
+
+    successful_tool_names = [
+        name for name in enabled_tools
+        if name not in [f['tool'] for f in failed_tools]
+    ]
+
     logger.info(
         "✓ 串行端口扫描执行完成 - 成功: %d/%d (成功: %s, 失败: %s)",
         len(tool_stats), len(enabled_tools),
         ', '.join(successful_tool_names) if successful_tool_names else '无',
         ', '.join([f['tool'] for f in failed_tools]) if failed_tools else '无'
     )
-    
+
     return tool_stats, processed_records, successful_tool_names, failed_tools
 
 
 @flow(
-    name="port_scan", 
+    name="port_scan",
     log_prints=True,
     on_running=[on_scan_flow_running],
     on_completion=[on_scan_flow_completed],
@@ -349,19 +305,19 @@ def port_scan_flow(
 ) -> dict:
     """
     端口扫描 Flow
-    
+
     主要功能：
         1. 扫描目标域名/IP 的开放端口
         2. 保存 host + ip + port 三元映射到 HostPortMapping 表
-    
+
     输出资产：
         - HostPortMapping：主机端口映射（host + ip + port 三元组）
-    
+
     工作流程：
         Step 0: 创建工作目录
         Step 1: 导出域名列表到文件（供扫描工具使用）
         Step 2: 解析配置，获取启用的工具
-        Step 3: 串行执行扫描工具，运行端口扫描工具并实时解析输出到数据库（→ HostPortMapping）
+        Step 3: 串行执行扫描工具，运行端口扫描工具并实时解析输出到数据库
 
     Args:
         scan_id: 扫描任务 ID
@@ -371,35 +327,15 @@ def port_scan_flow(
         enabled_tools: 启用的工具配置字典
 
     Returns:
-        dict: {
-            'success': bool,
-            'scan_id': int,
-            'target': str,
-            'scan_workspace_dir': str,
-            'domains_file': str,
-            'domain_count': int,
-            'processed_records': int,
-            'executed_tasks': list,
-            'tool_stats': {
-                'total': int,                    # 总工具数
-                'successful': int,               # 成功工具数
-                'failed': int,                   # 失败工具数
-                'successful_tools': list[str],   # 成功工具列表 ['naabu_active']
-                'failed_tools': list[dict],      # 失败工具列表 [{'tool': 'naabu_passive', 'reason': '超时'}]
-                'details': dict                  # 详细执行结果（保留向后兼容）
-            }
-        }
+        dict: 扫描结果
 
     Raises:
         ValueError: 配置错误
         RuntimeError: 执行失败
-    
-    Note:
-        端口扫描工具（如 naabu）会解析域名获取 IP，输出 host + ip + port 三元组。
-        同一 host 可能对应多个 IP（CDN、负载均衡），因此使用三元映射表存储。
     """
     try:
-        # 参数验证
+        wait_for_system_load(context="port_scan_flow")
+
         if scan_id is None:
             raise ValueError("scan_id 不能为空")
         if not target_name:
@@ -410,35 +346,33 @@ def port_scan_flow(
             raise ValueError("scan_workspace_dir 不能为空")
         if not enabled_tools:
             raise ValueError("enabled_tools 不能为空")
-        
+
         logger.info(
-            "="*60 + "\n" +
-            "开始端口扫描\n" +
-            f"  Scan ID: {scan_id}\n" +
-            f"  Target: {target_name}\n" +
-            f"  Workspace: {scan_workspace_dir}\n" +
-            "="*60
+            "开始端口扫描 - Scan ID: %s, Target: %s, Workspace: %s",
+            scan_id, target_name, scan_workspace_dir
         )
-        
+        user_log(scan_id, "port_scan", "Starting port scan")
+
         # Step 0: 创建工作目录
         from apps.scan.utils import setup_scan_directory
         port_scan_dir = setup_scan_directory(scan_workspace_dir, 'port_scan')
-        
-        # Step 1: 导出扫描目标列表到文件（根据 Target 类型自动决定内容）
-        targets_file, target_count, target_type = _export_scan_targets(target_id, port_scan_dir)
-        
-        if target_count == 0:
-            logger.warning("目标下没有可扫描的地址，跳过端口扫描")
+
+        # Step 1: 导出主机列表
+        hosts_file, host_count, target_type = _export_hosts(target_id, port_scan_dir)
+
+        if host_count == 0:
+            logger.warning("跳过端口扫描：没有主机可扫描 - Scan ID: %s", scan_id)
+            user_log(scan_id, "port_scan", "Skipped: no hosts to scan", "warning")
             return {
                 'success': True,
                 'scan_id': scan_id,
                 'target': target_name,
                 'scan_workspace_dir': scan_workspace_dir,
-                'targets_file': targets_file,
-                'target_count': 0,
+                'hosts_file': hosts_file,
+                'host_count': 0,
                 'target_type': target_type,
                 'processed_records': 0,
-                'executed_tasks': ['export_scan_targets'],
+                'executed_tasks': ['export_hosts'],
                 'tool_stats': {
                     'total': 0,
                     'successful': 0,
@@ -448,38 +382,35 @@ def port_scan_flow(
                     'details': {}
                 }
             }
-        
+
         # Step 2: 工具配置信息
         logger.info("Step 2: 工具配置信息")
-        logger.info(
-            "✓ 启用工具: %s",
-            ', '.join(enabled_tools.keys())
-        )
-        
+        logger.info("✓ 启用工具: %s", ', '.join(enabled_tools.keys()))
+
         # Step 3: 串行执行扫描工具
         logger.info("Step 3: 串行执行扫描工具")
         tool_stats, processed_records, successful_tool_names, failed_tools = _run_scans_sequentially(
             enabled_tools=enabled_tools,
-            domains_file=targets_file,  # 现在是 targets_file，兼容原参数名
+            domains_file=hosts_file,
             port_scan_dir=port_scan_dir,
             scan_id=scan_id,
             target_id=target_id,
             target_name=target_name
         )
-        
-        logger.info("="*60 + "\n✓ 端口扫描完成\n" + "="*60)
-        
-        # 动态生成已执行的任务列表
-        executed_tasks = ['export_scan_targets', 'parse_config']
-        executed_tasks.extend([f'run_and_stream_save_ports ({tool})' for tool in tool_stats.keys()])
-        
+
+        logger.info("✓ 端口扫描完成 - 发现端口: %d", processed_records)
+        user_log(scan_id, "port_scan", f"port_scan completed: found {processed_records} ports")
+
+        executed_tasks = ['export_hosts', 'parse_config']
+        executed_tasks.extend([f'run_and_stream_save_ports ({tool})' for tool in tool_stats])
+
         return {
             'success': True,
             'scan_id': scan_id,
             'target': target_name,
             'scan_workspace_dir': scan_workspace_dir,
-            'targets_file': targets_file,
-            'target_count': target_count,
+            'hosts_file': hosts_file,
+            'host_count': host_count,
             'target_type': target_type,
             'processed_records': processed_records,
             'executed_tasks': executed_tasks,
@@ -488,8 +419,8 @@ def port_scan_flow(
                 'successful': len(successful_tool_names),
                 'failed': len(failed_tools),
                 'successful_tools': successful_tool_names,
-                'failed_tools': failed_tools,  # [{'tool': 'naabu_active', 'reason': '超时'}]
-                'details': tool_stats  # 详细结果（保留向后兼容）
+                'failed_tools': failed_tools,
+                'details': tool_stats
             }
         }
 

backend/apps/scan/flows/screenshot_flow.py

@@ -0,0 +1,208 @@
+"""
+截图 Flow
+
+负责编排截图的完整流程：
+1. 从数据库获取 URL 列表（websites 和/或 endpoints）
+2. 批量截图并保存快照
+3. 同步到资产表
+
+支持两种模式：
+1. 传统模式（向后兼容）：使用 target_id 从数据库获取 URL
+2. Provider 模式：使用 TargetProvider 从任意数据源获取 URL
+"""
+
+import logging
+from typing import Optional
+
+from prefect import flow
+
+from apps.scan.handlers.scan_flow_handlers import (
+    on_scan_flow_completed,
+    on_scan_flow_failed,
+    on_scan_flow_running,
+)
+from apps.scan.providers import TargetProvider
+from apps.scan.services.target_export_service import DataSource, get_urls_with_fallback
+from apps.scan.tasks.screenshot import capture_screenshots_task
+from apps.scan.utils import user_log, wait_for_system_load
+
+logger = logging.getLogger(__name__)
+
+# URL 来源到 DataSource 的映射
+_SOURCE_MAPPING = {
+    'websites': DataSource.WEBSITE,
+    'endpoints': DataSource.ENDPOINT,
+}
+
+
+def _parse_screenshot_config(enabled_tools: dict) -> dict:
+    """解析截图配置"""
+    playwright_config = enabled_tools.get('playwright', {})
+    return {
+        'concurrency': playwright_config.get('concurrency', 5),
+        'url_sources': playwright_config.get('url_sources', ['websites'])
+    }
+
+
+def _map_url_sources_to_data_sources(url_sources: list[str]) -> list[str]:
+    """将配置中的 url_sources 映射为 DataSource 常量"""
+    sources = []
+    for source in url_sources:
+        if source in _SOURCE_MAPPING:
+            sources.append(_SOURCE_MAPPING[source])
+        else:
+            logger.warning("未知的 URL 来源: %s，跳过", source)
+
+    # 添加默认回退（从 subdomain 构造）
+    sources.append(DataSource.DEFAULT)
+    return sources
+
+
+def _collect_urls_from_provider(provider: TargetProvider) -> tuple[list[str], str, list[str]]:
+    """从 Provider 收集 URL"""
+    logger.info("使用 Provider 模式获取 URL - Provider: %s", type(provider).__name__)
+    urls = list(provider.iter_urls())
+
+    blacklist_filter = provider.get_blacklist_filter()
+    if blacklist_filter:
+        urls = [url for url in urls if blacklist_filter.is_allowed(url)]
+
+    return urls, 'provider', ['provider']
+
+
+def _collect_urls_from_database(
+    target_id: int,
+    url_sources: list[str]
+) -> tuple[list[str], str, list[str]]:
+    """从数据库收集 URL（带黑名单过滤和回退）"""
+    data_sources = _map_url_sources_to_data_sources(url_sources)
+    result = get_urls_with_fallback(target_id, sources=data_sources)
+    return result['urls'], result['source'], result['tried_sources']
+
+
+def _build_empty_result(scan_id: int, target_name: str) -> dict:
+    """构建空结果"""
+    return {
+        'success': True,
+        'scan_id': scan_id,
+        'target': target_name,
+        'total_urls': 0,
+        'successful': 0,
+        'failed': 0,
+        'synced': 0
+    }
+
+
+@flow(
+    name="screenshot",
+    log_prints=True,
+    on_running=[on_scan_flow_running],
+    on_completion=[on_scan_flow_completed],
+    on_failure=[on_scan_flow_failed],
+)
+def screenshot_flow(
+    scan_id: int,
+    target_name: str,
+    target_id: int,
+    scan_workspace_dir: str,
+    enabled_tools: dict,
+    provider: Optional[TargetProvider] = None
+) -> dict:
+    """
+    截图 Flow
+
+    支持两种模式：
+    1. 传统模式（向后兼容）：使用 target_id 从数据库获取 URL
+    2. Provider 模式：使用 TargetProvider 从任意数据源获取 URL
+
+    Args:
+        scan_id: 扫描任务 ID
+        target_name: 目标名称
+        target_id: 目标 ID
+        scan_workspace_dir: 扫描工作空间目录
+        enabled_tools: 启用的工具配置
+        provider: TargetProvider 实例（新模式，可选）
+
+    Returns:
+        截图结果字典
+    """
+    try:
+        # 负载检查：等待系统资源充足
+        wait_for_system_load(context="screenshot_flow")
+
+        mode = 'Provider' if provider else 'Legacy'
+        logger.info(
+            "开始截图扫描 - Scan ID: %s, Target: %s, Mode: %s",
+            scan_id, target_name, mode
+        )
+        user_log(scan_id, "screenshot", "Starting screenshot capture")
+
+        # Step 1: 解析配置
+        config = _parse_screenshot_config(enabled_tools)
+        concurrency = config['concurrency']
+        logger.info("截图配置 - 并发: %d, URL来源: %s", concurrency, config['url_sources'])
+
+        # Step 2: 收集 URL 列表
+        if provider is not None:
+            urls, source_info, tried_sources = _collect_urls_from_provider(provider)
+        else:
+            urls, source_info, tried_sources = _collect_urls_from_database(
+                target_id, config['url_sources']
+            )
+
+        logger.info(
+            "URL 收集完成 - 来源: %s, 数量: %d, 尝试过: %s",
+            source_info, len(urls), tried_sources
+        )
+
+        if not urls:
+            logger.warning("没有可截图的 URL，跳过截图任务")
+            user_log(scan_id, "screenshot", "Skipped: no URLs to capture", "warning")
+            return _build_empty_result(scan_id, target_name)
+
+        user_log(
+            scan_id, "screenshot",
+            f"Found {len(urls)} URLs to capture (source: {source_info})"
+        )
+
+        # Step 3: 批量截图
+        logger.info("批量截图 - %d 个 URL", len(urls))
+        capture_result = capture_screenshots_task(
+            urls=urls,
+            scan_id=scan_id,
+            target_id=target_id,
+            config={'concurrency': concurrency}
+        )
+
+        # Step 4: 同步到资产表
+        logger.info("同步截图到资产表")
+        from apps.asset.services.screenshot_service import ScreenshotService
+        synced = ScreenshotService().sync_screenshots_to_asset(scan_id, target_id)
+
+        total = capture_result['total']
+        successful = capture_result['successful']
+        failed = capture_result['failed']
+
+        logger.info(
+            "✓ 截图完成 - 总数: %d, 成功: %d, 失败: %d, 同步: %d",
+            total, successful, failed, synced
+        )
+        user_log(
+            scan_id, "screenshot",
+            f"Screenshot completed: {successful}/{total} captured, {synced} synced"
+        )
+
+        return {
+            'success': True,
+            'scan_id': scan_id,
+            'target': target_name,
+            'total_urls': total,
+            'successful': successful,
+            'failed': failed,
+            'synced': synced
+        }
+
+    except Exception:
+        logger.exception("截图 Flow 失败")
+        user_log(scan_id, "screenshot", "Screenshot failed", "error")
+        raise

backend/apps/scan/flows/site_scan_flow.py

@@ -1,4 +1,3 @@
-
 """
 站点扫描 Flow
 
@@ -11,295 +10,319 @@
 - 配置由 YAML 解析
 """
 
-# Django 环境初始化（导入即生效）
-from apps.common.prefect_django_setup import setup_django_for_prefect
-
 import logging
-import os
 import subprocess
+from dataclasses import dataclass
+from datetime import datetime
 from pathlib import Path
-from typing import Callable
+from typing import Optional
+
 from prefect import flow
-from apps.scan.tasks.site_scan import export_site_urls_task, run_and_stream_save_websites_task
+
+# Django 环境初始化（导入即生效）
+from apps.common.prefect_django_setup import setup_django_for_prefect  # noqa: F401
 from apps.scan.handlers.scan_flow_handlers import (
-    on_scan_flow_running,
     on_scan_flow_completed,
     on_scan_flow_failed,
+    on_scan_flow_running,
 )
-from apps.scan.utils import config_parser, build_scan_command
+from apps.scan.tasks.site_scan import (
+    export_site_urls_task,
+    run_and_stream_save_websites_task,
+)
+from apps.scan.utils import build_scan_command, user_log, wait_for_system_load
 
 logger = logging.getLogger(__name__)
 
 
-def calculate_timeout_by_line_count(
-    tool_config: dict,
-    file_path: str, 
-    base_per_time: int = 1,
-    min_timeout: int = 60
-) -> int:
-    """
-    根据文件行数计算 timeout
-    
-    使用 wc -l 统计文件行数，根据行数和每行基础时间计算 timeout
-    
-    Args:
-        tool_config: 工具配置字典（此函数未使用，但保持接口一致性）
-        file_path: 要统计行数的文件路径
-        base_per_time: 每行的基础时间（秒），默认1秒
-        min_timeout: 最小超时时间（秒），默认60秒
-    
-    Returns:
-        int: 计算出的超时时间（秒），不低于 min_timeout
-    
-    Example:
-        timeout = calculate_timeout_by_line_count(
-            tool_config={},
-            file_path='/path/to/urls.txt', 
-            base_per_time=2
-        )
-    """
+@dataclass
+class ScanContext:
+    """扫描上下文，封装扫描参数"""
+    scan_id: int
+    target_id: int
+    target_name: str
+    site_scan_dir: Path
+    urls_file: str
+    total_urls: int
+
+
+def _count_file_lines(file_path: str) -> int:
+    """使用 wc -l 统计文件行数"""
     try:
-        # 使用 wc -l 快速统计行数
         result = subprocess.run(
             ['wc', '-l', file_path],
             capture_output=True,
             text=True,
             check=True
         )
-        # wc -l 输出格式：行数 + 空格 + 文件名
-        line_count = int(result.stdout.strip().split()[0])
-        
-        # 计算 timeout：行数 × 每行基础时间，不低于最小值
-        timeout = max(line_count * base_per_time, min_timeout)
-        
-        logger.info(
-            f"timeout 自动计算: 文件={file_path}, "
-            f"行数={line_count}, 每行时间={base_per_time}秒, 最小值={min_timeout}秒, timeout={timeout}秒"
-        )
-        
-        return timeout
-        
-    except Exception as e:
-        # 如果 wc -l 失败，使用默认值
-        logger.warning(f"wc -l 计算行数失败: {e}，使用默认 timeout: {min_timeout}秒")
-        return min_timeout
+        return int(result.stdout.strip().split()[0])
+    except (subprocess.CalledProcessError, ValueError, IndexError) as e:
+        logger.warning("wc -l 计算行数失败: %s，返回 0", e)
+        return 0
 
 
+def _calculate_timeout_by_line_count(
+    file_path: str,
+    base_per_time: int = 1,
+    min_timeout: int = 60
+) -> int:
+    """
+    根据文件行数计算 timeout
+
+    Args:
+        file_path: 要统计行数的文件路径
+        base_per_time: 每行的基础时间（秒），默认1秒
+        min_timeout: 最小超时时间（秒），默认60秒
+
+    Returns:
+        int: 计算出的超时时间（秒），不低于 min_timeout
+    """
+    line_count = _count_file_lines(file_path)
+    timeout = max(line_count * base_per_time, min_timeout)
+
+    logger.info(
+        "timeout 自动计算: 文件=%s, 行数=%d, 每行时间=%d秒, timeout=%d秒",
+        file_path, line_count, base_per_time, timeout
+    )
+    return timeout
 
 
-
-def _export_site_urls(target_id: int, site_scan_dir: Path, target_name: str = None) -> tuple[str, int, int]:
+def _export_site_urls(
+    target_id: int,
+    site_scan_dir: Path
+) -> tuple[str, int, int]:
     """
     导出站点 URL 到文件
-    
+
     Args:
         target_id: 目标 ID
         site_scan_dir: 站点扫描目录
-        target_name: 目标名称（用于懒加载时写入默认值）
-        
+
     Returns:
         tuple: (urls_file, total_urls, association_count)
-        
-    Raises:
-        ValueError: URL 数量为 0
     """
     logger.info("Step 1: 导出站点URL列表")
-    
+
     urls_file = str(site_scan_dir / 'site_urls.txt')
     export_result = export_site_urls_task(
         target_id=target_id,
         output_file=urls_file,
-        batch_size=1000  # 每次处理1000个子域名
+        batch_size=1000
     )
-    
+
     total_urls = export_result['total_urls']
-    association_count = export_result['association_count']  # 主机端口关联数
-    
+    association_count = export_result['association_count']
+
     logger.info(
         "✓ 站点URL导出完成 - 文件: %s, URL数量: %d, 关联数: %d",
-        export_result['output_file'],
-        total_urls,
-        association_count
+        export_result['output_file'], total_urls, association_count
     )
-    
+
     if total_urls == 0:
         logger.warning("目标下没有可用的站点URL，无法执行站点扫描")
-        # 不抛出异常，由上层决定如何处理
-        # raise ValueError("目标下没有可用的站点URL，无法执行站点扫描")
-    
+
     return export_result['output_file'], total_urls, association_count
 
 
+def _get_tool_timeout(tool_config: dict, urls_file: str) -> int:
+    """获取工具超时时间（支持 'auto' 动态计算）"""
+    config_timeout = tool_config.get('timeout', 300)
+
+    if config_timeout == 'auto':
+        return _calculate_timeout_by_line_count(urls_file, base_per_time=1)
+
+    dynamic_timeout = _calculate_timeout_by_line_count(urls_file, base_per_time=1)
+    return max(dynamic_timeout, config_timeout)
+
+
+def _execute_single_tool(
+    tool_name: str,
+    tool_config: dict,
+    ctx: ScanContext
+) -> Optional[dict]:
+    """
+    执行单个扫描工具
+
+    Returns:
+        成功返回结果字典，失败返回 None
+    """
+    # 构建命令
+    try:
+        command = build_scan_command(
+            tool_name=tool_name,
+            scan_type='site_scan',
+            command_params={'url_file': ctx.urls_file},
+            tool_config=tool_config
+        )
+    except (ValueError, KeyError) as e:
+        logger.error("构建 %s 命令失败: %s", tool_name, e)
+        return None
+
+    timeout = _get_tool_timeout(tool_config, ctx.urls_file)
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    log_file = ctx.site_scan_dir / f"{tool_name}_{timestamp}.log"
+
+    logger.info(
+        "开始执行 %s 站点扫描 - URL数: %d, 超时: %ds",
+        tool_name, ctx.total_urls, timeout
+    )
+    user_log(ctx.scan_id, "site_scan", f"Running {tool_name}: {command}")
+
+    try:
+        result = run_and_stream_save_websites_task(
+            cmd=command,
+            tool_name=tool_name,
+            scan_id=ctx.scan_id,
+            target_id=ctx.target_id,
+            cwd=str(ctx.site_scan_dir),
+            shell=True,
+            timeout=timeout,
+            log_file=str(log_file)
+        )
+
+        tool_created = result.get('created_websites', 0)
+        skipped = result.get('skipped_no_subdomain', 0) + result.get('skipped_failed', 0)
+
+        logger.info(
+            "✓ 工具 %s 完成 - 处理: %d, 创建: %d, 跳过: %d",
+            tool_name, result.get('processed_records', 0), tool_created, skipped
+        )
+        user_log(
+            ctx.scan_id, "site_scan",
+            f"{tool_name} completed: found {tool_created} websites"
+        )
+
+        return {'command': command, 'result': result, 'timeout': timeout}
+
+    except subprocess.TimeoutExpired:
+        logger.warning(
+            "⚠️ 工具 %s 执行超时 - 超时配置: %d秒 (超时前数据已保存)",
+            tool_name, timeout
+        )
+        user_log(
+            ctx.scan_id, "site_scan",
+            f"{tool_name} failed: timeout after {timeout}s", "error"
+        )
+    except (OSError, RuntimeError) as exc:
+        logger.error("工具 %s 执行失败: %s", tool_name, exc, exc_info=True)
+        user_log(ctx.scan_id, "site_scan", f"{tool_name} failed: {exc}", "error")
+
+    return None
+
+
 def _run_scans_sequentially(
     enabled_tools: dict,
-    urls_file: str,
-    total_urls: int,
-    site_scan_dir: Path,
-    scan_id: int,
-    target_id: int,
-    target_name: str
+    ctx: ScanContext
 ) -> tuple[dict, int, list, list]:
     """
     串行执行站点扫描任务
-    
-    Args:
-        enabled_tools: 已启用的工具配置字典
-        urls_file: URL 文件路径
-        total_urls: URL 总数
-        site_scan_dir: 站点扫描目录
-        scan_id: 扫描任务 ID
-        target_id: 目标 ID
-        target_name: 目标名称（用于错误日志）
-        
+
     Returns:
-        tuple: (tool_stats, processed_records, successful_tool_names, failed_tools)
-        
-    Raises:
-        RuntimeError: 所有工具均失败
+        tuple: (tool_stats, processed_records, successful_tools, failed_tools)
     """
     tool_stats = {}
     processed_records = 0
     failed_tools = []
-    
+
     for tool_name, tool_config in enabled_tools.items():
-        # 1. 构建完整命令（变量替换）
-        try:
-            command = build_scan_command(
-                tool_name=tool_name,
-                scan_type='site_scan',
-                command_params={
-                    'url_file': urls_file
-                },
-                tool_config=tool_config
-            )
-        except Exception as e:
-            reason = f"命令构建失败: {str(e)}"
-            logger.error(f"构建 {tool_name} 命令失败: {e}")
-            failed_tools.append({'tool': tool_name, 'reason': reason})
-            continue
-        
-        # 2. 获取超时时间（支持 'auto' 动态计算）
-        config_timeout = tool_config.get('timeout', 300)
-        if config_timeout == 'auto':
-            # 动态计算超时时间
-            timeout = calculate_timeout_by_line_count(tool_config, urls_file, base_per_time=1)
-            logger.info(f"✓ 工具 {tool_name} 动态计算 timeout: {timeout}秒")
+        result = _execute_single_tool(tool_name, tool_config, ctx)
+
+        if result:
+            tool_stats[tool_name] = result
+            processed_records += result['result'].get('processed_records', 0)
         else:
-            # 使用配置的超时时间和动态计算的较大值
-            dynamic_timeout = calculate_timeout_by_line_count(tool_config, urls_file, base_per_time=1)
-            timeout = max(dynamic_timeout, config_timeout)
-        
-        # 2.1 生成日志文件路径（类似端口扫描）
-        from datetime import datetime
-        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-        log_file = site_scan_dir / f"{tool_name}_{timestamp}.log"
-        
-        logger.info(
-            "开始执行 %s 站点扫描 - URL数: %d, 最终超时: %ds",
-            tool_name, total_urls, timeout
-        )
-        
-        # 3. 执行扫描任务
-        try:
-            # 流式执行扫描并实时保存结果
-            result = run_and_stream_save_websites_task(
-                cmd=command,
-                tool_name=tool_name,
-                scan_id=scan_id,
-                target_id=target_id,
-                cwd=str(site_scan_dir),
-                shell=True,
-                timeout=timeout,
-                log_file=str(log_file)
-            )
-            
-            tool_stats[tool_name] = {
-                'command': command,
-                'result': result,
-                'timeout': timeout
-            }
-            processed_records += result.get('processed_records', 0)
-            
-            logger.info(
-                "✓ 工具 %s 流式处理完成 - 处理记录: %d, 创建站点: %d, 跳过: %d",
-                tool_name,
-                result.get('processed_records', 0),
-                result.get('created_websites', 0),
-                result.get('skipped_no_subdomain', 0) + result.get('skipped_failed', 0)
-            )
-            
-        except subprocess.TimeoutExpired as exc:
-            # 超时异常单独处理
-            reason = f"执行超时（配置: {timeout}秒）"
-            failed_tools.append({'tool': tool_name, 'reason': reason})
-            logger.warning(
-                "⚠️ 工具 %s 执行超时 - 超时配置: %d秒\n"
-                "注意：超时前已解析的站点数据已保存到数据库，但扫描未完全完成。",
-                tool_name, timeout
-            )
-        except Exception as exc:
-            # 其他异常
-            failed_tools.append({'tool': tool_name, 'reason': str(exc)})
-            logger.error("工具 %s 执行失败: %s", tool_name, exc, exc_info=True)
-    
+            failed_tools.append({'tool': tool_name, 'reason': '执行失败'})
+
     if failed_tools:
         logger.warning(
             "以下扫描工具执行失败: %s",
-            ', '.join([f['tool'] for f in failed_tools])
+            ', '.join(f['tool'] for f in failed_tools)
         )
-    
+
     if not tool_stats:
-        error_details = "; ".join([f"{f['tool']}: {f['reason']}" for f in failed_tools])
-        logger.warning("所有站点扫描工具均失败 - 目标: %s, 失败工具: %s", target_name, error_details)
-        # 返回空结果，不抛出异常，让扫描继续
+        logger.warning(
+            "所有站点扫描工具均失败 - 目标: %s", ctx.target_name
+        )
         return {}, 0, [], failed_tools
-    
-    # 动态计算成功的工具列表
-    successful_tool_names = [name for name in enabled_tools.keys() 
-                              if name not in [f['tool'] for f in failed_tools]]
-    
+
+    successful_tools = [
+        name for name in enabled_tools
+        if name not in {f['tool'] for f in failed_tools}
+    ]
+
     logger.info(
-        "✓ 串行站点扫描执行完成 - 成功: %d/%d (成功: %s, 失败: %s)",
-        len(tool_stats), len(enabled_tools),
-        ', '.join(successful_tool_names) if successful_tool_names else '无',
-        ', '.join([f['tool'] for f in failed_tools]) if failed_tools else '无'
+        "✓ 站点扫描执行完成 - 成功: %d/%d",
+        len(tool_stats), len(enabled_tools)
     )
-    
-    return tool_stats, processed_records, successful_tool_names, failed_tools
+
+    return tool_stats, processed_records, successful_tools, failed_tools
 
 
-def calculate_timeout(url_count: int, base: int = 600, per_url: int = 1) -> int:
-    """
-    根据 URL 数量动态计算扫描超时时间
+def _build_empty_result(
+    scan_id: int,
+    target_name: str,
+    scan_workspace_dir: str,
+    urls_file: str,
+    association_count: int
+) -> dict:
+    """构建空结果（无 URL 可扫描时）"""
+    return {
+        'success': True,
+        'scan_id': scan_id,
+        'target': target_name,
+        'scan_workspace_dir': scan_workspace_dir,
+        'urls_file': urls_file,
+        'total_urls': 0,
+        'association_count': association_count,
+        'processed_records': 0,
+        'created_websites': 0,
+        'skipped_no_subdomain': 0,
+        'skipped_failed': 0,
+        'executed_tasks': ['export_site_urls'],
+        'tool_stats': {
+            'total': 0,
+            'successful': 0,
+            'failed': 0,
+            'successful_tools': [],
+            'failed_tools': [],
+            'details': {}
+        }
+    }
 
-    规则：
-    - 基础时间：默认 600 秒（10 分钟）
-    - 每个 URL 额外增加：默认 1 秒
 
-    Args:
-        url_count: URL 数量，必须为正整数
-        base: 基础超时时间（秒），默认 600
-        per_url: 每个 URL 增加的时间（秒），默认 1
+def _aggregate_tool_results(tool_stats: dict) -> tuple[int, int, int]:
+    """汇总工具结果"""
+    total_created = sum(
+        s['result'].get('created_websites', 0) for s in tool_stats.values()
+    )
+    total_skipped_no_subdomain = sum(
+        s['result'].get('skipped_no_subdomain', 0) for s in tool_stats.values()
+    )
+    total_skipped_failed = sum(
+        s['result'].get('skipped_failed', 0) for s in tool_stats.values()
+    )
+    return total_created, total_skipped_no_subdomain, total_skipped_failed
 
-    Returns:
-        int: 计算得到的超时时间（秒），不超过 max_timeout
 
-    Raises:
-        ValueError: 当 url_count 为负数或 0 时抛出异常
-    """
-    if url_count < 0:
-        raise ValueError(f"URL数量不能为负数: {url_count}")
-    if url_count == 0:
-        raise ValueError("URL数量不能为0")
-
-    timeout = base + int(url_count * per_url)
-    
-    # 不设置上限，由调用方根据需要控制
-    return timeout
+def _validate_flow_params(
+    scan_id: int,
+    target_name: str,
+    target_id: int,
+    scan_workspace_dir: str
+) -> None:
+    """验证 Flow 参数"""
+    if scan_id is None:
+        raise ValueError("scan_id 不能为空")
+    if not target_name:
+        raise ValueError("target_name 不能为空")
+    if target_id is None:
+        raise ValueError("target_id 不能为空")
+    if not scan_workspace_dir:
+        raise ValueError("scan_workspace_dir 不能为空")
 
 
 @flow(
-    name="site_scan", 
+    name="site_scan",
     log_prints=True,
     on_running=[on_scan_flow_running],
     on_completion=[on_scan_flow_completed],
@@ -314,135 +337,83 @@ def site_scan_flow(
 ) -> dict:
     """
     站点扫描 Flow
-    
+
     主要功能：
         1. 从target获取所有子域名与其对应的端口号，拼接成URL写入文件
         2. 用httpx进行批量请求并实时保存到数据库（流式处理）
-    
-    工作流程：
-        Step 0: 创建工作目录
-        Step 1: 导出站点 URL 列表
-        Step 2: 解析配置，获取启用的工具
-        Step 3: 串行执行扫描工具并实时保存结果
-    
+
     Args:
         scan_id: 扫描任务 ID
         target_name: 目标名称
         target_id: 目标 ID
         scan_workspace_dir: 扫描工作空间目录
         enabled_tools: 启用的工具配置字典
-    
+
     Returns:
-        dict: {
-            'success': bool,
-            'scan_id': int,
-            'target': str,
-            'scan_workspace_dir': str,
-            'urls_file': str,
-            'total_urls': int,
-            'association_count': int,
-            'processed_records': int,
-            'created_websites': int,
-            'skipped_no_subdomain': int,
-            'skipped_failed': int,
-            'executed_tasks': list,
-            'tool_stats': {
-                'total': int,
-                'successful': int,
-                'failed': int,
-                'successful_tools': list[str],
-                'failed_tools': list[dict]
-            }
-        }
-        
+        dict: 扫描结果
+
     Raises:
         ValueError: 配置错误
         RuntimeError: 执行失败
     """
     try:
+        wait_for_system_load(context="site_scan_flow")
+
         logger.info(
-            "="*60 + "\n" +
-            "开始站点扫描\n" +
-            f"  Scan ID: {scan_id}\n" +
-            f"  Target: {target_name}\n" +
-            f"  Workspace: {scan_workspace_dir}\n" +
-            "="*60
+            "开始站点扫描 - Scan ID: %s, Target: %s, Workspace: %s",
+            scan_id, target_name, scan_workspace_dir
         )
-        
-        # 参数验证
-        if scan_id is None:
-            raise ValueError("scan_id 不能为空")
-        if not target_name:
-            raise ValueError("target_name 不能为空")
-        if target_id is None:
-            raise ValueError("target_id 不能为空")
-        if not scan_workspace_dir:
-            raise ValueError("scan_workspace_dir 不能为空")
-        
+
+        _validate_flow_params(scan_id, target_name, target_id, scan_workspace_dir)
+        user_log(scan_id, "site_scan", "Starting site scan")
+
         # Step 0: 创建工作目录
         from apps.scan.utils import setup_scan_directory
         site_scan_dir = setup_scan_directory(scan_workspace_dir, 'site_scan')
-        
+
         # Step 1: 导出站点 URL
         urls_file, total_urls, association_count = _export_site_urls(
-            target_id, site_scan_dir, target_name
+            target_id, site_scan_dir
         )
-        
+
         if total_urls == 0:
-            logger.warning("目标下没有可用的站点URL，跳过站点扫描")
-            return {
-                'success': True,
-                'scan_id': scan_id,
-                'target': target_name,
-                'scan_workspace_dir': scan_workspace_dir,
-                'urls_file': urls_file,
-                'total_urls': 0,
-                'association_count': association_count,
-                'processed_records': 0,
-                'created_websites': 0,
-                'skipped_no_subdomain': 0,
-                'skipped_failed': 0,
-                'executed_tasks': ['export_site_urls'],
-                'tool_stats': {
-                    'total': 0,
-                    'successful': 0,
-                    'failed': 0,
-                    'successful_tools': [],
-                    'failed_tools': [],
-                    'details': {}
-                }
-            }
-        
+            logger.warning("跳过站点扫描：没有站点 URL 可扫描 - Scan ID: %s", scan_id)
+            user_log(scan_id, "site_scan", "Skipped: no site URLs to scan", "warning")
+            return _build_empty_result(
+                scan_id, target_name, scan_workspace_dir, urls_file, association_count
+            )
+
         # Step 2: 工具配置信息
-        logger.info("Step 2: 工具配置信息")
-        logger.info(
-            "✓ 启用工具: %s",
-            ', '.join(enabled_tools.keys())
-        )
-        
+        logger.info("✓ 启用工具: %s", ', '.join(enabled_tools))
+
         # Step 3: 串行执行扫描工具
-        logger.info("Step 3: 串行执行扫描工具并实时保存结果")
-        tool_stats, processed_records, successful_tool_names, failed_tools = _run_scans_sequentially(
-            enabled_tools=enabled_tools,
-            urls_file=urls_file,
-            total_urls=total_urls,
-            site_scan_dir=site_scan_dir,
+        ctx = ScanContext(
             scan_id=scan_id,
             target_id=target_id,
-            target_name=target_name
+            target_name=target_name,
+            site_scan_dir=site_scan_dir,
+            urls_file=urls_file,
+            total_urls=total_urls
         )
-        
-        logger.info("="*60 + "\n✓ 站点扫描完成\n" + "="*60)
-        
-        # 动态生成已执行的任务列表
+
+        tool_stats, processed_records, successful_tools, failed_tools = \
+            _run_scans_sequentially(enabled_tools, ctx)
+
+        # 汇总结果
         executed_tasks = ['export_site_urls', 'parse_config']
-        executed_tasks.extend([f'run_and_stream_save_websites ({tool})' for tool in tool_stats.keys()])
-        
-        # 汇总所有工具的结果
-        total_created = sum(stats['result'].get('created_websites', 0) for stats in tool_stats.values())
-        total_skipped_no_subdomain = sum(stats['result'].get('skipped_no_subdomain', 0) for stats in tool_stats.values())
-        total_skipped_failed = sum(stats['result'].get('skipped_failed', 0) for stats in tool_stats.values())
-        
+        executed_tasks.extend(
+            f'run_and_stream_save_websites ({tool})' for tool in tool_stats
+        )
+
+        total_created, total_skipped_no_sub, total_skipped_failed = \
+            _aggregate_tool_results(tool_stats)
+
+        logger.info("✓ 站点扫描完成 - 创建站点: %d", total_created)
+        user_log(
+            scan_id, "site_scan",
+            f"site_scan completed: found {total_created} websites"
+        )
+
         return {
             'success': True,
             'scan_id': scan_id,
@@ -453,25 +424,20 @@ def site_scan_flow(
             'association_count': association_count,
             'processed_records': processed_records,
             'created_websites': total_created,
-            'skipped_no_subdomain': total_skipped_no_subdomain,
+            'skipped_no_subdomain': total_skipped_no_sub,
             'skipped_failed': total_skipped_failed,
             'executed_tasks': executed_tasks,
             'tool_stats': {
                 'total': len(enabled_tools),
-                'successful': len(successful_tool_names),
+                'successful': len(successful_tools),
                 'failed': len(failed_tools),
-                'successful_tools': successful_tool_names,
+                'successful_tools': successful_tools,
                 'failed_tools': failed_tools,
                 'details': tool_stats
             }
         }
-        
-    except ValueError as e:
-        logger.error("配置错误: %s", e)
+
+    except ValueError:
         raise
-    except RuntimeError as e:
-        logger.error("运行时错误: %s", e)
+    except RuntimeError:
         raise
-    except Exception as e:
-        logger.exception("站点扫描失败: %s", e)
-        raise

backend/apps/scan/flows/url_fetch/domain_name_url_fetch_flow.py

@@ -59,6 +59,8 @@ def domain_name_url_fetch_flow(
     - IP 和 CIDR 类型会自动跳过（waymore 等工具不支持）
     - 工具会自动收集 *.target_name 的所有历史 URL，无需遍历子域名
     """
+    from apps.scan.utils import user_log
+    
     try:
         output_path = Path(output_dir)
         output_path.mkdir(parents=True, exist_ok=True)
@@ -145,6 +147,9 @@ def domain_name_url_fetch_flow(
                 timeout,
             )
 
+            # 记录工具开始执行日志
+            user_log(scan_id, "url_fetch", f"Running {tool_name}: {command}")
+
             future = run_url_fetcher_task.submit(
                 tool_name=tool_name,
                 command=command,
@@ -163,22 +168,28 @@ def domain_name_url_fetch_flow(
                 if result and result.get("success"):
                     result_files.append(result["output_file"])
                     successful_tools.append(tool_name)
+                    url_count = result.get("url_count", 0)
                     logger.info(
                         "✓ 工具 %s 执行成功 - 发现 URL: %d",
                         tool_name,
-                        result.get("url_count", 0),
+                        url_count,
                     )
+                    user_log(scan_id, "url_fetch", f"{tool_name} completed: found {url_count} urls")
                 else:
+                    reason = "未生成结果或无有效 URL"
                     failed_tools.append(
                         {
                             "tool": tool_name,
-                            "reason": "未生成结果或无有效 URL",
+                            "reason": reason,
                         }
                     )
                     logger.warning("⚠️ 工具 %s 未生成有效结果", tool_name)
+                    user_log(scan_id, "url_fetch", f"{tool_name} failed: {reason}", "error")
             except Exception as e:
-                failed_tools.append({"tool": tool_name, "reason": str(e)})
+                reason = str(e)
+                failed_tools.append({"tool": tool_name, "reason": reason})
                 logger.warning("⚠️ 工具 %s 执行失败: %s", tool_name, e)
+                user_log(scan_id, "url_fetch", f"{tool_name} failed: {reason}", "error")
 
         logger.info(
             "基于 domain_name 的 URL 获取完成 - 成功工具: %s, 失败工具: %s",

backend/apps/scan/flows/url_fetch/main_flow.py

@@ -10,21 +10,18 @@ URL Fetch 主 Flow
 - 统一进行 httpx 验证（如果启用）
 """
 
-# Django 环境初始化
-from apps.common.prefect_django_setup import setup_django_for_prefect
-
 import logging
-import os
-from pathlib import Path
 from datetime import datetime
+from pathlib import Path
 
 from prefect import flow
 
 from apps.scan.handlers.scan_flow_handlers import (
-    on_scan_flow_running,
     on_scan_flow_completed,
     on_scan_flow_failed,
+    on_scan_flow_running,
 )
+from apps.scan.utils import user_log, wait_for_system_load
 
 from .domain_name_url_fetch_flow import domain_name_url_fetch_flow
 from .sites_url_fetch_flow import sites_url_fetch_flow
@@ -42,13 +39,10 @@ SITES_FILE_TOOLS = {'katana'}
 POST_PROCESS_TOOLS = {'uro', 'httpx'}
 
 
-
-
-
 def _classify_tools(enabled_tools: dict) -> tuple[dict, dict, dict, dict]:
     """
     将启用的工具按输入类型分类
-    
+
     Returns:
         tuple: (domain_name_tools, sites_file_tools, uro_config, httpx_config)
     """
@@ -75,23 +69,23 @@ def _classify_tools(enabled_tools: dict) -> tuple[dict, dict, dict, dict]:
 def _merge_and_deduplicate_urls(result_files: list, url_fetch_dir: Path) -> tuple[str, int]:
     """合并并去重 URL"""
     from apps.scan.tasks.url_fetch import merge_and_deduplicate_urls_task
-    
+
     merged_file = merge_and_deduplicate_urls_task(
         result_files=result_files,
         result_dir=str(url_fetch_dir)
     )
-    
+
     # 统计唯一 URL 数量
     unique_url_count = 0
     if Path(merged_file).exists():
-        with open(merged_file, 'r') as f:
+        with open(merged_file, 'r', encoding='utf-8') as f:
             unique_url_count = sum(1 for line in f if line.strip())
-    
+
     logger.info(
         "✓ URL 合并去重完成 - 合并文件: %s, 唯一 URL 数: %d",
         merged_file, unique_url_count
     )
-    
+
     return merged_file, unique_url_count
 
 
@@ -102,12 +96,12 @@ def _clean_urls_with_uro(
 ) -> tuple[str, int, int]:
     """使用 uro 清理合并后的 URL 列表"""
     from apps.scan.tasks.url_fetch import clean_urls_task
-    
+
     raw_timeout = uro_config.get('timeout', 60)
     whitelist = uro_config.get('whitelist')
     blacklist = uro_config.get('blacklist')
     filters = uro_config.get('filters')
-    
+
     # 计算超时时间
     if isinstance(raw_timeout, str) and raw_timeout == 'auto':
         timeout = calculate_timeout_by_line_count(
@@ -123,7 +117,7 @@ def _clean_urls_with_uro(
         except (TypeError, ValueError):
             logger.warning("uro timeout 配置无效(%s)，使用默认 60 秒", raw_timeout)
             timeout = 60
-    
+
     result = clean_urls_task(
         input_file=merged_file,
         output_dir=str(url_fetch_dir),
@@ -132,12 +126,12 @@ def _clean_urls_with_uro(
         blacklist=blacklist,
         filters=filters
     )
-    
+
     if result['success']:
         return result['output_file'], result['output_count'], result['removed_count']
-    else:
-        logger.warning("uro 清理失败: %s，使用原始合并文件", result.get('error', '未知错误'))
-        return merged_file, result['input_count'], 0
+
+    logger.warning("uro 清理失败: %s，使用原始合并文件", result.get('error', '未知错误'))
+    return merged_file, result['input_count'], 0
 
 
 def _validate_and_stream_save_urls(
@@ -150,25 +144,25 @@ def _validate_and_stream_save_urls(
     """使用 httpx 验证 URL 存活并流式保存到数据库"""
     from apps.scan.utils import build_scan_command
     from apps.scan.tasks.url_fetch import run_and_stream_save_urls_task
-    
+
     logger.info("开始使用 httpx 验证 URL 存活状态...")
-    
+
     # 统计待验证的 URL 数量
     try:
-        with open(merged_file, 'r') as f:
+        with open(merged_file, 'r', encoding='utf-8') as f:
             url_count = sum(1 for _ in f)
         logger.info("待验证 URL 数量: %d", url_count)
-    except Exception as e:
+    except OSError as e:
         logger.error("读取 URL 文件失败: %s", e)
         return 0
-    
+
     if url_count == 0:
         logger.warning("没有需要验证的 URL")
         return 0
-    
+
     # 构建 httpx 命令
     command_params = {'url_file': merged_file}
-    
+
     try:
         command = build_scan_command(
             tool_name='httpx',
@@ -176,21 +170,19 @@ def _validate_and_stream_save_urls(
             command_params=command_params,
             tool_config=httpx_config
         )
-    except Exception as e:
+    except (ValueError, KeyError) as e:
         logger.error("构建 httpx 命令失败: %s", e)
         logger.warning("降级处理：将直接保存所有 URL（不验证存活）")
         return _save_urls_to_database(merged_file, scan_id, target_id)
-    
+
     # 计算超时时间
     raw_timeout = httpx_config.get('timeout', 'auto')
-    timeout = 3600
     if isinstance(raw_timeout, str) and raw_timeout == 'auto':
         # 按 URL 行数计算超时时间：每行 3 秒，最小 60 秒
         timeout = max(60, url_count * 3)
         logger.info(
             "自动计算 httpx 超时时间(按行数，每行 3 秒，最小 60 秒): url_count=%d, timeout=%d 秒",
-            url_count,
-            timeout,
+            url_count, timeout
         )
     else:
         try:
@@ -198,49 +190,44 @@ def _validate_and_stream_save_urls(
         except (TypeError, ValueError):
             timeout = 3600
         logger.info("使用配置的 httpx 超时时间: %d 秒", timeout)
-    
+
     # 生成日志文件路径
     timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
     log_file = url_fetch_dir / f"httpx_validation_{timestamp}.log"
-    
+
     # 流式执行
-    try:
-        result = run_and_stream_save_urls_task(
-            cmd=command,
-            tool_name='httpx',
-            scan_id=scan_id,
-            target_id=target_id,
-            cwd=str(url_fetch_dir),
-            shell=True,
-            timeout=timeout,
-            log_file=str(log_file)
-        )
-        
-        saved = result.get('saved_urls', 0)
-        logger.info(
-            "✓ httpx 验证完成 - 存活 URL: %d (%.1f%%)",
-            saved, (saved / url_count * 100) if url_count > 0 else 0
-        )
-        return saved
-        
-    except Exception as e:
-        logger.error("httpx 流式验证失败: %s", e, exc_info=True)
-        raise
+    result = run_and_stream_save_urls_task(
+        cmd=command,
+        tool_name='httpx',
+        scan_id=scan_id,
+        target_id=target_id,
+        cwd=str(url_fetch_dir),
+        shell=True,
+        timeout=timeout,
+        log_file=str(log_file)
+    )
+
+    saved = result.get('saved_urls', 0)
+    logger.info(
+        "✓ httpx 验证完成 - 存活 URL: %d (%.1f%%)",
+        saved, (saved / url_count * 100) if url_count > 0 else 0
+    )
+    return saved
 
 
 def _save_urls_to_database(merged_file: str, scan_id: int, target_id: int) -> int:
     """保存 URL 到数据库（不验证存活）"""
     from apps.scan.tasks.url_fetch import save_urls_task
-    
+
     result = save_urls_task(
         urls_file=merged_file,
         scan_id=scan_id,
         target_id=target_id
     )
-    
+
     saved_count = result.get('saved_urls', 0)
     logger.info("✓ URL 保存完成 - 保存数量: %d", saved_count)
-    
+
     return saved_count
 
 
@@ -260,7 +247,7 @@ def url_fetch_flow(
 ) -> dict:
     """
     URL 获取主 Flow
-    
+
     执行流程：
     1. 准备工作目录
     2. 按输入类型分类工具（domain_name / sites_file / 后处理）
@@ -270,34 +257,32 @@ def url_fetch_flow(
     4. 合并所有子 Flow 的结果并去重
     5. uro 去重（如果启用）
     6. httpx 验证（如果启用）
-    
+
     Args:
         scan_id: 扫描 ID
         target_name: 目标名称
         target_id: 目标 ID
         scan_workspace_dir: 扫描工作目录
         enabled_tools: 启用的工具配置
-        
+
     Returns:
         dict: 扫描结果
     """
     try:
+        # 负载检查：等待系统资源充足
+        wait_for_system_load(context="url_fetch_flow")
+
         logger.info(
-            "="*60 + "\n" +
-            "开始 URL 获取扫描\n" +
-            f"  Scan ID: {scan_id}\n" +
-            f"  Target: {target_name}\n" +
-            f"  Workspace: {scan_workspace_dir}\n" +
-            "="*60
+            "开始 URL 获取扫描 - Scan ID: %s, Target: %s, Workspace: %s",
+            scan_id, target_name, scan_workspace_dir
         )
-        
+        user_log(scan_id, "url_fetch", "Starting URL fetch")
+
         # Step 1: 准备工作目录
-        logger.info("Step 1: 准备工作目录")
         from apps.scan.utils import setup_scan_directory
         url_fetch_dir = setup_scan_directory(scan_workspace_dir, 'url_fetch')
-        
+
         # Step 2: 分类工具（按输入类型）
-        logger.info("Step 2: 分类工具")
         domain_name_tools, sites_file_tools, uro_config, httpx_config = _classify_tools(enabled_tools)
 
         logger.info(
@@ -314,15 +299,14 @@ def url_fetch_flow(
                 "URL Fetch 流程需要至少启用一个 URL 获取工具（如 waymore, katana）。"
                 "httpx 和 uro 仅用于后处理，不能单独使用。"
             )
-        
-        # Step 3: 并行执行子 Flow
+
+        # Step 3: 执行子 Flow
         all_result_files = []
         all_failed_tools = []
         all_successful_tools = []
-        
-        # 3a: 基于 domain_name（target_name） 的 URL 被动收集（如 waymore）
+
+        # 3a: 基于 domain_name 的 URL 被动收集（如 waymore）
         if domain_name_tools:
-            logger.info("Step 3a: 执行基于 domain_name 的 URL 被动收集子 Flow")
             tn_result = domain_name_url_fetch_flow(
                 scan_id=scan_id,
                 target_id=target_id,
@@ -333,10 +317,9 @@ def url_fetch_flow(
             all_result_files.extend(tn_result.get('result_files', []))
             all_failed_tools.extend(tn_result.get('failed_tools', []))
             all_successful_tools.extend(tn_result.get('successful_tools', []))
-        
+
         # 3b: 爬虫（以 sites_file 为输入）
         if sites_file_tools:
-            logger.info("Step 3b: 执行爬虫子 Flow")
             crawl_result = sites_url_fetch_flow(
                 scan_id=scan_id,
                 target_id=target_id,
@@ -347,12 +330,13 @@ def url_fetch_flow(
             all_result_files.extend(crawl_result.get('result_files', []))
             all_failed_tools.extend(crawl_result.get('failed_tools', []))
             all_successful_tools.extend(crawl_result.get('successful_tools', []))
-        
+
         # 检查是否有成功的工具
         if not all_result_files:
-            error_details = "; ".join([f"{f['tool']}: {f['reason']}" for f in all_failed_tools])
+            error_details = "; ".join([
+                "%s: %s" % (f['tool'], f['reason']) for f in all_failed_tools
+            ])
             logger.warning("所有 URL 获取工具均失败 - 目标: %s, 失败详情: %s", target_name, error_details)
-            # 返回空结果，不抛出异常，让扫描继续
             return {
                 'success': True,
                 'scan_id': scan_id,
@@ -363,31 +347,24 @@ def url_fetch_flow(
                 'successful_tools': [],
                 'message': '所有 URL 获取工具均无结果'
             }
-        
+
         # Step 4: 合并并去重 URL
-        logger.info("Step 4: 合并并去重 URL")
-        merged_file, unique_url_count = _merge_and_deduplicate_urls(
+        merged_file, _ = _merge_and_deduplicate_urls(
             result_files=all_result_files,
             url_fetch_dir=url_fetch_dir
         )
-        
+
         # Step 5: 使用 uro 清理 URL（如果启用）
         url_file_for_validation = merged_file
-        uro_removed_count = 0
-        
         if uro_config and uro_config.get('enabled', False):
-            logger.info("Step 5: 使用 uro 清理 URL")
-            url_file_for_validation, cleaned_count, uro_removed_count = _clean_urls_with_uro(
+            url_file_for_validation, _, _ = _clean_urls_with_uro(
                 merged_file=merged_file,
                 uro_config=uro_config,
                 url_fetch_dir=url_fetch_dir
             )
-        else:
-            logger.info("Step 5: 跳过 uro 清理（未启用）")
-        
+
         # Step 6: 使用 httpx 验证存活并保存（如果启用）
         if httpx_config and httpx_config.get('enabled', False):
-            logger.info("Step 6: 使用 httpx 验证 URL 存活并流式保存")
             saved_count = _validate_and_stream_save_urls(
                 merged_file=url_file_for_validation,
                 httpx_config=httpx_config,
@@ -396,15 +373,16 @@ def url_fetch_flow(
                 target_id=target_id
             )
         else:
-            logger.info("Step 6: 保存到数据库（未启用 httpx 验证）")
             saved_count = _save_urls_to_database(
                 merged_file=url_file_for_validation,
                 scan_id=scan_id,
                 target_id=target_id
             )
-        
-        logger.info("="*60 + "\n✓ URL 获取扫描完成\n" + "="*60)
-        
+
+        # 记录 Flow 完成
+        logger.info("✓ URL 获取完成 - 保存 endpoints: %d", saved_count)
+        user_log(scan_id, "url_fetch", "url_fetch completed: found %d endpoints" % saved_count)
+
         # 构建已执行的任务列表
         executed_tasks = ['setup_directory', 'classify_tools']
         if domain_name_tools:
@@ -418,7 +396,7 @@ def url_fetch_flow(
             executed_tasks.append('httpx_validation_and_save')
         else:
             executed_tasks.append('save_urls')
-        
+
         return {
             'success': True,
             'scan_id': scan_id,
@@ -434,7 +412,7 @@ def url_fetch_flow(
                 'failed_tools': [f['tool'] for f in all_failed_tools]
             }
         }
-        
+
     except Exception as e:
         logger.error("URL 获取扫描失败: %s", e, exc_info=True)
         raise

backend/apps/scan/flows/url_fetch/sites_url_fetch_flow.py

@@ -116,7 +116,8 @@ def sites_url_fetch_flow(
             tools=enabled_tools,
             input_file=sites_file,
             input_type="sites_file",
-            output_dir=output_path
+            output_dir=output_path,
+            scan_id=scan_id
         )
         
         logger.info(

backend/apps/scan/flows/url_fetch/utils.py

@@ -152,7 +152,8 @@ def run_tools_parallel(
     tools: dict,
     input_file: str,
     input_type: str,
-    output_dir: Path
+    output_dir: Path,
+    scan_id: int
 ) -> tuple[list, list, list]:
     """
     并行执行工具列表
@@ -162,11 +163,13 @@ def run_tools_parallel(
         input_file: 输入文件路径
         input_type: 输入类型
         output_dir: 输出目录
+        scan_id: 扫描任务 ID（用于记录日志）
         
     Returns:
         tuple: (result_files, failed_tools, successful_tool_names)
     """
     from apps.scan.tasks.url_fetch import run_url_fetcher_task
+    from apps.scan.utils import user_log
 
     futures: dict[str, object] = {}
     failed_tools: list[dict] = []
@@ -192,6 +195,9 @@ def run_tools_parallel(
             exec_params["timeout"],
         )
 
+        # 记录工具开始执行日志
+        user_log(scan_id, "url_fetch", f"Running {tool_name}: {exec_params['command']}")
+
         # 提交并行任务
         future = run_url_fetcher_task.submit(
             tool_name=tool_name,
@@ -208,22 +214,28 @@ def run_tools_parallel(
             result = future.result()
             if result and result['success']:
                 result_files.append(result['output_file'])
+                url_count = result['url_count']
                 logger.info(
                     "✓ 工具 %s 执行成功 - 发现 URL: %d",
-                    tool_name, result['url_count']
+                    tool_name, url_count
                 )
+                user_log(scan_id, "url_fetch", f"{tool_name} completed: found {url_count} urls")
             else:
+                reason = '未生成结果或无有效URL'
                 failed_tools.append({
                     'tool': tool_name,
-                    'reason': '未生成结果或无有效URL'
+                    'reason': reason
                 })
                 logger.warning("⚠️ 工具 %s 未生成有效结果", tool_name)
+                user_log(scan_id, "url_fetch", f"{tool_name} failed: {reason}", "error")
         except Exception as e:
+            reason = str(e)
             failed_tools.append({
                 'tool': tool_name,
-                'reason': str(e)
+                'reason': reason
             })
             logger.warning("⚠️ 工具 %s 执行失败: %s", tool_name, e)
+            user_log(scan_id, "url_fetch", f"{tool_name} failed: {reason}", "error")
 
     # 计算成功的工具列表
     failed_tool_names = [f['tool'] for f in failed_tools]

backend/apps/scan/flows/vuln_scan/endpoints_vuln_scan_flow.py

@@ -12,7 +12,7 @@ from apps.scan.handlers.scan_flow_handlers import (
     on_scan_flow_completed,
     on_scan_flow_failed,
 )
-from apps.scan.utils import build_scan_command, ensure_nuclei_templates_local
+from apps.scan.utils import build_scan_command, ensure_nuclei_templates_local, user_log
 from apps.scan.tasks.vuln_scan import (
     export_endpoints_task,
     run_vuln_tool_task,
@@ -141,6 +141,7 @@ def endpoints_vuln_scan_flow(
             # Dalfox XSS 使用流式任务，一边解析一边保存漏洞结果
             if tool_name == "dalfox_xss":
                 logger.info("开始执行漏洞扫描工具 %s（流式保存漏洞结果，已提交任务）", tool_name)
+                user_log(scan_id, "vuln_scan", f"Running {tool_name}: {command}")
                 future = run_and_stream_save_dalfox_vulns_task.submit(
                     cmd=command,
                     tool_name=tool_name,
@@ -163,6 +164,7 @@ def endpoints_vuln_scan_flow(
             elif tool_name == "nuclei":
                 # Nuclei 使用流式任务
                 logger.info("开始执行漏洞扫描工具 %s（流式保存漏洞结果，已提交任务）", tool_name)
+                user_log(scan_id, "vuln_scan", f"Running {tool_name}: {command}")
                 future = run_and_stream_save_nuclei_vulns_task.submit(
                     cmd=command,
                     tool_name=tool_name,
@@ -185,6 +187,7 @@ def endpoints_vuln_scan_flow(
             else:
                 # 其他工具仍使用非流式执行逻辑
                 logger.info("开始执行漏洞扫描工具 %s（已提交任务）", tool_name)
+                user_log(scan_id, "vuln_scan", f"Running {tool_name}: {command}")
                 future = run_vuln_tool_task.submit(
                     tool_name=tool_name,
                     command=command,
@@ -203,24 +206,34 @@ def endpoints_vuln_scan_flow(
         # 统一收集所有工具的执行结果
         for tool_name, meta in tool_futures.items():
             future = meta["future"]
-            result = future.result()
+            try:
+                result = future.result()
 
-            if meta["mode"] == "streaming":
-                tool_results[tool_name] = {
-                    "command": meta["command"],
-                    "timeout": meta["timeout"],
-                    "processed_records": result.get("processed_records"),
-                    "created_vulns": result.get("created_vulns"),
-                    "command_log_file": meta["log_file"],
-                }
-            else:
-                tool_results[tool_name] = {
-                    "command": meta["command"],
-                    "timeout": meta["timeout"],
-                    "duration": result.get("duration"),
-                    "returncode": result.get("returncode"),
-                    "command_log_file": result.get("command_log_file"),
-                }
+                if meta["mode"] == "streaming":
+                    created_vulns = result.get("created_vulns", 0)
+                    tool_results[tool_name] = {
+                        "command": meta["command"],
+                        "timeout": meta["timeout"],
+                        "processed_records": result.get("processed_records"),
+                        "created_vulns": created_vulns,
+                        "command_log_file": meta["log_file"],
+                    }
+                    logger.info("✓ 工具 %s 执行完成 - 漏洞: %d", tool_name, created_vulns)
+                    user_log(scan_id, "vuln_scan", f"{tool_name} completed: found {created_vulns} vulnerabilities")
+                else:
+                    tool_results[tool_name] = {
+                        "command": meta["command"],
+                        "timeout": meta["timeout"],
+                        "duration": result.get("duration"),
+                        "returncode": result.get("returncode"),
+                        "command_log_file": result.get("command_log_file"),
+                    }
+                    logger.info("✓ 工具 %s 执行完成 - returncode=%s", tool_name, result.get("returncode"))
+                    user_log(scan_id, "vuln_scan", f"{tool_name} completed")
+            except Exception as e:
+                reason = str(e)
+                logger.error("工具 %s 执行失败: %s", tool_name, e, exc_info=True)
+                user_log(scan_id, "vuln_scan", f"{tool_name} failed: {reason}", "error")
 
         return {
             "success": True,

backend/apps/scan/flows/vuln_scan/main_flow.py

@@ -1,5 +1,6 @@
-from apps.common.prefect_django_setup import setup_django_for_prefect
-
+"""
+漏洞扫描主 Flow
+"""
 import logging
 from typing import Dict, Tuple
 
@@ -11,6 +12,7 @@ from apps.scan.handlers.scan_flow_handlers import (
     on_scan_flow_failed,
 )
 from apps.scan.configs.command_templates import get_command_template
+from apps.scan.utils import user_log, wait_for_system_load
 from .endpoints_vuln_scan_flow import endpoints_vuln_scan_flow
 
 
@@ -61,6 +63,9 @@ def vuln_scan_flow(
     - nuclei: 通用漏洞扫描（流式保存，支持模板 commit hash 同步）
     """
     try:
+        # 负载检查：等待系统资源充足
+        wait_for_system_load(context="vuln_scan_flow")
+
         if scan_id is None:
             raise ValueError("scan_id 不能为空")
         if not target_name:
@@ -72,6 +77,9 @@ def vuln_scan_flow(
         if not enabled_tools:
             raise ValueError("enabled_tools 不能为空")
 
+        logger.info("开始漏洞扫描 - Scan ID: %s, Target: %s", scan_id, target_name)
+        user_log(scan_id, "vuln_scan", "Starting vulnerability scan")
+
         # Step 1: 分类工具
         endpoints_tools, other_tools = _classify_vuln_tools(enabled_tools)
 
@@ -99,6 +107,14 @@ def vuln_scan_flow(
             enabled_tools=endpoints_tools,
         )
 
+        # 记录 Flow 完成
+        total_vulns = sum(
+            r.get("created_vulns", 0) 
+            for r in endpoint_result.get("tool_results", {}).values()
+        )
+        logger.info("✓ 漏洞扫描完成 - 新增漏洞: %d", total_vulns)
+        user_log(scan_id, "vuln_scan", f"vuln_scan completed: found {total_vulns} vulnerabilities")
+
         # 目前只有一个子 Flow，直接返回其结果
         return endpoint_result
 

backend/apps/scan/handlers/scan_flow_handlers.py

@@ -14,6 +14,7 @@ from prefect import Flow
 from prefect.client.schemas import FlowRun, State
 
 from apps.scan.utils.performance import FlowPerformanceTracker
+from apps.scan.utils import user_log
 
 logger = logging.getLogger(__name__)
 
@@ -136,6 +137,7 @@ def on_scan_flow_failed(flow: Flow, flow_run: FlowRun, state: State) -> None:
     - 更新阶段进度为 failed
     - 发送扫描失败通知
     - 记录性能指标（含错误信息）
+    - 写入 ScanLog 供前端显示
     
     Args:
         flow: Prefect Flow 对象
@@ -152,6 +154,11 @@ def on_scan_flow_failed(flow: Flow, flow_run: FlowRun, state: State) -> None:
     # 提取错误信息
     error_message = str(state.message) if state.message else "未知错误"
     
+    # 写入 ScanLog 供前端显示
+    stage = _get_stage_from_flow_name(flow.name)
+    if scan_id and stage:
+        user_log(scan_id, stage, f"Failed: {error_message}", "error")
+    
     # 记录性能指标（失败情况）
     tracker = _flow_trackers.pop(str(flow_run.id), None)
     if tracker:

backend/apps/scan/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.2.7 on 2026-01-02 04:45
+# Generated by Django 5.2.7 on 2026-01-06 00:55
 
 import django.contrib.postgres.fields
 import django.db.models.deletion
@@ -31,6 +31,20 @@ class Migration(migrations.Migration):
                 'db_table': 'notification_settings',
             },
         ),
+        migrations.CreateModel(
+            name='SubfinderProviderSettings',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('providers', models.JSONField(default=dict, help_text='各 Provider 的 API Key 配置')),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+            ],
+            options={
+                'verbose_name': 'Subfinder Provider 配置',
+                'verbose_name_plural': 'Subfinder Provider 配置',
+                'db_table': 'subfinder_provider_settings',
+            },
+        ),
         migrations.CreateModel(
             name='Notification',
             fields=[
@@ -57,7 +71,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(primary_key=True, serialize=False)),
                 ('engine_ids', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), default=list, help_text='引擎 ID 列表', size=None)),
                 ('engine_names', models.JSONField(default=list, help_text='引擎名称列表，如 ["引擎A", "引擎B"]')),
-                ('merged_configuration', models.TextField(default='', help_text='合并后的 YAML 配置')),
+                ('yaml_configuration', models.TextField(default='', help_text='YAML 格式的扫描配置')),
                 ('created_at', models.DateTimeField(auto_now_add=True, help_text='任务创建时间')),
                 ('stopped_at', models.DateTimeField(blank=True, help_text='扫描结束时间', null=True)),
                 ('status', models.CharField(choices=[('cancelled', '已取消'), ('completed', '已完成'), ('failed', '失败'), ('initiated', '初始化'), ('running', '运行中')], db_index=True, default='initiated', help_text='任务状态', max_length=20)),
@@ -87,7 +101,22 @@ class Migration(migrations.Migration):
                 'verbose_name_plural': '扫描任务',
                 'db_table': 'scan',
                 'ordering': ['-created_at'],
-                'indexes': [models.Index(fields=['-created_at'], name='scan_created_0bb6c7_idx'), models.Index(fields=['target'], name='scan_target__718b9d_idx'), models.Index(fields=['deleted_at', '-created_at'], name='scan_deleted_eb17e8_idx')],
+            },
+        ),
+        migrations.CreateModel(
+            name='ScanLog',
+            fields=[
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
+                ('level', models.CharField(choices=[('info', 'Info'), ('warning', 'Warning'), ('error', 'Error')], default='info', help_text='日志级别', max_length=10)),
+                ('content', models.TextField(help_text='日志内容')),
+                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True, help_text='创建时间')),
+                ('scan', models.ForeignKey(help_text='关联的扫描任务', on_delete=django.db.models.deletion.CASCADE, related_name='logs', to='scan.scan')),
+            ],
+            options={
+                'verbose_name': '扫描日志',
+                'verbose_name_plural': '扫描日志',
+                'db_table': 'scan_log',
+                'ordering': ['created_at'],
             },
         ),
         migrations.CreateModel(
@@ -97,7 +126,7 @@ class Migration(migrations.Migration):
                 ('name', models.CharField(help_text='任务名称', max_length=200)),
                 ('engine_ids', django.contrib.postgres.fields.ArrayField(base_field=models.IntegerField(), default=list, help_text='引擎 ID 列表', size=None)),
                 ('engine_names', models.JSONField(default=list, help_text='引擎名称列表，如 ["引擎A", "引擎B"]')),
-                ('merged_configuration', models.TextField(default='', help_text='合并后的 YAML 配置')),
+                ('yaml_configuration', models.TextField(default='', help_text='YAML 格式的扫描配置')),
                 ('cron_expression', models.CharField(default='0 2 * * *', help_text='Cron 表达式，格式：分 时 日 月 周', max_length=100)),
                 ('is_enabled', models.BooleanField(db_index=True, default=True, help_text='是否启用')),
                 ('run_count', models.IntegerField(default=0, help_text='已执行次数')),
@@ -113,7 +142,34 @@ class Migration(migrations.Migration):
                 'verbose_name_plural': '定时扫描任务',
                 'db_table': 'scheduled_scan',
                 'ordering': ['-created_at'],
-                'indexes': [models.Index(fields=['-created_at'], name='scheduled_s_created_9b9c2e_idx'), models.Index(fields=['is_enabled', '-created_at'], name='scheduled_s_is_enab_23d660_idx'), models.Index(fields=['name'], name='scheduled_s_name_bf332d_idx')],
             },
         ),
+        migrations.AddIndex(
+            model_name='scan',
+            index=models.Index(fields=['-created_at'], name='scan_created_0bb6c7_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scan',
+            index=models.Index(fields=['target'], name='scan_target__718b9d_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scan',
+            index=models.Index(fields=['deleted_at', '-created_at'], name='scan_deleted_eb17e8_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scanlog',
+            index=models.Index(fields=['scan', 'created_at'], name='scan_log_scan_id_c4814a_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scheduledscan',
+            index=models.Index(fields=['-created_at'], name='scheduled_s_created_9b9c2e_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scheduledscan',
+            index=models.Index(fields=['is_enabled', '-created_at'], name='scheduled_s_is_enab_23d660_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='scheduledscan',
+            index=models.Index(fields=['name'], name='scheduled_s_name_bf332d_idx'),
+        ),
     ]

backend/apps/scan/migrations/0002_add_cached_screenshots_count.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.7 on 2026-01-07 14:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scan', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='scan',
+            name='cached_screenshots_count',
+            field=models.IntegerField(default=0, help_text='缓存的截图数量'),
+        ),
+    ]

backend/apps/scan/migrations/0003_add_wecom_fields.py

@@ -0,0 +1,23 @@
+# Generated manually for WeCom notification support
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scan', '0002_add_cached_screenshots_count'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='notificationsettings',
+            name='wecom_enabled',
+            field=models.BooleanField(default=False, help_text='是否启用企业微信通知'),
+        ),
+        migrations.AddField(
+            model_name='notificationsettings',
+            name='wecom_webhook_url',
+            field=models.URLField(blank=True, default='', help_text='企业微信机器人 Webhook URL'),
+        ),
+    ]

backend/apps/scan/models/__init__.py

@@ -0,0 +1,18 @@
+"""Scan Models - 统一导出"""
+
+from .scan_models import Scan, SoftDeleteManager
+from .scan_log_model import ScanLog
+from .scheduled_scan_model import ScheduledScan
+from .subfinder_provider_settings_model import SubfinderProviderSettings
+
+# 兼容旧名称（已废弃，请使用 SubfinderProviderSettings）
+ProviderSettings = SubfinderProviderSettings
+
+__all__ = [
+    'Scan',
+    'ScanLog',
+    'ScheduledScan',
+    'SoftDeleteManager',
+    'SubfinderProviderSettings',
+    'ProviderSettings',  # 兼容旧名称
+]

backend/apps/scan/models/scan_log_model.py

@@ -0,0 +1,41 @@
+"""扫描日志模型"""
+
+from django.db import models
+
+
+class ScanLog(models.Model):
+    """扫描日志模型"""
+    
+    class Level(models.TextChoices):
+        INFO = 'info', 'Info'
+        WARNING = 'warning', 'Warning'
+        ERROR = 'error', 'Error'
+    
+    id = models.BigAutoField(primary_key=True)
+    scan = models.ForeignKey(
+        'Scan',
+        on_delete=models.CASCADE,
+        related_name='logs',
+        db_index=True,
+        help_text='关联的扫描任务'
+    )
+    level = models.CharField(
+        max_length=10,
+        choices=Level.choices,
+        default=Level.INFO,
+        help_text='日志级别'
+    )
+    content = models.TextField(help_text='日志内容')
+    created_at = models.DateTimeField(auto_now_add=True, db_index=True, help_text='创建时间')
+    
+    class Meta:
+        db_table = 'scan_log'
+        verbose_name = '扫描日志'
+        verbose_name_plural = '扫描日志'
+        ordering = ['created_at']
+        indexes = [
+            models.Index(fields=['scan', 'created_at']),
+        ]
+    
+    def __str__(self):
+        return f"[{self.level}] {self.content[:50]}"

backend/apps/scan/models/scan_models.py

@@ -1,9 +1,9 @@
+"""扫描相关模型"""
+
 from django.db import models
 from django.contrib.postgres.fields import ArrayField
 
-from ..common.definitions import ScanStatus
-
-
+from apps.common.definitions import ScanStatus
 
 
 class SoftDeleteManager(models.Manager):
@@ -30,9 +30,9 @@ class Scan(models.Model):
         default=list,
         help_text='引擎名称列表，如 ["引擎A", "引擎B"]'
     )
-    merged_configuration = models.TextField(
+    yaml_configuration = models.TextField(
         default='',
-        help_text='合并后的 YAML 配置'
+        help_text='YAML 格式的扫描配置'
     )
 
     created_at = models.DateTimeField(auto_now_add=True, help_text='任务创建时间')
@@ -84,6 +84,7 @@ class Scan(models.Model):
     cached_endpoints_count = models.IntegerField(default=0, help_text='缓存的端点数量')
     cached_ips_count = models.IntegerField(default=0, help_text='缓存的IP地址数量')
     cached_directories_count = models.IntegerField(default=0, help_text='缓存的目录数量')
+    cached_screenshots_count = models.IntegerField(default=0, help_text='缓存的截图数量')
     cached_vulns_total = models.IntegerField(default=0, help_text='缓存的漏洞总数')
     cached_vulns_critical = models.IntegerField(default=0, help_text='缓存的严重漏洞数量')
     cached_vulns_high = models.IntegerField(default=0, help_text='缓存的高危漏洞数量')
@@ -97,99 +98,10 @@ class Scan(models.Model):
         verbose_name_plural = '扫描任务'
         ordering = ['-created_at']
         indexes = [
-            models.Index(fields=['-created_at']),  # 优化按创建时间降序排序（list 查询的默认排序）
-            models.Index(fields=['target']),  # 优化按目标查询扫描任务
-            models.Index(fields=['deleted_at', '-created_at']),  # 软删除 + 时间索引
+            models.Index(fields=['-created_at']),
+            models.Index(fields=['target']),
+            models.Index(fields=['deleted_at', '-created_at']),
         ]
 
     def __str__(self):
         return f"Scan #{self.id} - {self.target.name}"
-
-
-class ScheduledScan(models.Model):
-    """
-    定时扫描任务模型
-    
-    调度机制：
-    - APScheduler 每分钟检查 next_run_time
-    - 到期任务通过 task_distributor 分发到 Worker 执行
-    - 支持 cron 表达式进行灵活调度
-    
-    扫描模式（二选一）：
-    - 组织扫描：设置 organization，执行时动态获取组织下所有目标
-    - 目标扫描：设置 target，扫描单个目标
-    - organization 优先级高于 target
-    """
-    
-    id = models.AutoField(primary_key=True)
-    
-    # 基本信息
-    name = models.CharField(max_length=200, help_text='任务名称')
-    
-    # 多引擎支持字段
-    engine_ids = ArrayField(
-        models.IntegerField(),
-        default=list,
-        help_text='引擎 ID 列表'
-    )
-    engine_names = models.JSONField(
-        default=list,
-        help_text='引擎名称列表，如 ["引擎A", "引擎B"]'
-    )
-    merged_configuration = models.TextField(
-        default='',
-        help_text='合并后的 YAML 配置'
-    )
-    
-    # 关联的组织（组织扫描模式：执行时动态获取组织下所有目标）
-    organization = models.ForeignKey(
-        'targets.Organization',
-        on_delete=models.CASCADE,
-        related_name='scheduled_scans',
-        null=True,
-        blank=True,
-        help_text='扫描组织（设置后执行时动态获取组织下所有目标）'
-    )
-    
-    # 关联的目标（目标扫描模式：扫描单个目标）
-    target = models.ForeignKey(
-        'targets.Target',
-        on_delete=models.CASCADE,
-        related_name='scheduled_scans',
-        null=True,
-        blank=True,
-        help_text='扫描单个目标（与 organization 二选一）'
-    )
-    
-    # 调度配置 - 直接使用 Cron 表达式
-    cron_expression = models.CharField(
-        max_length=100,
-        default='0 2 * * *',
-        help_text='Cron 表达式，格式：分 时 日 月 周'
-    )
-    
-    # 状态
-    is_enabled = models.BooleanField(default=True, db_index=True, help_text='是否启用')
-    
-    # 执行统计
-    run_count = models.IntegerField(default=0, help_text='已执行次数')
-    last_run_time = models.DateTimeField(null=True, blank=True, help_text='上次执行时间')
-    next_run_time = models.DateTimeField(null=True, blank=True, help_text='下次执行时间')
-    
-    # 时间戳
-    created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
-    updated_at = models.DateTimeField(auto_now=True, help_text='更新时间')
-    
-    class Meta:
-        db_table = 'scheduled_scan'
-        verbose_name = '定时扫描任务'
-        verbose_name_plural = '定时扫描任务'
-        ordering = ['-created_at']
-        indexes = [
-            models.Index(fields=['-created_at']),
-            models.Index(fields=['is_enabled', '-created_at']),
-            models.Index(fields=['name']),  # 优化 name 搜索
-        ]
-    
-    def __str__(self):
-        return f"ScheduledScan #{self.id} - {self.name}"

backend/apps/scan/models/scheduled_scan_model.py

@@ -0,0 +1,73 @@
+"""定时扫描任务模型"""
+
+from django.db import models
+from django.contrib.postgres.fields import ArrayField
+
+
+class ScheduledScan(models.Model):
+    """定时扫描任务模型"""
+    
+    id = models.AutoField(primary_key=True)
+    
+    name = models.CharField(max_length=200, help_text='任务名称')
+    
+    engine_ids = ArrayField(
+        models.IntegerField(),
+        default=list,
+        help_text='引擎 ID 列表'
+    )
+    engine_names = models.JSONField(
+        default=list,
+        help_text='引擎名称列表，如 ["引擎A", "引擎B"]'
+    )
+    yaml_configuration = models.TextField(
+        default='',
+        help_text='YAML 格式的扫描配置'
+    )
+    
+    organization = models.ForeignKey(
+        'targets.Organization',
+        on_delete=models.CASCADE,
+        related_name='scheduled_scans',
+        null=True,
+        blank=True,
+        help_text='扫描组织（设置后执行时动态获取组织下所有目标）'
+    )
+    
+    target = models.ForeignKey(
+        'targets.Target',
+        on_delete=models.CASCADE,
+        related_name='scheduled_scans',
+        null=True,
+        blank=True,
+        help_text='扫描单个目标（与 organization 二选一）'
+    )
+    
+    cron_expression = models.CharField(
+        max_length=100,
+        default='0 2 * * *',
+        help_text='Cron 表达式，格式：分 时 日 月 周'
+    )
+    
+    is_enabled = models.BooleanField(default=True, db_index=True, help_text='是否启用')
+    
+    run_count = models.IntegerField(default=0, help_text='已执行次数')
+    last_run_time = models.DateTimeField(null=True, blank=True, help_text='上次执行时间')
+    next_run_time = models.DateTimeField(null=True, blank=True, help_text='下次执行时间')
+    
+    created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
+    updated_at = models.DateTimeField(auto_now=True, help_text='更新时间')
+    
+    class Meta:
+        db_table = 'scheduled_scan'
+        verbose_name = '定时扫描任务'
+        verbose_name_plural = '定时扫描任务'
+        ordering = ['-created_at']
+        indexes = [
+            models.Index(fields=['-created_at']),
+            models.Index(fields=['is_enabled', '-created_at']),
+            models.Index(fields=['name']),
+        ]
+    
+    def __str__(self):
+        return f"ScheduledScan #{self.id} - {self.name}"

backend/apps/scan/models/subfinder_provider_settings_model.py

@@ -0,0 +1,64 @@
+"""Subfinder Provider 配置模型（单例模式）
+
+用于存储 subfinder 第三方数据源的 API Key 配置
+"""
+
+from django.db import models
+
+
+class SubfinderProviderSettings(models.Model):
+    """
+    Subfinder Provider 配置（单例模式）
+    存储第三方数据源的 API Key 配置，用于 subfinder 子域名发现
+    
+    支持的 Provider:
+    - fofa: email + api_key (composite)
+    - censys: api_id + api_secret (composite)
+    - hunter, shodan, zoomeye, securitytrails, threatbook, quake: api_key (single)
+    """
+    
+    providers = models.JSONField(
+        default=dict,
+        help_text='各 Provider 的 API Key 配置'
+    )
+    
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+    
+    class Meta:
+        db_table = 'subfinder_provider_settings'
+        verbose_name = 'Subfinder Provider 配置'
+        verbose_name_plural = 'Subfinder Provider 配置'
+    
+    DEFAULT_PROVIDERS = {
+        'fofa': {'enabled': False, 'email': '', 'api_key': ''},
+        'hunter': {'enabled': False, 'api_key': ''},
+        'shodan': {'enabled': False, 'api_key': ''},
+        'censys': {'enabled': False, 'api_id': '', 'api_secret': ''},
+        'zoomeye': {'enabled': False, 'api_key': ''},
+        'securitytrails': {'enabled': False, 'api_key': ''},
+        'threatbook': {'enabled': False, 'api_key': ''},
+        'quake': {'enabled': False, 'api_key': ''},
+    }
+    
+    def save(self, *args, **kwargs):
+        self.pk = 1
+        super().save(*args, **kwargs)
+    
+    @classmethod
+    def get_instance(cls) -> 'SubfinderProviderSettings':
+        """获取或创建单例实例"""
+        obj, _ = cls.objects.get_or_create(
+            pk=1,
+            defaults={'providers': cls.DEFAULT_PROVIDERS.copy()}
+        )
+        return obj
+    
+    def get_provider_config(self, provider: str) -> dict:
+        """获取指定 Provider 的配置"""
+        return self.providers.get(provider, self.DEFAULT_PROVIDERS.get(provider, {}))
+    
+    def is_provider_enabled(self, provider: str) -> bool:
+        """检查指定 Provider 是否启用"""
+        config = self.get_provider_config(provider)
+        return config.get('enabled', False)

backend/apps/scan/notifications/models.py

@@ -1,8 +1,14 @@
 """通知系统数据模型"""
 
-from django.db import models
+import logging
+from datetime import timedelta
 
-from .types import NotificationLevel, NotificationCategory
+from django.db import models
+from django.utils import timezone
+
+from .types import NotificationCategory, NotificationLevel
+
+logger = logging.getLogger(__name__)
 
 
 class NotificationSettings(models.Model):
@@ -10,31 +16,34 @@ class NotificationSettings(models.Model):
     通知设置（单例模型）
     存储 Discord webhook 配置和各分类的通知开关
     """
-    
+
     # Discord 配置
     discord_enabled = models.BooleanField(default=False, help_text='是否启用 Discord 通知')
     discord_webhook_url = models.URLField(blank=True, default='', help_text='Discord Webhook URL')
-    
+
+    # 企业微信配置
+    wecom_enabled = models.BooleanField(default=False, help_text='是否启用企业微信通知')
+    wecom_webhook_url = models.URLField(blank=True, default='', help_text='企业微信机器人 Webhook URL')
+
     # 分类开关（使用 JSONField 存储）
     categories = models.JSONField(
         default=dict,
         help_text='各分类通知开关，如 {"scan": true, "vulnerability": true, "asset": true, "system": false}'
     )
-    
+
     # 时间信息
     created_at = models.DateTimeField(auto_now_add=True)
     updated_at = models.DateTimeField(auto_now=True)
-    
+
     class Meta:
         db_table = 'notification_settings'
         verbose_name = '通知设置'
         verbose_name_plural = '通知设置'
-    
+
     def save(self, *args, **kwargs):
-        # 单例模式：强制只有一条记录
-        self.pk = 1
+        self.pk = 1  # 单例模式
         super().save(*args, **kwargs)
-    
+
     @classmethod
     def get_instance(cls) -> 'NotificationSettings':
         """获取或创建单例实例"""
@@ -52,7 +61,7 @@ class NotificationSettings(models.Model):
             }
         )
         return obj
-    
+
     def is_category_enabled(self, category: str) -> bool:
         """检查指定分类是否启用通知"""
         return self.categories.get(category, False)
@@ -60,10 +69,9 @@ class NotificationSettings(models.Model):
 
 class Notification(models.Model):
     """通知模型"""
-    
+
     id = models.AutoField(primary_key=True)
-    
-    # 通知分类
+
     category = models.CharField(
         max_length=20,
         choices=NotificationCategory.choices,
@@ -71,8 +79,7 @@ class Notification(models.Model):
         db_index=True,
         help_text='通知分类'
     )
-    
-    # 通知级别
+
     level = models.CharField(
         max_length=20,
         choices=NotificationLevel.choices,
@@ -80,16 +87,15 @@ class Notification(models.Model):
         db_index=True,
         help_text='通知级别'
     )
-    
+
     title = models.CharField(max_length=200, help_text='通知标题')
     message = models.CharField(max_length=2000, help_text='通知内容')
-    
-    # 时间信息
+
     created_at = models.DateTimeField(auto_now_add=True, help_text='创建时间')
-    
+
     is_read = models.BooleanField(default=False, help_text='是否已读')
     read_at = models.DateTimeField(null=True, blank=True, help_text='阅读时间')
-    
+
     class Meta:
         db_table = 'notification'
         verbose_name = '通知'
@@ -101,44 +107,26 @@ class Notification(models.Model):
             models.Index(fields=['level', '-created_at']),
             models.Index(fields=['is_read', '-created_at']),
         ]
-    
+
     def __str__(self):
         return f"{self.get_level_display()} - {self.title}"
-    
+
     @classmethod
-    def cleanup_old_notifications(cls):
-        """
-        清理超过15天的旧通知（硬编码）
-        
-        Returns:
-            int: 删除的通知数量
-        """
-        from datetime import timedelta
-        from django.utils import timezone
-        
-        # 硬编码：只保留最近15天的通知
+    def cleanup_old_notifications(cls) -> int:
+        """清理超过15天的旧通知"""
         cutoff_date = timezone.now() - timedelta(days=15)
-        delete_result = cls.objects.filter(created_at__lt=cutoff_date).delete()
-        
-        return delete_result[0] if delete_result[0] else 0
-    
+        deleted_count, _ = cls.objects.filter(created_at__lt=cutoff_date).delete()
+        return deleted_count or 0
+
     def save(self, *args, **kwargs):
-        """
-        重写save方法，在创建新通知时自动清理旧通知
-        """
+        """重写save方法，在创建新通知时自动清理旧通知"""
         is_new = self.pk is None
         super().save(*args, **kwargs)
-        
-        # 只在创建新通知时执行清理（自动清理超过15天的通知）
+
         if is_new:
             try:
                 deleted_count = self.__class__.cleanup_old_notifications()
                 if deleted_count > 0:
-                    import logging
-                    logger = logging.getLogger(__name__)
-                    logger.info(f"自动清理了 {deleted_count} 条超过15天的旧通知")
-            except Exception as e:
-                # 清理失败不应影响通知创建
-                import logging
-                logger = logging.getLogger(__name__)
-                logger.warning(f"通知自动清理失败: {e}")
+                    logger.info("自动清理了 %d 条超过15天的旧通知", deleted_count)
+            except Exception:
+                logger.warning("通知自动清理失败", exc_info=True)

backend/apps/scan/notifications/repositories.py

@@ -1,52 +1,70 @@
+"""通知系统仓储层模块"""
+
 import logging
-from typing import TypedDict
+from dataclasses import dataclass
+from typing import Optional
+
+from django.db.models import QuerySet
 from django.utils import timezone
 
 from apps.common.decorators import auto_ensure_db_connection
-from .models import Notification, NotificationSettings
 
+from .models import Notification, NotificationSettings
 
 logger = logging.getLogger(__name__)
 
 
-class NotificationSettingsData(TypedDict):
-    """通知设置数据结构"""
+@dataclass
+class NotificationSettingsData:
+    """通知设置更新数据"""
+
     discord_enabled: bool
     discord_webhook_url: str
     categories: dict[str, bool]
+    wecom_enabled: bool = False
+    wecom_webhook_url: str = ''
 
 
 @auto_ensure_db_connection
 class NotificationSettingsRepository:
     """通知设置仓储层"""
-    
+
     def get_settings(self) -> NotificationSettings:
         """获取通知设置单例"""
         return NotificationSettings.get_instance()
-    
-    def update_settings(
-        self,
-        discord_enabled: bool,
-        discord_webhook_url: str,
-        categories: dict[str, bool]
-    ) -> NotificationSettings:
+
+    def update_settings(self, data: NotificationSettingsData) -> NotificationSettings:
         """更新通知设置"""
         settings = NotificationSettings.get_instance()
-        settings.discord_enabled = discord_enabled
-        settings.discord_webhook_url = discord_webhook_url
-        settings.categories = categories
+        settings.discord_enabled = data.discord_enabled
+        settings.discord_webhook_url = data.discord_webhook_url
+        settings.wecom_enabled = data.wecom_enabled
+        settings.wecom_webhook_url = data.wecom_webhook_url
+        settings.categories = data.categories
         settings.save()
         return settings
-    
+
     def is_category_enabled(self, category: str) -> bool:
         """检查指定分类是否启用"""
-        settings = self.get_settings()
-        return settings.is_category_enabled(category)
+        return self.get_settings().is_category_enabled(category)
 
 
 @auto_ensure_db_connection
 class DjangoNotificationRepository:
-    def get_filtered(self, level: str | None = None, unread: bool | None = None):
+    """通知数据仓储层"""
+
+    def get_filtered(
+        self,
+        level: Optional[str] = None,
+        unread: Optional[bool] = None
+    ) -> QuerySet[Notification]:
+        """
+        获取过滤后的通知列表
+
+        Args:
+            level: 通知级别过滤
+            unread: 已读状态过滤 (True=未读, False=已读, None=全部)
+        """
         queryset = Notification.objects.all()
 
         if level:
@@ -60,16 +78,24 @@ class DjangoNotificationRepository:
         return queryset.order_by("-created_at")
 
     def get_unread_count(self) -> int:
+        """获取未读通知数量"""
         return Notification.objects.filter(is_read=False).count()
 
     def mark_all_as_read(self) -> int:
-        updated = Notification.objects.filter(is_read=False).update(
+        """标记所有通知为已读，返回更新数量"""
+        return Notification.objects.filter(is_read=False).update(
             is_read=True,
             read_at=timezone.now(),
         )
-        return updated
 
-    def create(self, title: str, message: str, level: str, category: str = 'system') -> Notification:
+    def create(
+        self,
+        title: str,
+        message: str,
+        level: str,
+        category: str = 'system'
+    ) -> Notification:
+        """创建新通知"""
         return Notification.objects.create(
             category=category,
             level=level,

backend/apps/scan/notifications/services.py

@@ -60,13 +60,12 @@ def push_to_external_channels(notification: Notification) -> None:
         except Exception as e:
             logger.warning(f"Discord 推送失败: {e}")
     
-    # 未来扩展：Slack
-    # if settings.slack_enabled and settings.slack_webhook_url:
-    #     _send_slack(notification, settings.slack_webhook_url)
-    
-    # 未来扩展：Telegram
-    # if settings.telegram_enabled and settings.telegram_bot_token:
-    #     _send_telegram(notification, settings.telegram_chat_id)
+    # 企业微信渠道
+    if settings.wecom_enabled and settings.wecom_webhook_url:
+        try:
+            _send_wecom(notification, settings.wecom_webhook_url)
+        except Exception as e:
+            logger.warning(f"企业微信推送失败: {e}")
 
 
 def _send_discord(notification: Notification, webhook_url: str) -> bool:
@@ -103,6 +102,41 @@ def _send_discord(notification: Notification, webhook_url: str) -> bool:
         return False
 
 
+def _send_wecom(notification: Notification, webhook_url: str) -> bool:
+    """发送到企业微信机器人 Webhook"""
+    try:
+        emoji = CATEGORY_EMOJI.get(notification.category, '📢')
+
+        # 企业微信 Markdown 格式
+        content = f"""**{emoji} {notification.title}**
+> 级别：{notification.get_level_display()}
+> 分类：{notification.get_category_display()}
+
+{notification.message}"""
+
+        payload = {
+            'msgtype': 'markdown',
+            'markdown': {'content': content}
+        }
+
+        response = requests.post(webhook_url, json=payload, timeout=10)
+
+        if response.status_code == 200:
+            result = response.json()
+            if result.get('errcode') == 0:
+                logger.info(f"企业微信通知发送成功 - {notification.title}")
+                return True
+            logger.warning(f"企业微信发送失败 - errcode: {result.get('errcode')}, errmsg: {result.get('errmsg')}")
+            return False
+
+        logger.warning(f"企业微信发送失败 - 状态码: {response.status_code}")
+        return False
+
+    except requests.RequestException as e:
+        logger.error(f"企业微信网络错误: {e}")
+        return False
+
+
 # ============================================================
 # 设置服务
 # ============================================================
@@ -121,31 +155,43 @@ class NotificationSettingsService:
                 'enabled': settings.discord_enabled,
                 'webhookUrl': settings.discord_webhook_url,
             },
+            'wecom': {
+                'enabled': settings.wecom_enabled,
+                'webhookUrl': settings.wecom_webhook_url,
+            },
             'categories': settings.categories,
         }
     
     def update_settings(self, data: dict) -> dict:
         """更新通知设置
-        
+
         注意：DRF CamelCaseJSONParser 会将前端的 webhookUrl 转换为 webhook_url
         """
         discord_data = data.get('discord', {})
+        wecom_data = data.get('wecom', {})
         categories = data.get('categories', {})
-        
+
         # CamelCaseJSONParser 转换后的字段名是 webhook_url
-        webhook_url = discord_data.get('webhook_url', '')
-        
+        discord_webhook_url = discord_data.get('webhook_url', '')
+        wecom_webhook_url = wecom_data.get('webhook_url', '')
+
         settings = self.repo.update_settings(
             discord_enabled=discord_data.get('enabled', False),
-            discord_webhook_url=webhook_url,
+            discord_webhook_url=discord_webhook_url,
+            wecom_enabled=wecom_data.get('enabled', False),
+            wecom_webhook_url=wecom_webhook_url,
             categories=categories,
         )
-        
+
         return {
             'discord': {
                 'enabled': settings.discord_enabled,
                 'webhookUrl': settings.discord_webhook_url,
             },
+            'wecom': {
+                'enabled': settings.wecom_enabled,
+                'webhookUrl': settings.wecom_webhook_url,
+            },
             'categories': settings.categories,
         }
 

backend/apps/scan/notifications/urls.py

@@ -21,9 +21,6 @@ urlpatterns = [
 
     # 标记全部已读
     path('mark-all-as-read/', NotificationMarkAllAsReadView.as_view(), name='mark-all-as-read'),
-    
-    # 测试通知
-    path('test/', views.notifications_test, name='test'),
 ]
 
 # WebSocket 实时通知路由在 routing.py 中定义：ws://host/ws/notifications/

backend/apps/scan/notifications/views.py

@@ -23,45 +23,7 @@ from .services import NotificationService, NotificationSettingsService
 logger = logging.getLogger(__name__)
 
 
-def notifications_test(request):
-    """
-    测试通知推送
-    """
-    try:
-        from .services import create_notification
-        from django.http import JsonResponse
 
-        level_param = request.GET.get('level', NotificationLevel.LOW)
-        try:
-            level_choice = NotificationLevel(level_param)
-        except ValueError:
-            level_choice = NotificationLevel.LOW
-
-        title = request.GET.get('title') or "测试通知"
-        message = request.GET.get('message') or "这是一条测试通知消息"
-
-        # 创建测试通知
-        notification = create_notification(
-            title=title,
-            message=message,
-            level=level_choice
-        )
-        
-        return JsonResponse({
-            'success': True,
-            'message': '测试通知已发送',
-            'notification_id': notification.id
-        })
-        
-    except Exception as e:
-        logger.error(f"发送测试通知失败: {e}")
-        return JsonResponse({
-            'success': False,
-            'error': str(e)
-        }, status=500)
-
-
-# build_api_response 已废弃，请使用 success_response/error_response
 
 
 def _parse_bool(value: str | None) -> bool | None:

backend/apps/scan/orchestrators/flow_orchestrator.py

@@ -147,10 +147,10 @@ class FlowOrchestrator:
                     return True
             return False
         
-        # 其他扫描类型：检查 tools
+        # 其他扫描类型（包括 screenshot）：检查 tools
         tools = scan_config.get('tools', {})
         for tool_config in tools.values():
-            if tool_config.get('enabled', False):
+            if isinstance(tool_config, dict) and tool_config.get('enabled', False):
                 return True
                 
         return False
@@ -222,6 +222,10 @@ class FlowOrchestrator:
             from apps.scan.flows.vuln_scan import vuln_scan_flow
             return vuln_scan_flow
         
+        elif scan_type == 'screenshot':
+            from apps.scan.flows.screenshot_flow import screenshot_flow
+            return screenshot_flow
+        
         else:
             logger.warning(f"未实现的扫描类型: {scan_type}")
             return None

backend/apps/scan/providers/__init__.py

@@ -0,0 +1,56 @@
+"""
+扫描目标提供者模块
+
+提供统一的目标获取接口，支持多种数据源：
+- DatabaseTargetProvider: 从数据库查询（完整扫描）
+- ListTargetProvider: 使用内存列表（快速扫描阶段1）
+- SnapshotTargetProvider: 从快照表读取（快速扫描阶段2+）
+- PipelineTargetProvider: 使用管道输出（Phase 2）
+
+使用方式：
+    from apps.scan.providers import (
+        DatabaseTargetProvider,
+        ListTargetProvider,
+        SnapshotTargetProvider,
+        ProviderContext
+    )
+    
+    # 数据库模式（完整扫描）
+    provider = DatabaseTargetProvider(target_id=123)
+    
+    # 列表模式（快速扫描阶段1）
+    context = ProviderContext(target_id=1, scan_id=100)
+    provider = ListTargetProvider(
+        targets=["a.test.com"],
+        context=context
+    )
+    
+    # 快照模式（快速扫描阶段2+）
+    context = ProviderContext(target_id=1, scan_id=100)
+    provider = SnapshotTargetProvider(
+        scan_id=100,
+        snapshot_type="subdomain",
+        context=context
+    )
+    
+    # 使用 Provider
+    for host in provider.iter_hosts():
+        scan(host)
+"""
+
+from .base import TargetProvider, ProviderContext
+from .list_provider import ListTargetProvider
+from .database_provider import DatabaseTargetProvider
+from .snapshot_provider import SnapshotTargetProvider, SnapshotType
+from .pipeline_provider import PipelineTargetProvider, StageOutput
+
+__all__ = [
+    'TargetProvider',
+    'ProviderContext',
+    'ListTargetProvider',
+    'DatabaseTargetProvider',
+    'SnapshotTargetProvider',
+    'SnapshotType',
+    'PipelineTargetProvider',
+    'StageOutput',
+]

backend/apps/scan/providers/base.py

@@ -0,0 +1,115 @@
+"""
+扫描目标提供者基础模块
+
+定义 ProviderContext 数据类和 TargetProvider 抽象基类。
+"""
+
+import ipaddress
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Iterator, Optional
+
+if TYPE_CHECKING:
+    from apps.common.utils import BlacklistFilter
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ProviderContext:
+    """
+    Provider 上下文，携带元数据
+
+    Attributes:
+        target_id: 关联的 Target ID（用于结果保存），None 表示临时扫描（不保存）
+        scan_id: 扫描任务 ID
+    """
+    target_id: Optional[int] = None
+    scan_id: Optional[int] = None
+
+
+class TargetProvider(ABC):
+    """
+    扫描目标提供者抽象基类
+
+    职责：
+    - 提供扫描目标（域名、IP、URL 等）的迭代器
+    - 提供黑名单过滤器
+    - 携带上下文信息（target_id, scan_id 等）
+    - 自动展开 CIDR（子类无需关心）
+
+    使用方式：
+        provider = create_target_provider(target_id=123)
+        for host in provider.iter_hosts():
+            print(host)
+    """
+
+    def __init__(self, context: Optional[ProviderContext] = None):
+        self._context = context or ProviderContext()
+
+    @property
+    def context(self) -> ProviderContext:
+        """返回 Provider 上下文"""
+        return self._context
+
+    @staticmethod
+    def _expand_host(host: str) -> Iterator[str]:
+        """
+        展开主机（如果是 CIDR 则展开为多个 IP，否则直接返回）
+
+        示例：
+            "192.168.1.0/30" → "192.168.1.1", "192.168.1.2"
+            "192.168.1.1" → "192.168.1.1"
+            "example.com" → "example.com"
+        """
+        from apps.common.validators import detect_target_type
+        from apps.targets.models import Target
+
+        host = host.strip()
+        if not host:
+            return
+
+        try:
+            target_type = detect_target_type(host)
+
+            if target_type == Target.TargetType.CIDR:
+                network = ipaddress.ip_network(host, strict=False)
+                if network.num_addresses == 1:
+                    yield str(network.network_address)
+                else:
+                    yield from (str(ip) for ip in network.hosts())
+            elif target_type in (Target.TargetType.IP, Target.TargetType.DOMAIN):
+                yield host
+        except ValueError as e:
+            logger.warning("跳过无效的主机格式 '%s': %s", host, str(e))
+
+    def iter_hosts(self) -> Iterator[str]:
+        """迭代主机列表（域名/IP），自动展开 CIDR"""
+        for host in self._iter_raw_hosts():
+            yield from self._expand_host(host)
+
+    @abstractmethod
+    def _iter_raw_hosts(self) -> Iterator[str]:
+        """迭代原始主机列表（可能包含 CIDR），子类实现"""
+        pass
+
+    @abstractmethod
+    def iter_urls(self) -> Iterator[str]:
+        """迭代 URL 列表"""
+        pass
+
+    @abstractmethod
+    def get_blacklist_filter(self) -> Optional['BlacklistFilter']:
+        """获取黑名单过滤器，返回 None 表示不过滤"""
+        pass
+
+    @property
+    def target_id(self) -> Optional[int]:
+        """返回关联的 target_id，临时扫描返回 None"""
+        return self._context.target_id
+
+    @property
+    def scan_id(self) -> Optional[int]:
+        """返回关联的 scan_id"""
+        return self._context.scan_id

backend/apps/scan/providers/database_provider.py

@@ -0,0 +1,93 @@
+"""
+数据库目标提供者模块
+
+提供基于数据库查询的目标提供者实现。
+"""
+
+import logging
+from typing import TYPE_CHECKING, Iterator, Optional
+
+from .base import ProviderContext, TargetProvider
+
+if TYPE_CHECKING:
+    from apps.common.utils import BlacklistFilter
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseTargetProvider(TargetProvider):
+    """
+    数据库目标提供者 - 从 Target 表及关联资产表查询
+
+    数据来源：
+    - iter_hosts(): 根据 Target 类型返回域名/IP
+    - iter_urls(): WebSite/Endpoint 表，带回退链
+
+    使用方式：
+        provider = DatabaseTargetProvider(target_id=123)
+        for host in provider.iter_hosts():
+            scan(host)
+    """
+
+    def __init__(self, target_id: int, context: Optional[ProviderContext] = None):
+        ctx = context or ProviderContext()
+        ctx.target_id = target_id
+        super().__init__(ctx)
+        self._blacklist_filter: Optional['BlacklistFilter'] = None
+
+    def iter_hosts(self) -> Iterator[str]:
+        """从数据库查询主机列表，自动展开 CIDR 并应用黑名单过滤"""
+        blacklist = self.get_blacklist_filter()
+
+        for host in self._iter_raw_hosts():
+            for expanded_host in self._expand_host(host):
+                if not blacklist or blacklist.is_allowed(expanded_host):
+                    yield expanded_host
+
+    def _iter_raw_hosts(self) -> Iterator[str]:
+        """从数据库查询原始主机列表（可能包含 CIDR）"""
+        from apps.asset.services.asset.subdomain_service import SubdomainService
+        from apps.targets.models import Target
+        from apps.targets.services import TargetService
+
+        target = TargetService().get_target(self.target_id)
+        if not target:
+            logger.warning("Target ID %d 不存在", self.target_id)
+            return
+
+        if target.type == Target.TargetType.DOMAIN:
+            yield target.name
+            for domain in SubdomainService().iter_subdomain_names_by_target(
+                target_id=self.target_id,
+                chunk_size=1000
+            ):
+                if domain != target.name:
+                    yield domain
+
+        elif target.type in (Target.TargetType.IP, Target.TargetType.CIDR):
+            yield target.name
+
+    def iter_urls(self) -> Iterator[str]:
+        """从数据库查询 URL 列表，使用回退链：Endpoint → WebSite → Default"""
+        from apps.scan.services.target_export_service import (
+            DataSource,
+            _iter_urls_with_fallback,
+        )
+
+        blacklist = self.get_blacklist_filter()
+
+        for url, _ in _iter_urls_with_fallback(
+            target_id=self.target_id,
+            sources=[DataSource.ENDPOINT, DataSource.WEBSITE, DataSource.DEFAULT],
+            blacklist_filter=blacklist
+        ):
+            yield url
+
+    def get_blacklist_filter(self) -> Optional['BlacklistFilter']:
+        """获取黑名单过滤器（延迟加载）"""
+        if self._blacklist_filter is None:
+            from apps.common.services import BlacklistService
+            from apps.common.utils import BlacklistFilter
+            rules = BlacklistService().get_rules(self.target_id)
+            self._blacklist_filter = BlacklistFilter(rules)
+        return self._blacklist_filter

backend/apps/scan/providers/list_provider.py

@@ -0,0 +1,84 @@
+"""
+列表目标提供者模块
+
+提供基于内存列表的目标提供者实现。
+"""
+
+from typing import Iterator, Optional, List
+
+from .base import TargetProvider, ProviderContext
+
+
+class ListTargetProvider(TargetProvider):
+    """
+    列表目标提供者 - 直接使用内存中的列表
+    
+    用于快速扫描、临时扫描等场景，只扫描用户指定的目标。
+    
+    特点：
+    - 不查询数据库
+    - 不应用黑名单过滤（用户明确指定的目标）
+    - 不关联 target_id（由调用方负责创建 Target）
+    - 自动检测输入类型（URL/域名/IP/CIDR）
+    - 自动展开 CIDR
+    
+    使用方式：
+        # 快速扫描：用户提供目标，自动识别类型
+        provider = ListTargetProvider(targets=[
+            "example.com",              # 域名
+            "192.168.1.0/24",           # CIDR（自动展开）
+            "https://api.example.com"   # URL
+        ])
+        for host in provider.iter_hosts():
+            scan(host)
+    """
+    
+    def __init__(
+        self,
+        targets: Optional[List[str]] = None,
+        context: Optional[ProviderContext] = None
+    ):
+        """
+        初始化列表目标提供者
+        
+        Args:
+            targets: 目标列表（自动识别类型：URL/域名/IP/CIDR）
+            context: Provider 上下文
+        """
+        from apps.common.validators import detect_input_type
+        
+        ctx = context or ProviderContext()
+        super().__init__(ctx)
+        
+        # 自动分类目标
+        self._hosts = []
+        self._urls = []
+        
+        if targets:
+            for target in targets:
+                target = target.strip()
+                if not target:
+                    continue
+                
+                try:
+                    input_type = detect_input_type(target)
+                    if input_type == 'url':
+                        self._urls.append(target)
+                    else:
+                        # domain/ip/cidr 都作为 host
+                        self._hosts.append(target)
+                except ValueError:
+                    # 无法识别类型，默认作为 host
+                    self._hosts.append(target)
+    
+    def _iter_raw_hosts(self) -> Iterator[str]:
+        """迭代原始主机列表（可能包含 CIDR）"""
+        yield from self._hosts
+    
+    def iter_urls(self) -> Iterator[str]:
+        """迭代 URL 列表"""
+        yield from self._urls
+    
+    def get_blacklist_filter(self) -> None:
+        """列表模式不使用黑名单过滤"""
+        return None

backend/apps/scan/providers/pipeline_provider.py

@@ -0,0 +1,91 @@
+"""
+管道目标提供者模块
+
+提供基于管道阶段输出的目标提供者实现。
+用于 Phase 2 管道模式的阶段间数据传递。
+"""
+
+from dataclasses import dataclass, field
+from typing import Iterator, Optional, List, Dict, Any
+
+from .base import TargetProvider, ProviderContext
+
+
+@dataclass
+class StageOutput:
+    """
+    阶段输出数据
+    
+    用于在管道阶段之间传递数据。
+    
+    Attributes:
+        hosts: 主机列表（域名/IP）
+        urls: URL 列表
+        new_targets: 新发现的目标列表
+        stats: 统计信息
+        success: 是否成功
+        error: 错误信息
+    """
+    hosts: List[str] = field(default_factory=list)
+    urls: List[str] = field(default_factory=list)
+    new_targets: List[str] = field(default_factory=list)
+    stats: Dict[str, Any] = field(default_factory=dict)
+    success: bool = True
+    error: Optional[str] = None
+
+
+class PipelineTargetProvider(TargetProvider):
+    """
+    管道目标提供者 - 使用上一阶段的输出
+    
+    用于 Phase 2 管道模式的阶段间数据传递。
+    
+    特点：
+    - 不查询数据库
+    - 不应用黑名单过滤（数据已在上一阶段过滤）
+    - 直接使用 StageOutput 中的数据
+    
+    使用方式（Phase 2）：
+        stage1_output = stage1.run(input)
+        provider = PipelineTargetProvider(
+            previous_output=stage1_output,
+            target_id=123
+        )
+        for host in provider.iter_hosts():
+            stage2.scan(host)
+    """
+    
+    def __init__(
+        self,
+        previous_output: StageOutput,
+        target_id: Optional[int] = None,
+        context: Optional[ProviderContext] = None
+    ):
+        """
+        初始化管道目标提供者
+        
+        Args:
+            previous_output: 上一阶段的输出
+            target_id: 可选，关联到某个 Target（用于保存结果）
+            context: Provider 上下文
+        """
+        ctx = context or ProviderContext(target_id=target_id)
+        super().__init__(ctx)
+        self._previous_output = previous_output
+    
+    def _iter_raw_hosts(self) -> Iterator[str]:
+        """迭代上一阶段输出的原始主机（可能包含 CIDR）"""
+        yield from self._previous_output.hosts
+    
+    def iter_urls(self) -> Iterator[str]:
+        """迭代上一阶段输出的 URL"""
+        yield from self._previous_output.urls
+    
+    def get_blacklist_filter(self) -> None:
+        """管道传递的数据已经过滤过了"""
+        return None
+    
+    @property
+    def previous_output(self) -> StageOutput:
+        """返回上一阶段的输出"""
+        return self._previous_output

backend/apps/scan/providers/snapshot_provider.py

@@ -0,0 +1,175 @@
+"""
+快照目标提供者模块
+
+提供基于快照表的目标提供者实现。
+用于快速扫描的阶段间数据传递。
+"""
+
+import logging
+from typing import Iterator, Optional, Literal
+
+from .base import TargetProvider, ProviderContext
+
+logger = logging.getLogger(__name__)
+
+# 快照类型定义
+SnapshotType = Literal["subdomain", "website", "endpoint", "host_port"]
+
+
+class SnapshotTargetProvider(TargetProvider):
+    """
+    快照目标提供者 - 从快照表读取本次扫描的数据
+    
+    用于快速扫描的阶段间数据传递，解决精确扫描控制问题。
+    
+    核心价值：
+    - 只返回本次扫描（scan_id）发现的资产
+    - 避免扫描历史数据（DatabaseTargetProvider 会扫描所有历史资产）
+    
+    特点：
+    - 通过 scan_id 过滤快照表
+    - 不应用黑名单过滤（数据已在上一阶段过滤）
+    - 支持多种快照类型（subdomain/website/endpoint/host_port）
+    
+    使用场景：
+        # 快速扫描流程
+        用户输入: a.test.com
+        创建 Target: test.com (id=1)
+        创建 Scan: scan_id=100
+        
+        # 阶段1: 子域名发现
+        provider = ListTargetProvider(
+            targets=["a.test.com"],
+            context=ProviderContext(target_id=1, scan_id=100)
+        )
+        # 发现: b.a.test.com, c.a.test.com
+        # 保存: SubdomainSnapshot(scan_id=100) + Subdomain(target_id=1)
+        
+        # 阶段2: 端口扫描
+        provider = SnapshotTargetProvider(
+            scan_id=100,
+            snapshot_type="subdomain",
+            context=ProviderContext(target_id=1, scan_id=100)
+        )
+        # 只返回: b.a.test.com, c.a.test.com（本次扫描发现的）
+        # 不返回: www.test.com, api.test.com（历史数据）
+        
+        # 阶段3: 网站扫描
+        provider = SnapshotTargetProvider(
+            scan_id=100,
+            snapshot_type="host_port",
+            context=ProviderContext(target_id=1, scan_id=100)
+        )
+        # 只返回本次扫描发现的 IP:Port
+    """
+    
+    def __init__(
+        self,
+        scan_id: int,
+        snapshot_type: SnapshotType,
+        context: Optional[ProviderContext] = None
+    ):
+        """
+        初始化快照目标提供者
+        
+        Args:
+            scan_id: 扫描任务 ID（必需）
+            snapshot_type: 快照类型
+                - "subdomain": 子域名快照（SubdomainSnapshot）
+                - "website": 网站快照（WebsiteSnapshot）
+                - "endpoint": 端点快照（EndpointSnapshot）
+                - "host_port": 主机端口映射快照（HostPortMappingSnapshot）
+            context: Provider 上下文
+        """
+        ctx = context or ProviderContext()
+        ctx.scan_id = scan_id
+        super().__init__(ctx)
+        self._scan_id = scan_id
+        self._snapshot_type = snapshot_type
+    
+    def _iter_raw_hosts(self) -> Iterator[str]:
+        """
+        从快照表迭代主机列表
+        
+        根据 snapshot_type 选择不同的快照表：
+        - subdomain: SubdomainSnapshot.name
+        - host_port: HostPortMappingSnapshot.host (返回 host:port 格式，不经过验证)
+        """
+        if self._snapshot_type == "subdomain":
+            from apps.asset.services.snapshot import SubdomainSnapshotsService
+            service = SubdomainSnapshotsService()
+            yield from service.iter_subdomain_names_by_scan(
+                scan_id=self._scan_id,
+                chunk_size=1000
+            )
+        
+        elif self._snapshot_type == "host_port":
+            # host_port 类型不使用 _iter_raw_hosts，直接在 iter_hosts 中处理
+            # 这里返回空，避免被基类的 iter_hosts 调用
+            return
+        
+        else:
+            # 其他类型暂不支持 iter_hosts
+            logger.warning(
+                "快照类型 '%s' 不支持 iter_hosts，返回空迭代器",
+                self._snapshot_type
+            )
+            return
+    
+    def iter_hosts(self) -> Iterator[str]:
+        """
+        迭代主机列表
+        
+        对于 host_port 类型，返回 host:port 格式，不经过 CIDR 展开验证
+        """
+        if self._snapshot_type == "host_port":
+            # host_port 类型直接返回 host:port，不经过 _expand_host 验证
+            from apps.asset.services.snapshot import HostPortMappingSnapshotsService
+            service = HostPortMappingSnapshotsService()
+            queryset = service.get_by_scan(scan_id=self._scan_id)
+            for mapping in queryset.iterator(chunk_size=1000):
+                yield f"{mapping.host}:{mapping.port}"
+        else:
+            # 其他类型使用基类的 iter_hosts（会调用 _iter_raw_hosts 并展开 CIDR）
+            yield from super().iter_hosts()
+    
+    def iter_urls(self) -> Iterator[str]:
+        """
+        从快照表迭代 URL 列表
+        
+        根据 snapshot_type 选择不同的快照表：
+        - website: WebsiteSnapshot.url
+        - endpoint: EndpointSnapshot.url
+        """
+        if self._snapshot_type == "website":
+            from apps.asset.services.snapshot import WebsiteSnapshotsService
+            service = WebsiteSnapshotsService()
+            yield from service.iter_website_urls_by_scan(
+                scan_id=self._scan_id,
+                chunk_size=1000
+            )
+        
+        elif self._snapshot_type == "endpoint":
+            from apps.asset.services.snapshot import EndpointSnapshotsService
+            service = EndpointSnapshotsService()
+            # 从快照表获取端点 URL
+            queryset = service.get_by_scan(scan_id=self._scan_id)
+            for endpoint in queryset.iterator(chunk_size=1000):
+                yield endpoint.url
+        
+        else:
+            # 其他类型暂不支持 iter_urls
+            logger.warning(
+                "快照类型 '%s' 不支持 iter_urls，返回空迭代器",
+                self._snapshot_type
+            )
+            return
+    
+    def get_blacklist_filter(self) -> None:
+        """快照数据已在上一阶段过滤过了"""
+        return None
+    
+    @property
+    def snapshot_type(self) -> SnapshotType:
+        """返回快照类型"""
+        return self._snapshot_type

backend/apps/scan/providers/tests/__init__.py

@@ -0,0 +1,3 @@
+"""
+扫描目标提供者测试模块
+"""

backend/apps/scan/providers/tests/test_common_properties.py

@@ -0,0 +1,256 @@
+"""
+通用属性测试
+
+包含跨多个 Provider 的通用属性测试：
+- Property 4: Context Propagation
+- Property 5: Non-Database Provider Blacklist Filter
+- Property 7: CIDR Expansion Consistency
+"""
+
+import pytest
+from hypothesis import given, strategies as st, settings
+from ipaddress import IPv4Network
+
+from apps.scan.providers import (
+    ProviderContext,
+    ListTargetProvider,
+    DatabaseTargetProvider,
+    PipelineTargetProvider,
+    SnapshotTargetProvider
+)
+from apps.scan.providers.pipeline_provider import StageOutput
+
+
+class TestContextPropagation:
+    """
+    Property 4: Context Propagation
+    
+    *For any* ProviderContext，传入 Provider 构造函数后，
+    Provider 的 target_id 和 scan_id 属性应该与 context 中的值一致。
+    
+    **Validates: Requirements 1.3, 1.5, 7.4, 7.5**
+    """
+    
+    @given(
+        target_id=st.integers(min_value=1, max_value=10000),
+        scan_id=st.integers(min_value=1, max_value=10000)
+    )
+    @settings(max_examples=100)
+    def test_property_4_list_provider_context_propagation(self, target_id, scan_id):
+        """
+        Property 4: Context Propagation (ListTargetProvider)
+        
+        Feature: scan-target-provider, Property 4: Context Propagation
+        **Validates: Requirements 1.3, 1.5, 7.4, 7.5**
+        """
+        ctx = ProviderContext(target_id=target_id, scan_id=scan_id)
+        provider = ListTargetProvider(targets=["example.com"], context=ctx)
+        
+        assert provider.target_id == target_id
+        assert provider.scan_id == scan_id
+        assert provider.context.target_id == target_id
+        assert provider.context.scan_id == scan_id
+    
+    @given(
+        target_id=st.integers(min_value=1, max_value=10000),
+        scan_id=st.integers(min_value=1, max_value=10000)
+    )
+    @settings(max_examples=100)
+    def test_property_4_database_provider_context_propagation(self, target_id, scan_id):
+        """
+        Property 4: Context Propagation (DatabaseTargetProvider)
+        
+        Feature: scan-target-provider, Property 4: Context Propagation
+        **Validates: Requirements 1.3, 1.5, 7.4, 7.5**
+        """
+        ctx = ProviderContext(target_id=999, scan_id=scan_id)
+        # DatabaseTargetProvider 会覆盖 context 中的 target_id
+        provider = DatabaseTargetProvider(target_id=target_id, context=ctx)
+        
+        assert provider.target_id == target_id  # 使用构造函数参数
+        assert provider.scan_id == scan_id  # 使用 context 中的值
+        assert provider.context.target_id == target_id
+        assert provider.context.scan_id == scan_id
+    
+    @given(
+        target_id=st.integers(min_value=1, max_value=10000),
+        scan_id=st.integers(min_value=1, max_value=10000)
+    )
+    @settings(max_examples=100)
+    def test_property_4_pipeline_provider_context_propagation(self, target_id, scan_id):
+        """
+        Property 4: Context Propagation (PipelineTargetProvider)
+        
+        Feature: scan-target-provider, Property 4: Context Propagation
+        **Validates: Requirements 1.3, 1.5, 7.4, 7.5**
+        """
+        ctx = ProviderContext(target_id=target_id, scan_id=scan_id)
+        stage_output = StageOutput(hosts=["example.com"])
+        provider = PipelineTargetProvider(previous_output=stage_output, context=ctx)
+        
+        assert provider.target_id == target_id
+        assert provider.scan_id == scan_id
+        assert provider.context.target_id == target_id
+        assert provider.context.scan_id == scan_id
+    
+    @given(
+        target_id=st.integers(min_value=1, max_value=10000),
+        scan_id=st.integers(min_value=1, max_value=10000)
+    )
+    @settings(max_examples=100)
+    def test_property_4_snapshot_provider_context_propagation(self, target_id, scan_id):
+        """
+        Property 4: Context Propagation (SnapshotTargetProvider)
+        
+        Feature: scan-target-provider, Property 4: Context Propagation
+        **Validates: Requirements 1.3, 1.5, 7.4, 7.5**
+        """
+        ctx = ProviderContext(target_id=target_id, scan_id=999)
+        # SnapshotTargetProvider 会覆盖 context 中的 scan_id
+        provider = SnapshotTargetProvider(
+            scan_id=scan_id,
+            snapshot_type="subdomain",
+            context=ctx
+        )
+        
+        assert provider.target_id == target_id  # 使用 context 中的值
+        assert provider.scan_id == scan_id  # 使用构造函数参数
+        assert provider.context.target_id == target_id
+        assert provider.context.scan_id == scan_id
+
+
+class TestNonDatabaseProviderBlacklistFilter:
+    """
+    Property 5: Non-Database Provider Blacklist Filter
+    
+    *For any* ListTargetProvider 或 PipelineTargetProvider 实例，
+    get_blacklist_filter() 方法应该返回 None。
+    
+    **Validates: Requirements 3.4, 9.4, 9.5**
+    """
+    
+    @given(targets=st.lists(st.text(min_size=1, max_size=20), max_size=10))
+    @settings(max_examples=100)
+    def test_property_5_list_provider_no_blacklist(self, targets):
+        """
+        Property 5: Non-Database Provider Blacklist Filter (ListTargetProvider)
+        
+        Feature: scan-target-provider, Property 5: Non-Database Provider Blacklist Filter
+        **Validates: Requirements 3.4, 9.4, 9.5**
+        """
+        provider = ListTargetProvider(targets=targets)
+        assert provider.get_blacklist_filter() is None
+    
+    @given(hosts=st.lists(st.text(min_size=1, max_size=20), max_size=10))
+    @settings(max_examples=100)
+    def test_property_5_pipeline_provider_no_blacklist(self, hosts):
+        """
+        Property 5: Non-Database Provider Blacklist Filter (PipelineTargetProvider)
+        
+        Feature: scan-target-provider, Property 5: Non-Database Provider Blacklist Filter
+        **Validates: Requirements 3.4, 9.4, 9.5**
+        """
+        stage_output = StageOutput(hosts=hosts)
+        provider = PipelineTargetProvider(previous_output=stage_output)
+        assert provider.get_blacklist_filter() is None
+    
+    def test_property_5_snapshot_provider_no_blacklist(self):
+        """
+        Property 5: Non-Database Provider Blacklist Filter (SnapshotTargetProvider)
+        
+        Feature: scan-target-provider, Property 5: Non-Database Provider Blacklist Filter
+        **Validates: Requirements 3.4, 9.4, 9.5**
+        """
+        provider = SnapshotTargetProvider(scan_id=1, snapshot_type="subdomain")
+        assert provider.get_blacklist_filter() is None
+
+
+class TestCIDRExpansionConsistency:
+    """
+    Property 7: CIDR Expansion Consistency
+    
+    *For any* CIDR 字符串（如 "192.168.1.0/24"），所有 Provider 的 iter_hosts() 
+    方法应该将其展开为相同的单个 IP 地址列表。
+    
+    **Validates: Requirements 1.1, 3.6**
+    """
+    
+    @given(
+        # 生成小的 CIDR 范围以避免测试超时
+        network_prefix=st.integers(min_value=1, max_value=254),
+        cidr_suffix=st.integers(min_value=28, max_value=30)  # /28 = 16 IPs, /30 = 4 IPs
+    )
+    @settings(max_examples=50, deadline=None)
+    def test_property_7_cidr_expansion_consistency(self, network_prefix, cidr_suffix):
+        """
+        Property 7: CIDR Expansion Consistency
+        
+        Feature: scan-target-provider, Property 7: CIDR Expansion Consistency
+        **Validates: Requirements 1.1, 3.6**
+        
+        For any CIDR string, all Providers should expand it to the same IP list.
+        """
+        cidr = f"192.168.{network_prefix}.0/{cidr_suffix}"
+        
+        # 计算预期的 IP 列表
+        network = IPv4Network(cidr, strict=False)
+        # 排除网络地址和广播地址
+        expected_ips = [str(ip) for ip in network.hosts()]
+        
+        # 如果 CIDR 太小（/31 或 /32），使用所有地址
+        if not expected_ips:
+            expected_ips = [str(ip) for ip in network]
+        
+        # ListTargetProvider
+        list_provider = ListTargetProvider(targets=[cidr])
+        list_result = list(list_provider.iter_hosts())
+        
+        # PipelineTargetProvider
+        stage_output = StageOutput(hosts=[cidr])
+        pipeline_provider = PipelineTargetProvider(previous_output=stage_output)
+        pipeline_result = list(pipeline_provider.iter_hosts())
+        
+        # 验证：所有 Provider 展开的结果应该一致
+        assert list_result == expected_ips, f"ListProvider CIDR expansion mismatch for {cidr}"
+        assert pipeline_result == expected_ips, f"PipelineProvider CIDR expansion mismatch for {cidr}"
+        assert list_result == pipeline_result, f"Providers produce different results for {cidr}"
+    
+    def test_cidr_expansion_with_multiple_cidrs(self):
+        """测试多个 CIDR 的展开一致性"""
+        cidrs = ["192.168.1.0/30", "10.0.0.0/30"]
+        
+        # 计算预期结果
+        expected_ips = []
+        for cidr in cidrs:
+            network = IPv4Network(cidr, strict=False)
+            expected_ips.extend([str(ip) for ip in network.hosts()])
+        
+        # ListTargetProvider
+        list_provider = ListTargetProvider(targets=cidrs)
+        list_result = list(list_provider.iter_hosts())
+        
+        # PipelineTargetProvider
+        stage_output = StageOutput(hosts=cidrs)
+        pipeline_provider = PipelineTargetProvider(previous_output=stage_output)
+        pipeline_result = list(pipeline_provider.iter_hosts())
+        
+        # 验证
+        assert list_result == expected_ips
+        assert pipeline_result == expected_ips
+        assert list_result == pipeline_result
+    
+    def test_mixed_hosts_and_cidrs(self):
+        """测试混合主机和 CIDR 的处理"""
+        targets = ["example.com", "192.168.1.0/30", "test.com"]
+        
+        # 计算预期结果
+        network = IPv4Network("192.168.1.0/30", strict=False)
+        cidr_ips = [str(ip) for ip in network.hosts()]
+        expected = ["example.com"] + cidr_ips + ["test.com"]
+        
+        # ListTargetProvider
+        list_provider = ListTargetProvider(targets=targets)
+        list_result = list(list_provider.iter_hosts())
+        
+        # 验证
+        assert list_result == expected

backend/apps/scan/providers/tests/test_database_provider.py

@@ -0,0 +1,158 @@
+"""
+DatabaseTargetProvider 属性测试
+
+Property 7: DatabaseTargetProvider Blacklist Application
+*For any* 带有黑名单规则的 target_id，DatabaseTargetProvider 的 iter_hosts() 和 iter_urls() 
+应该过滤掉匹配黑名单规则的目标。
+
+**Validates: Requirements 2.3, 10.1, 10.2, 10.3**
+"""
+
+import pytest
+from unittest.mock import patch, MagicMock
+from hypothesis import given, strategies as st, settings
+
+from apps.scan.providers.database_provider import DatabaseTargetProvider
+from apps.scan.providers.base import ProviderContext
+
+
+# 生成有效域名的策略
+def valid_domain_strategy():
+    """生成有效的域名"""
+    label = st.text(
+        alphabet=st.characters(whitelist_categories=('L',), min_codepoint=97, max_codepoint=122),
+        min_size=2,
+        max_size=10
+    )
+    return st.builds(
+        lambda a, b, c: f"{a}.{b}.{c}",
+        label, label, st.sampled_from(['com', 'net', 'org', 'io'])
+    )
+
+
+class MockBlacklistFilter:
+    """模拟黑名单过滤器"""
+    
+    def __init__(self, blocked_patterns: list):
+        self.blocked_patterns = blocked_patterns
+    
+    def is_allowed(self, target: str) -> bool:
+        """检查目标是否被允许（不在黑名单中）"""
+        for pattern in self.blocked_patterns:
+            if pattern in target:
+                return False
+        return True
+
+
+class TestDatabaseTargetProviderProperties:
+    """DatabaseTargetProvider 属性测试类"""
+    
+    @given(
+        hosts=st.lists(valid_domain_strategy(), min_size=1, max_size=20),
+        blocked_keyword=st.text(
+            alphabet=st.characters(whitelist_categories=('L',), min_codepoint=97, max_codepoint=122),
+            min_size=2,
+            max_size=5
+        )
+    )
+    @settings(max_examples=100)
+    def test_property_7_blacklist_filters_hosts(self, hosts, blocked_keyword):
+        """
+        Property 7: DatabaseTargetProvider Blacklist Application (hosts)
+        
+        Feature: scan-target-provider, Property 7: DatabaseTargetProvider Blacklist Application
+        **Validates: Requirements 2.3, 10.1, 10.2, 10.3**
+        
+        For any set of hosts and a blacklist keyword, the provider should filter out
+        all hosts containing the blocked keyword.
+        """
+        # 创建模拟的黑名单过滤器
+        mock_filter = MockBlacklistFilter([blocked_keyword])
+        
+        # 创建 provider 并注入模拟的黑名单过滤器
+        provider = DatabaseTargetProvider(target_id=1)
+        provider._blacklist_filter = mock_filter
+        
+        # 模拟 Target 和 SubdomainService
+        mock_target = MagicMock()
+        mock_target.type = 'domain'
+        mock_target.name = hosts[0] if hosts else 'example.com'
+        
+        with patch('apps.targets.services.TargetService') as mock_target_service, \
+             patch('apps.asset.services.asset.subdomain_service.SubdomainService') as mock_subdomain_service:
+            
+            mock_target_service.return_value.get_target.return_value = mock_target
+            mock_subdomain_service.return_value.iter_subdomain_names_by_target.return_value = iter(hosts[1:] if len(hosts) > 1 else [])
+            
+            # 获取结果
+            result = list(provider.iter_hosts())
+            
+            # 验证：所有结果都不包含被阻止的关键词
+            for host in result:
+                assert blocked_keyword not in host, f"Host '{host}' should be filtered by blacklist keyword '{blocked_keyword}'"
+            
+            # 验证：所有不包含关键词的主机都应该在结果中
+            if hosts:
+                all_hosts = [hosts[0]] + [h for h in hosts[1:] if h != hosts[0]]
+                expected_allowed = [h for h in all_hosts if blocked_keyword not in h]
+            else:
+                expected_allowed = []
+            
+            assert set(result) == set(expected_allowed)
+
+
+class TestDatabaseTargetProviderUnit:
+    """DatabaseTargetProvider 单元测试类"""
+    
+    def test_target_id_in_context(self):
+        """测试 target_id 正确设置到上下文中"""
+        provider = DatabaseTargetProvider(target_id=123)
+        assert provider.target_id == 123
+        assert provider.context.target_id == 123
+    
+    def test_context_propagation(self):
+        """测试上下文传递"""
+        ctx = ProviderContext(scan_id=789)
+        provider = DatabaseTargetProvider(target_id=123, context=ctx)
+        
+        assert provider.target_id == 123  # target_id 被覆盖
+        assert provider.scan_id == 789
+    
+    def test_blacklist_filter_lazy_loading(self):
+        """测试黑名单过滤器延迟加载"""
+        provider = DatabaseTargetProvider(target_id=123)
+        
+        # 初始时 _blacklist_filter 为 None
+        assert provider._blacklist_filter is None
+        
+        # 模拟 BlacklistService
+        with patch('apps.common.services.BlacklistService') as mock_service, \
+             patch('apps.common.utils.BlacklistFilter') as mock_filter_class:
+            
+            mock_service.return_value.get_rules.return_value = []
+            mock_filter_instance = MagicMock()
+            mock_filter_class.return_value = mock_filter_instance
+            
+            # 第一次调用
+            result1 = provider.get_blacklist_filter()
+            assert result1 == mock_filter_instance
+            
+            # 第二次调用应该返回缓存的实例
+            result2 = provider.get_blacklist_filter()
+            assert result2 == mock_filter_instance
+            
+            # BlacklistService 只应该被调用一次
+            mock_service.return_value.get_rules.assert_called_once_with(123)
+    
+    def test_nonexistent_target_returns_empty(self):
+        """测试不存在的 target 返回空迭代器"""
+        provider = DatabaseTargetProvider(target_id=99999)
+        
+        with patch('apps.targets.services.TargetService') as mock_service, \
+             patch('apps.common.services.BlacklistService') as mock_blacklist_service:
+            
+            mock_service.return_value.get_target.return_value = None
+            mock_blacklist_service.return_value.get_rules.return_value = []
+            
+            result = list(provider.iter_hosts())
+            assert result == []

backend/apps/scan/providers/tests/test_list_provider.py

@@ -0,0 +1,152 @@
+"""
+ListTargetProvider 属性测试
+
+Property 1: ListTargetProvider Round-Trip
+*For any* 主机列表和 URL 列表，创建 ListTargetProvider 后迭代 iter_hosts() 和 iter_urls() 
+应该返回与输入相同的元素（顺序相同）。
+
+**Validates: Requirements 3.1, 3.2**
+"""
+
+import pytest
+from hypothesis import given, strategies as st, settings, assume
+
+from apps.scan.providers.list_provider import ListTargetProvider
+from apps.scan.providers.base import ProviderContext
+
+
+# 生成有效域名的策略
+def valid_domain_strategy():
+    """生成有效的域名"""
+    # 生成简单的域名格式: subdomain.domain.tld
+    label = st.text(
+        alphabet=st.characters(whitelist_categories=('L',), min_codepoint=97, max_codepoint=122),
+        min_size=2,
+        max_size=10
+    )
+    return st.builds(
+        lambda a, b, c: f"{a}.{b}.{c}",
+        label, label, st.sampled_from(['com', 'net', 'org', 'io'])
+    )
+
+# 生成有效 IP 地址的策略
+def valid_ip_strategy():
+    """生成有效的 IPv4 地址"""
+    octet = st.integers(min_value=1, max_value=254)
+    return st.builds(
+        lambda a, b, c, d: f"{a}.{b}.{c}.{d}",
+        octet, octet, octet, octet
+    )
+
+# 组合策略：域名或 IP
+host_strategy = st.one_of(valid_domain_strategy(), valid_ip_strategy())
+
+# 生成有效 URL 的策略
+def valid_url_strategy():
+    """生成有效的 URL"""
+    domain = valid_domain_strategy()
+    return st.builds(
+        lambda d, path: f"https://{d}/{path}" if path else f"https://{d}",
+        domain,
+        st.one_of(
+            st.just(""),
+            st.text(
+                alphabet=st.characters(whitelist_categories=('L',), min_codepoint=97, max_codepoint=122),
+                min_size=1,
+                max_size=10
+            )
+        )
+    )
+
+url_strategy = valid_url_strategy()
+
+
+class TestListTargetProviderProperties:
+    """ListTargetProvider 属性测试类"""
+    
+    @given(hosts=st.lists(host_strategy, max_size=50))
+    @settings(max_examples=100)
+    def test_property_1_hosts_round_trip(self, hosts):
+        """
+        Property 1: ListTargetProvider Round-Trip (hosts)
+        
+        Feature: scan-target-provider, Property 1: ListTargetProvider Round-Trip
+        **Validates: Requirements 3.1, 3.2**
+        
+        For any host list, creating a ListTargetProvider and iterating iter_hosts()
+        should return the same elements in the same order.
+        """
+        # ListTargetProvider 使用 targets 参数，自动分类为 hosts/urls
+        provider = ListTargetProvider(targets=hosts)
+        result = list(provider.iter_hosts())
+        assert result == hosts
+    
+    @given(urls=st.lists(url_strategy, max_size=50))
+    @settings(max_examples=100)
+    def test_property_1_urls_round_trip(self, urls):
+        """
+        Property 1: ListTargetProvider Round-Trip (urls)
+        
+        Feature: scan-target-provider, Property 1: ListTargetProvider Round-Trip
+        **Validates: Requirements 3.1, 3.2**
+        
+        For any URL list, creating a ListTargetProvider and iterating iter_urls()
+        should return the same elements in the same order.
+        """
+        # ListTargetProvider 使用 targets 参数，自动分类为 hosts/urls
+        provider = ListTargetProvider(targets=urls)
+        result = list(provider.iter_urls())
+        assert result == urls
+    
+    @given(
+        hosts=st.lists(host_strategy, max_size=30),
+        urls=st.lists(url_strategy, max_size=30)
+    )
+    @settings(max_examples=100)
+    def test_property_1_combined_round_trip(self, hosts, urls):
+        """
+        Property 1: ListTargetProvider Round-Trip (combined)
+        
+        Feature: scan-target-provider, Property 1: ListTargetProvider Round-Trip
+        **Validates: Requirements 3.1, 3.2**
+        
+        For any combination of hosts and URLs, both should round-trip correctly.
+        """
+        # 合并 hosts 和 urls，ListTargetProvider 会自动分类
+        combined = hosts + urls
+        provider = ListTargetProvider(targets=combined)
+        
+        hosts_result = list(provider.iter_hosts())
+        urls_result = list(provider.iter_urls())
+        
+        assert hosts_result == hosts
+        assert urls_result == urls
+
+
+class TestListTargetProviderUnit:
+    """ListTargetProvider 单元测试类"""
+    
+    def test_empty_lists(self):
+        """测试空列表返回空迭代器 - Requirements 3.5"""
+        provider = ListTargetProvider()
+        assert list(provider.iter_hosts()) == []
+        assert list(provider.iter_urls()) == []
+    
+    def test_blacklist_filter_returns_none(self):
+        """测试黑名单过滤器返回 None - Requirements 3.4"""
+        provider = ListTargetProvider(targets=["example.com"])
+        assert provider.get_blacklist_filter() is None
+    
+    def test_target_id_association(self):
+        """测试 target_id 关联 - Requirements 3.3"""
+        ctx = ProviderContext(target_id=123)
+        provider = ListTargetProvider(targets=["example.com"], context=ctx)
+        assert provider.target_id == 123
+    
+    def test_context_propagation(self):
+        """测试上下文传递"""
+        ctx = ProviderContext(target_id=456, scan_id=789)
+        provider = ListTargetProvider(targets=["example.com"], context=ctx)
+        
+        assert provider.target_id == 456
+        assert provider.scan_id == 789

backend/apps/scan/providers/tests/test_pipeline_provider.py

@@ -0,0 +1,180 @@
+"""
+PipelineTargetProvider 属性测试
+
+Property 3: PipelineTargetProvider Round-Trip
+*For any* StageOutput 对象，PipelineTargetProvider 的 iter_hosts() 和 iter_urls() 
+应该返回与 StageOutput 中 hosts 和 urls 列表相同的元素。
+
+**Validates: Requirements 5.1, 5.2**
+"""
+
+import pytest
+from hypothesis import given, strategies as st, settings
+
+from apps.scan.providers.pipeline_provider import PipelineTargetProvider, StageOutput
+from apps.scan.providers.base import ProviderContext
+
+
+# 生成有效域名的策略
+def valid_domain_strategy():
+    """生成有效的域名"""
+    label = st.text(
+        alphabet=st.characters(whitelist_categories=('L',), min_codepoint=97, max_codepoint=122),
+        min_size=2,
+        max_size=10
+    )
+    return st.builds(
+        lambda a, b, c: f"{a}.{b}.{c}",
+        label, label, st.sampled_from(['com', 'net', 'org', 'io'])
+    )
+
+# 生成有效 IP 地址的策略
+def valid_ip_strategy():
+    """生成有效的 IPv4 地址"""
+    octet = st.integers(min_value=1, max_value=254)
+    return st.builds(
+        lambda a, b, c, d: f"{a}.{b}.{c}.{d}",
+        octet, octet, octet, octet
+    )
+
+# 组合策略：域名或 IP
+host_strategy = st.one_of(valid_domain_strategy(), valid_ip_strategy())
+
+# 生成有效 URL 的策略
+def valid_url_strategy():
+    """生成有效的 URL"""
+    domain = valid_domain_strategy()
+    return st.builds(
+        lambda d, path: f"https://{d}/{path}" if path else f"https://{d}",
+        domain,
+        st.one_of(
+            st.just(""),
+            st.text(
+                alphabet=st.characters(whitelist_categories=('L',), min_codepoint=97, max_codepoint=122),
+                min_size=1,
+                max_size=10
+            )
+        )
+    )
+
+url_strategy = valid_url_strategy()
+
+
+class TestPipelineTargetProviderProperties:
+    """PipelineTargetProvider 属性测试类"""
+    
+    @given(hosts=st.lists(host_strategy, max_size=50))
+    @settings(max_examples=100)
+    def test_property_3_hosts_round_trip(self, hosts):
+        """
+        Property 3: PipelineTargetProvider Round-Trip (hosts)
+        
+        Feature: scan-target-provider, Property 3: PipelineTargetProvider Round-Trip
+        **Validates: Requirements 5.1, 5.2**
+        
+        For any StageOutput with hosts, PipelineTargetProvider should return
+        the same hosts in the same order.
+        """
+        stage_output = StageOutput(hosts=hosts)
+        provider = PipelineTargetProvider(previous_output=stage_output)
+        result = list(provider.iter_hosts())
+        assert result == hosts
+    
+    @given(urls=st.lists(url_strategy, max_size=50))
+    @settings(max_examples=100)
+    def test_property_3_urls_round_trip(self, urls):
+        """
+        Property 3: PipelineTargetProvider Round-Trip (urls)
+        
+        Feature: scan-target-provider, Property 3: PipelineTargetProvider Round-Trip
+        **Validates: Requirements 5.1, 5.2**
+        
+        For any StageOutput with urls, PipelineTargetProvider should return
+        the same urls in the same order.
+        """
+        stage_output = StageOutput(urls=urls)
+        provider = PipelineTargetProvider(previous_output=stage_output)
+        result = list(provider.iter_urls())
+        assert result == urls
+    
+    @given(
+        hosts=st.lists(host_strategy, max_size=30),
+        urls=st.lists(url_strategy, max_size=30)
+    )
+    @settings(max_examples=100)
+    def test_property_3_combined_round_trip(self, hosts, urls):
+        """
+        Property 3: PipelineTargetProvider Round-Trip (combined)
+        
+        Feature: scan-target-provider, Property 3: PipelineTargetProvider Round-Trip
+        **Validates: Requirements 5.1, 5.2**
+        
+        For any StageOutput with both hosts and urls, both should round-trip correctly.
+        """
+        stage_output = StageOutput(hosts=hosts, urls=urls)
+        provider = PipelineTargetProvider(previous_output=stage_output)
+        
+        hosts_result = list(provider.iter_hosts())
+        urls_result = list(provider.iter_urls())
+        
+        assert hosts_result == hosts
+        assert urls_result == urls
+
+
+class TestPipelineTargetProviderUnit:
+    """PipelineTargetProvider 单元测试类"""
+    
+    def test_empty_stage_output(self):
+        """测试空 StageOutput 返回空迭代器 - Requirements 5.5"""
+        stage_output = StageOutput()
+        provider = PipelineTargetProvider(previous_output=stage_output)
+        
+        assert list(provider.iter_hosts()) == []
+        assert list(provider.iter_urls()) == []
+    
+    def test_blacklist_filter_returns_none(self):
+        """测试黑名单过滤器返回 None - Requirements 5.3"""
+        stage_output = StageOutput(hosts=["example.com"])
+        provider = PipelineTargetProvider(previous_output=stage_output)
+        assert provider.get_blacklist_filter() is None
+    
+    def test_target_id_association(self):
+        """测试 target_id 关联 - Requirements 5.4"""
+        stage_output = StageOutput(hosts=["example.com"])
+        provider = PipelineTargetProvider(previous_output=stage_output, target_id=123)
+        assert provider.target_id == 123
+    
+    def test_context_propagation(self):
+        """测试上下文传递"""
+        ctx = ProviderContext(target_id=456, scan_id=789)
+        stage_output = StageOutput(hosts=["example.com"])
+        provider = PipelineTargetProvider(previous_output=stage_output, context=ctx)
+        
+        assert provider.target_id == 456
+        assert provider.scan_id == 789
+    
+    def test_previous_output_property(self):
+        """测试 previous_output 属性"""
+        stage_output = StageOutput(hosts=["example.com"], urls=["https://example.com"])
+        provider = PipelineTargetProvider(previous_output=stage_output)
+        
+        assert provider.previous_output is stage_output
+        assert provider.previous_output.hosts == ["example.com"]
+        assert provider.previous_output.urls == ["https://example.com"]
+    
+    def test_stage_output_with_metadata(self):
+        """测试带元数据的 StageOutput"""
+        stage_output = StageOutput(
+            hosts=["example.com"],
+            urls=["https://example.com"],
+            new_targets=["new.example.com"],
+            stats={"count": 1},
+            success=True,
+            error=None
+        )
+        provider = PipelineTargetProvider(previous_output=stage_output)
+        
+        assert list(provider.iter_hosts()) == ["example.com"]
+        assert list(provider.iter_urls()) == ["https://example.com"]
+        assert provider.previous_output.new_targets == ["new.example.com"]
+        assert provider.previous_output.stats == {"count": 1}

backend/apps/scan/providers/tests/test_snapshot_provider.py

@@ -0,0 +1,191 @@
+"""
+SnapshotTargetProvider 单元测试
+"""
+
+import pytest
+from unittest.mock import Mock, patch
+
+from apps.scan.providers import SnapshotTargetProvider, ProviderContext
+
+
+class TestSnapshotTargetProvider:
+    """SnapshotTargetProvider 测试类"""
+    
+    def test_init_with_scan_id_and_type(self):
+        """测试初始化"""
+        provider = SnapshotTargetProvider(
+            scan_id=100,
+            snapshot_type="subdomain"
+        )
+        
+        assert provider.scan_id == 100
+        assert provider.snapshot_type == "subdomain"
+        assert provider.target_id is None  # 默认 context
+    
+    def test_init_with_context(self):
+        """测试带 context 初始化"""
+        ctx = ProviderContext(target_id=1, scan_id=100)
+        provider = SnapshotTargetProvider(
+            scan_id=100,
+            snapshot_type="subdomain",
+            context=ctx
+        )
+        
+        assert provider.scan_id == 100
+        assert provider.target_id == 1
+        assert provider.snapshot_type == "subdomain"
+    
+    @patch('apps.asset.services.snapshot.SubdomainSnapshotsService')
+    def test_iter_hosts_subdomain(self, mock_service_class):
+        """测试从子域名快照迭代主机"""
+        # Mock service
+        mock_service = Mock()
+        mock_service.iter_subdomain_names_by_scan.return_value = iter([
+            "a.example.com",
+            "b.example.com"
+        ])
+        mock_service_class.return_value = mock_service
+        
+        # 创建 provider
+        provider = SnapshotTargetProvider(
+            scan_id=100,
+            snapshot_type="subdomain"
+        )
+        
+        # 迭代主机
+        hosts = list(provider.iter_hosts())
+        
+        assert hosts == ["a.example.com", "b.example.com"]
+        mock_service.iter_subdomain_names_by_scan.assert_called_once_with(
+            scan_id=100,
+            chunk_size=1000
+        )
+    
+    @patch('apps.asset.services.snapshot.HostPortMappingSnapshotsService')
+    def test_iter_hosts_host_port(self, mock_service_class):
+        """测试从主机端口映射快照迭代主机"""
+        # Mock queryset
+        mock_mapping1 = Mock()
+        mock_mapping1.host = "example.com"
+        mock_mapping1.port = 80
+        
+        mock_mapping2 = Mock()
+        mock_mapping2.host = "example.com"
+        mock_mapping2.port = 443
+        
+        mock_queryset = Mock()
+        mock_queryset.iterator.return_value = iter([mock_mapping1, mock_mapping2])
+        
+        # Mock service
+        mock_service = Mock()
+        mock_service.get_by_scan.return_value = mock_queryset
+        mock_service_class.return_value = mock_service
+        
+        # 创建 provider
+        provider = SnapshotTargetProvider(
+            scan_id=100,
+            snapshot_type="host_port"
+        )
+        
+        # 迭代主机
+        hosts = list(provider.iter_hosts())
+        
+        assert hosts == ["example.com:80", "example.com:443"]
+        mock_service.get_by_scan.assert_called_once_with(scan_id=100)
+    
+    @patch('apps.asset.services.snapshot.WebsiteSnapshotsService')
+    def test_iter_urls_website(self, mock_service_class):
+        """测试从网站快照迭代 URL"""
+        # Mock service
+        mock_service = Mock()
+        mock_service.iter_website_urls_by_scan.return_value = iter([
+            "http://example.com",
+            "https://example.com"
+        ])
+        mock_service_class.return_value = mock_service
+        
+        # 创建 provider
+        provider = SnapshotTargetProvider(
+            scan_id=100,
+            snapshot_type="website"
+        )
+        
+        # 迭代 URL
+        urls = list(provider.iter_urls())
+        
+        assert urls == ["http://example.com", "https://example.com"]
+        mock_service.iter_website_urls_by_scan.assert_called_once_with(
+            scan_id=100,
+            chunk_size=1000
+        )
+    
+    @patch('apps.asset.services.snapshot.EndpointSnapshotsService')
+    def test_iter_urls_endpoint(self, mock_service_class):
+        """测试从端点快照迭代 URL"""
+        # Mock queryset
+        mock_endpoint1 = Mock()
+        mock_endpoint1.url = "http://example.com/api/v1"
+        
+        mock_endpoint2 = Mock()
+        mock_endpoint2.url = "http://example.com/api/v2"
+        
+        mock_queryset = Mock()
+        mock_queryset.iterator.return_value = iter([mock_endpoint1, mock_endpoint2])
+        
+        # Mock service
+        mock_service = Mock()
+        mock_service.get_by_scan.return_value = mock_queryset
+        mock_service_class.return_value = mock_service
+        
+        # 创建 provider
+        provider = SnapshotTargetProvider(
+            scan_id=100,
+            snapshot_type="endpoint"
+        )
+        
+        # 迭代 URL
+        urls = list(provider.iter_urls())
+        
+        assert urls == ["http://example.com/api/v1", "http://example.com/api/v2"]
+        mock_service.get_by_scan.assert_called_once_with(scan_id=100)
+    
+    def test_iter_hosts_unsupported_type(self):
+        """测试不支持的快照类型（iter_hosts）"""
+        provider = SnapshotTargetProvider(
+            scan_id=100,
+            snapshot_type="website"  # website 不支持 iter_hosts
+        )
+        
+        hosts = list(provider.iter_hosts())
+        assert hosts == []
+    
+    def test_iter_urls_unsupported_type(self):
+        """测试不支持的快照类型（iter_urls）"""
+        provider = SnapshotTargetProvider(
+            scan_id=100,
+            snapshot_type="subdomain"  # subdomain 不支持 iter_urls
+        )
+        
+        urls = list(provider.iter_urls())
+        assert urls == []
+    
+    def test_get_blacklist_filter(self):
+        """测试黑名单过滤器（快照模式不使用黑名单）"""
+        provider = SnapshotTargetProvider(
+            scan_id=100,
+            snapshot_type="subdomain"
+        )
+        
+        assert provider.get_blacklist_filter() is None
+    
+    def test_context_propagation(self):
+        """测试上下文传递"""
+        ctx = ProviderContext(target_id=456, scan_id=789)
+        provider = SnapshotTargetProvider(
+            scan_id=100,  # 会被 context 覆盖
+            snapshot_type="subdomain",
+            context=ctx
+        )
+        
+        assert provider.target_id == 456
+        assert provider.scan_id == 100  # scan_id 在 __init__ 中被设置

backend/apps/scan/repositories/django_scan_repository.py

@@ -104,7 +104,7 @@ class DjangoScanRepository:
         target: Target,
         engine_ids: List[int],
         engine_names: List[str],
-        merged_configuration: str,
+        yaml_configuration: str,
         results_dir: str,
         status: ScanStatus = ScanStatus.INITIATED
     ) -> Scan:
@@ -115,7 +115,7 @@ class DjangoScanRepository:
             target: 扫描目标
             engine_ids: 引擎 ID 列表
             engine_names: 引擎名称列表
-            merged_configuration: 合并后的 YAML 配置
+            yaml_configuration: YAML 格式的扫描配置
             results_dir: 结果目录
             status: 初始状态
         
@@ -126,7 +126,7 @@ class DjangoScanRepository:
             target=target,
             engine_ids=engine_ids,
             engine_names=engine_names,
-            merged_configuration=merged_configuration,
+            yaml_configuration=yaml_configuration,
             results_dir=results_dir,
             status=status,
             container_ids=[]
@@ -464,6 +464,7 @@ class DjangoScanRepository:
                 'endpoints': scan.endpoint_snapshots.count(),
                 'ips': ips_count,
                 'directories': scan.directory_snapshots.count(),
+                'screenshots': scan.screenshot_snapshots.count(),
                 'vulns_total': total_vulns,
                 'vulns_critical': severity_stats['critical'],
                 'vulns_high': severity_stats['high'],
@@ -478,6 +479,7 @@ class DjangoScanRepository:
                 'cached_endpoints_count': stats['endpoints'],
                 'cached_ips_count': stats['ips'],
                 'cached_directories_count': stats['directories'],
+                'cached_screenshots_count': stats['screenshots'],
                 'cached_vulns_total': stats['vulns_total'],
                 'cached_vulns_critical': stats['vulns_critical'],
                 'cached_vulns_high': stats['vulns_high'],

backend/apps/scan/repositories/scheduled_scan_repository.py

@@ -31,7 +31,7 @@ class ScheduledScanDTO:
     name: str = ''
     engine_ids: List[int] = None  # 多引擎支持
     engine_names: List[str] = None  # 引擎名称列表
-    merged_configuration: str = ''  # 合并后的配置
+    yaml_configuration: str = ''  # YAML 格式的扫描配置
     organization_id: Optional[int] = None  # 组织扫描模式
     target_id: Optional[int] = None  # 目标扫描模式
     cron_expression: Optional[str] = None
@@ -114,7 +114,7 @@ class DjangoScheduledScanRepository:
                 name=dto.name,
                 engine_ids=dto.engine_ids,
                 engine_names=dto.engine_names,
-                merged_configuration=dto.merged_configuration,
+                yaml_configuration=dto.yaml_configuration,
                 organization_id=dto.organization_id,  # 组织扫描模式
                 target_id=dto.target_id if not dto.organization_id else None,  # 目标扫描模式
                 cron_expression=dto.cron_expression,
@@ -147,8 +147,8 @@ class DjangoScheduledScanRepository:
                     scheduled_scan.engine_ids = dto.engine_ids
                 if dto.engine_names is not None:
                     scheduled_scan.engine_names = dto.engine_names
-                if dto.merged_configuration is not None:
-                    scheduled_scan.merged_configuration = dto.merged_configuration
+                if dto.yaml_configuration is not None:
+                    scheduled_scan.yaml_configuration = dto.yaml_configuration
                 if dto.cron_expression is not None:
                     scheduled_scan.cron_expression = dto.cron_expression
                 if dto.is_enabled is not None:

backend/apps/scan/serializers.py

@@ -1,266 +0,0 @@
-from rest_framework import serializers
-from django.db.models import Count
-
-from .models import Scan, ScheduledScan
-
-
-class ScanSerializer(serializers.ModelSerializer):
-    """扫描任务序列化器"""
-    target_name = serializers.SerializerMethodField()
-    
-    class Meta:
-        model = Scan
-        fields = [
-            'id', 'target', 'target_name', 'engine_ids', 'engine_names',
-            'created_at', 'stopped_at', 'status', 'results_dir',
-            'container_ids', 'error_message'
-        ]
-        read_only_fields = [
-            'id', 'created_at', 'stopped_at', 'results_dir',
-            'container_ids', 'error_message', 'status'
-        ]
-    
-    def get_target_name(self, obj):
-        """获取目标名称"""
-        return obj.target.name if obj.target else None
-
-
-class ScanHistorySerializer(serializers.ModelSerializer):
-    """扫描历史列表专用序列化器
-    
-    为前端扫描历史页面提供优化的数据格式，包括：
-    - 扫描汇总统计（子域名、端点、漏洞数量）
-    - 进度百分比和当前阶段
-    - 执行节点信息
-    """
-    
-    # 字段映射
-    target_name = serializers.CharField(source='target.name', read_only=True)
-    worker_name = serializers.CharField(source='worker.name', read_only=True, allow_null=True)
-    
-    # 计算字段
-    summary = serializers.SerializerMethodField()
-    
-    # 进度跟踪字段（直接从模型读取）
-    progress = serializers.IntegerField(read_only=True)
-    current_stage = serializers.CharField(read_only=True)
-    stage_progress = serializers.JSONField(read_only=True)
-    
-    class Meta:
-        model = Scan
-        fields = [
-            'id', 'target', 'target_name', 'engine_ids', 'engine_names', 
-            'worker_name', 'created_at', 'status', 'error_message', 'summary', 
-            'progress', 'current_stage', 'stage_progress'
-        ]
-    
-    def get_summary(self, obj):
-        """获取扫描汇总数据。
-
-        设计原则：
-        - 子域名/网站/端点/IP/目录使用缓存字段（避免实时 COUNT）
-        - 漏洞统计使用 Scan 上的缓存字段，在扫描结束时统一聚合
-        """
-        # 1. 使用缓存字段构建基础统计（子域名、网站、端点、IP、目录）
-        summary = {
-            'subdomains': obj.cached_subdomains_count or 0,
-            'websites': obj.cached_websites_count or 0,
-            'endpoints': obj.cached_endpoints_count or 0,
-            'ips': obj.cached_ips_count or 0,
-            'directories': obj.cached_directories_count or 0,
-        }
-
-        # 2. 使用 Scan 模型上的缓存漏洞统计（按严重性聚合）
-        summary['vulnerabilities'] = {
-            'total': obj.cached_vulns_total or 0,
-            'critical': obj.cached_vulns_critical or 0,
-            'high': obj.cached_vulns_high or 0,
-            'medium': obj.cached_vulns_medium or 0,
-            'low': obj.cached_vulns_low or 0,
-        }
-
-        return summary
-
-
-class QuickScanSerializer(serializers.Serializer):
-    """
-    快速扫描序列化器
-    
-    功能：
-    - 接收目标列表和引擎配置
-    - 自动创建/获取目标
-    - 立即发起扫描
-    """
-    
-    # 批量创建的最大数量限制
-    MAX_BATCH_SIZE = 1000
-    
-    # 目标列表
-    targets = serializers.ListField(
-        child=serializers.DictField(),
-        help_text='目标列表，每个目标包含 name 字段'
-    )
-    
-    # 扫描引擎 ID 列表
-    engine_ids = serializers.ListField(
-        child=serializers.IntegerField(),
-        required=True,
-        help_text='使用的扫描引擎 ID 列表 (必填)'
-    )
-    
-    def validate_targets(self, value):
-        """验证目标列表"""
-        if not value:
-            raise serializers.ValidationError("目标列表不能为空")
-        
-        # 检查数量限制，防止服务器过载
-        if len(value) > self.MAX_BATCH_SIZE:
-            raise serializers.ValidationError(
-                f"快速扫描最多支持 {self.MAX_BATCH_SIZE} 个目标，当前提交了 {len(value)} 个"
-            )
-        
-        # 验证每个目标的必填字段
-        for idx, target in enumerate(value):
-            if 'name' not in target:
-                raise serializers.ValidationError(f"第 {idx + 1} 个目标缺少 name 字段")
-            if not target['name']:
-                raise serializers.ValidationError(f"第 {idx + 1} 个目标的 name 不能为空")
-        
-        return value
-    
-    def validate_engine_ids(self, value):
-        """验证引擎 ID 列表"""
-        if not value:
-            raise serializers.ValidationError("engine_ids 不能为空")
-        return value
-
-
-# ==================== 定时扫描序列化器 ====================
-
-class ScheduledScanSerializer(serializers.ModelSerializer):
-    """定时扫描任务序列化器（用于列表和详情）"""
-    
-    # 关联字段
-    organization_id = serializers.IntegerField(source='organization.id', read_only=True, allow_null=True)
-    organization_name = serializers.CharField(source='organization.name', read_only=True, allow_null=True)
-    target_id = serializers.IntegerField(source='target.id', read_only=True, allow_null=True)
-    target_name = serializers.CharField(source='target.name', read_only=True, allow_null=True)
-    scan_mode = serializers.SerializerMethodField()
-    
-    class Meta:
-        model = ScheduledScan
-        fields = [
-            'id', 'name',
-            'engine_ids', 'engine_names',
-            'organization_id', 'organization_name',
-            'target_id', 'target_name',
-            'scan_mode',
-            'cron_expression',
-            'is_enabled',
-            'run_count', 'last_run_time', 'next_run_time',
-            'created_at', 'updated_at'
-        ]
-        read_only_fields = [
-            'id', 'run_count',
-            'last_run_time', 'next_run_time',
-            'created_at', 'updated_at'
-        ]
-    
-    def get_scan_mode(self, obj):
-        """获取扫描模式：organization 或 target"""
-        return 'organization' if obj.organization_id else 'target'
-
-
-class CreateScheduledScanSerializer(serializers.Serializer):
-    """创建定时扫描任务序列化器
-    
-    扫描模式（二选一）：
-    - 组织扫描：提供 organization_id，执行时动态获取组织下所有目标
-    - 目标扫描：提供 target_id，扫描单个目标
-    """
-    
-    name = serializers.CharField(max_length=200, help_text='任务名称')
-    engine_ids = serializers.ListField(
-        child=serializers.IntegerField(),
-        help_text='扫描引擎 ID 列表'
-    )
-    
-    # 组织扫描模式
-    organization_id = serializers.IntegerField(
-        required=False,
-        allow_null=True,
-        help_text='组织 ID（组织扫描模式：执行时动态获取组织下所有目标）'
-    )
-    
-    # 目标扫描模式
-    target_id = serializers.IntegerField(
-        required=False,
-        allow_null=True,
-        help_text='目标 ID（目标扫描模式：扫描单个目标）'
-    )
-    
-    cron_expression = serializers.CharField(
-        max_length=100,
-        default='0 2 * * *',
-        help_text='Cron 表达式，格式：分 时 日 月 周'
-    )
-    is_enabled = serializers.BooleanField(default=True, help_text='是否立即启用')
-    
-    def validate_engine_ids(self, value):
-        """验证引擎 ID 列表"""
-        if not value:
-            raise serializers.ValidationError("engine_ids 不能为空")
-        return value
-    
-    def validate(self, data):
-        """验证 organization_id 和 target_id 互斥"""
-        organization_id = data.get('organization_id')
-        target_id = data.get('target_id')
-        
-        if not organization_id and not target_id:
-            raise serializers.ValidationError('必须提供 organization_id 或 target_id 其中之一')
-        
-        if organization_id and target_id:
-            raise serializers.ValidationError('organization_id 和 target_id 只能提供其中之一')
-        
-        return data
-
-
-class UpdateScheduledScanSerializer(serializers.Serializer):
-    """更新定时扫描任务序列化器"""
-    
-    name = serializers.CharField(max_length=200, required=False, help_text='任务名称')
-    engine_ids = serializers.ListField(
-        child=serializers.IntegerField(),
-        required=False,
-        help_text='扫描引擎 ID 列表'
-    )
-    
-    # 组织扫描模式
-    organization_id = serializers.IntegerField(
-        required=False,
-        allow_null=True,
-        help_text='组织 ID（设置后清空 target_id）'
-    )
-    
-    # 目标扫描模式
-    target_id = serializers.IntegerField(
-        required=False,
-        allow_null=True,
-        help_text='目标 ID（设置后清空 organization_id）'
-    )
-    
-    cron_expression = serializers.CharField(max_length=100, required=False, help_text='Cron 表达式')
-    is_enabled = serializers.BooleanField(required=False, help_text='是否启用')
-    
-    def validate_engine_ids(self, value):
-        """验证引擎 ID 列表"""
-        if value is not None and not value:
-            raise serializers.ValidationError("engine_ids 不能为空")
-        return value
-
-
-class ToggleScheduledScanSerializer(serializers.Serializer):
-    """切换定时扫描启用状态序列化器"""
-    
-    is_enabled = serializers.BooleanField(help_text='是否启用')

backend/apps/scan/serializers/__init__.py

@@ -0,0 +1,40 @@
+"""Scan Serializers - 统一导出"""
+
+from .mixins import ScanConfigValidationMixin
+from .scan_serializers import (
+    ScanSerializer,
+    ScanHistorySerializer,
+    QuickScanSerializer,
+    InitiateScanSerializer,
+)
+from .scan_log_serializers import ScanLogSerializer
+from .scheduled_scan_serializers import (
+    ScheduledScanSerializer,
+    CreateScheduledScanSerializer,
+    UpdateScheduledScanSerializer,
+    ToggleScheduledScanSerializer,
+)
+from .subfinder_provider_settings_serializers import SubfinderProviderSettingsSerializer
+
+# 兼容旧名称
+ProviderSettingsSerializer = SubfinderProviderSettingsSerializer
+
+__all__ = [
+    # Mixins
+    'ScanConfigValidationMixin',
+    # Scan
+    'ScanSerializer',
+    'ScanHistorySerializer',
+    'QuickScanSerializer',
+    'InitiateScanSerializer',
+    # ScanLog
+    'ScanLogSerializer',
+    # Scheduled Scan
+    'ScheduledScanSerializer',
+    'CreateScheduledScanSerializer',
+    'UpdateScheduledScanSerializer',
+    'ToggleScheduledScanSerializer',
+    # Subfinder Provider Settings
+    'SubfinderProviderSettingsSerializer',
+    'ProviderSettingsSerializer',  # 兼容旧名称
+]

backend/apps/scan/serializers/mixins.py

@@ -0,0 +1,57 @@
+"""序列化器通用 Mixin 和工具类"""
+
+from rest_framework import serializers
+import yaml
+
+
+class DuplicateKeyLoader(yaml.SafeLoader):
+    """自定义 YAML Loader，检测重复 key"""
+    pass
+
+
+def _check_duplicate_keys(loader, node, deep=False):
+    """检测 YAML mapping 中的重复 key"""
+    mapping = {}
+    for key_node, value_node in node.value:
+        key = loader.construct_object(key_node, deep=deep)
+        if key in mapping:
+            raise yaml.constructor.ConstructorError(
+                "while constructing a mapping", node.start_mark,
+                f"发现重复的配置项 '{key}'，后面的配置会覆盖前面的配置，请删除重复项", key_node.start_mark
+            )
+        mapping[key] = loader.construct_object(value_node, deep=deep)
+    return mapping
+
+
+DuplicateKeyLoader.add_constructor(
+    yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+    _check_duplicate_keys
+)
+
+
+class ScanConfigValidationMixin:
+    """扫描配置验证 Mixin"""
+    
+    def validate_configuration(self, value):
+        """验证 YAML 配置格式"""
+        if not value or not value.strip():
+            raise serializers.ValidationError("configuration 不能为空")
+        
+        try:
+            yaml.load(value, Loader=DuplicateKeyLoader)
+        except yaml.YAMLError as e:
+            raise serializers.ValidationError(f"无效的 YAML 格式: {str(e)}")
+        
+        return value
+    
+    def validate_engine_ids(self, value):
+        """验证引擎 ID 列表"""
+        if not value:
+            raise serializers.ValidationError("engine_ids 不能为空，请至少选择一个扫描引擎")
+        return value
+    
+    def validate_engine_names(self, value):
+        """验证引擎名称列表"""
+        if not value:
+            raise serializers.ValidationError("engine_names 不能为空")
+        return value

backend/apps/scan/serializers/scan_log_serializers.py

@@ -0,0 +1,13 @@
+"""扫描日志序列化器"""
+
+from rest_framework import serializers
+
+from ..models import ScanLog
+
+
+class ScanLogSerializer(serializers.ModelSerializer):
+    """扫描日志序列化器"""
+    
+    class Meta:
+        model = ScanLog
+        fields = ['id', 'level', 'content', 'created_at']

backend/apps/scan/serializers/scan_serializers.py

@@ -0,0 +1,112 @@
+"""扫描任务序列化器"""
+
+from rest_framework import serializers
+
+from ..models import Scan
+from .mixins import ScanConfigValidationMixin
+
+
+class ScanSerializer(serializers.ModelSerializer):
+    """扫描任务序列化器"""
+    target_name = serializers.SerializerMethodField()
+    
+    class Meta:
+        model = Scan
+        fields = [
+            'id', 'target', 'target_name', 'engine_ids', 'engine_names',
+            'created_at', 'stopped_at', 'status', 'results_dir',
+            'container_ids', 'error_message'
+        ]
+        read_only_fields = [
+            'id', 'created_at', 'stopped_at', 'results_dir',
+            'container_ids', 'error_message', 'status'
+        ]
+    
+    def get_target_name(self, obj):
+        return obj.target.name if obj.target else None
+
+
+class ScanHistorySerializer(serializers.ModelSerializer):
+    """扫描历史列表序列化器"""
+    
+    target_name = serializers.CharField(source='target.name', read_only=True)
+    worker_name = serializers.CharField(source='worker.name', read_only=True, allow_null=True)
+    summary = serializers.SerializerMethodField()
+    progress = serializers.IntegerField(read_only=True)
+    current_stage = serializers.CharField(read_only=True)
+    stage_progress = serializers.JSONField(read_only=True)
+    
+    class Meta:
+        model = Scan
+        fields = [
+            'id', 'target', 'target_name', 'engine_ids', 'engine_names', 
+            'worker_name', 'created_at', 'status', 'error_message', 'summary', 
+            'progress', 'current_stage', 'stage_progress', 'yaml_configuration'
+        ]
+    
+    def get_summary(self, obj):
+        summary = {
+            'subdomains': obj.cached_subdomains_count or 0,
+            'websites': obj.cached_websites_count or 0,
+            'endpoints': obj.cached_endpoints_count or 0,
+            'ips': obj.cached_ips_count or 0,
+            'directories': obj.cached_directories_count or 0,
+            'screenshots': obj.cached_screenshots_count or 0,
+        }
+        summary['vulnerabilities'] = {
+            'total': obj.cached_vulns_total or 0,
+            'critical': obj.cached_vulns_critical or 0,
+            'high': obj.cached_vulns_high or 0,
+            'medium': obj.cached_vulns_medium or 0,
+            'low': obj.cached_vulns_low or 0,
+        }
+        return summary
+
+
+class QuickScanSerializer(ScanConfigValidationMixin, serializers.Serializer):
+    """快速扫描序列化器"""
+    
+    MAX_BATCH_SIZE = 5000
+    
+    targets = serializers.ListField(
+        child=serializers.DictField(),
+        help_text='目标列表，每个目标包含 name 字段'
+    )
+    configuration = serializers.CharField(required=True, help_text='YAML 格式的扫描配置')
+    engine_ids = serializers.ListField(child=serializers.IntegerField(), required=True)
+    engine_names = serializers.ListField(child=serializers.CharField(), required=True)
+    
+    def validate_targets(self, value):
+        if not value:
+            raise serializers.ValidationError("目标列表不能为空")
+        if len(value) > self.MAX_BATCH_SIZE:
+            raise serializers.ValidationError(
+                f"快速扫描最多支持 {self.MAX_BATCH_SIZE} 个目标，当前提交了 {len(value)} 个"
+            )
+        for idx, target in enumerate(value):
+            if 'name' not in target:
+                raise serializers.ValidationError(f"第 {idx + 1} 个目标缺少 name 字段")
+            if not target['name']:
+                raise serializers.ValidationError(f"第 {idx + 1} 个目标的 name 不能为空")
+        return value
+
+
+class InitiateScanSerializer(ScanConfigValidationMixin, serializers.Serializer):
+    """发起扫描任务序列化器"""
+    
+    configuration = serializers.CharField(required=True, help_text='YAML 格式的扫描配置')
+    engine_ids = serializers.ListField(child=serializers.IntegerField(), required=True)
+    engine_names = serializers.ListField(child=serializers.CharField(), required=True)
+    organization_id = serializers.IntegerField(required=False, allow_null=True)
+    target_id = serializers.IntegerField(required=False, allow_null=True)
+    
+    def validate(self, data):
+        organization_id = data.get('organization_id')
+        target_id = data.get('target_id')
+        
+        if not organization_id and not target_id:
+            raise serializers.ValidationError('必须提供 organization_id 或 target_id 其中之一')
+        if organization_id and target_id:
+            raise serializers.ValidationError('organization_id 和 target_id 只能提供其中之一')
+        
+        return data

backend/apps/scan/serializers/scheduled_scan_serializers.py

@@ -0,0 +1,84 @@
+"""定时扫描序列化器"""
+
+from rest_framework import serializers
+
+from ..models import ScheduledScan
+from .mixins import ScanConfigValidationMixin
+
+
+class ScheduledScanSerializer(serializers.ModelSerializer):
+    """定时扫描任务序列化器（用于列表和详情）"""
+    
+    organization_id = serializers.IntegerField(source='organization.id', read_only=True, allow_null=True)
+    organization_name = serializers.CharField(source='organization.name', read_only=True, allow_null=True)
+    target_id = serializers.IntegerField(source='target.id', read_only=True, allow_null=True)
+    target_name = serializers.CharField(source='target.name', read_only=True, allow_null=True)
+    scan_mode = serializers.SerializerMethodField()
+    
+    class Meta:
+        model = ScheduledScan
+        fields = [
+            'id', 'name',
+            'engine_ids', 'engine_names',
+            'organization_id', 'organization_name',
+            'target_id', 'target_name',
+            'scan_mode',
+            'cron_expression',
+            'is_enabled',
+            'run_count', 'last_run_time', 'next_run_time',
+            'created_at', 'updated_at'
+        ]
+        read_only_fields = [
+            'id', 'run_count',
+            'last_run_time', 'next_run_time',
+            'created_at', 'updated_at'
+        ]
+    
+    def get_scan_mode(self, obj):
+        return 'organization' if obj.organization_id else 'target'
+
+
+class CreateScheduledScanSerializer(ScanConfigValidationMixin, serializers.Serializer):
+    """创建定时扫描任务序列化器"""
+    
+    name = serializers.CharField(max_length=200, help_text='任务名称')
+    configuration = serializers.CharField(required=True, help_text='YAML 格式的扫描配置')
+    engine_ids = serializers.ListField(child=serializers.IntegerField(), required=True)
+    engine_names = serializers.ListField(child=serializers.CharField(), required=True)
+    organization_id = serializers.IntegerField(required=False, allow_null=True)
+    target_id = serializers.IntegerField(required=False, allow_null=True)
+    cron_expression = serializers.CharField(max_length=100, default='0 2 * * *')
+    is_enabled = serializers.BooleanField(default=True)
+    
+    def validate(self, data):
+        organization_id = data.get('organization_id')
+        target_id = data.get('target_id')
+        
+        if not organization_id and not target_id:
+            raise serializers.ValidationError('必须提供 organization_id 或 target_id 其中之一')
+        if organization_id and target_id:
+            raise serializers.ValidationError('organization_id 和 target_id 只能提供其中之一')
+        
+        return data
+
+
+class UpdateScheduledScanSerializer(serializers.Serializer):
+    """更新定时扫描任务序列化器"""
+    
+    name = serializers.CharField(max_length=200, required=False)
+    engine_ids = serializers.ListField(child=serializers.IntegerField(), required=False)
+    organization_id = serializers.IntegerField(required=False, allow_null=True)
+    target_id = serializers.IntegerField(required=False, allow_null=True)
+    cron_expression = serializers.CharField(max_length=100, required=False)
+    is_enabled = serializers.BooleanField(required=False)
+    
+    def validate_engine_ids(self, value):
+        if value is not None and not value:
+            raise serializers.ValidationError("engine_ids 不能为空")
+        return value
+
+
+class ToggleScheduledScanSerializer(serializers.Serializer):
+    """切换定时扫描启用状态序列化器"""
+    
+    is_enabled = serializers.BooleanField(help_text='是否启用')

backend/apps/scan/serializers/subfinder_provider_settings_serializers.py

@@ -0,0 +1,55 @@
+"""Subfinder Provider 配置序列化器"""
+
+from rest_framework import serializers
+
+
+class SubfinderProviderSettingsSerializer(serializers.Serializer):
+    """Subfinder Provider 配置序列化器
+    
+    支持的 Provider:
+    - fofa: email + api_key (composite)
+    - censys: api_id + api_secret (composite)
+    - hunter, shodan, zoomeye, securitytrails, threatbook, quake: api_key (single)
+    
+    注意：djangorestframework-camel-case 会自动处理 camelCase <-> snake_case 转换
+    所以这里统一使用 snake_case
+    """
+    
+    VALID_PROVIDERS = {
+        'fofa', 'hunter', 'shodan', 'censys', 
+        'zoomeye', 'securitytrails', 'threatbook', 'quake'
+    }
+    
+    def to_internal_value(self, data):
+        """验证并转换输入数据"""
+        if not isinstance(data, dict):
+            raise serializers.ValidationError('Expected a dictionary')
+        
+        result = {}
+        for provider, config in data.items():
+            if provider not in self.VALID_PROVIDERS:
+                continue
+            
+            if not isinstance(config, dict):
+                continue
+            
+            db_config = {'enabled': bool(config.get('enabled', False))}
+            
+            if provider == 'fofa':
+                db_config['email'] = str(config.get('email', ''))
+                db_config['api_key'] = str(config.get('api_key', ''))
+            elif provider == 'censys':
+                db_config['api_id'] = str(config.get('api_id', ''))
+                db_config['api_secret'] = str(config.get('api_secret', ''))
+            else:
+                db_config['api_key'] = str(config.get('api_key', ''))
+            
+            result[provider] = db_config
+        
+        return result
+    
+    def to_representation(self, instance):
+        """输出数据（数据库格式，camel-case 中间件会自动转换）"""
+        if isinstance(instance, dict):
+            return instance
+        return instance.providers if hasattr(instance, 'providers') else {}

backend/apps/scan/services/__init__.py

@@ -17,8 +17,12 @@ from .scan_state_service import ScanStateService
 from .scan_control_service import ScanControlService
 from .scan_stats_service import ScanStatsService
 from .scheduled_scan_service import ScheduledScanService
-from .blacklist_service import BlacklistService
-from .target_export_service import TargetExportService
+from .target_export_service import (
+    TargetExportService,
+    create_export_service,
+    export_urls_with_fallback,
+    DataSource,
+)
 
 __all__ = [
     'ScanService',           # 主入口（向后兼容）
@@ -27,7 +31,9 @@ __all__ = [
     'ScanControlService',
     'ScanStatsService',
     'ScheduledScanService',
-    'BlacklistService',      # 黑名单过滤服务
     'TargetExportService',   # 目标导出服务
+    'create_export_service',
+    'export_urls_with_fallback',
+    'DataSource',
 ]
 

backend/apps/scan/services/blacklist_service.py

@@ -1,82 +0,0 @@
-"""
-黑名单过滤服务
-
-过滤敏感域名（如 .gov、.edu、.mil 等）
-
-当前版本使用默认规则，后续将支持从前端配置加载。
-"""
-
-from typing import List, Optional
-from django.db.models import QuerySet
-import re
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-class BlacklistService:
-    """
-    黑名单过滤服务 - 过滤敏感域名
-    
-    TODO: 后续版本支持从前端配置加载黑名单规则
-    - 用户在开始扫描时配置黑名单 URL、域名、IP
-    - 黑名单规则存储在数据库中，与 Scan 或 Engine 关联
-    """
-    
-    # 默认黑名单正则规则
-    DEFAULT_PATTERNS = [
-        r'\.gov$',           # .gov 结尾
-        r'\.gov\.[a-z]{2}$', # .gov.cn, .gov.uk 等
-    ]
-    
-    def __init__(self, patterns: Optional[List[str]] = None):
-        """
-        初始化黑名单服务
-        
-        Args:
-            patterns: 正则表达式列表，None 使用默认规则
-        """
-        self.patterns = patterns or self.DEFAULT_PATTERNS
-        self._compiled_patterns = [re.compile(p) for p in self.patterns]
-    
-    def filter_queryset(
-        self,
-        queryset: QuerySet,
-        url_field: str = 'url'
-    ) -> QuerySet:
-        """
-        数据库层面过滤 queryset
-        
-        使用 PostgreSQL 正则表达式排除黑名单 URL
-        
-        Args:
-            queryset: 原始 queryset
-            url_field: URL 字段名
-            
-        Returns:
-            QuerySet: 过滤后的 queryset
-        """
-        for pattern in self.patterns:
-            queryset = queryset.exclude(**{f'{url_field}__regex': pattern})
-        return queryset
-    
-    def filter_url(self, url: str) -> bool:
-        """
-        检查单个 URL 是否通过黑名单过滤
-        
-        Args:
-            url: 要检查的 URL
-            
-        Returns:
-            bool: True 表示通过（不在黑名单），False 表示被过滤
-        """
-        for pattern in self._compiled_patterns:
-            if pattern.search(url):
-                return False
-        return True
-    
-    # TODO: 后续版本实现
-    # @classmethod
-    # def from_scan(cls, scan_id: int) -> 'BlacklistService':
-    #     """从数据库加载扫描配置的黑名单规则"""
-    #     pass

backend/apps/scan/services/scan_creation_service.py

@@ -282,7 +282,7 @@ class ScanCreationService:
         targets: List[Target],
         engine_ids: List[int],
         engine_names: List[str],
-        merged_configuration: str,
+        yaml_configuration: str,
         scheduled_scan_name: str | None = None
     ) -> List[Scan]:
         """
@@ -292,7 +292,7 @@ class ScanCreationService:
             targets: 目标列表
             engine_ids: 引擎 ID 列表
             engine_names: 引擎名称列表
-            merged_configuration: 合并后的 YAML 配置
+            yaml_configuration: YAML 格式的扫描配置
             scheduled_scan_name: 定时扫描任务名称（可选，用于通知显示）
         
         Returns:
@@ -312,7 +312,7 @@ class ScanCreationService:
                     target=target,
                     engine_ids=engine_ids,
                     engine_names=engine_names,
-                    merged_configuration=merged_configuration,
+                    yaml_configuration=yaml_configuration,
                     results_dir=scan_workspace_dir,
                     status=ScanStatus.INITIATED,
                     container_ids=[],

backend/apps/scan/services/scan_service.py

@@ -117,12 +117,12 @@ class ScanService:
         targets: List[Target],
         engine_ids: List[int],
         engine_names: List[str],
-        merged_configuration: str,
+        yaml_configuration: str,
         scheduled_scan_name: str | None = None
     ) -> List[Scan]:
         """批量创建扫描任务（委托给 ScanCreationService）"""
         return self.creation_service.create_scans(
-            targets, engine_ids, engine_names, merged_configuration, scheduled_scan_name
+            targets, engine_ids, engine_names, yaml_configuration, scheduled_scan_name
         )
     
     # ==================== 状态管理方法（委托给 ScanStateService） ====================

backend/apps/scan/services/scheduled_scan_service.py

@@ -54,7 +54,7 @@ class ScheduledScanService:
     
     def create(self, dto: ScheduledScanDTO) -> ScheduledScan:
         """
-        创建定时扫描任务
+        创建定时扫描任务（使用引擎 ID 合并配置）
         
         流程：
         1. 验证参数
@@ -88,7 +88,7 @@ class ScheduledScanService:
         
         # 设置 DTO 的合并配置和引擎名称
         dto.engine_names = engine_names
-        dto.merged_configuration = merged_configuration
+        dto.yaml_configuration = merged_configuration
         
         # 3. 创建数据库记录
         scheduled_scan = self.repo.create(dto)
@@ -107,12 +107,49 @@ class ScheduledScanService:
         
         return scheduled_scan
     
-    def _validate_create_dto(self, dto: ScheduledScanDTO) -> None:
-        """验证创建 DTO"""
-        from apps.targets.repositories import DjangoOrganizationRepository
+    def create_with_configuration(self, dto: ScheduledScanDTO) -> ScheduledScan:
+        """
+        创建定时扫描任务（直接使用前端传递的配置）
         
-        if not dto.name:
-            raise ValidationError('任务名称不能为空')
+        流程：
+        1. 验证参数
+        2. 直接使用 dto.yaml_configuration
+        3. 创建数据库记录
+        4. 计算并设置 next_run_time
+        
+        Args:
+            dto: 定时扫描 DTO（必须包含 yaml_configuration）
+        
+        Returns:
+            创建的 ScheduledScan 对象
+        
+        Raises:
+            ValidationError: 参数验证失败
+        """
+        # 1. 验证参数
+        self._validate_create_dto_with_configuration(dto)
+        
+        # 2. 创建数据库记录（直接使用 dto 中的配置）
+        scheduled_scan = self.repo.create(dto)
+        
+        # 3. 如果有 cron 表达式且已启用，计算下次执行时间
+        if scheduled_scan.cron_expression and scheduled_scan.is_enabled:
+            next_run_time = self._calculate_next_run_time(scheduled_scan)
+            if next_run_time:
+                self.repo.update_next_run_time(scheduled_scan.id, next_run_time)
+                scheduled_scan.next_run_time = next_run_time
+        
+        logger.info(
+            "创建定时扫描任务 - ID: %s, 名称: %s, 下次执行: %s",
+            scheduled_scan.id, scheduled_scan.name, scheduled_scan.next_run_time
+        )
+        
+        return scheduled_scan
+    
+    def _validate_create_dto(self, dto: ScheduledScanDTO) -> None:
+        """验证创建 DTO（使用引擎 ID）"""
+        # 基础验证
+        self._validate_base_dto(dto)
         
         if not dto.engine_ids:
             raise ValidationError('必须选择扫描引擎')
@@ -121,6 +158,21 @@ class ScheduledScanService:
         for engine_id in dto.engine_ids:
             if not self.engine_repo.get_by_id(engine_id):
                 raise ValidationError(f'扫描引擎 ID {engine_id} 不存在')
+    
+    def _validate_create_dto_with_configuration(self, dto: ScheduledScanDTO) -> None:
+        """验证创建 DTO（使用前端传递的配置）"""
+        # 基础验证
+        self._validate_base_dto(dto)
+        
+        if not dto.yaml_configuration:
+            raise ValidationError('配置不能为空')
+    
+    def _validate_base_dto(self, dto: ScheduledScanDTO) -> None:
+        """验证 DTO 的基础字段（公共逻辑）"""
+        from apps.targets.repositories import DjangoOrganizationRepository
+        
+        if not dto.name:
+            raise ValidationError('任务名称不能为空')
         
         # 验证扫描模式（organization_id 和 target_id 互斥）
         if not dto.organization_id and not dto.target_id:
@@ -178,7 +230,7 @@ class ScheduledScanService:
             
             merged_configuration = merge_engine_configs(engines)
             dto.engine_names = engine_names
-            dto.merged_configuration = merged_configuration
+            dto.yaml_configuration = merged_configuration
         
         # 更新数据库记录
         scheduled_scan = self.repo.update(scheduled_scan_id, dto)
@@ -329,7 +381,7 @@ class ScheduledScanService:
         立即触发扫描（支持组织扫描和目标扫描两种模式）
         
         复用 ScanService 的逻辑，与 API 调用保持一致。
-        使用存储的 merged_configuration 而不是重新合并。
+        使用存储的 yaml_configuration 而不是重新合并。
         """
         from apps.scan.services.scan_service import ScanService
         
@@ -347,7 +399,7 @@ class ScheduledScanService:
             targets=targets,
             engine_ids=scheduled_scan.engine_ids,
             engine_names=scheduled_scan.engine_names,
-            merged_configuration=scheduled_scan.merged_configuration,
+            yaml_configuration=scheduled_scan.yaml_configuration,
             scheduled_scan_name=scheduled_scan.name
         )
         

backend/apps/scan/services/subfinder_provider_config_service.py

@@ -0,0 +1,138 @@
+"""Subfinder Provider 配置文件生成服务
+
+负责生成 subfinder 的 provider-config.yaml 配置文件
+"""
+
+import logging
+import os
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+from ..models import SubfinderProviderSettings
+
+logger = logging.getLogger(__name__)
+
+
+class SubfinderProviderConfigService:
+    """Subfinder Provider 配置文件生成服务"""
+    
+    # Provider 格式定义
+    PROVIDER_FORMATS = {
+        'fofa': {'type': 'composite', 'format': '{email}:{api_key}'},
+        'censys': {'type': 'composite', 'format': '{api_id}:{api_secret}'},
+        'hunter': {'type': 'single', 'field': 'api_key'},
+        'shodan': {'type': 'single', 'field': 'api_key'},
+        'zoomeye': {'type': 'single', 'field': 'api_key'},
+        'securitytrails': {'type': 'single', 'field': 'api_key'},
+        'threatbook': {'type': 'single', 'field': 'api_key'},
+        'quake': {'type': 'single', 'field': 'api_key'},
+    }
+    
+    def generate(self, output_dir: str) -> Optional[str]:
+        """
+        生成 provider-config.yaml 文件
+        
+        Args:
+            output_dir: 输出目录路径
+            
+        Returns:
+            生成的配置文件路径，如果没有启用的 provider 则返回 None
+        """
+        settings = SubfinderProviderSettings.get_instance()
+        
+        config = {}
+        has_enabled = False
+        
+        for provider, format_info in self.PROVIDER_FORMATS.items():
+            provider_config = settings.providers.get(provider, {})
+            
+            if not provider_config.get('enabled'):
+                config[provider] = []
+                continue
+            
+            value = self._build_provider_value(provider, provider_config)
+            if value:
+                config[provider] = [value]  # 单个 key 放入数组
+                has_enabled = True
+                logger.debug(f"Provider {provider} 已启用")
+            else:
+                config[provider] = []
+        
+        # 检查是否有任何启用的 provider
+        if not has_enabled:
+            logger.info("没有启用的 Provider，跳过配置文件生成")
+            return None
+        
+        # 确保输出目录存在
+        output_path = Path(output_dir) / 'provider-config.yaml'
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        # 写入 YAML 文件（使用默认列表格式，和 subfinder 一致）
+        with open(output_path, 'w', encoding='utf-8') as f:
+            yaml.dump(config, f, default_flow_style=False, allow_unicode=True)
+        
+        # 设置文件权限为 600（仅所有者可读写）
+        os.chmod(output_path, 0o600)
+        
+        logger.info(f"Provider 配置文件已生成: {output_path}")
+        return str(output_path)
+    
+    def _build_provider_value(self, provider: str, config: dict) -> Optional[str]:
+        """根据 provider 格式规则构建配置值
+        
+        Args:
+            provider: provider 名称
+            config: provider 配置字典
+            
+        Returns:
+            构建的配置值字符串，如果配置不完整则返回 None
+        """
+        format_info = self.PROVIDER_FORMATS.get(provider)
+        if not format_info:
+            return None
+        
+        if format_info['type'] == 'composite':
+            # 复合格式：需要多个字段
+            format_str = format_info['format']
+            try:
+                # 提取格式字符串中的字段名
+                # 例如 '{email}:{api_key}' -> ['email', 'api_key']
+                import re
+                fields = re.findall(r'\{(\w+)\}', format_str)
+                
+                # 检查所有字段是否都有值
+                values = {}
+                for field in fields:
+                    value = config.get(field, '').strip()
+                    if not value:
+                        logger.debug(f"Provider {provider} 缺少字段 {field}")
+                        return None
+                    values[field] = value
+                
+                return format_str.format(**values)
+            except (KeyError, ValueError) as e:
+                logger.warning(f"构建 {provider} 配置值失败: {e}")
+                return None
+        else:
+            # 单字段格式
+            field = format_info['field']
+            value = config.get(field, '').strip()
+            if not value:
+                logger.debug(f"Provider {provider} 缺少字段 {field}")
+                return None
+            return value
+    
+    def cleanup(self, config_path: str) -> None:
+        """清理配置文件
+        
+        Args:
+            config_path: 配置文件路径
+        """
+        try:
+            if config_path and Path(config_path).exists():
+                Path(config_path).unlink()
+                logger.debug(f"已清理配置文件: {config_path}")
+        except Exception as e:
+            logger.warning(f"清理配置文件失败: {config_path} - {e}")

backend/apps/scan/services/target_export_service.py

@@ -2,7 +2,9 @@
 目标导出服务
 
 提供统一的目标提取和文件导出功能，支持：
-- URL 导出（流式写入 + 默认值回退）
+- URL 导出（纯导出，不做隐式回退）
+- 默认 URL 生成（独立方法）
+- 带回退链的 URL 导出（用例层编排）
 - 域名/IP 导出（用于端口扫描）
 - 黑名单过滤集成
 """
@@ -10,37 +12,326 @@
 import ipaddress
 import logging
 from pathlib import Path
-from typing import Dict, Any, Optional, Iterator
+from typing import Dict, Any, Optional, List, Iterator, Tuple
 
 from django.db.models import QuerySet
 
-from .blacklist_service import BlacklistService
+from apps.common.utils import BlacklistFilter
 
 logger = logging.getLogger(__name__)
 
 
+class DataSource:
+    """数据源类型常量"""
+    ENDPOINT = "endpoint"
+    WEBSITE = "website"
+    HOST_PORT = "host_port"
+    DEFAULT = "default"
+
+
+def create_export_service(target_id: int) -> 'TargetExportService':
+    """
+    工厂函数：创建带黑名单过滤的导出服务
+    
+    Args:
+        target_id: 目标 ID，用于加载黑名单规则
+        
+    Returns:
+        TargetExportService: 配置好黑名单过滤器的导出服务实例
+    """
+    from apps.common.services import BlacklistService
+    
+    rules = BlacklistService().get_rules(target_id)
+    blacklist_filter = BlacklistFilter(rules)
+    return TargetExportService(blacklist_filter=blacklist_filter)
+
+
+def _iter_default_urls_from_target(
+    target_id: int,
+    blacklist_filter: Optional[BlacklistFilter] = None
+) -> Iterator[str]:
+    """
+    内部生成器：从 Target 本身生成默认 URL
+    
+    根据 Target 类型生成 URL：
+    - DOMAIN: http(s)://domain
+    - IP: http(s)://ip
+    - CIDR: 展开为所有 IP 的 http(s)://ip
+    - URL: 直接使用目标 URL
+    
+    Args:
+        target_id: 目标 ID
+        blacklist_filter: 黑名单过滤器
+        
+    Yields:
+        str: URL
+    """
+    from apps.targets.services import TargetService
+    from apps.targets.models import Target
+    
+    target_service = TargetService()
+    target = target_service.get_target(target_id)
+    
+    if not target:
+        logger.warning("Target ID %d 不存在，无法生成默认 URL", target_id)
+        return
+    
+    target_name = target.name
+    target_type = target.type
+    
+    # 根据 Target 类型生成 URL
+    if target_type == Target.TargetType.DOMAIN:
+        urls = [f"http://{target_name}", f"https://{target_name}"]
+    elif target_type == Target.TargetType.IP:
+        urls = [f"http://{target_name}", f"https://{target_name}"]
+    elif target_type == Target.TargetType.CIDR:
+        try:
+            network = ipaddress.ip_network(target_name, strict=False)
+            urls = []
+            for ip in network.hosts():
+                urls.extend([f"http://{ip}", f"https://{ip}"])
+            # /32 或 /128 特殊处理
+            if not urls:
+                ip = str(network.network_address)
+                urls = [f"http://{ip}", f"https://{ip}"]
+        except ValueError as e:
+            logger.error("CIDR 解析失败: %s - %s", target_name, e)
+            return
+    elif target_type == Target.TargetType.URL:
+        urls = [target_name]
+    else:
+        logger.warning("不支持的 Target 类型: %s", target_type)
+        return
+    
+    # 过滤并产出
+    for url in urls:
+        if blacklist_filter and not blacklist_filter.is_allowed(url):
+            continue
+        yield url
+
+
+def _iter_urls_with_fallback(
+    target_id: int,
+    sources: List[str],
+    blacklist_filter: Optional[BlacklistFilter] = None,
+    batch_size: int = 1000,
+    tried_sources: Optional[List[str]] = None
+) -> Iterator[Tuple[str, str]]:
+    """
+    内部生成器：流式产出 URL（带回退链）
+    
+    按 sources 顺序尝试每个数据源，直到有数据返回。
+    
+    回退逻辑：
+    - 数据源有数据且通过过滤 → 产出 URL，停止回退
+    - 数据源有数据但全被过滤 → 不回退，停止（避免意外暴露）
+    - 数据源为空 → 继续尝试下一个
+    
+    Args:
+        target_id: 目标 ID
+        sources: 数据源优先级列表
+        blacklist_filter: 黑名单过滤器
+        batch_size: 批次大小
+        tried_sources: 可选，用于记录尝试过的数据源（外部传入列表，会被修改）
+        
+    Yields:
+        Tuple[str, str]: (url, source) - URL 和来源标识
+    """
+    from apps.asset.models import Endpoint, WebSite
+    
+    for source in sources:
+        if tried_sources is not None:
+            tried_sources.append(source)
+        
+        has_output = False  # 是否有输出（通过过滤的）
+        has_raw_data = False  # 是否有原始数据（过滤前）
+        
+        if source == DataSource.DEFAULT:
+            # 默认 URL 生成（从 Target 本身构造，复用共用生成器）
+            for url in _iter_default_urls_from_target(target_id, blacklist_filter):
+                has_raw_data = True
+                has_output = True
+                yield url, source
+            
+            # 检查是否有原始数据（需要单独判断，因为生成器可能被过滤后为空）
+            if not has_raw_data:
+                # 再次检查 Target 是否存在
+                from apps.targets.services import TargetService
+                target = TargetService().get_target(target_id)
+                has_raw_data = target is not None
+            
+            if has_raw_data:
+                if not has_output:
+                    logger.info("%s 有数据但全被黑名单过滤，不回退", source)
+                return
+            continue
+        
+        # 构建对应数据源的 queryset
+        if source == DataSource.ENDPOINT:
+            queryset = Endpoint.objects.filter(target_id=target_id).values_list('url', flat=True)
+        elif source == DataSource.WEBSITE:
+            queryset = WebSite.objects.filter(target_id=target_id).values_list('url', flat=True)
+        else:
+            logger.warning("未知的数据源类型: %s，跳过", source)
+            continue
+        
+        for url in queryset.iterator(chunk_size=batch_size):
+            if url:
+                has_raw_data = True
+                if blacklist_filter and not blacklist_filter.is_allowed(url):
+                    continue
+                has_output = True
+                yield url, source
+        
+        # 有原始数据就停止（不管是否被过滤）
+        if has_raw_data:
+            if not has_output:
+                logger.info("%s 有数据但全被黑名单过滤，不回退", source)
+            return
+        
+        logger.info("%s 为空，尝试下一个数据源", source)
+
+
+def get_urls_with_fallback(
+    target_id: int,
+    sources: List[str],
+    batch_size: int = 1000
+) -> Dict[str, Any]:
+    """
+    带回退链的 URL 获取用例函数（返回列表）
+    
+    按 sources 顺序尝试每个数据源，直到有数据返回。
+    
+    Args:
+        target_id: 目标 ID
+        sources: 数据源优先级列表，如 ["website", "endpoint", "default"]
+        batch_size: 批次大小
+        
+    Returns:
+        dict: {
+            'success': bool,
+            'urls': List[str],
+            'total_count': int,
+            'source': str,  # 实际使用的数据源
+            'tried_sources': List[str],  # 尝试过的数据源
+        }
+    """
+    from apps.common.services import BlacklistService
+    
+    rules = BlacklistService().get_rules(target_id)
+    blacklist_filter = BlacklistFilter(rules)
+    
+    urls = []
+    actual_source = 'none'
+    tried_sources = []
+    
+    for url, source in _iter_urls_with_fallback(target_id, sources, blacklist_filter, batch_size, tried_sources):
+        urls.append(url)
+        actual_source = source
+    
+    if urls:
+        logger.info("从 %s 获取 %d 条 URL", actual_source, len(urls))
+    else:
+        logger.warning("所有数据源都为空，无法获取 URL")
+    
+    return {
+        'success': True,
+        'urls': urls,
+        'total_count': len(urls),
+        'source': actual_source,
+        'tried_sources': tried_sources,
+    }
+
+
+def export_urls_with_fallback(
+    target_id: int,
+    output_file: str,
+    sources: List[str],
+    batch_size: int = 1000
+) -> Dict[str, Any]:
+    """
+    带回退链的 URL 导出用例函数（写入文件）
+    
+    按 sources 顺序尝试每个数据源，直到有数据返回。
+    流式写入，内存占用 O(1)。
+    
+    Args:
+        target_id: 目标 ID
+        output_file: 输出文件路径
+        sources: 数据源优先级列表，如 ["endpoint", "website", "default"]
+        batch_size: 批次大小
+        
+    Returns:
+        dict: {
+            'success': bool,
+            'output_file': str,
+            'total_count': int,
+            'source': str,  # 实际使用的数据源
+            'tried_sources': List[str],  # 尝试过的数据源
+        }
+    """
+    from apps.common.services import BlacklistService
+    
+    rules = BlacklistService().get_rules(target_id)
+    blacklist_filter = BlacklistFilter(rules)
+    
+    output_path = Path(output_file)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    
+    total_count = 0
+    actual_source = 'none'
+    tried_sources = []
+    
+    with open(output_path, 'w', encoding='utf-8', buffering=8192) as f:
+        for url, source in _iter_urls_with_fallback(target_id, sources, blacklist_filter, batch_size, tried_sources):
+            f.write(f"{url}\n")
+            total_count += 1
+            actual_source = source
+            
+            if total_count % 10000 == 0:
+                logger.info("已导出 %d 个 URL...", total_count)
+    
+    if total_count > 0:
+        logger.info("从 %s 导出 %d 条 URL 到 %s", actual_source, total_count, output_file)
+    else:
+        logger.warning("所有数据源都为空，无法导出 URL")
+    
+    return {
+        'success': True,
+        'output_file': str(output_path),
+        'total_count': total_count,
+        'source': actual_source,
+        'tried_sources': tried_sources,
+    }
+
+
 class TargetExportService:
     """
     目标导出服务 - 提供统一的目标提取和文件导出功能
     
     使用方式：
-        # Task 层决定数据源
-        queryset = WebSite.objects.filter(target_id=target_id).values_list('url', flat=True)
+        # 方式 1：使用用例函数（推荐）
+        from apps.scan.services.target_export_service import export_urls_with_fallback, DataSource
         
-        # 使用导出服务
-        blacklist_service = BlacklistService()
-        export_service = TargetExportService(blacklist_service=blacklist_service)
+        result = export_urls_with_fallback(
+            target_id=1,
+            output_file='/path/to/output.txt',
+            sources=[DataSource.ENDPOINT, DataSource.WEBSITE, DataSource.DEFAULT]
+        )
+        
+        # 方式 2：直接使用 Service（纯导出，不带回退）
+        export_service = create_export_service(target_id)
         result = export_service.export_urls(target_id, output_path, queryset)
     """
     
-    def __init__(self, blacklist_service: Optional[BlacklistService] = None):
+    def __init__(self, blacklist_filter: Optional[BlacklistFilter] = None):
         """
         初始化导出服务
         
         Args:
-            blacklist_service: 黑名单过滤服务，None 表示禁用过滤
+            blacklist_filter: 黑名单过滤器，None 表示禁用过滤
         """
-        self.blacklist_service = blacklist_service
+        self.blacklist_filter = blacklist_filter
     
     def export_urls(
         self,
@@ -51,16 +342,14 @@ class TargetExportService:
         batch_size: int = 1000
     ) -> Dict[str, Any]:
         """
-        统一 URL 导出函数
+        纯 URL 导出函数 - 只负责将 queryset 数据写入文件
         
-        自动判断数据库有无数据：
-        - 有数据：流式写入数据库数据到文件
-        - 无数据：调用默认值生成器生成 URL
+        不做任何隐式回退或默认 URL 生成。
         
         Args:
             target_id: 目标 ID
             output_path: 输出文件路径
-            queryset: 数据源 queryset（由 Task 层构建，应为 values_list flat=True）
+            queryset: 数据源 queryset（由调用方构建，应为 values_list flat=True）
             url_field: URL 字段名（用于黑名单过滤）
             batch_size: 批次大小
             
@@ -68,7 +357,9 @@ class TargetExportService:
             dict: {
                 'success': bool,
                 'output_file': str,
-                'total_count': int
+                'total_count': int,        # 实际写入数量
+                'queryset_count': int,     # 原始数据数量（迭代计数）
+                'filtered_count': int,     # 被黑名单过滤的数量
             }
             
         Raises:
@@ -79,19 +370,18 @@ class TargetExportService:
         
         logger.info("开始导出 URL - target_id=%s, output=%s", target_id, output_path)
         
-        # 应用黑名单过滤（数据库层面）
-        if self.blacklist_service:
-            # 注意：queryset 应该是原始 queryset，不是 values_list
-            # 这里假设 Task 层传入的是 values_list，需要在 Task 层处理过滤
-            pass
-        
         total_count = 0
+        filtered_count = 0
+        queryset_count = 0
+        
         try:
             with open(output_file, 'w', encoding='utf-8', buffering=8192) as f:
                 for url in queryset.iterator(chunk_size=batch_size):
+                    queryset_count += 1
                     if url:
-                        # Python 层面黑名单过滤
-                        if self.blacklist_service and not self.blacklist_service.filter_url(url):
+                        # 黑名单过滤
+                        if self.blacklist_filter and not self.blacklist_filter.is_allowed(url):
+                            filtered_count += 1
                             continue
                         f.write(f"{url}\n")
                         total_count += 1
@@ -102,25 +392,29 @@ class TargetExportService:
             logger.error("文件写入失败: %s - %s", output_path, e)
             raise
         
-        # 默认值回退模式
-        if total_count == 0:
-            total_count = self._generate_default_urls(target_id, output_file)
+        if filtered_count > 0:
+            logger.info("黑名单过滤: 过滤 %d 个 URL", filtered_count)
         
-        logger.info("✓ URL 导出完成 - 数量: %d, 文件: %s", total_count, output_path)
+        logger.info(
+            "✓ URL 导出完成 - 写入: %d, 原始: %d, 过滤: %d, 文件: %s",
+            total_count, queryset_count, filtered_count, output_path
+        )
         
         return {
             'success': True,
             'output_file': str(output_file),
-            'total_count': total_count
+            'total_count': total_count,
+            'queryset_count': queryset_count,
+            'filtered_count': filtered_count,
         }
 
-    def _generate_default_urls(
+    def generate_default_urls(
         self,
         target_id: int,
-        output_path: Path
-    ) -> int:
+        output_path: str
+    ) -> Dict[str, Any]:
         """
-        默认值生成器（内部函数）
+        默认 URL 生成器
         
         根据 Target 类型生成默认 URL：
         - DOMAIN: http(s)://domain
@@ -133,91 +427,43 @@ class TargetExportService:
             output_path: 输出文件路径
             
         Returns:
-            int: 写入的 URL 总数
+            dict: {
+                'success': bool,
+                'output_file': str,
+                'total_count': int,
+            }
         """
-        from apps.targets.services import TargetService
-        from apps.targets.models import Target
+        output_file = Path(output_path)
+        output_file.parent.mkdir(parents=True, exist_ok=True)
         
-        target_service = TargetService()
-        target = target_service.get_target(target_id)
-        
-        if not target:
-            logger.warning("Target ID %d 不存在，无法生成默认 URL", target_id)
-            return 0
-        
-        target_name = target.name
-        target_type = target.type
-        
-        logger.info("懒加载模式：Target 类型=%s, 名称=%s", target_type, target_name)
+        logger.info("生成默认 URL - target_id=%d", target_id)
         
         total_urls = 0
         
-        with open(output_path, 'w', encoding='utf-8', buffering=8192) as f:
-            if target_type == Target.TargetType.DOMAIN:
-                urls = [f"http://{target_name}", f"https://{target_name}"]
-                for url in urls:
-                    if self._should_write_url(url):
-                        f.write(f"{url}\n")
-                        total_urls += 1
-                        
-            elif target_type == Target.TargetType.IP:
-                urls = [f"http://{target_name}", f"https://{target_name}"]
-                for url in urls:
-                    if self._should_write_url(url):
-                        f.write(f"{url}\n")
-                        total_urls += 1
-                        
-            elif target_type == Target.TargetType.CIDR:
-                try:
-                    network = ipaddress.ip_network(target_name, strict=False)
-                    
-                    for ip in network.hosts():
-                        urls = [f"http://{ip}", f"https://{ip}"]
-                        for url in urls:
-                            if self._should_write_url(url):
-                                f.write(f"{url}\n")
-                                total_urls += 1
-                        
-                        if total_urls % 10000 == 0:
-                            logger.info("已生成 %d 个 URL...", total_urls)
-                    
-                    # /32 或 /128 特殊处理
-                    if total_urls == 0:
-                        ip = str(network.network_address)
-                        urls = [f"http://{ip}", f"https://{ip}"]
-                        for url in urls:
-                            if self._should_write_url(url):
-                                f.write(f"{url}\n")
-                                total_urls += 1
-                                
-                except ValueError as e:
-                    logger.error("CIDR 解析失败: %s - %s", target_name, e)
-                    raise ValueError(f"无效的 CIDR: {target_name}") from e
-                    
-            elif target_type == Target.TargetType.URL:
-                if self._should_write_url(target_name):
-                    f.write(f"{target_name}\n")
-                    total_urls = 1
-            else:
-                logger.warning("不支持的 Target 类型: %s", target_type)
+        with open(output_file, 'w', encoding='utf-8', buffering=8192) as f:
+            for url in _iter_default_urls_from_target(target_id, self.blacklist_filter):
+                f.write(f"{url}\n")
+                total_urls += 1
+                
+                if total_urls % 10000 == 0:
+                    logger.info("已生成 %d 个 URL...", total_urls)
         
-        logger.info("✓ 懒加载生成默认 URL - 数量: %d", total_urls)
-        return total_urls
-    
-    def _should_write_url(self, url: str) -> bool:
-        """检查 URL 是否应该写入（通过黑名单过滤）"""
-        if self.blacklist_service:
-            return self.blacklist_service.filter_url(url)
-        return True
+        logger.info("✓ 默认 URL 生成完成 - 数量: %d", total_urls)
+        
+        return {
+            'success': True,
+            'output_file': str(output_file),
+            'total_count': total_urls,
+        }
 
-    def export_targets(
+    def export_hosts(
         self,
         target_id: int,
         output_path: str,
         batch_size: int = 1000
     ) -> Dict[str, Any]:
         """
-        域名/IP 导出函数（用于端口扫描）
+        主机列表导出函数（用于端口扫描）
         
         根据 Target 类型选择导出逻辑：
         - DOMAIN: 从 Subdomain 表流式导出子域名
@@ -239,8 +485,7 @@ class TargetExportService:
         """
         from apps.targets.services import TargetService
         from apps.targets.models import Target
-        from apps.asset.services.asset.subdomain_service import SubdomainService
-        
+
         output_file = Path(output_path)
         output_file.parent.mkdir(parents=True, exist_ok=True)
         
@@ -255,7 +500,7 @@ class TargetExportService:
         target_name = target.name
         
         logger.info(
-            "开始导出扫描目标 - Target ID: %d, Name: %s, Type: %s, 输出文件: %s",
+            "开始导出主机列表 - Target ID: %d, Name: %s, Type: %s, 输出文件: %s",
             target_id, target_name, target_type, output_path
         )
         
@@ -277,7 +522,7 @@ class TargetExportService:
             raise ValueError(f"不支持的目标类型: {target_type}")
         
         logger.info(
-            "✓ 扫描目标导出完成 - 类型: %s, 总数: %d, 文件: %s",
+            "✓ 主机列表导出完成 - 类型: %s, 总数: %d, 文件: %s",
             type_desc, total_count, output_path
         )
         
@@ -295,7 +540,7 @@ class TargetExportService:
         output_path: Path,
         batch_size: int
     ) -> int:
-        """导出域名类型目标的子域名"""
+        """导出域名类型目标的根域名 + 子域名"""
         from apps.asset.services.asset.subdomain_service import SubdomainService
         
         subdomain_service = SubdomainService()
@@ -305,23 +550,27 @@ class TargetExportService:
         )
         
         total_count = 0
+        written_domains = set()  # 去重（子域名表可能已包含根域名）
+        
         with open(output_path, 'w', encoding='utf-8', buffering=8192) as f:
+            # 1. 先写入根域名
+            if self._should_write_target(target_name):
+                f.write(f"{target_name}\n")
+                written_domains.add(target_name)
+                total_count += 1
+            
+            # 2. 再写入子域名（跳过已写入的根域名）
             for domain_name in domain_iterator:
+                if domain_name in written_domains:
+                    continue
                 if self._should_write_target(domain_name):
                     f.write(f"{domain_name}\n")
+                    written_domains.add(domain_name)
                     total_count += 1
                     
                     if total_count % 10000 == 0:
                         logger.info("已导出 %d 个域名...", total_count)
         
-        # 默认值模式：如果没有子域名，使用根域名
-        if total_count == 0:
-            logger.info("采用默认域名：%s (target_id=%d)", target_name, target_id)
-            if self._should_write_target(target_name):
-                with open(output_path, 'w', encoding='utf-8') as f:
-                    f.write(f"{target_name}\n")
-                total_count = 1
-        
         return total_count
     
     def _export_ip(self, target_name: str, output_path: Path) -> int:
@@ -359,6 +608,6 @@ class TargetExportService:
     
     def _should_write_target(self, target: str) -> bool:
         """检查目标是否应该写入（通过黑名单过滤）"""
-        if self.blacklist_service:
-            return self.blacklist_service.filter_url(target)
+        if self.blacklist_filter:
+            return self.blacklist_filter.is_allowed(target)
         return True

backend/apps/scan/tasks/directory_scan/export_sites_task.py

@@ -1,38 +1,48 @@
 """
 导出站点 URL 到 TXT 文件的 Task
 
-使用 TargetExportService 统一处理导出逻辑和默认值回退
-数据源: WebSite.url
+支持两种模式：
+1. 传统模式（向后兼容）：使用 target_id 从数据库导出
+2. Provider 模式：使用 TargetProvider 从任意数据源导出
+
+数据源: WebSite.url → Default
 """
 import logging
+from typing import Optional
+from pathlib import Path
 from prefect import task
 
-from apps.asset.models import WebSite
-from apps.scan.services import TargetExportService, BlacklistService
+from apps.scan.services.target_export_service import (
+    export_urls_with_fallback,
+    DataSource,
+)
+from apps.scan.providers import TargetProvider
 
 logger = logging.getLogger(__name__)
 
 
 @task(name="export_sites")
 def export_sites_task(
-    target_id: int,
-    output_file: str,
+    target_id: Optional[int] = None,
+    output_file: str = "",
+    provider: Optional[TargetProvider] = None,
     batch_size: int = 1000,
 ) -> dict:
     """
     导出目标下的所有站点 URL 到 TXT 文件
 
-    数据源: WebSite.url
-    
-    懒加载模式：
-    - 如果数据库为空，根据 Target 类型生成默认 URL
-    - DOMAIN: http(s)://domain
-    - IP: http(s)://ip
-    - CIDR: 展开为所有 IP 的 URL
+    支持两种模式：
+    1. 传统模式（向后兼容）：传入 target_id，从数据库导出
+    2. Provider 模式：传入 provider，从任意数据源导出
+
+    数据源优先级（回退链，仅传统模式）：
+    1. WebSite 表 - 站点级别 URL
+    2. 默认生成 - 根据 Target 类型生成 http(s)://target_name
 
     Args:
-        target_id: 目标 ID
+        target_id: 目标 ID（传统模式，向后兼容）
         output_file: 输出文件路径（绝对路径）
+        provider: TargetProvider 实例（新模式）
         batch_size: 每次读取的批次大小，默认 1000
 
     Returns:
@@ -46,26 +56,61 @@ def export_sites_task(
         ValueError: 参数错误
         IOError: 文件写入失败
     """
-    # 构建数据源 queryset（Task 层决定数据源）
-    queryset = WebSite.objects.filter(target_id=target_id).values_list('url', flat=True)
+    # 参数验证：至少提供一个
+    if target_id is None and provider is None:
+        raise ValueError("必须提供 target_id 或 provider 参数之一")
     
-    # 使用 TargetExportService 处理导出
-    blacklist_service = BlacklistService()
-    export_service = TargetExportService(blacklist_service=blacklist_service)
+    # Provider 模式：使用 TargetProvider 导出
+    if provider is not None:
+        logger.info("使用 Provider 模式 - Provider: %s", type(provider).__name__)
+        return _export_with_provider(output_file, provider)
     
-    result = export_service.export_urls(
+    # 传统模式：使用 export_urls_with_fallback
+    logger.info("使用传统模式 - Target ID: %d", target_id)
+    result = export_urls_with_fallback(
         target_id=target_id,
-        output_path=output_file,
-        queryset=queryset,
-        batch_size=batch_size
+        output_file=output_file,
+        sources=[DataSource.WEBSITE, DataSource.DEFAULT],
+        batch_size=batch_size,
+    )
+    
+    logger.info(
+        "站点 URL 导出完成 - source=%s, count=%d",
+        result['source'], result['total_count']
     )
     
     # 保持返回值格式不变（向后兼容）
     return {
         'success': result['success'],
         'output_file': result['output_file'],
-        'total_count': result['total_count']
+        'total_count': result['total_count'],
     }
 
 
-
+def _export_with_provider(output_file: str, provider: TargetProvider) -> dict:
+    """使用 Provider 导出 URL"""
+    output_path = Path(output_file)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    
+    total_count = 0
+    blacklist_filter = provider.get_blacklist_filter()
+    
+    with open(output_path, 'w', encoding='utf-8', buffering=8192) as f:
+        for url in provider.iter_urls():
+            # 应用黑名单过滤（如果有）
+            if blacklist_filter and not blacklist_filter.is_allowed(url):
+                continue
+            
+            f.write(f"{url}\n")
+            total_count += 1
+            
+            if total_count % 1000 == 0:
+                logger.info("已导出 %d 个 URL...", total_count)
+    
+    logger.info("✓ URL 导出完成 - 总数: %d, 文件: %s", total_count, str(output_path))
+    
+    return {
+        'success': True,
+        'output_file': str(output_path),
+        'total_count': total_count,
+    }