更新锁定镜像

- Remove pg_trgm and pg_ivm extension setup from AssetConfig.ready() method
- Move extension creation to migration 0002 using RunSQL operations
- Add pg_trgm extension creation for text search index support
- Add pg_ivm extension creation for IMMV incremental maintenance
- Generate unique cursor names in search_service to prevent concurrent request conflicts
- Add @transaction.non_atomic_requests decorator to export view for server-side cursor compatibility
- Simplify app initialization by delegating extension setup to database migrations
- Improve thread safety and concurrency handling for streaming exports

.github/workflows/docker-build.yml

@@ -19,7 +19,8 @@ permissions:
   contents: write
 
 jobs:
-  build:
+  # AMD64 构建（原生 x64 runner）
+  build-amd64:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -27,39 +28,30 @@ jobs:
           - image: xingrin-server
             dockerfile: docker/server/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
           - image: xingrin-frontend
             dockerfile: docker/frontend/Dockerfile
             context: .
-            platforms: linux/amd64  # ARM64 构建时 Next.js 在 QEMU 下会崩溃
           - image: xingrin-worker
             dockerfile: docker/worker/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
           - image: xingrin-nginx
             dockerfile: docker/nginx/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
           - image: xingrin-agent
             dockerfile: docker/agent/Dockerfile
             context: .
-            platforms: linux/amd64,linux/arm64
+          - image: xingrin-postgres
+            dockerfile: docker/postgres/Dockerfile
+            context: docker/postgres
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Free disk space (for large builds like worker)
+      - name: Free disk space
         run: |
-          echo "=== Before cleanup ==="
-          df -h
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /usr/local/lib/android
-          sudo rm -rf /opt/ghc
-          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
           sudo docker image prune -af
-          echo "=== After cleanup ==="
-          df -h
 
       - name: Generate SSL certificates for nginx build
         if: matrix.image == 'xingrin-nginx'
@@ -69,10 +61,6 @@ jobs:
             -keyout docker/nginx/ssl/privkey.pem \
             -out docker/nginx/ssl/fullchain.pem \
             -subj "/CN=localhost"
-          echo "SSL certificates generated for CI build"
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -83,7 +71,120 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Get version from git tag
+      - name: Get version
+        id: version
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+          else
+            echo "VERSION=dev-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build and push AMD64
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
+          platforms: linux/amd64
+          push: true
+          tags: ${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}-amd64
+          build-args: IMAGE_TAG=${{ steps.version.outputs.VERSION }}
+          cache-from: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-amd64
+          cache-to: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-amd64,mode=max
+          provenance: false
+          sbom: false
+
+  # ARM64 构建（原生 ARM64 runner）
+  build-arm64:
+    runs-on: ubuntu-22.04-arm
+    strategy:
+      matrix:
+        include:
+          - image: xingrin-server
+            dockerfile: docker/server/Dockerfile
+            context: .
+          - image: xingrin-frontend
+            dockerfile: docker/frontend/Dockerfile
+            context: .
+          - image: xingrin-worker
+            dockerfile: docker/worker/Dockerfile
+            context: .
+          - image: xingrin-nginx
+            dockerfile: docker/nginx/Dockerfile
+            context: .
+          - image: xingrin-agent
+            dockerfile: docker/agent/Dockerfile
+            context: .
+          - image: xingrin-postgres
+            dockerfile: docker/postgres/Dockerfile
+            context: docker/postgres
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Generate SSL certificates for nginx build
+        if: matrix.image == 'xingrin-nginx'
+        run: |
+          mkdir -p docker/nginx/ssl
+          openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
+            -keyout docker/nginx/ssl/privkey.pem \
+            -out docker/nginx/ssl/fullchain.pem \
+            -subj "/CN=localhost"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get version
+        id: version
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+          else
+            echo "VERSION=dev-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build and push ARM64
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
+          platforms: linux/arm64
+          push: true
+          tags: ${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}-arm64
+          build-args: IMAGE_TAG=${{ steps.version.outputs.VERSION }}
+          cache-from: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-arm64
+          cache-to: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache-arm64,mode=max
+          provenance: false
+          sbom: false
+
+  # 合并多架构 manifest
+  merge-manifests:
+    runs-on: ubuntu-latest
+    needs: [build-amd64, build-arm64]
+    strategy:
+      matrix:
+        image:
+          - xingrin-server
+          - xingrin-frontend
+          - xingrin-worker
+          - xingrin-nginx
+          - xingrin-agent
+          - xingrin-postgres
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get version
         id: version
         run: |
           if [[ $GITHUB_REF == refs/tags/* ]]; then
@@ -94,28 +195,27 @@ jobs:
             echo "IS_RELEASE=false" >> $GITHUB_OUTPUT
           fi
 
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: ${{ matrix.context }}
-          file: ${{ matrix.dockerfile }}
-          platforms: ${{ matrix.platforms }}
-          push: true
-          tags: |
-            ${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:${{ steps.version.outputs.VERSION }}
-            ${{ steps.version.outputs.IS_RELEASE == 'true' && format('{0}/{1}:latest', env.IMAGE_PREFIX, matrix.image) || '' }}
-          build-args: |
-            IMAGE_TAG=${{ steps.version.outputs.VERSION }}
-          cache-from: type=gha,scope=${{ matrix.image }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.image }}
-          provenance: false
-          sbom: false
+      - name: Create and push multi-arch manifest
+        run: |
+          VERSION=${{ steps.version.outputs.VERSION }}
+          IMAGE=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}
+          
+          docker manifest create ${IMAGE}:${VERSION} \
+            ${IMAGE}:${VERSION}-amd64 \
+            ${IMAGE}:${VERSION}-arm64
+          docker manifest push ${IMAGE}:${VERSION}
+          
+          if [[ "${{ steps.version.outputs.IS_RELEASE }}" == "true" ]]; then
+            docker manifest create ${IMAGE}:latest \
+              ${IMAGE}:${VERSION}-amd64 \
+              ${IMAGE}:${VERSION}-arm64
+            docker manifest push ${IMAGE}:latest
+          fi
 
-  # 所有镜像构建成功后，更新 VERSION 文件
-  # 根据 tag 所在的分支更新对应分支的 VERSION 文件
+  # 更新 VERSION 文件
   update-version:
     runs-on: ubuntu-latest
-    needs: build
+    needs: merge-manifests
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
       - name: Checkout repository

README.md

@@ -189,6 +189,7 @@ url="/api/v1" && status!="404"
 ### 📊 可视化界面
 - **数据统计** - 资产/漏洞统计仪表盘
 - **实时通知** - WebSocket 消息推送
+- **通知推送** - 实时企业微信，tg，discard消息推送服务
 
 ---
 

VERSION

@@ -1 +1 @@
-v1.3.2-dev
+v1.3.8-dev

backend/apps/asset/apps.py

@@ -1,106 +1,6 @@
-import logging
-import sys
-
 from django.apps import AppConfig
 
-logger = logging.getLogger(__name__)
-
 
 class AssetConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'apps.asset'
-    
-    def ready(self):
-        # 导入所有模型以确保Django发现并注册
-        from . import models
-        
-        # 启用 pg_trgm 扩展（用于文本模糊搜索索引）
-        # 用于已有数据库升级场景
-        self._ensure_pg_trgm_extension()
-        
-        # 验证 pg_ivm 扩展是否可用（用于 IMMV 增量维护）
-        self._verify_pg_ivm_extension()
-    
-    def _ensure_pg_trgm_extension(self):
-        """
-        确保 pg_trgm 扩展已启用。
-        该扩展用于 response_body 和 response_headers 字段的 GIN 索引，
-        支持高效的文本模糊搜索。
-        """
-        from django.db import connection
-        
-        # 检查是否为 PostgreSQL 数据库
-        if connection.vendor != 'postgresql':
-            logger.debug("跳过 pg_trgm 扩展：当前数据库不是 PostgreSQL")
-            return
-        
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
-                logger.debug("pg_trgm 扩展已启用")
-        except Exception as e:
-            # 记录错误但不阻止应用启动
-            # 常见原因：权限不足（需要超级用户权限）
-            logger.warning(
-                "无法创建 pg_trgm 扩展: %s。"
-                "这可能导致 response_body 和 response_headers 字段的 GIN 索引无法正常工作。"
-                "请手动执行: CREATE EXTENSION IF NOT EXISTS pg_trgm;",
-                str(e)
-            )
-    
-    def _verify_pg_ivm_extension(self):
-        """
-        验证 pg_ivm 扩展是否可用。
-        pg_ivm 用于 IMMV（增量维护物化视图），是系统必需的扩展。
-        如果不可用，将记录错误并退出。
-        """
-        from django.db import connection
-        
-        # 检查是否为 PostgreSQL 数据库
-        if connection.vendor != 'postgresql':
-            logger.debug("跳过 pg_ivm 验证：当前数据库不是 PostgreSQL")
-            return
-        
-        # 跳过某些管理命令（如 migrate、makemigrations）
-        import sys
-        if len(sys.argv) > 1 and sys.argv[1] in ('migrate', 'makemigrations', 'collectstatic', 'check'):
-            logger.debug("跳过 pg_ivm 验证：当前为管理命令")
-            return
-        
-        try:
-            with connection.cursor() as cursor:
-                # 检查 pg_ivm 扩展是否已安装
-                cursor.execute("""
-                    SELECT COUNT(*) FROM pg_extension WHERE extname = 'pg_ivm'
-                """)
-                count = cursor.fetchone()[0]
-                
-                if count > 0:
-                    logger.info("✓ pg_ivm 扩展已启用")
-                else:
-                    # 尝试创建扩展
-                    try:
-                        cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_ivm;")
-                        logger.info("✓ pg_ivm 扩展已创建并启用")
-                    except Exception as create_error:
-                        logger.error(
-                            "=" * 60 + "\n"
-                            "错误: pg_ivm 扩展未安装\n"
-                            "=" * 60 + "\n"
-                            "pg_ivm 是系统必需的扩展，用于增量维护物化视图。\n\n"
-                            "请在 PostgreSQL 服务器上安装 pg_ivm：\n"
-                            "  curl -sSL https://raw.githubusercontent.com/yyhuni/xingrin/main/docker/scripts/install-pg-ivm.sh | sudo bash\n\n"
-                            "或手动安装：\n"
-                            "  1. apt install build-essential postgresql-server-dev-15 git\n"
-                            "  2. git clone https://github.com/sraoss/pg_ivm.git && cd pg_ivm && make && make install\n"
-                            "  3. 在 postgresql.conf 中添加: shared_preload_libraries = 'pg_ivm'\n"
-                            "  4. 重启 PostgreSQL\n"
-                            "=" * 60
-                        )
-                        # 在生产环境中退出，开发环境中仅警告
-                        from django.conf import settings
-                        if not settings.DEBUG:
-                            sys.exit(1)
-                        
-        except Exception as e:
-            logger.error(f"pg_ivm 扩展验证失败: {e}")

backend/apps/asset/migrations/0002_create_search_materialized_view.py

@@ -18,7 +18,13 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        # 1. 确保 pg_ivm 扩展已启用
+        # 1. 确保 pg_trgm 扩展已启用（用于文本模糊搜索索引）
+        migrations.RunSQL(
+            sql="CREATE EXTENSION IF NOT EXISTS pg_trgm;",
+            reverse_sql="-- pg_trgm extension kept for other uses"
+        ),
+        
+        # 2. 确保 pg_ivm 扩展已启用（用于 IMMV 增量维护）
         migrations.RunSQL(
             sql="CREATE EXTENSION IF NOT EXISTS pg_ivm;",
             reverse_sql="-- pg_ivm extension kept for other uses"

backend/apps/asset/services/search_service.py

@@ -11,7 +11,8 @@
 
 import logging
 import re
-from typing import Optional, List, Dict, Any, Tuple, Literal
+import uuid
+from typing import Optional, List, Dict, Any, Tuple, Literal, Iterator
 
 from django.db import connection
 
@@ -400,7 +401,7 @@ class AssetSearchService:
         query: str, 
         asset_type: AssetType = 'website',
         batch_size: int = 1000
-    ):
+    ) -> Iterator[Dict[str, Any]]:
         """
         流式搜索资产（使用服务端游标，内存友好）
         
@@ -425,9 +426,12 @@ class AssetSearchService:
             ORDER BY created_at DESC
         """
         
+        # 生成唯一的游标名称，避免并发请求冲突
+        cursor_name = f'export_cursor_{uuid.uuid4().hex[:8]}'
+        
         try:
             # 使用服务端游标，避免一次性加载所有数据到内存
-            with connection.cursor(name='export_cursor') as cursor:
+            with connection.cursor(name=cursor_name) as cursor:
                 cursor.itersize = batch_size
                 cursor.execute(sql, params)
                 columns = [col[0] for col in cursor.description]

backend/apps/asset/views/search_views.py

@@ -34,7 +34,7 @@ from rest_framework import status
 from rest_framework.views import APIView
 from rest_framework.request import Request
 from django.http import StreamingHttpResponse
-from django.db import connection
+from django.db import connection, transaction
 
 from apps.common.response_helpers import success_response, error_response
 from apps.common.error_codes import ErrorCodes
@@ -286,6 +286,9 @@ class AssetSearchExportView(APIView):
     
     Response:
         CSV 文件流（使用服务端游标，支持大数据量导出）
+    
+    注意：使用 @transaction.non_atomic_requests 装饰器，
+    因为服务端游标不能在事务块内使用。
     """
     
     def __init__(self, **kwargs):
@@ -312,6 +315,7 @@ class AssetSearchExportView(APIView):
         
         return headers, formatters
     
+    @transaction.non_atomic_requests
     def get(self, request: Request):
         """导出搜索结果为 CSV（流式导出，无数量限制）"""
         from apps.common.utils import generate_csv_rows

backend/config/settings.py

@@ -219,6 +219,8 @@ REST_FRAMEWORK = {
 # 允许所有来源（前后端分离项目，安全性由认证系统保障）
 CORS_ALLOW_ALL_ORIGINS = os.getenv('CORS_ALLOW_ALL_ORIGINS', 'True').lower() == 'true'
 CORS_ALLOW_CREDENTIALS = True
+# 暴露额外的响应头给前端（Content-Disposition 用于文件下载获取文件名）
+CORS_EXPOSE_HEADERS = ['Content-Disposition']
 
 # ==================== CSRF 配置 ====================
 CSRF_TRUSTED_ORIGINS = os.getenv('CSRF_TRUSTED_ORIGINS', 'http://localhost:3000,http://127.0.0.1:3000').split(',')

docker/docker-compose.dev.yml

@@ -8,7 +8,7 @@ services:
     build:
       context: ./postgres
       dockerfile: Dockerfile
-    image: ${DOCKER_USER:-yyhuni}/xingrin-postgres:15
+    image: ${DOCKER_USER:-yyhuni}/xingrin-postgres:${IMAGE_TAG:-dev}
     restart: always
     environment:
       POSTGRES_DB: ${DB_NAME}

docker/docker-compose.yml

@@ -14,7 +14,7 @@ services:
     build:
       context: ./postgres
       dockerfile: Dockerfile
-    image: ${DOCKER_USER:-yyhuni}/xingrin-postgres:15
+    image: ${DOCKER_USER:-yyhuni}/xingrin-postgres:${IMAGE_TAG:?IMAGE_TAG is required}
     restart: always
     environment:
       POSTGRES_DB: ${DB_NAME}

docker/worker/Dockerfile

@@ -1,6 +1,5 @@
 # 第一阶段：使用 Go 官方镜像编译工具
-# 锁定 digest 避免上游更新导致缓存失效
-FROM golang:1.24@sha256:7e050c14ae9ca5ae56408a288336545b18632f51402ab0ec8e7be0e649a1fc42 AS go-builder
+FROM golang:1.24 AS go-builder
 
 ENV GOPROXY=https://goproxy.cn,direct
 # Naabu 需要 CGO 和 libpcap
@@ -37,8 +36,7 @@ RUN CGO_ENABLED=0 go install -v github.com/owasp-amass/amass/v5/cmd/amass@main
 RUN go install github.com/hahwul/dalfox/v2@latest
 
 # 第二阶段：运行时镜像
-# 锁定 digest 避免上游更新导致缓存失效
-FROM ubuntu:24.04@sha256:4fdf0125919d24aec972544669dcd7d6a26a8ad7e6561c73d5549bd6db258ac2
+FROM ubuntu:24.04
 
 # 避免交互式提示
 ENV DEBIAN_FRONTEND=noninteractive