refactor(asset): move database extension initialization to migrations

- Remove pg_trgm and pg_ivm extension setup from AssetConfig.ready() method
- Move extension creation to migration 0002 using RunSQL operations
- Add pg_trgm extension creation for text search index support
- Add pg_ivm extension creation for IMMV incremental maintenance
- Generate unique cursor names in search_service to prevent concurrent request conflicts
- Add @transaction.non_atomic_requests decorator to export view for server-side cursor compatibility
- Simplify app initialization by delegating extension setup to database migrations
- Improve thread safety and concurrency handling for streaming exports

.github/workflows/docker-build.yml

@@ -44,6 +44,10 @@ jobs:
             dockerfile: docker/agent/Dockerfile
             context: .
             platforms: linux/amd64,linux/arm64
+          - image: xingrin-postgres
+            dockerfile: docker/postgres/Dockerfile
+            context: docker/postgres
+            platforms: linux/amd64,linux/arm64
 
     steps:
       - name: Checkout
@@ -106,8 +110,8 @@ jobs:
             ${{ steps.version.outputs.IS_RELEASE == 'true' && format('{0}/{1}:latest', env.IMAGE_PREFIX, matrix.image) || '' }}
           build-args: |
             IMAGE_TAG=${{ steps.version.outputs.VERSION }}
-          cache-from: type=gha,scope=${{ matrix.image }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.image }}
+          cache-from: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache
+          cache-to: type=registry,ref=${{ env.IMAGE_PREFIX }}/${{ matrix.image }}:cache,mode=max
           provenance: false
           sbom: false
 

README.md

@@ -189,6 +189,7 @@ url="/api/v1" && status!="404"
 ### 📊 可视化界面
 - **数据统计** - 资产/漏洞统计仪表盘
 - **实时通知** - WebSocket 消息推送
+- **通知推送** - 实时企业微信，tg，discard消息推送服务
 
 ---
 

VERSION

@@ -1 +1 @@
-v1.3.2-dev
+v1.3.7-dev

backend/apps/asset/apps.py

@@ -1,106 +1,6 @@
-import logging
-import sys
-
 from django.apps import AppConfig
 
-logger = logging.getLogger(__name__)
-
 
 class AssetConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'apps.asset'
-    
-    def ready(self):
-        # 导入所有模型以确保Django发现并注册
-        from . import models
-        
-        # 启用 pg_trgm 扩展（用于文本模糊搜索索引）
-        # 用于已有数据库升级场景
-        self._ensure_pg_trgm_extension()
-        
-        # 验证 pg_ivm 扩展是否可用（用于 IMMV 增量维护）
-        self._verify_pg_ivm_extension()
-    
-    def _ensure_pg_trgm_extension(self):
-        """
-        确保 pg_trgm 扩展已启用。
-        该扩展用于 response_body 和 response_headers 字段的 GIN 索引，
-        支持高效的文本模糊搜索。
-        """
-        from django.db import connection
-        
-        # 检查是否为 PostgreSQL 数据库
-        if connection.vendor != 'postgresql':
-            logger.debug("跳过 pg_trgm 扩展：当前数据库不是 PostgreSQL")
-            return
-        
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm;")
-                logger.debug("pg_trgm 扩展已启用")
-        except Exception as e:
-            # 记录错误但不阻止应用启动
-            # 常见原因：权限不足（需要超级用户权限）
-            logger.warning(
-                "无法创建 pg_trgm 扩展: %s。"
-                "这可能导致 response_body 和 response_headers 字段的 GIN 索引无法正常工作。"
-                "请手动执行: CREATE EXTENSION IF NOT EXISTS pg_trgm;",
-                str(e)
-            )
-    
-    def _verify_pg_ivm_extension(self):
-        """
-        验证 pg_ivm 扩展是否可用。
-        pg_ivm 用于 IMMV（增量维护物化视图），是系统必需的扩展。
-        如果不可用，将记录错误并退出。
-        """
-        from django.db import connection
-        
-        # 检查是否为 PostgreSQL 数据库
-        if connection.vendor != 'postgresql':
-            logger.debug("跳过 pg_ivm 验证：当前数据库不是 PostgreSQL")
-            return
-        
-        # 跳过某些管理命令（如 migrate、makemigrations）
-        import sys
-        if len(sys.argv) > 1 and sys.argv[1] in ('migrate', 'makemigrations', 'collectstatic', 'check'):
-            logger.debug("跳过 pg_ivm 验证：当前为管理命令")
-            return
-        
-        try:
-            with connection.cursor() as cursor:
-                # 检查 pg_ivm 扩展是否已安装
-                cursor.execute("""
-                    SELECT COUNT(*) FROM pg_extension WHERE extname = 'pg_ivm'
-                """)
-                count = cursor.fetchone()[0]
-                
-                if count > 0:
-                    logger.info("✓ pg_ivm 扩展已启用")
-                else:
-                    # 尝试创建扩展
-                    try:
-                        cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_ivm;")
-                        logger.info("✓ pg_ivm 扩展已创建并启用")
-                    except Exception as create_error:
-                        logger.error(
-                            "=" * 60 + "\n"
-                            "错误: pg_ivm 扩展未安装\n"
-                            "=" * 60 + "\n"
-                            "pg_ivm 是系统必需的扩展，用于增量维护物化视图。\n\n"
-                            "请在 PostgreSQL 服务器上安装 pg_ivm：\n"
-                            "  curl -sSL https://raw.githubusercontent.com/yyhuni/xingrin/main/docker/scripts/install-pg-ivm.sh | sudo bash\n\n"
-                            "或手动安装：\n"
-                            "  1. apt install build-essential postgresql-server-dev-15 git\n"
-                            "  2. git clone https://github.com/sraoss/pg_ivm.git && cd pg_ivm && make && make install\n"
-                            "  3. 在 postgresql.conf 中添加: shared_preload_libraries = 'pg_ivm'\n"
-                            "  4. 重启 PostgreSQL\n"
-                            "=" * 60
-                        )
-                        # 在生产环境中退出，开发环境中仅警告
-                        from django.conf import settings
-                        if not settings.DEBUG:
-                            sys.exit(1)
-                        
-        except Exception as e:
-            logger.error(f"pg_ivm 扩展验证失败: {e}")

backend/apps/asset/migrations/0002_create_search_materialized_view.py

@@ -18,7 +18,13 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        # 1. 确保 pg_ivm 扩展已启用
+        # 1. 确保 pg_trgm 扩展已启用（用于文本模糊搜索索引）
+        migrations.RunSQL(
+            sql="CREATE EXTENSION IF NOT EXISTS pg_trgm;",
+            reverse_sql="-- pg_trgm extension kept for other uses"
+        ),
+        
+        # 2. 确保 pg_ivm 扩展已启用（用于 IMMV 增量维护）
         migrations.RunSQL(
             sql="CREATE EXTENSION IF NOT EXISTS pg_ivm;",
             reverse_sql="-- pg_ivm extension kept for other uses"

backend/apps/asset/services/search_service.py

@@ -11,7 +11,8 @@
 
 import logging
 import re
-from typing import Optional, List, Dict, Any, Tuple, Literal
+import uuid
+from typing import Optional, List, Dict, Any, Tuple, Literal, Iterator
 
 from django.db import connection
 
@@ -400,7 +401,7 @@ class AssetSearchService:
         query: str, 
         asset_type: AssetType = 'website',
         batch_size: int = 1000
-    ):
+    ) -> Iterator[Dict[str, Any]]:
         """
         流式搜索资产（使用服务端游标，内存友好）
         
@@ -425,9 +426,12 @@ class AssetSearchService:
             ORDER BY created_at DESC
         """
         
+        # 生成唯一的游标名称，避免并发请求冲突
+        cursor_name = f'export_cursor_{uuid.uuid4().hex[:8]}'
+        
         try:
             # 使用服务端游标，避免一次性加载所有数据到内存
-            with connection.cursor(name='export_cursor') as cursor:
+            with connection.cursor(name=cursor_name) as cursor:
                 cursor.itersize = batch_size
                 cursor.execute(sql, params)
                 columns = [col[0] for col in cursor.description]

backend/apps/asset/views/search_views.py

@@ -34,7 +34,7 @@ from rest_framework import status
 from rest_framework.views import APIView
 from rest_framework.request import Request
 from django.http import StreamingHttpResponse
-from django.db import connection
+from django.db import connection, transaction
 
 from apps.common.response_helpers import success_response, error_response
 from apps.common.error_codes import ErrorCodes
@@ -286,6 +286,9 @@ class AssetSearchExportView(APIView):
     
     Response:
         CSV 文件流（使用服务端游标，支持大数据量导出）
+    
+    注意：使用 @transaction.non_atomic_requests 装饰器，
+    因为服务端游标不能在事务块内使用。
     """
     
     def __init__(self, **kwargs):
@@ -312,6 +315,7 @@ class AssetSearchExportView(APIView):
         
         return headers, formatters
     
+    @transaction.non_atomic_requests
     def get(self, request: Request):
         """导出搜索结果为 CSV（流式导出，无数量限制）"""
         from apps.common.utils import generate_csv_rows

backend/config/settings.py

@@ -219,6 +219,8 @@ REST_FRAMEWORK = {
 # 允许所有来源（前后端分离项目，安全性由认证系统保障）
 CORS_ALLOW_ALL_ORIGINS = os.getenv('CORS_ALLOW_ALL_ORIGINS', 'True').lower() == 'true'
 CORS_ALLOW_CREDENTIALS = True
+# 暴露额外的响应头给前端（Content-Disposition 用于文件下载获取文件名）
+CORS_EXPOSE_HEADERS = ['Content-Disposition']
 
 # ==================== CSRF 配置 ====================
 CSRF_TRUSTED_ORIGINS = os.getenv('CSRF_TRUSTED_ORIGINS', 'http://localhost:3000,http://127.0.0.1:3000').split(',')

docker/docker-compose.dev.yml

@@ -8,7 +8,7 @@ services:
     build:
       context: ./postgres
       dockerfile: Dockerfile
-    image: ${DOCKER_USER:-yyhuni}/xingrin-postgres:15
+    image: ${DOCKER_USER:-yyhuni}/xingrin-postgres:${IMAGE_TAG:-dev}
     restart: always
     environment:
       POSTGRES_DB: ${DB_NAME}

docker/docker-compose.yml

@@ -14,7 +14,7 @@ services:
     build:
       context: ./postgres
       dockerfile: Dockerfile
-    image: ${DOCKER_USER:-yyhuni}/xingrin-postgres:15
+    image: ${DOCKER_USER:-yyhuni}/xingrin-postgres:${IMAGE_TAG:?IMAGE_TAG is required}
     restart: always
     environment:
       POSTGRES_DB: ${DB_NAME}