优化：卸载脚本兼容性，防止清理数据库不干净

2026-01-31 11:46:16 +08:00 · 2025-12-19 10:20:04 +08:00
parent 3b014bd04c
commit bed80e4ba7
11 changed files with 193 additions and 42 deletions
--- a/backend/apps/common/container_bootstrap.py
+++ b/backend/apps/common/container_bootstrap.py
@@ -31,16 +31,22 @@ def fetch_config_and_setup_django():
        sys.exit(1)
    
    config_url = f"{server_url}/api/workers/config/"
+    print(f"[CONFIG] 正在从配置中心获取配置: {config_url}")
    try:
        resp = requests.get(config_url, timeout=10)
        resp.raise_for_status()
        config = resp.json()
        
        # 数据库配置（必需）
-        os.environ.setdefault("DB_HOST", config['db']['host'])
-        os.environ.setdefault("DB_PORT", config['db']['port'])
-        os.environ.setdefault("DB_NAME", config['db']['name'])
-        os.environ.setdefault("DB_USER", config['db']['user'])
+        db_host = config['db']['host']
+        db_port = config['db']['port']
+        db_name = config['db']['name']
+        db_user = config['db']['user']
+        
+        os.environ.setdefault("DB_HOST", db_host)
+        os.environ.setdefault("DB_PORT", db_port)
+        os.environ.setdefault("DB_NAME", db_name)
+        os.environ.setdefault("DB_USER", db_user)
        os.environ.setdefault("DB_PASSWORD", config['db']['password'])
        
        # Redis 配置
@@ -52,7 +58,12 @@ def fetch_config_and_setup_django():
        os.environ.setdefault("ENABLE_COMMAND_LOGGING", str(config['logging']['enableCommandLogging']).lower())
        os.environ.setdefault("DEBUG", str(config['debug']))
        
-        print(f"[CONFIG] 从配置中心获取配置成功: {config_url}")
+        print(f"[CONFIG] ✓ 配置获取成功")
+        print(f"[CONFIG]   DB_HOST: {db_host}")
+        print(f"[CONFIG]   DB_PORT: {db_port}")
+        print(f"[CONFIG]   DB_NAME: {db_name}")
+        print(f"[CONFIG]   DB_USER: {db_user}")
+        print(f"[CONFIG]   REDIS_URL: {config['redisUrl']}")
        
    except Exception as e:
        print(f"[ERROR] 获取配置失败: {config_url} - {e}", file=sys.stderr)
--- a/backend/apps/engine/services/nuclei_template_repo_service.py
+++ b/backend/apps/engine/services/nuclei_template_repo_service.py
@@ -198,9 +198,27 @@ class NucleiTemplateRepoService:

        # 判断是 clone 还是 pull
        if git_dir.is_dir():
-            # 已有仓库，执行 pull
-            cmd = ["git", "-C", str(local_path), "pull", "--ff-only"]
-            action = "pull"
+            # 检查远程地址是否变化
+            current_remote = subprocess.run(
+                ["git", "-C", str(local_path), "remote", "get-url", "origin"],
+                check=False,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+            )
+            current_url = current_remote.stdout.strip() if current_remote.returncode == 0 else ""
+            
+            if current_url != obj.repo_url:
+                # 远程地址变化，删除旧目录重新 clone
+                logger.info("nuclei 模板仓库 %s 远程地址变化，重新 clone: %s -> %s", obj.id, current_url, obj.repo_url)
+                shutil.rmtree(local_path)
+                local_path.mkdir(parents=True, exist_ok=True)
+                cmd = ["git", "clone", "--depth", "1", obj.repo_url, str(local_path)]
+                action = "clone"
+            else:
+                # 已有仓库且地址未变，执行 pull
+                cmd = ["git", "-C", str(local_path), "pull", "--ff-only"]
+                action = "pull"
        else:
            # 新仓库，执行 clone
            if local_path.exists() and not local_path.is_dir():
--- a/backend/apps/engine/services/task_distributor.py
+++ b/backend/apps/engine/services/task_distributor.py
@@ -407,8 +407,20 @@ class TaskDistributor:
        Note:
            engine_config 由 Flow 内部通过 scan_id 查询数据库获取
        """
+        logger.info("="*60)
+        logger.info("execute_scan_flow 开始")
+        logger.info("  scan_id: %s", scan_id)
+        logger.info("  target_name: %s", target_name)
+        logger.info("  target_id: %s", target_id)
+        logger.info("  scan_workspace_dir: %s", scan_workspace_dir)
+        logger.info("  engine_name: %s", engine_name)
+        logger.info("  docker_image: %s", self.docker_image)
+        logger.info("="*60)
+        
        # 1. 等待提交间隔（后台线程执行，不阻塞 API）
+        logger.info("等待提交间隔...")
        self._wait_for_submit_interval()
+        logger.info("提交间隔等待完成")
        
        # 2. 选择最佳 Worker
        worker = self.select_best_worker()
--- a/backend/apps/scan/scripts/run_initiate_scan.py
+++ b/backend/apps/scan/scripts/run_initiate_scan.py
@@ -6,14 +6,32 @@
 必须在 Django 导入之前获取配置并设置环境变量。
 """
 import argparse
-from apps.common.container_bootstrap import fetch_config_and_setup_django
+import sys
+import os
+import traceback


 def main():
+    print("="*60)
+    print("run_initiate_scan.py 启动")
+    print(f"  Python: {sys.version}")
+    print(f"  CWD: {os.getcwd()}")
+    print(f"  SERVER_URL: {os.environ.get('SERVER_URL', 'NOT SET')}")
+    print("="*60)
+    
    # 1. 从配置中心获取配置并初始化 Django（必须在 Django 导入之前）
-    fetch_config_and_setup_django()
+    print("[1/4] 从配置中心获取配置...")
+    try:
+        from apps.common.container_bootstrap import fetch_config_and_setup_django
+        fetch_config_and_setup_django()
+        print("[1/4] ✓ 配置获取成功")
+    except Exception as e:
+        print(f"[1/4] ✗ 配置获取失败: {e}")
+        traceback.print_exc()
+        sys.exit(1)
    
    # 2. 解析命令行参数
+    print("[2/4] 解析命令行参数...")
    parser = argparse.ArgumentParser(description="执行扫描初始化 Flow")
    parser.add_argument("--scan_id", type=int, required=True, help="扫描任务 ID")
    parser.add_argument("--target_name", type=str, required=True, help="目标名称")
@@ -23,21 +41,41 @@ def main():
    parser.add_argument("--scheduled_scan_name", type=str, default=None, help="定时扫描任务名称（可选）")
    
    args = parser.parse_args()
+    print(f"[2/4] ✓ 参数解析成功:")
+    print(f"       scan_id: {args.scan_id}")
+    print(f"       target_name: {args.target_name}")
+    print(f"       target_id: {args.target_id}")
+    print(f"       scan_workspace_dir: {args.scan_workspace_dir}")
+    print(f"       engine_name: {args.engine_name}")
+    print(f"       scheduled_scan_name: {args.scheduled_scan_name}")
    
    # 3. 现在可以安全导入 Django 相关模块
-    from apps.scan.flows.initiate_scan_flow import initiate_scan_flow
+    print("[3/4] 导入 initiate_scan_flow...")
+    try:
+        from apps.scan.flows.initiate_scan_flow import initiate_scan_flow
+        print("[3/4] ✓ 导入成功")
+    except Exception as e:
+        print(f"[3/4] ✗ 导入失败: {e}")
+        traceback.print_exc()
+        sys.exit(1)
    
    # 4. 执行 Flow
-    result = initiate_scan_flow(
-        scan_id=args.scan_id,
-        target_name=args.target_name,
-        target_id=args.target_id,
-        scan_workspace_dir=args.scan_workspace_dir,
-        engine_name=args.engine_name,
-        scheduled_scan_name=args.scheduled_scan_name,
-    )
-    
-    print(f"Flow 执行完成: {result}")
+    print("[4/4] 执行 initiate_scan_flow...")
+    try:
+        result = initiate_scan_flow(
+            scan_id=args.scan_id,
+            target_name=args.target_name,
+            target_id=args.target_id,
+            scan_workspace_dir=args.scan_workspace_dir,
+            engine_name=args.engine_name,
+            scheduled_scan_name=args.scheduled_scan_name,
+        )
+        print("[4/4] ✓ Flow 执行完成")
+        print(f"结果: {result}")
+    except Exception as e:
+        print(f"[4/4] ✗ Flow 执行失败: {e}")
+        traceback.print_exc()
+        sys.exit(1)


 if __name__ == "__main__":
--- a/backend/apps/scan/services/scan_creation_service.py
+++ b/backend/apps/scan/services/scan_creation_service.py
@@ -266,15 +266,26 @@ class ScanCreationService:
        Args:
            scan_data: 扫描任务数据列表
        """
+        logger.info("="*60)
+        logger.info("开始分发扫描任务到 Workers - 数量: %d", len(scan_data))
+        logger.info("="*60)
+        
        # 后台线程需要新的数据库连接
        connection.close()
+        logger.info("已关闭旧数据库连接，准备获取新连接")
        
        distributor = get_task_distributor()
+        logger.info("TaskDistributor 初始化完成")
+        
        scan_repo = DjangoScanRepository()
+        logger.info("ScanRepository 初始化完成")
        
        for data in scan_data:
            scan_id = data['scan_id']
+            logger.info("-"*40)
+            logger.info("准备分发扫描任务 - Scan ID: %s, Target: %s", scan_id, data['target_name'])
            try:
+                logger.info("调用 distributor.execute_scan_flow...")
                success, message, container_id, worker_id = distributor.execute_scan_flow(
                    scan_id=scan_id,
                    target_name=data['target_name'],
@@ -284,20 +295,29 @@ class ScanCreationService:
                    scheduled_scan_name=data.get('scheduled_scan_name'),
                )
                
+                logger.info(
+                    "execute_scan_flow 返回 - success: %s, message: %s, container_id: %s, worker_id: %s",
+                    success, message, container_id, worker_id
+                )
+                
                if success:
                    if container_id:
                        scan_repo.append_container_id(scan_id, container_id)
+                        logger.info("已记录 container_id: %s", container_id)
                    if worker_id:
                        scan_repo.update_worker(scan_id, worker_id)
+                        logger.info("已记录 worker_id: %s", worker_id)
                    logger.info(
                        "✓ 扫描任务已提交 - Scan ID: %s, Worker: %s",
                        scan_id, worker_id
                    )
                else:
+                    logger.error("execute_scan_flow 返回失败 - message: %s", message)
                    raise Exception(message)
                    
            except Exception as e:
                logger.error("提交扫描任务失败 - Scan ID: %s, 错误: %s", scan_id, e)
+                logger.exception("详细堆栈:")
                try:
                    scan_repo.update_status(
                        scan_id,
--- a/frontend/components/settings/system-logs/system-logs-view.tsx
+++ b/frontend/components/settings/system-logs/system-logs-view.tsx
@@ -9,7 +9,7 @@ import { useSystemLogs } from "@/hooks/use-system-logs"

 export function SystemLogsView() {
  const { theme } = useTheme()
-  const { data } = useSystemLogs({ lines: 200 })
+  const { data } = useSystemLogs({ lines: 500 })

  const content = useMemo(() => data?.content ?? "", [data?.content])

--- a/frontend/hooks/use-nuclei-repos.ts
+++ b/frontend/hooks/use-nuclei-repos.ts
@@ -62,7 +62,7 @@ export function useUpdateNucleiRepo() {
    mutationFn: (data: {
      id: number
      repoUrl?: string
-    }) => nucleiRepoApi.updateRepo(data.id, data),
+    }) => nucleiRepoApi.updateRepo(data.id, { repoUrl: data.repoUrl }),
    onSuccess: (_data, variables) => {
      toast.success("仓库配置已更新")
      queryClient.invalidateQueries({ queryKey: ["nuclei-repos"] })
--- a/frontend/services/nuclei-repo.api.ts
+++ b/frontend/services/nuclei-repo.api.ts
@@ -75,9 +75,9 @@ export const nucleiRepoApi = {
    return response.data
  },

-  /** 更新仓库 */
+  /** 更新仓库（部分更新） */
  updateRepo: async (repoId: number, payload: UpdateRepoPayload): Promise<NucleiRepoResponse> => {
-    const response = await api.put<NucleiRepoResponse>(`${BASE_URL}${repoId}/`, payload)
+    const response = await api.patch<NucleiRepoResponse>(`${BASE_URL}${repoId}/`, payload)
    return response.data
  },

--- a/install.sh
+++ b/install.sh
@@ -75,7 +75,12 @@ fi

 # 获取真实用户（通过 sudo 运行时 $SUDO_USER 是真实用户）
 REAL_USER="${SUDO_USER:-$USER}"
-REAL_HOME=$(getent passwd "$REAL_USER" | cut -d: -f6)
+# macOS 没有 getent，使用 dscl 或 ~$USER 替代
+if command -v getent &>/dev/null; then
+    REAL_HOME=$(getent passwd "$REAL_USER" | cut -d: -f6)
+else
+    REAL_HOME=$(eval echo "~$REAL_USER")
+fi

 # 项目根目录
 ROOT_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -110,13 +115,22 @@ generate_random_string() {
    fi
 }

+# 跨平台 sed -i（兼容 macOS 和 Linux）
+sed_inplace() {
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        sed -i '' "$@"
+    else
+        sed -i "$@"
+    fi
+}
+
 # 更新 .env 文件中的某个键
 update_env_var() {
    local file="$1"
    local key="$2"
    local value="$3"
    if grep -q "^$key=" "$file"; then
-        sed -i -e "s|^$key=.*|$key=$value|" "$file"
+        sed_inplace "s|^$key=.*|$key=$value|" "$file"
    else
        echo "$key=$value" >> "$file"
    fi
@@ -357,10 +371,10 @@ if [ -f "$DOCKER_DIR/.env.example" ]; then
            -c "CREATE DATABASE $prefect_db;" 2>/dev/null || true
        success "数据库准备完成"
        
-        sed -i "s/^DB_HOST=.*/DB_HOST=$db_host/" "$DOCKER_DIR/.env"
-        sed -i "s/^DB_PORT=.*/DB_PORT=$db_port/" "$DOCKER_DIR/.env"
-        sed -i "s/^DB_USER=.*/DB_USER=$db_user/" "$DOCKER_DIR/.env"
-        sed -i "s/^DB_PASSWORD=.*/DB_PASSWORD=$db_password/" "$DOCKER_DIR/.env"
+        sed_inplace "s/^DB_HOST=.*/DB_HOST=$db_host/" "$DOCKER_DIR/.env"
+        sed_inplace "s/^DB_PORT=.*/DB_PORT=$db_port/" "$DOCKER_DIR/.env"
+        sed_inplace "s/^DB_USER=.*/DB_USER=$db_user/" "$DOCKER_DIR/.env"
+        sed_inplace "s/^DB_PASSWORD=.*/DB_PASSWORD=$db_password/" "$DOCKER_DIR/.env"
        success "已配置远程数据库: $db_user@$db_host:$db_port"
    else
        info "使用本地 PostgreSQL 容器"
--- a/uninstall.sh
+++ b/uninstall.sh
@@ -80,12 +80,12 @@ if [[ $ans_stop =~ ^[Yy]$ ]]; then
    # 先强制停止并删除可能占用网络的容器（xingrin-agent 等）
    docker rm -f xingrin-agent xingrin-watchdog 2>/dev/null || true
    
-    # 停止两种模式的容器
+    # 停止两种模式的容器（不带 -v，volume 在第 5 步单独处理）
    [ -f "docker-compose.yml" ] && ${COMPOSE_CMD} -f docker-compose.yml down 2>/dev/null || true
    [ -f "docker-compose.dev.yml" ] && ${COMPOSE_CMD} -f docker-compose.dev.yml down 2>/dev/null || true
    
    # 手动删除网络（以防 compose 未能删除）
-    docker network rm xingrin_network 2>/dev/null || true
+    docker network rm xingrin_network docker_default 2>/dev/null || true
    
    success "容器和网络已停止/删除（如存在）。"
 else
@@ -156,19 +156,28 @@ ans_db=${ans_db:-Y}

 if [[ $ans_db =~ ^[Yy]$ ]]; then
    info "尝试删除与 XingRin 相关的 Postgres 容器和数据卷..."
-    # docker-compose 项目名为 docker，常见资源名如下（忽略不存在的情况）：
-    # - 容器: docker-postgres-1
-    # - 数据卷: docker_postgres_data（对应 compose 中的 postgres_data 卷）
-    docker rm -f docker-postgres-1 2>/dev/null || true
-    docker volume rm docker_postgres_data 2>/dev/null || true
-    success "本地 Postgres 容器及数据卷已尝试删除（不存在会自动忽略）。"
+    # 删除可能的容器名（不同 compose 版本命名不同）
+    docker rm -f docker-postgres-1 xingrin-postgres postgres 2>/dev/null || true
+    
+    # 删除可能的 volume 名（取决于项目名和 compose 配置）
+    # 先列出要删除的 volume
+    for vol in postgres_data docker_postgres_data xingrin_postgres_data; do
+        if docker volume inspect "$vol" >/dev/null 2>&1; then
+            if docker volume rm "$vol" 2>/dev/null; then
+                success "已删除 volume: $vol"
+            else
+                warn "无法删除 volume: $vol（可能正在被使用，请先停止所有容器）"
+            fi
+        fi
+    done
+    success "本地 Postgres 数据卷清理完成。"
 else
    warn "已保留本地 Postgres 容器和 volume。"
 fi

-step "[6/6] 是否删除与 XingRin 相关的 Docker 镜像？(y/N)"
+step "[6/6] 是否删除与 XingRin 相关的 Docker 镜像？(Y/n)"
 read -r ans_images
-ans_images=${ans_images:-N}
+ans_images=${ans_images:-Y}

 if [[ $ans_images =~ ^[Yy]$ ]]; then
    info "正在删除 Docker 镜像..."
@@ -199,9 +208,29 @@ if [[ $ans_images =~ ^[Yy]$ ]]; then
    fi
    
    docker rmi redis:7-alpine 2>/dev/null || true
+    
+    # 删除本地构建的开发镜像
+    docker rmi docker-server docker-frontend docker-nginx docker-agent docker-worker 2>/dev/null || true
+    docker rmi "docker-worker:${IMAGE_TAG}-dev" 2>/dev/null || true
+    
    success "Docker 镜像已删除（如存在）。"
 else
    warn "已保留 Docker 镜像。"
 fi

+# 清理构建缓存（可选，会导致下次构建变慢）
+echo ""
+echo -n -e "${BOLD}${CYAN}[?] 是否清理 Docker 构建缓存？(y/N) ${RESET}"
+echo -e "${YELLOW}（清理后下次构建会很慢，一般不需要）${RESET}"
+read -r ans_cache
+ans_cache=${ans_cache:-N}
+
+if [[ $ans_cache =~ ^[Yy]$ ]]; then
+    info "清理 Docker 构建缓存..."
+    docker builder prune -af 2>/dev/null || true
+    success "构建缓存已清理。"
+else
+    warn "已保留构建缓存（推荐）。"
+fi
+
 success "卸载流程已完成。"
--- a/update.sh
+++ b/update.sh
@@ -18,6 +18,15 @@

 cd "$(dirname "$0")"

+# 跨平台 sed -i（兼容 macOS 和 Linux）
+sed_inplace() {
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        sed -i '' "$@"
+    else
+        sed -i "$@"
+    fi
+}
+
 # 解析参数判断模式
 DEV_MODE=false
 for arg in "$@"; do
@@ -92,7 +101,7 @@ if [ -f "VERSION" ]; then
    if [ -n "$NEW_VERSION" ]; then
        # 更新 .env 中的 IMAGE_TAG（所有节点将使用此版本的镜像）
        if grep -q "^IMAGE_TAG=" "docker/.env"; then
-            sed -i "s/^IMAGE_TAG=.*/IMAGE_TAG=$NEW_VERSION/" "docker/.env"
+            sed_inplace "s/^IMAGE_TAG=.*/IMAGE_TAG=$NEW_VERSION/" "docker/.env"
            echo -e "    ${GREEN}+${NC} 版本同步: IMAGE_TAG=$NEW_VERSION"
        else
            echo "IMAGE_TAG=$NEW_VERSION" >> "docker/.env"