diff --git a/backend/apps/common/container_bootstrap.py b/backend/apps/common/container_bootstrap.py index 27adab29..a66f8704 100644 --- a/backend/apps/common/container_bootstrap.py +++ b/backend/apps/common/container_bootstrap.py @@ -31,16 +31,22 @@ def fetch_config_and_setup_django(): sys.exit(1) config_url = f"{server_url}/api/workers/config/" + print(f"[CONFIG] 正在从配置中心获取配置: {config_url}") try: resp = requests.get(config_url, timeout=10) resp.raise_for_status() config = resp.json() # 数据库配置(必需) - os.environ.setdefault("DB_HOST", config['db']['host']) - os.environ.setdefault("DB_PORT", config['db']['port']) - os.environ.setdefault("DB_NAME", config['db']['name']) - os.environ.setdefault("DB_USER", config['db']['user']) + db_host = config['db']['host'] + db_port = config['db']['port'] + db_name = config['db']['name'] + db_user = config['db']['user'] + + os.environ.setdefault("DB_HOST", db_host) + os.environ.setdefault("DB_PORT", db_port) + os.environ.setdefault("DB_NAME", db_name) + os.environ.setdefault("DB_USER", db_user) os.environ.setdefault("DB_PASSWORD", config['db']['password']) # Redis 配置 @@ -52,7 +58,12 @@ def fetch_config_and_setup_django(): os.environ.setdefault("ENABLE_COMMAND_LOGGING", str(config['logging']['enableCommandLogging']).lower()) os.environ.setdefault("DEBUG", str(config['debug'])) - print(f"[CONFIG] 从配置中心获取配置成功: {config_url}") + print(f"[CONFIG] ✓ 配置获取成功") + print(f"[CONFIG] DB_HOST: {db_host}") + print(f"[CONFIG] DB_PORT: {db_port}") + print(f"[CONFIG] DB_NAME: {db_name}") + print(f"[CONFIG] DB_USER: {db_user}") + print(f"[CONFIG] REDIS_URL: {config['redisUrl']}") except Exception as e: print(f"[ERROR] 获取配置失败: {config_url} - {e}", file=sys.stderr) diff --git a/backend/apps/engine/services/nuclei_template_repo_service.py b/backend/apps/engine/services/nuclei_template_repo_service.py index 6f837a78..7913c6ee 100644 --- a/backend/apps/engine/services/nuclei_template_repo_service.py +++ b/backend/apps/engine/services/nuclei_template_repo_service.py @@ -198,9 +198,27 @@ class NucleiTemplateRepoService: # 判断是 clone 还是 pull if git_dir.is_dir(): - # 已有仓库,执行 pull - cmd = ["git", "-C", str(local_path), "pull", "--ff-only"] - action = "pull" + # 检查远程地址是否变化 + current_remote = subprocess.run( + ["git", "-C", str(local_path), "remote", "get-url", "origin"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + current_url = current_remote.stdout.strip() if current_remote.returncode == 0 else "" + + if current_url != obj.repo_url: + # 远程地址变化,删除旧目录重新 clone + logger.info("nuclei 模板仓库 %s 远程地址变化,重新 clone: %s -> %s", obj.id, current_url, obj.repo_url) + shutil.rmtree(local_path) + local_path.mkdir(parents=True, exist_ok=True) + cmd = ["git", "clone", "--depth", "1", obj.repo_url, str(local_path)] + action = "clone" + else: + # 已有仓库且地址未变,执行 pull + cmd = ["git", "-C", str(local_path), "pull", "--ff-only"] + action = "pull" else: # 新仓库,执行 clone if local_path.exists() and not local_path.is_dir(): diff --git a/backend/apps/engine/services/task_distributor.py b/backend/apps/engine/services/task_distributor.py index 94d8f157..628c63b7 100644 --- a/backend/apps/engine/services/task_distributor.py +++ b/backend/apps/engine/services/task_distributor.py @@ -407,8 +407,20 @@ class TaskDistributor: Note: engine_config 由 Flow 内部通过 scan_id 查询数据库获取 """ + logger.info("="*60) + logger.info("execute_scan_flow 开始") + logger.info(" scan_id: %s", scan_id) + logger.info(" target_name: %s", target_name) + logger.info(" target_id: %s", target_id) + logger.info(" scan_workspace_dir: %s", scan_workspace_dir) + logger.info(" engine_name: %s", engine_name) + logger.info(" docker_image: %s", self.docker_image) + logger.info("="*60) + # 1. 等待提交间隔(后台线程执行,不阻塞 API) + logger.info("等待提交间隔...") self._wait_for_submit_interval() + logger.info("提交间隔等待完成") # 2. 选择最佳 Worker worker = self.select_best_worker() diff --git a/backend/apps/scan/scripts/run_initiate_scan.py b/backend/apps/scan/scripts/run_initiate_scan.py index 67a52aac..31d072be 100644 --- a/backend/apps/scan/scripts/run_initiate_scan.py +++ b/backend/apps/scan/scripts/run_initiate_scan.py @@ -6,14 +6,32 @@ 必须在 Django 导入之前获取配置并设置环境变量。 """ import argparse -from apps.common.container_bootstrap import fetch_config_and_setup_django +import sys +import os +import traceback def main(): + print("="*60) + print("run_initiate_scan.py 启动") + print(f" Python: {sys.version}") + print(f" CWD: {os.getcwd()}") + print(f" SERVER_URL: {os.environ.get('SERVER_URL', 'NOT SET')}") + print("="*60) + # 1. 从配置中心获取配置并初始化 Django(必须在 Django 导入之前) - fetch_config_and_setup_django() + print("[1/4] 从配置中心获取配置...") + try: + from apps.common.container_bootstrap import fetch_config_and_setup_django + fetch_config_and_setup_django() + print("[1/4] ✓ 配置获取成功") + except Exception as e: + print(f"[1/4] ✗ 配置获取失败: {e}") + traceback.print_exc() + sys.exit(1) # 2. 解析命令行参数 + print("[2/4] 解析命令行参数...") parser = argparse.ArgumentParser(description="执行扫描初始化 Flow") parser.add_argument("--scan_id", type=int, required=True, help="扫描任务 ID") parser.add_argument("--target_name", type=str, required=True, help="目标名称") @@ -23,21 +41,41 @@ def main(): parser.add_argument("--scheduled_scan_name", type=str, default=None, help="定时扫描任务名称(可选)") args = parser.parse_args() + print(f"[2/4] ✓ 参数解析成功:") + print(f" scan_id: {args.scan_id}") + print(f" target_name: {args.target_name}") + print(f" target_id: {args.target_id}") + print(f" scan_workspace_dir: {args.scan_workspace_dir}") + print(f" engine_name: {args.engine_name}") + print(f" scheduled_scan_name: {args.scheduled_scan_name}") # 3. 现在可以安全导入 Django 相关模块 - from apps.scan.flows.initiate_scan_flow import initiate_scan_flow + print("[3/4] 导入 initiate_scan_flow...") + try: + from apps.scan.flows.initiate_scan_flow import initiate_scan_flow + print("[3/4] ✓ 导入成功") + except Exception as e: + print(f"[3/4] ✗ 导入失败: {e}") + traceback.print_exc() + sys.exit(1) # 4. 执行 Flow - result = initiate_scan_flow( - scan_id=args.scan_id, - target_name=args.target_name, - target_id=args.target_id, - scan_workspace_dir=args.scan_workspace_dir, - engine_name=args.engine_name, - scheduled_scan_name=args.scheduled_scan_name, - ) - - print(f"Flow 执行完成: {result}") + print("[4/4] 执行 initiate_scan_flow...") + try: + result = initiate_scan_flow( + scan_id=args.scan_id, + target_name=args.target_name, + target_id=args.target_id, + scan_workspace_dir=args.scan_workspace_dir, + engine_name=args.engine_name, + scheduled_scan_name=args.scheduled_scan_name, + ) + print("[4/4] ✓ Flow 执行完成") + print(f"结果: {result}") + except Exception as e: + print(f"[4/4] ✗ Flow 执行失败: {e}") + traceback.print_exc() + sys.exit(1) if __name__ == "__main__": diff --git a/backend/apps/scan/services/scan_creation_service.py b/backend/apps/scan/services/scan_creation_service.py index 55ce395f..7535f8be 100644 --- a/backend/apps/scan/services/scan_creation_service.py +++ b/backend/apps/scan/services/scan_creation_service.py @@ -266,15 +266,26 @@ class ScanCreationService: Args: scan_data: 扫描任务数据列表 """ + logger.info("="*60) + logger.info("开始分发扫描任务到 Workers - 数量: %d", len(scan_data)) + logger.info("="*60) + # 后台线程需要新的数据库连接 connection.close() + logger.info("已关闭旧数据库连接,准备获取新连接") distributor = get_task_distributor() + logger.info("TaskDistributor 初始化完成") + scan_repo = DjangoScanRepository() + logger.info("ScanRepository 初始化完成") for data in scan_data: scan_id = data['scan_id'] + logger.info("-"*40) + logger.info("准备分发扫描任务 - Scan ID: %s, Target: %s", scan_id, data['target_name']) try: + logger.info("调用 distributor.execute_scan_flow...") success, message, container_id, worker_id = distributor.execute_scan_flow( scan_id=scan_id, target_name=data['target_name'], @@ -284,20 +295,29 @@ class ScanCreationService: scheduled_scan_name=data.get('scheduled_scan_name'), ) + logger.info( + "execute_scan_flow 返回 - success: %s, message: %s, container_id: %s, worker_id: %s", + success, message, container_id, worker_id + ) + if success: if container_id: scan_repo.append_container_id(scan_id, container_id) + logger.info("已记录 container_id: %s", container_id) if worker_id: scan_repo.update_worker(scan_id, worker_id) + logger.info("已记录 worker_id: %s", worker_id) logger.info( "✓ 扫描任务已提交 - Scan ID: %s, Worker: %s", scan_id, worker_id ) else: + logger.error("execute_scan_flow 返回失败 - message: %s", message) raise Exception(message) except Exception as e: logger.error("提交扫描任务失败 - Scan ID: %s, 错误: %s", scan_id, e) + logger.exception("详细堆栈:") try: scan_repo.update_status( scan_id, diff --git a/frontend/components/settings/system-logs/system-logs-view.tsx b/frontend/components/settings/system-logs/system-logs-view.tsx index 9632fbf0..77695941 100644 --- a/frontend/components/settings/system-logs/system-logs-view.tsx +++ b/frontend/components/settings/system-logs/system-logs-view.tsx @@ -9,7 +9,7 @@ import { useSystemLogs } from "@/hooks/use-system-logs" export function SystemLogsView() { const { theme } = useTheme() - const { data } = useSystemLogs({ lines: 200 }) + const { data } = useSystemLogs({ lines: 500 }) const content = useMemo(() => data?.content ?? "", [data?.content]) diff --git a/frontend/hooks/use-nuclei-repos.ts b/frontend/hooks/use-nuclei-repos.ts index 37e2eca1..97663472 100644 --- a/frontend/hooks/use-nuclei-repos.ts +++ b/frontend/hooks/use-nuclei-repos.ts @@ -62,7 +62,7 @@ export function useUpdateNucleiRepo() { mutationFn: (data: { id: number repoUrl?: string - }) => nucleiRepoApi.updateRepo(data.id, data), + }) => nucleiRepoApi.updateRepo(data.id, { repoUrl: data.repoUrl }), onSuccess: (_data, variables) => { toast.success("仓库配置已更新") queryClient.invalidateQueries({ queryKey: ["nuclei-repos"] }) diff --git a/frontend/services/nuclei-repo.api.ts b/frontend/services/nuclei-repo.api.ts index 076eb903..a77eab17 100644 --- a/frontend/services/nuclei-repo.api.ts +++ b/frontend/services/nuclei-repo.api.ts @@ -75,9 +75,9 @@ export const nucleiRepoApi = { return response.data }, - /** 更新仓库 */ + /** 更新仓库(部分更新) */ updateRepo: async (repoId: number, payload: UpdateRepoPayload): Promise => { - const response = await api.put(`${BASE_URL}${repoId}/`, payload) + const response = await api.patch(`${BASE_URL}${repoId}/`, payload) return response.data }, diff --git a/install.sh b/install.sh index c5d6adaf..d04f3078 100755 --- a/install.sh +++ b/install.sh @@ -75,7 +75,12 @@ fi # 获取真实用户(通过 sudo 运行时 $SUDO_USER 是真实用户) REAL_USER="${SUDO_USER:-$USER}" -REAL_HOME=$(getent passwd "$REAL_USER" | cut -d: -f6) +# macOS 没有 getent,使用 dscl 或 ~$USER 替代 +if command -v getent &>/dev/null; then + REAL_HOME=$(getent passwd "$REAL_USER" | cut -d: -f6) +else + REAL_HOME=$(eval echo "~$REAL_USER") +fi # 项目根目录 ROOT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -110,13 +115,22 @@ generate_random_string() { fi } +# 跨平台 sed -i(兼容 macOS 和 Linux) +sed_inplace() { + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} + # 更新 .env 文件中的某个键 update_env_var() { local file="$1" local key="$2" local value="$3" if grep -q "^$key=" "$file"; then - sed -i -e "s|^$key=.*|$key=$value|" "$file" + sed_inplace "s|^$key=.*|$key=$value|" "$file" else echo "$key=$value" >> "$file" fi @@ -357,10 +371,10 @@ if [ -f "$DOCKER_DIR/.env.example" ]; then -c "CREATE DATABASE $prefect_db;" 2>/dev/null || true success "数据库准备完成" - sed -i "s/^DB_HOST=.*/DB_HOST=$db_host/" "$DOCKER_DIR/.env" - sed -i "s/^DB_PORT=.*/DB_PORT=$db_port/" "$DOCKER_DIR/.env" - sed -i "s/^DB_USER=.*/DB_USER=$db_user/" "$DOCKER_DIR/.env" - sed -i "s/^DB_PASSWORD=.*/DB_PASSWORD=$db_password/" "$DOCKER_DIR/.env" + sed_inplace "s/^DB_HOST=.*/DB_HOST=$db_host/" "$DOCKER_DIR/.env" + sed_inplace "s/^DB_PORT=.*/DB_PORT=$db_port/" "$DOCKER_DIR/.env" + sed_inplace "s/^DB_USER=.*/DB_USER=$db_user/" "$DOCKER_DIR/.env" + sed_inplace "s/^DB_PASSWORD=.*/DB_PASSWORD=$db_password/" "$DOCKER_DIR/.env" success "已配置远程数据库: $db_user@$db_host:$db_port" else info "使用本地 PostgreSQL 容器" diff --git a/uninstall.sh b/uninstall.sh index 01aff6bb..a026eca2 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -80,12 +80,12 @@ if [[ $ans_stop =~ ^[Yy]$ ]]; then # 先强制停止并删除可能占用网络的容器(xingrin-agent 等) docker rm -f xingrin-agent xingrin-watchdog 2>/dev/null || true - # 停止两种模式的容器 + # 停止两种模式的容器(不带 -v,volume 在第 5 步单独处理) [ -f "docker-compose.yml" ] && ${COMPOSE_CMD} -f docker-compose.yml down 2>/dev/null || true [ -f "docker-compose.dev.yml" ] && ${COMPOSE_CMD} -f docker-compose.dev.yml down 2>/dev/null || true # 手动删除网络(以防 compose 未能删除) - docker network rm xingrin_network 2>/dev/null || true + docker network rm xingrin_network docker_default 2>/dev/null || true success "容器和网络已停止/删除(如存在)。" else @@ -156,19 +156,28 @@ ans_db=${ans_db:-Y} if [[ $ans_db =~ ^[Yy]$ ]]; then info "尝试删除与 XingRin 相关的 Postgres 容器和数据卷..." - # docker-compose 项目名为 docker,常见资源名如下(忽略不存在的情况): - # - 容器: docker-postgres-1 - # - 数据卷: docker_postgres_data(对应 compose 中的 postgres_data 卷) - docker rm -f docker-postgres-1 2>/dev/null || true - docker volume rm docker_postgres_data 2>/dev/null || true - success "本地 Postgres 容器及数据卷已尝试删除(不存在会自动忽略)。" + # 删除可能的容器名(不同 compose 版本命名不同) + docker rm -f docker-postgres-1 xingrin-postgres postgres 2>/dev/null || true + + # 删除可能的 volume 名(取决于项目名和 compose 配置) + # 先列出要删除的 volume + for vol in postgres_data docker_postgres_data xingrin_postgres_data; do + if docker volume inspect "$vol" >/dev/null 2>&1; then + if docker volume rm "$vol" 2>/dev/null; then + success "已删除 volume: $vol" + else + warn "无法删除 volume: $vol(可能正在被使用,请先停止所有容器)" + fi + fi + done + success "本地 Postgres 数据卷清理完成。" else warn "已保留本地 Postgres 容器和 volume。" fi -step "[6/6] 是否删除与 XingRin 相关的 Docker 镜像?(y/N)" +step "[6/6] 是否删除与 XingRin 相关的 Docker 镜像?(Y/n)" read -r ans_images -ans_images=${ans_images:-N} +ans_images=${ans_images:-Y} if [[ $ans_images =~ ^[Yy]$ ]]; then info "正在删除 Docker 镜像..." @@ -199,9 +208,29 @@ if [[ $ans_images =~ ^[Yy]$ ]]; then fi docker rmi redis:7-alpine 2>/dev/null || true + + # 删除本地构建的开发镜像 + docker rmi docker-server docker-frontend docker-nginx docker-agent docker-worker 2>/dev/null || true + docker rmi "docker-worker:${IMAGE_TAG}-dev" 2>/dev/null || true + success "Docker 镜像已删除(如存在)。" else warn "已保留 Docker 镜像。" fi +# 清理构建缓存(可选,会导致下次构建变慢) +echo "" +echo -n -e "${BOLD}${CYAN}[?] 是否清理 Docker 构建缓存?(y/N) ${RESET}" +echo -e "${YELLOW}(清理后下次构建会很慢,一般不需要)${RESET}" +read -r ans_cache +ans_cache=${ans_cache:-N} + +if [[ $ans_cache =~ ^[Yy]$ ]]; then + info "清理 Docker 构建缓存..." + docker builder prune -af 2>/dev/null || true + success "构建缓存已清理。" +else + warn "已保留构建缓存(推荐)。" +fi + success "卸载流程已完成。" diff --git a/update.sh b/update.sh index 9bb578ec..1b99f993 100755 --- a/update.sh +++ b/update.sh @@ -18,6 +18,15 @@ cd "$(dirname "$0")" +# 跨平台 sed -i(兼容 macOS 和 Linux) +sed_inplace() { + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} + # 解析参数判断模式 DEV_MODE=false for arg in "$@"; do @@ -92,7 +101,7 @@ if [ -f "VERSION" ]; then if [ -n "$NEW_VERSION" ]; then # 更新 .env 中的 IMAGE_TAG(所有节点将使用此版本的镜像) if grep -q "^IMAGE_TAG=" "docker/.env"; then - sed -i "s/^IMAGE_TAG=.*/IMAGE_TAG=$NEW_VERSION/" "docker/.env" + sed_inplace "s/^IMAGE_TAG=.*/IMAGE_TAG=$NEW_VERSION/" "docker/.env" echo -e " ${GREEN}+${NC} 版本同步: IMAGE_TAG=$NEW_VERSION" else echo "IMAGE_TAG=$NEW_VERSION" >> "docker/.env"