diff --git a/backend/apps/common/services/system_log_service.py b/backend/apps/common/services/system_log_service.py index 1cc10f91..a5effe2a 100644 --- a/backend/apps/common/services/system_log_service.py +++ b/backend/apps/common/services/system_log_service.py @@ -21,8 +21,8 @@ class SystemLogService: """ def __init__(self): - # 日志文件路径（容器内路径，通过 volume 挂载到宿主机 /opt/xingrin/logs） - self.log_file = "/app/backend/logs/xingrin.log" + # 日志文件路径（统一使用 /opt/xingrin/logs） + self.log_file = "/opt/xingrin/logs/xingrin.log" self.default_lines = 200 # 默认返回行数 self.max_lines = 10000 # 最大返回行数限制 self.timeout_seconds = 3 # tail 命令超时时间 diff --git a/backend/apps/engine/management/commands/init_nuclei_templates.py b/backend/apps/engine/management/commands/init_nuclei_templates.py index 53b8ecad..3215f542 100644 --- a/backend/apps/engine/management/commands/init_nuclei_templates.py +++ b/backend/apps/engine/management/commands/init_nuclei_templates.py @@ -3,12 +3,17 @@ 项目安装后执行此命令，自动创建官方模板仓库记录。 使用方式： - python manage.py init_nuclei_templates # 只创建记录 + python manage.py init_nuclei_templates # 只创建记录（检测本地已有仓库） python manage.py init_nuclei_templates --sync # 创建并同步（git clone） """ import logging +import subprocess +from pathlib import Path + +from django.conf import settings from django.core.management.base import BaseCommand +from django.utils import timezone from apps.engine.models import NucleiTemplateRepo from apps.engine.services import NucleiTemplateRepoService @@ -26,6 +31,20 @@ DEFAULT_REPOS = [ ] +def get_local_commit_hash(local_path: Path) -> str: + """获取本地 Git 仓库的 commit hash""" + if not (local_path / ".git").is_dir(): + return "" + result = subprocess.run( + ["git", "-C", str(local_path), "rev-parse", "HEAD"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + return result.stdout.strip() if result.returncode == 0 else "" + + class Command(BaseCommand): help = "初始化 Nuclei 模板仓库（创建官方模板仓库记录）" @@ -46,6 +65,8 @@ class Command(BaseCommand): force = options.get("force", False) service = NucleiTemplateRepoService() + base_dir = Path(getattr(settings, "NUCLEI_TEMPLATES_REPOS_BASE_DIR", "/opt/xingrin/nuclei-repos")) + created = 0 skipped = 0 synced = 0 @@ -87,20 +108,30 @@ class Command(BaseCommand): # 创建新仓库记录 try: + # 检查本地是否已有仓库（由 install.sh 预下载） + local_path = base_dir / name + local_commit = get_local_commit_hash(local_path) + repo = NucleiTemplateRepo.objects.create( name=name, repo_url=repo_url, + local_path=str(local_path) if local_commit else "", + commit_hash=local_commit, + last_synced_at=timezone.now() if local_commit else None, ) - self.stdout.write(self.style.SUCCESS( - f"[{name}] 创建成功: id={repo.id}" - )) + + if local_commit: + self.stdout.write(self.style.SUCCESS( + f"[{name}] 创建成功（检测到本地仓库）: commit={local_commit[:8]}" + )) + else: + self.stdout.write(self.style.SUCCESS( + f"[{name}] 创建成功: id={repo.id}" + )) created += 1 - # 初始化本地路径 - service.ensure_local_path(repo) - - # 如果需要同步 - if do_sync: + # 如果本地没有仓库且需要同步 + if not local_commit and do_sync: try: self.stdout.write(self.style.WARNING( f"[{name}] 正在同步（首次可能需要几分钟）..." diff --git a/backend/apps/engine/management/commands/init_wordlists.py b/backend/apps/engine/management/commands/init_wordlists.py index ba4cb82f..11dedf6d 100644 --- a/backend/apps/engine/management/commands/init_wordlists.py +++ b/backend/apps/engine/management/commands/init_wordlists.py @@ -1,7 +1,8 @@ """初始化所有内置字典 Wordlist 记录 -- 目录扫描默认字典: dir_default.txt -> /app/backend/wordlist/dir_default.txt -- 子域名爆破默认字典: subdomains-top1million-110000.txt -> /app/backend/wordlist/subdomains-top1million-110000.txt +内置字典从镜像内 /app/backend/wordlist/ 复制到运行时目录 /opt/xingrin/wordlists/： +- 目录扫描默认字典: dir_default.txt +- 子域名爆破默认字典: subdomains-top1million-110000.txt 可重复执行：如果已存在同名记录且文件有效则跳过，只在缺失或文件丢失时创建/修复。 """ diff --git a/backend/apps/engine/services/task_distributor.py b/backend/apps/engine/services/task_distributor.py index ccdc2ae3..fc4071fd 100644 --- a/backend/apps/engine/services/task_distributor.py +++ b/backend/apps/engine/services/task_distributor.py @@ -76,8 +76,8 @@ class TaskDistributor: self.docker_image = settings.TASK_EXECUTOR_IMAGE if not self.docker_image: raise ValueError("TASK_EXECUTOR_IMAGE 未配置，请确保 IMAGE_TAG 环境变量已设置") - self.results_mount = getattr(settings, 'CONTAINER_RESULTS_MOUNT', '/app/backend/results') - self.logs_mount = getattr(settings, 'CONTAINER_LOGS_MOUNT', '/app/backend/logs') + # 统一使用 /opt/xingrin 下的路径 + self.logs_mount = "/opt/xingrin/logs" self.submit_interval = getattr(settings, 'TASK_SUBMIT_INTERVAL', 5) def get_online_workers(self) -> list[WorkerNode]: @@ -274,11 +274,8 @@ class TaskDistributor: network_arg = "" server_url = f"https://{settings.PUBLIC_HOST}:{settings.PUBLIC_PORT}" - # 挂载路径（所有节点统一使用固定路径） - host_results_dir = settings.HOST_RESULTS_DIR # /opt/xingrin/results - host_logs_dir = settings.HOST_LOGS_DIR # /opt/xingrin/logs - host_fingerprints_dir = settings.HOST_FINGERPRINTS_DIR # /opt/xingrin/fingerprints - host_wordlists_dir = settings.HOST_WORDLISTS_DIR # /opt/xingrin/wordlists + # 挂载路径（统一挂载 /opt/xingrin） + host_xingrin_dir = "/opt/xingrin" # 环境变量：SERVER_URL + IS_LOCAL，其他配置容器启动时从配置中心获取 # IS_LOCAL 用于 Worker 向配置中心声明身份，决定返回的数据库地址 @@ -294,12 +291,9 @@ class TaskDistributor: "-e PREFECT_LOGGING_LEVEL=WARNING", # 日志级别（减少 DEBUG 噪音） ] - # 挂载卷 + # 挂载卷（统一挂载整个 /opt/xingrin 目录） volumes = [ - f"-v {host_results_dir}:{self.results_mount}", - f"-v {host_logs_dir}:{self.logs_mount}", - f"-v {host_fingerprints_dir}:{host_fingerprints_dir}", - f"-v {host_wordlists_dir}:{host_wordlists_dir}", + f"-v {host_xingrin_dir}:{host_xingrin_dir}", ] # 构建命令行参数 @@ -560,7 +554,7 @@ class TaskDistributor: try: # 构建 docker run 命令（清理过期扫描结果目录） script_args = { - 'results_dir': '/app/backend/results', + 'results_dir': '/opt/xingrin/results', 'retention_days': retention_days, } diff --git a/backend/apps/engine/views/worker_views.py b/backend/apps/engine/views/worker_views.py index 06914023..0af47e20 100644 --- a/backend/apps/engine/views/worker_views.py +++ b/backend/apps/engine/views/worker_views.py @@ -390,8 +390,8 @@ class WorkerNodeViewSet(viewsets.ModelViewSet): }, 'redisUrl': worker_redis_url, 'paths': { - 'results': getattr(settings, 'CONTAINER_RESULTS_MOUNT', '/app/backend/results'), - 'logs': getattr(settings, 'CONTAINER_LOGS_MOUNT', '/app/backend/logs'), + 'results': getattr(settings, 'CONTAINER_RESULTS_MOUNT', '/opt/xingrin/results'), + 'logs': getattr(settings, 'CONTAINER_LOGS_MOUNT', '/opt/xingrin/logs'), }, 'logging': { 'level': os.getenv('LOG_LEVEL', 'INFO'), diff --git a/backend/apps/scan/scripts/run_cleanup.py b/backend/apps/scan/scripts/run_cleanup.py index 89715baf..e95caaad 100644 --- a/backend/apps/scan/scripts/run_cleanup.py +++ b/backend/apps/scan/scripts/run_cleanup.py @@ -83,7 +83,7 @@ def cleanup_results(results_dir: str, retention_days: int) -> dict: def main(): parser = argparse.ArgumentParser(description="清理任务") - parser.add_argument("--results_dir", type=str, default="/app/backend/results", help="扫描结果目录") + parser.add_argument("--results_dir", type=str, default="/opt/xingrin/results", help="扫描结果目录") parser.add_argument("--retention_days", type=int, default=7, help="保留天数") args = parser.parse_args() diff --git a/backend/config/settings.py b/backend/config/settings.py index 793cb34d..59f84644 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -275,13 +275,22 @@ LOGGING = get_logging_config(debug=DEBUG) # 命令执行日志开关（供 apps.scan.utils.command_executor 使用） ENABLE_COMMAND_LOGGING = get_bool_env('ENABLE_COMMAND_LOGGING', True) -# 扫描工具基础路径（后端和 Worker 统一使用该路径前缀存放三方工具等文件） +# ==================== 数据目录配置（统一使用 /opt/xingrin） ==================== +# 所有数据目录统一挂载到 /opt/xingrin，便于管理和备份 + +# 扫描工具基础路径 SCAN_TOOLS_BASE_PATH = os.getenv('SCAN_TOOLS_PATH', '/opt/xingrin/tools') -# 字典文件基础路径（后端和 Worker 统一使用该路径前缀存放字典文件） +# 字典文件基础路径 WORDLISTS_BASE_PATH = os.getenv('WORDLISTS_PATH', '/opt/xingrin/wordlists') -# Nuclei 模板基础路径（custom / public 两类模板目录） +# 指纹库基础路径 +FINGERPRINTS_BASE_PATH = os.getenv('FINGERPRINTS_PATH', '/opt/xingrin/fingerprints') + +# Nuclei 模板仓库根目录（存放 git clone 的仓库） +NUCLEI_TEMPLATES_REPOS_BASE_DIR = os.getenv('NUCLEI_TEMPLATES_REPOS_DIR', '/opt/xingrin/nuclei-repos') + +# Nuclei 模板基础路径（custom / public 两类模板目录，已废弃，保留兼容） NUCLEI_CUSTOM_TEMPLATES_DIR = os.getenv('NUCLEI_CUSTOM_TEMPLATES_DIR', '/opt/xingrin/nuclei-templates/custom') NUCLEI_PUBLIC_TEMPLATES_DIR = os.getenv('NUCLEI_PUBLIC_TEMPLATES_DIR', '/opt/xingrin/nuclei-templates/public') @@ -336,7 +345,7 @@ TASK_SUBMIT_INTERVAL = int(os.getenv('TASK_SUBMIT_INTERVAL', '6')) DOCKER_NETWORK_NAME = os.getenv('DOCKER_NETWORK_NAME', 'xingrin_network') # 宿主机挂载源路径（所有节点统一使用固定路径） -# 部署前需创建：mkdir -p /opt/xingrin/{results,logs,fingerprints,wordlists} +# 部署前需创建：mkdir -p /opt/xingrin HOST_RESULTS_DIR = '/opt/xingrin/results' HOST_LOGS_DIR = '/opt/xingrin/logs' HOST_FINGERPRINTS_DIR = '/opt/xingrin/fingerprints' @@ -364,6 +373,6 @@ WORKER_REDIS_URL = os.getenv( 'redis://redis:6379/0' if _is_internal_public else f'redis://{PUBLIC_HOST}:6379/0' ) -# 容器内挂载目标路径（固定值，不需要修改） -CONTAINER_RESULTS_MOUNT = '/app/backend/results' -CONTAINER_LOGS_MOUNT = '/app/backend/logs' +# 容器内挂载目标路径（统一使用 /opt/xingrin） +CONTAINER_RESULTS_MOUNT = '/opt/xingrin/results' +CONTAINER_LOGS_MOUNT = '/opt/xingrin/logs' diff --git a/docker/.env.example b/docker/.env.example index 165156cc..da9a2011 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -37,10 +37,10 @@ CORS_ALLOWED_ORIGINS=http://localhost:3000 # ==================== 路径配置（容器内路径） ==================== # 扫描结果保存目录 -SCAN_RESULTS_DIR=/app/backend/results +SCAN_RESULTS_DIR=/opt/xingrin/results # Django 日志目录 # 注意：如果留空或删除此变量，日志将只输出到 Docker 控制台（标准输出），不写入文件 -LOG_DIR=/app/backend/logs +LOG_DIR=/opt/xingrin/logs # ==================== 日志级别配置 ==================== # 应用日志级别：DEBUG / INFO / WARNING / ERROR diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 184f5b70..951e1dde 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -45,9 +45,8 @@ services: redis: condition: service_healthy volumes: - # 统一使用固定路径（开发环境：ln -s ~/project/backend/results /opt/xingrin/results） - - /opt/xingrin/results:/app/backend/results - - /opt/xingrin/logs:/app/backend/logs + # 统一挂载数据目录 + - /opt/xingrin:/opt/xingrin - /var/run/docker.sock:/var/run/docker.sock healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8888/api/"] @@ -110,10 +109,7 @@ services: image: docker-worker:${IMAGE_TAG:-latest}-dev restart: "no" volumes: - - /opt/xingrin/results:/app/backend/results - - /opt/xingrin/logs:/app/backend/logs - - /opt/xingrin/fingerprints:/opt/xingrin/fingerprints - - /opt/xingrin/wordlists:/opt/xingrin/wordlists + - /opt/xingrin:/opt/xingrin command: echo "Worker image built for development" volumes: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 06142aa3..ab0653b4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -47,11 +47,8 @@ services: redis: condition: service_healthy volumes: - # 统一使用固定路径（部署时需创建：mkdir -p /opt/xingrin/{results,logs,fingerprints,wordlists}） - - /opt/xingrin/results:/app/backend/results - - /opt/xingrin/logs:/app/backend/logs - - /opt/xingrin/fingerprints:/opt/xingrin/fingerprints - - /opt/xingrin/wordlists:/opt/xingrin/wordlists + # 统一挂载数据目录 + - /opt/xingrin:/opt/xingrin # Docker Socket 挂载：允许 Django 服务器执行本地 docker 命令（用于本地 Worker 任务分发） - /var/run/docker.sock:/var/run/docker.sock healthcheck: diff --git a/docker/scripts/init-data.sh b/docker/scripts/init-data.sh index bd86139a..2f5e725e 100755 --- a/docker/scripts/init-data.sh +++ b/docker/scripts/init-data.sh @@ -121,7 +121,8 @@ init_fingerprints() { # 初始化 Nuclei 模板仓库 init_nuclei_templates() { log_step "初始化 Nuclei 模板仓库..." - docker compose exec -T server python backend/manage.py init_nuclei_templates --sync + # 只创建数据库记录，git clone 由 install.sh 在容器外完成（支持 xget 加速） + docker compose exec -T server python backend/manage.py init_nuclei_templates log_info "Nuclei 模板仓库初始化完成" } diff --git a/install.sh b/install.sh index ec8f693e..acd5ea01 100755 --- a/install.sh +++ b/install.sh @@ -449,9 +449,9 @@ step "[3/3] 初始化配置" # 创建数据目录 info "创建数据目录..." -mkdir -p /opt/xingrin/{results,logs,fingerprints,wordlists} +mkdir -p /opt/xingrin/{results,logs,fingerprints,wordlists,nuclei-repos} chmod -R 777 /opt/xingrin -success "数据目录已创建: /opt/xingrin/{results,logs,fingerprints,wordlists}" +success "数据目录已创建: /opt/xingrin/" DOCKER_DIR="$ROOT_DIR/docker" if [ ! -d "$DOCKER_DIR" ]; then @@ -636,6 +636,36 @@ else fi fi +# ============================================================================== +# 预下载 Nuclei 模板仓库（在容器外执行，支持 xget 加速） +# ============================================================================== +step "预下载 Nuclei 模板仓库..." +NUCLEI_TEMPLATES_DIR="/opt/xingrin/nuclei-repos/nuclei-templates" +NUCLEI_TEMPLATES_REPO="https://github.com/projectdiscovery/nuclei-templates.git" + +# 确保目录存在 +mkdir -p /opt/xingrin/nuclei-repos + +if [ -d "$NUCLEI_TEMPLATES_DIR/.git" ]; then + info "Nuclei 模板仓库已存在，跳过下载" +else + # 构建 clone URL（如果启用了 xget 加速） + if [ -n "$XGET_MIRROR" ]; then + CLONE_URL="${XGET_MIRROR}/gh/${NUCLEI_TEMPLATES_REPO}" + info "使用 Xget 加速下载: $CLONE_URL" + else + CLONE_URL="$NUCLEI_TEMPLATES_REPO" + fi + + # 执行 git clone + if git clone --depth 1 "$CLONE_URL" "$NUCLEI_TEMPLATES_DIR"; then + success "Nuclei 模板仓库下载完成" + else + warn "Nuclei 模板仓库下载失败，将在服务启动后重试" + warn "您也可以稍后在 Web 界面「Nuclei 模板」中手动同步" + fi +fi + # ============================================================================== # 启动服务 # ============================================================================== diff --git a/uninstall.sh b/uninstall.sh index de242064..b5f3926d 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -64,7 +64,7 @@ fi # ============================================================================== # 1. 停止并删除全部容器/网络 # ============================================================================== -step "[1/6] 是否停止并删除全部容器/网络？(Y/n)" +step "[1/5] 是否停止并删除全部容器/网络？(Y/n)" read -r ans_stop ans_stop=${ans_stop:-Y} @@ -96,48 +96,30 @@ else fi # ============================================================================== -# 2. 删除扫描日志和结果目录 +# 2. 删除 /opt/xingrin 数据目录 # ============================================================================== -OPT_LOGS_DIR="/opt/xingrin/logs" -OPT_RESULTS_DIR="/opt/xingrin/results" +OPT_XINGRIN_DIR="/opt/xingrin" -step "[2/6] 是否删除扫描日志和结果目录 ($OPT_LOGS_DIR, $OPT_RESULTS_DIR)？(Y/n)" -read -r ans_logs -ans_logs=${ans_logs:-Y} +step "[2/5] 是否删除数据目录 ($OPT_XINGRIN_DIR)？(Y/n)" +echo -e " ${YELLOW}包含：扫描结果、日志、指纹库、字典、Nuclei 模板等${RESET}" +read -r ans_data +ans_data=${ans_data:-Y} -if [[ $ans_logs =~ ^[Yy]$ ]]; then - info "正在删除日志和结果目录..." - rm -rf "$OPT_LOGS_DIR" "$OPT_RESULTS_DIR" - success "已删除日志和结果目录。" +if [[ $ans_data =~ ^[Yy]$ ]]; then + info "正在删除数据目录..." + rm -rf "$OPT_XINGRIN_DIR" + success "已删除 $OPT_XINGRIN_DIR" else - warn "已保留日志和结果目录。" + warn "已保留数据目录。" fi # ============================================================================== -# 3. 删除 /opt/xingrin/tools 和 /opt/xingrin/wordlists -# ============================================================================== -TOOLS_DIR="/opt/xingrin/tools" -WORDLISTS_DIR="/opt/xingrin/wordlists" - -step "[3/6] 是否删除工具目录和字典目录 ($TOOLS_DIR, $WORDLISTS_DIR)？(Y/n)" -read -r ans_tools -ans_tools=${ans_tools:-Y} - -if [[ $ans_tools =~ ^[Yy]$ ]]; then - info "正在删除工具和字典目录..." - rm -rf "$TOOLS_DIR" "$WORDLISTS_DIR" - success "已删除 /opt/xingrin/tools 和 /opt/xingrin/wordlists。" -else - warn "已保留 /opt/xingrin/tools 和 /opt/xingrin/wordlists。" -fi - -# ============================================================================== -# 4. 删除 docker/.env 配置文件 +# 3. 删除 docker/.env 配置文件 # ============================================================================== ENV_FILE="$DOCKER_DIR/.env" -step "[4/6] 是否删除配置文件 ($ENV_FILE)？(Y/n)" +step "[3/5] 是否删除配置文件 ($ENV_FILE)？(Y/n)" echo -e " ${YELLOW}注意：删除后下次安装将生成新的随机密码。${RESET}" read -r ans_env ans_env=${ans_env:-Y} @@ -151,9 +133,9 @@ else fi # ============================================================================== -# 5. 删除本地 Postgres 容器及数据卷（如果使用本地 DB） +# 4. 删除本地 Postgres 容器及数据卷（如果使用本地 DB） # ============================================================================== -step "[5/6] 若使用本地 PostgreSQL 容器：是否删除数据库容器和 volume？(Y/n)" +step "[4/5] 若使用本地 PostgreSQL 容器：是否删除数据库容器和 volume？(Y/n)" read -r ans_db ans_db=${ans_db:-Y} @@ -178,7 +160,7 @@ else warn "已保留本地 Postgres 容器和 volume。" fi -step "[6/6] 是否删除与 XingRin 相关的 Docker 镜像？(Y/n)" +step "[5/5] 是否删除与 XingRin 相关的 Docker 镜像？(Y/n)" read -r ans_images ans_images=${ans_images:-Y}