mirror of
https://github.com/yyhuni/xingrin.git
synced 2026-01-31 19:53:11 +08:00
Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e9a58e89aa | ||
|
|
3d9d520dc7 | ||
|
|
8d814b5864 | ||
|
|
c16b7afabe | ||
|
|
fa55167989 | ||
|
|
55a2762c71 | ||
|
|
5532f1e63a | ||
|
|
948568e950 | ||
|
|
873b6893f1 | ||
|
|
dbb30f7c78 | ||
|
|
38eced3814 | ||
|
|
68fc7cee3b | ||
|
|
6e23824a45 | ||
|
|
a88cceb4f4 |
@@ -8,13 +8,32 @@
|
||||
2. 选择负载最低的 Worker(可能是本地或远程)
|
||||
3. 本地 Worker:直接执行 docker run
|
||||
4. 远程 Worker:通过 SSH 执行 docker run
|
||||
5. 任务执行完自动销毁容器
|
||||
5. 任务执行完自动销毁容器(--rm)
|
||||
|
||||
镜像版本管理:
|
||||
- 版本锁定:使用 settings.IMAGE_TAG 确保 server 和 worker 版本一致
|
||||
- 预拉取策略:安装时预拉取镜像,执行时使用 --pull=missing
|
||||
- 本地开发:可通过 TASK_EXECUTOR_IMAGE 环境变量指向本地镜像
|
||||
|
||||
环境变量注入:
|
||||
- Worker 容器不使用 env_file,通过 docker run -e 动态注入
|
||||
- 只注入 SERVER_URL,容器启动后从配置中心获取完整配置
|
||||
- 本地 Worker:SERVER_URL = http://server:{port}(Docker 内部网络)
|
||||
- 远程 Worker:SERVER_URL = http://{public_host}:{port}(公网地址)
|
||||
|
||||
任务启动流程:
|
||||
1. Server 调用 execute_scan_flow() 等方法提交任务
|
||||
2. select_best_worker() 从 Redis 读取心跳数据,选择负载最低的节点
|
||||
3. _build_docker_command() 构建完整的 docker run 命令:
|
||||
- 设置网络(本地加入 Docker 网络,远程不指定)
|
||||
- 注入环境变量(-e SERVER_URL=...)
|
||||
- 挂载结果和日志目录(-v)
|
||||
- 指定执行脚本(python -m apps.scan.scripts.xxx)
|
||||
4. _execute_docker_command() 执行命令:
|
||||
- 本地:subprocess.run() 直接执行
|
||||
- 远程:paramiko SSH 执行
|
||||
5. docker run -d 立即返回容器 ID,任务在后台执行
|
||||
|
||||
特点:
|
||||
- 负载感知:任务优先分发到最空闲的机器
|
||||
- 统一调度:本地和远程 Worker 使用相同的选择逻辑
|
||||
@@ -203,7 +222,12 @@ class TaskDistributor:
|
||||
host_logs_dir = settings.HOST_LOGS_DIR # /opt/xingrin/logs
|
||||
|
||||
# 环境变量:只需 SERVER_URL,其他配置容器启动时从配置中心获取
|
||||
env_vars = [f"-e SERVER_URL={shlex.quote(server_url)}"]
|
||||
# Prefect 本地模式配置:禁用 API server 和事件系统
|
||||
env_vars = [
|
||||
f"-e SERVER_URL={shlex.quote(server_url)}",
|
||||
"-e PREFECT_API_URL=", # 禁用 API server
|
||||
"-e PREFECT_LOGGING_EXTRA_LOGGERS=", # 禁用 Prefect 的额外内部日志器
|
||||
]
|
||||
|
||||
# 挂载卷
|
||||
volumes = [
|
||||
|
||||
@@ -157,6 +157,51 @@ class ScanService:
|
||||
"""取消所有正在运行的阶段(委托给 ScanStateService)"""
|
||||
return self.state_service.cancel_running_stages(scan_id, final_status)
|
||||
|
||||
# todo:待接入
|
||||
def add_command_to_scan(self, scan_id: int, stage_name: str, tool_name: str, command: str) -> bool:
|
||||
"""
|
||||
增量添加命令到指定扫描阶段
|
||||
|
||||
Args:
|
||||
scan_id: 扫描任务ID
|
||||
stage_name: 阶段名称(如 'subdomain_discovery', 'port_scan')
|
||||
tool_name: 工具名称
|
||||
command: 执行命令
|
||||
|
||||
Returns:
|
||||
bool: 是否成功添加
|
||||
"""
|
||||
try:
|
||||
scan = self.get_scan(scan_id, prefetch_relations=False)
|
||||
if not scan:
|
||||
logger.error(f"扫描任务不存在: {scan_id}")
|
||||
return False
|
||||
|
||||
stage_progress = scan.stage_progress or {}
|
||||
|
||||
# 确保指定阶段存在
|
||||
if stage_name not in stage_progress:
|
||||
stage_progress[stage_name] = {'status': 'running', 'commands': []}
|
||||
|
||||
# 确保 commands 列表存在
|
||||
if 'commands' not in stage_progress[stage_name]:
|
||||
stage_progress[stage_name]['commands'] = []
|
||||
|
||||
# 增量添加命令
|
||||
command_entry = f"{tool_name}: {command}"
|
||||
stage_progress[stage_name]['commands'].append(command_entry)
|
||||
|
||||
scan.stage_progress = stage_progress
|
||||
scan.save(update_fields=['stage_progress'])
|
||||
|
||||
command_count = len(stage_progress[stage_name]['commands'])
|
||||
logger.info(f"✓ 记录命令: {stage_name}.{tool_name} (总计: {command_count})")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"记录命令失败: {e}")
|
||||
return False
|
||||
|
||||
# ==================== 删除和控制方法(委托给 ScanControlService) ====================
|
||||
|
||||
def delete_scans_two_phase(self, scan_ids: List[int]) -> dict:
|
||||
|
||||
@@ -225,6 +225,13 @@ def _parse_and_validate_line(line: str) -> Optional[PortScanRecord]:
|
||||
ip = line_data.get('ip', '').strip()
|
||||
port = line_data.get('port')
|
||||
|
||||
logger.debug("解析到的主机名: %s, IP: %s, 端口: %s", host, ip, port)
|
||||
|
||||
if not host and ip:
|
||||
host = ip
|
||||
logger.debug("主机名为空,使用 IP 作为 host")
|
||||
|
||||
|
||||
# 步骤 4: 验证字段不为空
|
||||
if not host or not ip or port is None:
|
||||
logger.warning(
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 目前采用github action自动版本构建,用
|
||||
# git tag v1.0.9
|
||||
# git push origin v1.0.9
|
||||
# ============================================
|
||||
# Docker Hub 镜像推送脚本
|
||||
# 用途:构建并推送所有服务镜像到 Docker Hub
|
||||
|
||||
@@ -101,6 +101,18 @@ services:
|
||||
# SSL 证书挂载(方便更新)
|
||||
- ./nginx/ssl:/etc/nginx/ssl:ro
|
||||
|
||||
# Worker:扫描任务执行容器(开发模式下构建)
|
||||
worker:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/worker/Dockerfile
|
||||
image: docker-worker:latest
|
||||
restart: "no"
|
||||
volumes:
|
||||
- /opt/xingrin/results:/app/backend/results
|
||||
- /opt/xingrin/logs:/app/backend/logs
|
||||
command: echo "Worker image built for development"
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
|
||||
|
||||
@@ -27,10 +27,10 @@ check_docker() {
|
||||
|
||||
# ==================== Docker Compose 命令检测 ====================
|
||||
detect_compose_cmd() {
|
||||
if command -v docker-compose >/dev/null 2>&1; then
|
||||
COMPOSE_CMD="docker-compose"
|
||||
elif docker compose version >/dev/null 2>&1; then
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
COMPOSE_CMD="docker compose"
|
||||
elif command -v docker-compose >/dev/null 2>&1; then
|
||||
COMPOSE_CMD="docker-compose"
|
||||
else
|
||||
log_error "未检测到 docker-compose 或 docker compose。"
|
||||
exit 1
|
||||
|
||||
@@ -42,10 +42,10 @@ if ! docker info >/dev/null 2>&1; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if command -v docker-compose >/dev/null 2>&1; then
|
||||
COMPOSE_CMD="docker-compose"
|
||||
elif docker compose version >/dev/null 2>&1; then
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
COMPOSE_CMD="docker compose"
|
||||
elif command -v docker-compose >/dev/null 2>&1; then
|
||||
COMPOSE_CMD="docker-compose"
|
||||
else
|
||||
echo -e "${RED}[ERROR]${NC} 未检测到 docker compose,请先安装"
|
||||
exit 1
|
||||
|
||||
@@ -79,10 +79,10 @@ ENV GOPATH=/root/go
|
||||
ENV PATH=/usr/local/go/bin:$PATH:$GOPATH/bin
|
||||
ENV GOPROXY=https://goproxy.cn,direct
|
||||
|
||||
# 5. 安装 uv(超快的 Python 包管理器)
|
||||
# 5. 安装 uv( Python 包管理器)
|
||||
RUN pip install uv --break-system-packages
|
||||
|
||||
# 安装 Python 依赖(使用 uv 并行下载,速度快 10-100 倍)
|
||||
# 安装 Python 依赖(使用 uv 并行下载)
|
||||
COPY backend/requirements.txt .
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install --system -r requirements.txt --break-system-packages && \
|
||||
|
||||
@@ -78,21 +78,21 @@ export function createIPAddressColumns(params: {
|
||||
enableSorting: false,
|
||||
enableHiding: false,
|
||||
},
|
||||
// IP 地址列
|
||||
// IP 列
|
||||
{
|
||||
accessorKey: "ip",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="IP 地址" />
|
||||
<DataTableColumnHeader column={column} title="IP Address" />
|
||||
),
|
||||
cell: ({ row }) => (
|
||||
<TruncatedCell value={row.original.ip} maxLength="ip" mono />
|
||||
),
|
||||
},
|
||||
// 关联主机名列
|
||||
// host 列
|
||||
{
|
||||
accessorKey: "hosts",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="关联主机" />
|
||||
<DataTableColumnHeader column={column} title="Hosts" />
|
||||
),
|
||||
cell: ({ getValue }) => {
|
||||
const hosts = getValue<string[]>()
|
||||
@@ -107,7 +107,7 @@ export function createIPAddressColumns(params: {
|
||||
return (
|
||||
<div className="flex flex-col gap-1">
|
||||
{displayHosts.map((host, index) => (
|
||||
<span key={index} className="text-sm font-mono">{host}</span>
|
||||
<TruncatedCell key={index} value={host} maxLength="host" mono />
|
||||
))}
|
||||
{hasMore && (
|
||||
<Badge variant="secondary" className="text-xs w-fit">
|
||||
@@ -118,11 +118,11 @@ export function createIPAddressColumns(params: {
|
||||
)
|
||||
},
|
||||
},
|
||||
// 发现时间列
|
||||
// discoveredAt 列
|
||||
{
|
||||
accessorKey: "discoveredAt",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="发现时间" />
|
||||
<DataTableColumnHeader column={column} title="Discovered At" />
|
||||
),
|
||||
cell: ({ getValue }) => {
|
||||
const value = getValue<string | undefined>()
|
||||
@@ -133,7 +133,7 @@ export function createIPAddressColumns(params: {
|
||||
{
|
||||
accessorKey: "ports",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="开放端口" />
|
||||
<DataTableColumnHeader column={column} title="Open Ports" />
|
||||
),
|
||||
cell: ({ getValue }) => {
|
||||
const ports = getValue<number[]>()
|
||||
@@ -191,7 +191,7 @@ export function createIPAddressColumns(params: {
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="w-80 p-3">
|
||||
<div className="space-y-2">
|
||||
<h4 className="font-medium text-sm">所有开放端口 ({sortedPorts.length})</h4>
|
||||
<h4 className="font-medium text-sm">All Open Ports ({sortedPorts.length})</h4>
|
||||
<div className="flex flex-wrap gap-1 max-h-32 overflow-y-auto">
|
||||
{sortedPorts.map((port, index) => (
|
||||
<Badge
|
||||
|
||||
@@ -267,7 +267,7 @@ export const createTargetColumns = ({
|
||||
{
|
||||
accessorKey: "name",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="目标名称" />
|
||||
<DataTableColumnHeader column={column} title="Target Name" />
|
||||
),
|
||||
cell: ({ row }) => (
|
||||
<TargetNameCell
|
||||
@@ -282,7 +282,7 @@ export const createTargetColumns = ({
|
||||
{
|
||||
accessorKey: "type",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="类型" />
|
||||
<DataTableColumnHeader column={column} title="Type" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const type = row.getValue("type") as string | null
|
||||
|
||||
@@ -188,7 +188,7 @@ export const createEngineColumns = ({
|
||||
{
|
||||
accessorKey: "name",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="引擎名称" />
|
||||
<DataTableColumnHeader column={column} title="Engine Name" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const name = row.getValue("name") as string
|
||||
|
||||
@@ -180,7 +180,7 @@ export const createScheduledScanColumns = ({
|
||||
{
|
||||
accessorKey: "name",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="任务名称" />
|
||||
<DataTableColumnHeader column={column} title="Task Name" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const name = row.getValue("name") as string
|
||||
@@ -216,7 +216,7 @@ export const createScheduledScanColumns = ({
|
||||
{
|
||||
accessorKey: "engineName",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="扫描引擎" />
|
||||
<DataTableColumnHeader column={column} title="Scan Engine" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const engineName = row.getValue("engineName") as string
|
||||
@@ -283,7 +283,7 @@ export const createScheduledScanColumns = ({
|
||||
{
|
||||
accessorKey: "isEnabled",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="状态" />
|
||||
<DataTableColumnHeader column={column} title="Status" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const isEnabled = row.getValue("isEnabled") as boolean
|
||||
@@ -308,7 +308,7 @@ export const createScheduledScanColumns = ({
|
||||
{
|
||||
accessorKey: "nextRunTime",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="下次执行" />
|
||||
<DataTableColumnHeader column={column} title="Next Run" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const nextRunTime = row.getValue("nextRunTime") as string | undefined
|
||||
@@ -324,7 +324,7 @@ export const createScheduledScanColumns = ({
|
||||
{
|
||||
accessorKey: "runCount",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="执行次数" />
|
||||
<DataTableColumnHeader column={column} title="Run Count" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const count = row.getValue("runCount") as number
|
||||
@@ -338,7 +338,7 @@ export const createScheduledScanColumns = ({
|
||||
{
|
||||
accessorKey: "lastRunTime",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="上次执行" />
|
||||
<DataTableColumnHeader column={column} title="Last Run" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const lastRunTime = row.getValue("lastRunTime") as string | undefined
|
||||
|
||||
@@ -100,7 +100,7 @@ export const createSubdomainColumns = ({
|
||||
{
|
||||
accessorKey: "discoveredAt",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="发现时间" />
|
||||
<DataTableColumnHeader column={column} title="Discovered At" />
|
||||
),
|
||||
cell: ({ getValue }) => {
|
||||
const value = getValue<string | undefined>()
|
||||
|
||||
@@ -95,7 +95,7 @@ export const commandColumns: ColumnDef<Command>[] = [
|
||||
{
|
||||
accessorKey: "displayName",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="名称" />
|
||||
<DataTableColumnHeader column={column} title="Name" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const displayName = row.getValue("displayName") as string
|
||||
@@ -136,7 +136,7 @@ export const commandColumns: ColumnDef<Command>[] = [
|
||||
{
|
||||
accessorKey: "tool",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="所属工具" />
|
||||
<DataTableColumnHeader column={column} title="Tool" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const tool = row.original.tool
|
||||
@@ -156,7 +156,7 @@ export const commandColumns: ColumnDef<Command>[] = [
|
||||
{
|
||||
accessorKey: "commandTemplate",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="命令模板" />
|
||||
<DataTableColumnHeader column={column} title="Command Template" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const template = row.getValue("commandTemplate") as string
|
||||
@@ -192,7 +192,7 @@ export const commandColumns: ColumnDef<Command>[] = [
|
||||
{
|
||||
accessorKey: "description",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="描述" />
|
||||
<DataTableColumnHeader column={column} title="Description" />
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const description = row.getValue("description") as string
|
||||
@@ -217,7 +217,7 @@ export const commandColumns: ColumnDef<Command>[] = [
|
||||
{
|
||||
accessorKey: "updatedAt",
|
||||
header: ({ column }) => (
|
||||
<DataTableColumnHeader column={column} title="更新时间" />
|
||||
<DataTableColumnHeader column={column} title="Updated At" />
|
||||
),
|
||||
cell: ({ row }) => (
|
||||
<div className="text-sm text-muted-foreground">
|
||||
|
||||
44
install.sh
44
install.sh
@@ -126,7 +126,7 @@ update_env_var() {
|
||||
GENERATED_DB_PASSWORD=""
|
||||
GENERATED_DJANGO_KEY=""
|
||||
|
||||
# 生成自签 HTTPS 证书(无域名场景)
|
||||
# 生成自签 HTTPS 证书(使用容器,避免宿主机 openssl 兼容性问题)
|
||||
generate_self_signed_cert() {
|
||||
local ssl_dir="$DOCKER_DIR/nginx/ssl"
|
||||
local fullchain="$ssl_dir/fullchain.pem"
|
||||
@@ -139,14 +139,18 @@ generate_self_signed_cert() {
|
||||
|
||||
info "未检测到 HTTPS 证书,正在生成自签证书(localhost)..."
|
||||
mkdir -p "$ssl_dir"
|
||||
if openssl req -x509 -nodes -newkey rsa:2048 -days 365 \
|
||||
-keyout "$privkey" \
|
||||
-out "$fullchain" \
|
||||
|
||||
# 使用容器生成证书,避免依赖宿主机 openssl 版本
|
||||
if docker run --rm -v "$ssl_dir:/ssl" alpine/openssl \
|
||||
req -x509 -nodes -newkey rsa:2048 -days 365 \
|
||||
-keyout /ssl/privkey.pem \
|
||||
-out /ssl/fullchain.pem \
|
||||
-subj "/C=CN/ST=NA/L=NA/O=XingRin/CN=localhost" \
|
||||
-addext "subjectAltName=DNS:localhost,IP:127.0.0.1" >/dev/null 2>&1; then
|
||||
-addext "subjectAltName=DNS:localhost,IP:127.0.0.1" \
|
||||
>/dev/null 2>&1; then
|
||||
success "自签证书已生成: $ssl_dir"
|
||||
else
|
||||
warn "自签证书生成失败,请检查 openssl 是否可用,或手动放置证书到 $ssl_dir"
|
||||
warn "自签证书生成失败,请手动放置证书到 $ssl_dir"
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -225,7 +229,7 @@ show_summary() {
|
||||
|
||||
step "[1/3] 检查基础命令"
|
||||
MISSING_CMDS=()
|
||||
for cmd in git curl jq openssl; do
|
||||
for cmd in git curl; do
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
MISSING_CMDS+=("$cmd")
|
||||
warn "未安装: $cmd"
|
||||
@@ -396,11 +400,29 @@ DOCKER_USER=$(grep "^DOCKER_USER=" "$DOCKER_DIR/.env" 2>/dev/null | cut -d= -f2)
|
||||
DOCKER_USER=${DOCKER_USER:-yyhuni}
|
||||
WORKER_IMAGE="${DOCKER_USER}/xingrin-worker:${APP_VERSION}"
|
||||
|
||||
info "正在拉取: $WORKER_IMAGE"
|
||||
if docker pull "$WORKER_IMAGE"; then
|
||||
success "Worker 镜像拉取完成"
|
||||
# 开发模式下构建本地 worker 镜像
|
||||
if [ "$DEV_MODE" = true ]; then
|
||||
info "开发模式:构建本地 Worker 镜像..."
|
||||
if docker compose -f "$DOCKER_DIR/docker-compose.dev.yml" build worker; then
|
||||
# 设置 TASK_EXECUTOR_IMAGE 环境变量指向本地构建的镜像
|
||||
update_env_var "$DOCKER_DIR/.env" "TASK_EXECUTOR_IMAGE" "docker-worker:latest"
|
||||
success "本地 Worker 镜像构建完成,并设置为默认使用镜像"
|
||||
else
|
||||
warn "本地 Worker 镜像构建失败,将使用远程镜像"
|
||||
info "正在拉取: $WORKER_IMAGE"
|
||||
if docker pull "$WORKER_IMAGE"; then
|
||||
success "Worker 镜像拉取完成"
|
||||
else
|
||||
warn "Worker 镜像拉取失败,扫描时会自动重试拉取"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
warn "Worker 镜像拉取失败,扫描时会自动重试拉取"
|
||||
info "正在拉取: $WORKER_IMAGE"
|
||||
if docker pull "$WORKER_IMAGE"; then
|
||||
success "Worker 镜像拉取完成"
|
||||
else
|
||||
warn "Worker 镜像拉取失败,扫描时会自动重试拉取"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================
|
||||
|
||||
Reference in New Issue
Block a user