mirror of
https://github.com/yyhuni/xingrin.git
synced 2026-01-31 19:53:11 +08:00
Compare commits
2 Commits
v1.5.10-de
...
v1.5.11-de
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d75a3f6882 | ||
|
|
59e48e5b15 |
@@ -156,10 +156,10 @@ class TaskDistributor:
|
||||
# 降级策略:如果没有正常负载的,循环等待后重新检测
|
||||
if not scored_workers:
|
||||
if high_load_workers:
|
||||
# 高负载等待参数(默认每 60 秒检测一次,最多 10 次)
|
||||
high_load_wait = getattr(settings, 'HIGH_LOAD_WAIT_SECONDS', 60)
|
||||
high_load_max_retries = getattr(settings, 'HIGH_LOAD_MAX_RETRIES', 10)
|
||||
|
||||
# 高负载等待参数(每 2 分钟检测一次,最多等待 2 小时)
|
||||
high_load_wait = getattr(settings, 'HIGH_LOAD_WAIT_SECONDS', 120)
|
||||
high_load_max_retries = getattr(settings, 'HIGH_LOAD_MAX_RETRIES', 60)
|
||||
|
||||
# 开始等待前发送高负载通知
|
||||
high_load_workers.sort(key=lambda x: x[1])
|
||||
_, _, first_cpu, first_mem = high_load_workers[0]
|
||||
@@ -170,51 +170,51 @@ class TaskDistributor:
|
||||
cpu=first_cpu,
|
||||
mem=first_mem
|
||||
)
|
||||
|
||||
|
||||
for retry in range(high_load_max_retries):
|
||||
logger.warning(
|
||||
"所有 Worker 高负载,等待 %d 秒后重试... (%d/%d)",
|
||||
high_load_wait, retry + 1, high_load_max_retries
|
||||
)
|
||||
time.sleep(high_load_wait)
|
||||
|
||||
|
||||
# 重新获取负载数据
|
||||
loads = worker_load_service.get_all_loads(worker_ids)
|
||||
|
||||
|
||||
# 重新评估
|
||||
scored_workers = []
|
||||
high_load_workers = []
|
||||
|
||||
|
||||
for worker in workers:
|
||||
load = loads.get(worker.id)
|
||||
if not load:
|
||||
continue
|
||||
|
||||
|
||||
cpu = load.get('cpu', 0)
|
||||
mem = load.get('mem', 0)
|
||||
score = cpu * 0.7 + mem * 0.3
|
||||
|
||||
|
||||
if cpu > 85 or mem > 85:
|
||||
high_load_workers.append((worker, score, cpu, mem))
|
||||
else:
|
||||
scored_workers.append((worker, score, cpu, mem))
|
||||
|
||||
|
||||
# 如果有正常负载的 Worker,跳出循环
|
||||
if scored_workers:
|
||||
logger.info("检测到正常负载 Worker,结束等待")
|
||||
break
|
||||
|
||||
# 超时或仍然高负载,选择负载最低的
|
||||
|
||||
# 超时后强制派发到负载最低的 Worker
|
||||
if not scored_workers and high_load_workers:
|
||||
high_load_workers.sort(key=lambda x: x[1])
|
||||
best_worker, _, cpu, mem = high_load_workers[0]
|
||||
|
||||
|
||||
logger.warning(
|
||||
"等待超时,强制分发到高负载 Worker: %s (CPU: %.1f%%, MEM: %.1f%%)",
|
||||
"等待 %d 分钟后仍高负载,强制分发到 Worker: %s (CPU: %.1f%%, MEM: %.1f%%)",
|
||||
(high_load_wait * high_load_max_retries) // 60,
|
||||
best_worker.name, cpu, mem
|
||||
)
|
||||
return best_worker
|
||||
return best_worker
|
||||
else:
|
||||
logger.warning("没有可用的 Worker")
|
||||
return None
|
||||
|
||||
@@ -87,7 +87,7 @@ def on_all_workers_high_load(sender, worker_name, cpu, mem, **kwargs):
|
||||
"""所有 Worker 高负载时的通知处理"""
|
||||
create_notification(
|
||||
title="系统负载较高",
|
||||
message=f"所有节点负载较高(最低负载节点 CPU: {cpu:.1f}%, 内存: {mem:.1f}%),系统将等待最多 10 分钟后分发任务,扫描速度可能受影响",
|
||||
message=f"所有节点负载较高(最低负载节点 CPU: {cpu:.1f}%, 内存: {mem:.1f}%),系统将每 2 分钟检测一次,最多等待 2 小时后分发任务",
|
||||
level=NotificationLevel.MEDIUM,
|
||||
category=NotificationCategory.SYSTEM
|
||||
)
|
||||
|
||||
@@ -703,7 +703,6 @@ else
|
||||
warn "镜像加速已配置,但拉取仍然失败,可能原因:"
|
||||
echo -e " 1. 镜像源暂时不可用,请稍后重试"
|
||||
echo -e " 2. 网络连接问题"
|
||||
echo -e " 3. 镜像不存在或版本错误"
|
||||
fi
|
||||
echo
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user