414 lines
15 KiB
Python
414 lines
15 KiB
Python
# coding: utf-8
|
||
# -------------------------------------------------------------------
|
||
# YakPanel
|
||
# -------------------------------------------------------------------
|
||
# Copyright (c) 2014-2099 YakPanel(www.yakpanel.com) All rights reserved.
|
||
# -------------------------------------------------------------------
|
||
# Author: yakpanel
|
||
# -------------------------------------------------------------------
|
||
# ------------------------------
|
||
# task app
|
||
# ------------------------------
|
||
import os
|
||
import sys
|
||
from typing import Callable, Type, TypeVar, List, Dict
|
||
|
||
sys.path.insert(0, "/www/server/panel/class/")
|
||
|
||
import time
|
||
import threading
|
||
import heapq
|
||
import psutil
|
||
from dataclasses import dataclass, field, fields
|
||
from concurrent.futures import ThreadPoolExecutor, Future
|
||
from YakTask.conf import logger, CHILD_PID_PATH
|
||
|
||
T = TypeVar("T", bound="TaskInfo")
|
||
|
||
CPU_COUNT = psutil.cpu_count()
|
||
|
||
|
||
@dataclass(order=True)
|
||
class TaskInfo:
|
||
# dont init next_run
|
||
next_run: float = field(init=False)
|
||
task_id: str
|
||
# dont compare
|
||
func: Callable = field(compare=False)
|
||
interval: int | float
|
||
is_core: bool
|
||
loop: bool = True
|
||
|
||
@classmethod
|
||
def from_dict(cls: Type[T], data: dict, next_run: float) -> T:
|
||
class_fields = {f.name for f in fields(cls)}
|
||
filtered_data = {k: v for k, v in data.items() if k in class_fields}
|
||
instance = cls(**filtered_data)
|
||
instance.next_run = next_run
|
||
return instance
|
||
|
||
|
||
class SimpleBrain:
|
||
MAXFACTOR = 5.0 # 最大延迟因子
|
||
MEM_LIMIT_MB = 100 # 内存限制
|
||
MAX_WORKER = 16 # 最大线程数
|
||
DEFAULT_WORKER = max(4, CPU_COUNT) # 默认线程数
|
||
CHECKER_INTERVAL = 10 # 检查器间隔时间/秒
|
||
TIMEOUT_FACTOR = 2 # 任务超时倍率
|
||
TASK_MAX_TIMEOUT = 6 * 3600.0 # 任务最大超时6小时
|
||
|
||
__slots__ = (
|
||
"cpu_max", "workers", "executor", "task_queue", "queue_lock",
|
||
"task_status", "status_lock", "core_status_lock", "core_tasks", "delay_factor",
|
||
"shutdown", "mem_limit", "pool_full_count",
|
||
)
|
||
|
||
def __init__(self, cpu_max: float = 30.0, workers: int = None, mem_limit: int = None):
|
||
self.cpu_max = cpu_max
|
||
self.workers = min(self.MAX_WORKER, workers or self.DEFAULT_WORKER)
|
||
self.mem_limit = mem_limit or self.MEM_LIMIT_MB
|
||
logger.debug(f"max_workers = {self.workers}, mem_limit = {self.mem_limit}MB")
|
||
self.task_queue: List[TaskInfo] = []
|
||
self.queue_lock = threading.RLock()
|
||
self.status_lock = threading.RLock()
|
||
self.core_status_lock = threading.RLock()
|
||
self.core_tasks: Dict[str, tuple[threading.Thread, tuple[Callable, int]]] = {}
|
||
self.task_status: Dict[str, tuple[float, float, Future]] = {}
|
||
self.shutdown = False
|
||
self.delay_factor = 0.0
|
||
self.pool_full_count = 0
|
||
self.executor = ThreadPoolExecutor(max_workers=self.workers)
|
||
self._start_checker()
|
||
logger.debug("SimpleBrain initialized")
|
||
|
||
def _auto_pool(self):
|
||
"""自适应扩容池"""
|
||
cpu = psutil.cpu_percent(interval=0.1)
|
||
new_size = None
|
||
# 扩容, 任务堆积
|
||
if all([
|
||
self.pool_full_count >= (60 // self.CHECKER_INTERVAL), # 1min内连续满载
|
||
cpu < 80, # 在CPU未过载时进行线程扩容
|
||
self.workers < self.MAX_WORKER
|
||
]):
|
||
new_size = min(self.workers + 2, self.MAX_WORKER)
|
||
# else: # 缩容
|
||
|
||
if new_size and new_size != self.workers:
|
||
logger.warning(f"Resize Task Pool from {self.workers} to {new_size}...")
|
||
self._shutdown()
|
||
os.system(
|
||
f"nohup /www/server/panel/Yak-Task --max-workers {new_size} >> /www/server/panel/logs/task.log 2>&1 &"
|
||
)
|
||
os._exit(0)
|
||
|
||
@property
|
||
def queue_size(self) -> int:
|
||
with self.queue_lock:
|
||
return len(self.task_queue)
|
||
|
||
@property
|
||
def get_current_process_memory(self):
|
||
"""返回当前进程内存MB"""
|
||
return round(psutil.Process().memory_info().rss / 1024 / 1024, 2)
|
||
|
||
def __mem_limit(self, current_mem: float = None):
|
||
if not current_mem:
|
||
current_mem = self.get_current_process_memory
|
||
if current_mem >= self.mem_limit:
|
||
logger.warning("Memory exceed limit...Restart brain task")
|
||
self._shutdown()
|
||
os.system(
|
||
"nohup /www/server/panel/Yak-Task >> /www/server/panel/logs/task.log 2>&1 &"
|
||
)
|
||
os._exit(0)
|
||
|
||
def __core_alive(self):
|
||
with self.core_status_lock:
|
||
dead_tasks = [
|
||
task_id for task_id, (thread, _) in self.core_tasks.items()
|
||
if not thread.is_alive()
|
||
]
|
||
for task_id in dead_tasks:
|
||
logger.warning(f"Core task [{task_id}] is dead. Restarting...")
|
||
_, args = self.core_tasks[task_id]
|
||
thread = threading.Thread(
|
||
target=self._core_task_runner,
|
||
args=args,
|
||
daemon=True,
|
||
name=f"CoreTask-{task_id}"
|
||
)
|
||
thread.start()
|
||
self.core_tasks[task_id] = (thread, args)
|
||
|
||
def __normal_task_process(self, now_time: float) -> int:
|
||
"""
|
||
- 对于普通任务 deadline_ts != -1，如果超时则移除。
|
||
- 对于循环任务 deadline_ts == -1，如果其 future非running，则移除。
|
||
返回当前运行中的所有普通任务数
|
||
"""
|
||
|
||
def _pop_task(tid_str: str):
|
||
self.clean_child(os.path.join(CHILD_PID_PATH, f"{tid_str}.pid"))
|
||
self.task_status.pop(tid_str, None)
|
||
|
||
with self.status_lock:
|
||
for tid, info in list(self.task_status.items()):
|
||
if not info or len(info) != 3:
|
||
_pop_task(tid)
|
||
continue
|
||
|
||
start_time, deadline_ts, future = info
|
||
if deadline_ts != -1:
|
||
if now_time > deadline_ts:
|
||
# 普通任务超时静默移除
|
||
_pop_task(tid)
|
||
continue
|
||
|
||
if future and not future.running():
|
||
# loop任务异常移除
|
||
_pop_task(tid)
|
||
return len(self.task_status)
|
||
|
||
def __checker(self):
|
||
resize_check_interval = 60 # 每1min检查一次是否需要扩容
|
||
last_check_time = time.monotonic()
|
||
last_resize_check = time.monotonic()
|
||
|
||
while not self.shutdown:
|
||
try:
|
||
cpu = psutil.cpu_percent(interval=0.2)
|
||
now = time.monotonic()
|
||
# 每10s
|
||
now_time = time.time()
|
||
if now - last_check_time > self.CHECKER_INTERVAL:
|
||
last_check_time = now
|
||
mem = self.get_current_process_memory
|
||
self.__mem_limit(mem) # 内存阈值检测
|
||
self.__core_alive()
|
||
running_tasks = self.__normal_task_process(now_time)
|
||
rate = (running_tasks / self.workers) * 100
|
||
logger.debug(
|
||
f"Mem: {mem} MB, CPU: {cpu}%, "
|
||
f"Pool: {running_tasks}/{self.workers} ({rate:.1f}%)"
|
||
)
|
||
if rate >= 90.0: # 高负荷计数
|
||
self.pool_full_count += 1
|
||
else: # 重置计数
|
||
self.pool_full_count = 0
|
||
|
||
# 每1min
|
||
if now - last_resize_check > resize_check_interval:
|
||
last_resize_check = now
|
||
self._auto_pool()
|
||
|
||
if cpu > self.cpu_max:
|
||
self.delay_factor = min(
|
||
self.MAXFACTOR,
|
||
self.MAXFACTOR * (cpu - self.cpu_max) / (100.0 - self.cpu_max)
|
||
)
|
||
elif self.delay_factor > 0:
|
||
self.delay_factor = max(0.0, self.delay_factor - 0.1)
|
||
|
||
except Exception as e:
|
||
logger.error(f"cpu mem checker error: {e}")
|
||
time.sleep(5)
|
||
|
||
def _start_checker(self):
|
||
threading.Thread(
|
||
target=self.__checker, daemon=True, name="BrainChecker"
|
||
).start()
|
||
|
||
def _callback(self, task_id: str, future: Future):
|
||
try:
|
||
future.result()
|
||
except Exception as e:
|
||
logger.error(f"task error [{task_id}]: {str(e)}")
|
||
finally:
|
||
with self.status_lock:
|
||
info = self.task_status.pop(task_id, None)
|
||
if not info:
|
||
logger.info(f"task done [{task_id}]")
|
||
return
|
||
start_time, _, _ = info
|
||
logger.debug(
|
||
f"task done [{task_id}] use time: {round(time.time() - start_time, 2)}s"
|
||
)
|
||
|
||
def _run_func_safe(self, func: Callable):
|
||
"""捕获异常"""
|
||
try:
|
||
func()
|
||
except Exception:
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
def _core_task_runner(self, funcs: Callable | List[Callable], interval: float):
|
||
funcs = funcs if isinstance(funcs, list) else [funcs]
|
||
while 1:
|
||
for f in funcs:
|
||
self._run_func_safe(f)
|
||
time.sleep(interval)
|
||
|
||
def _submit_task(self, tasks: List[TaskInfo]):
|
||
for task in tasks:
|
||
start_time = time.time()
|
||
# 超时值: 自身任务间隔 * 超时倍率, 最大限制
|
||
if not task.loop:
|
||
timeout_sec = min(
|
||
self.TASK_MAX_TIMEOUT, (max(1.0, float(task.interval) * self.TIMEOUT_FACTOR))
|
||
)
|
||
deadline_ts = start_time + timeout_sec
|
||
else:
|
||
deadline_ts = -1 # 无限期
|
||
|
||
future = None
|
||
try:
|
||
with self.status_lock:
|
||
future = self.executor.submit(task.func)
|
||
self.task_status[task.task_id] = (
|
||
start_time, deadline_ts, future
|
||
)
|
||
future.add_done_callback(
|
||
lambda f, tid=task.task_id: self._callback(tid, f)
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"Failed to submit task [{task.task_id}]: {e}")
|
||
# 无条件确保清理状态
|
||
with self.status_lock:
|
||
self.task_status.pop(task.task_id, None)
|
||
continue
|
||
|
||
# ========================== Public =======================================
|
||
def register_task(self, **kwargs):
|
||
task_id = kwargs["task_id"]
|
||
if kwargs.get("is_core"):
|
||
# core task, daemon thread
|
||
if task_id in self.core_tasks:
|
||
return
|
||
|
||
args = (kwargs["func"], kwargs["interval"])
|
||
core_thread = threading.Thread(
|
||
target=self._core_task_runner,
|
||
args=args,
|
||
daemon=True,
|
||
name=f"CoreTask-{task_id}"
|
||
)
|
||
core_thread.start()
|
||
logger.debug(
|
||
f"registe core task [{task_id}], interval [{kwargs['interval']}]"
|
||
)
|
||
with self.core_status_lock:
|
||
self.core_tasks[task_id] = (core_thread, args)
|
||
|
||
else: # not core task, heapq, pool
|
||
with self.queue_lock:
|
||
if task_id not in {t.task_id for t in self.task_queue}:
|
||
task = TaskInfo.from_dict(kwargs, next_run=time.monotonic())
|
||
heapq.heappush(self.task_queue, task)
|
||
logger.debug(
|
||
f"register normal task [{task_id}], interval [{task.interval}s]"
|
||
)
|
||
|
||
@staticmethod
|
||
def clean_child(pid_path: str = None):
|
||
# 清理进程, 防泄漏
|
||
def _clean_single(path):
|
||
if not os.path.exists(path) or not path.endswith(".pid"):
|
||
return
|
||
try:
|
||
with open(path, "r") as pf:
|
||
pid = int(pf.read().strip())
|
||
psutil.Process(pid).kill()
|
||
except (psutil.NoSuchProcess, psutil.AccessDenied, FileNotFoundError, ValueError):
|
||
pass
|
||
except Exception:
|
||
pass
|
||
finally:
|
||
try:
|
||
os.remove(path)
|
||
except Exception:
|
||
pass
|
||
|
||
if not pid_path:
|
||
# 清理所有子进程
|
||
if not os.path.exists(CHILD_PID_PATH):
|
||
return
|
||
for pid_file in os.listdir(CHILD_PID_PATH):
|
||
_clean_single(os.path.join(CHILD_PID_PATH, pid_file))
|
||
else: # 清理指定子进程
|
||
_clean_single(pid_path)
|
||
|
||
def run(self):
|
||
self.clean_child()
|
||
while not self.shutdown:
|
||
now = time.monotonic()
|
||
with self.queue_lock:
|
||
ready_tasks = []
|
||
while (
|
||
self.task_queue
|
||
and self.task_queue[0].next_run <= now
|
||
and len(ready_tasks) < 1
|
||
):
|
||
ready_tasks.append(heapq.heappop(self.task_queue))
|
||
next_run_time = self.task_queue[0].next_run if self.task_queue else now + 1.0
|
||
|
||
submit = []
|
||
requeue = []
|
||
for task in ready_tasks:
|
||
with self.status_lock:
|
||
is_running = task.task_id in self.task_status
|
||
|
||
if is_running:
|
||
logger.debug(f"still running, {task.task_id}")
|
||
task.next_run = now + task.interval
|
||
requeue.append(task)
|
||
continue
|
||
|
||
if self.delay_factor > 0:
|
||
# 指数退避
|
||
logger.debug(f"cpu overload, requeue [{task.task_id}]")
|
||
task.next_run = now + 1.5 * (1.0 + self.delay_factor)
|
||
requeue.append(task)
|
||
continue
|
||
|
||
# 更新间隔, 准备提交, 归队
|
||
task.next_run = now + task.interval
|
||
requeue.append(task)
|
||
submit.append(task)
|
||
|
||
if requeue:
|
||
with self.queue_lock:
|
||
for task in requeue:
|
||
heapq.heappush(self.task_queue, task)
|
||
next_run_time = self.task_queue[0].next_run
|
||
|
||
if submit:
|
||
self._submit_task(submit)
|
||
time.sleep(1) # 使相同间隔的任务加大错峰
|
||
|
||
time.sleep(max(0.01, min(1.0, next_run_time - now)))
|
||
|
||
def _shutdown(self):
|
||
self.shutdown = True
|
||
logger.warning("shutdown...")
|
||
|
||
with self.queue_lock:
|
||
pending = len(self.task_queue)
|
||
self.task_queue.clear()
|
||
|
||
with self.status_lock:
|
||
running = len(self.task_status)
|
||
self.task_status.clear()
|
||
|
||
with self.core_status_lock:
|
||
self.core_tasks.clear()
|
||
|
||
logger.warning(
|
||
f"shutdown complete, {running} tasks cancelled, {pending} tasks discarded"
|
||
)
|
||
|
||
self.executor.shutdown(wait=False)
|
||
self.clean_child()
|
||
logger.warning("released")
|