#!/bin/python # coding: utf-8 # +------------------------------------------------------------------- # | YakPanel # +------------------------------------------------------------------- # | Copyright (c) 2015-2016 YakPanel(www.yakpanel.com) All rights reserved. # +------------------------------------------------------------------- # | Author: yakpanel # +------------------------------------------------------------------- # ------------------------------ # YakPanel background scheduled tasks # ------------------------------ import gc import os import shutil import sqlite3 import subprocess import sys import threading import time import traceback from datetime import datetime, timedelta from typing import Optional, Tuple import psutil try: import ujson as json except ImportError: try: os.system("btpip install ujson") import ujson as json except: import json sys.path.insert(0, "/www/server/panel/class/") try: from public.hook_import import hook_import hook_import() except: pass import db from panelTask import bt_task from script.restart_services import RestartServices from YakTask.brain import SimpleBrain from YakTask.conf import ( BASE_PATH, PYTHON_BIN, exlogPath, isTask, logger, ) def write_file(path: str, content: str, mode='w'): try: fp = open(path, mode) fp.write(content) fp.close() return True except: try: fp = open(path, mode, encoding="utf-8") fp.write(content) fp.close() return True except: return False def read_file(filename: str): fp = None try: fp = open(filename, "rb") f_body_bytes: bytes = fp.read() f_body = f_body_bytes.decode("utf-8", errors='ignore') fp.close() return f_body except Exception: return False finally: if fp and not fp.closed: fp.close() def exec_shell(cmdstring, timeout=None, shell=True, cwd=None): """ @name 执行命令 @param cmdstring 命令 [必传] @param timeout 超时时间 @param shell 是否通过shell运行 @return 命令执行结果 """ try: result = subprocess.run( cmdstring, shell=shell, cwd=cwd, timeout=timeout, capture_output=True, text=True, # 直接以文本模式处理输出 encoding='utf-8', errors='ignore', env=os.environ ) return result.stdout, result.stderr except subprocess.TimeoutExpired: return 'Timed out', '' except Exception: return '', traceback.format_exc() task_obj = bt_task() task_obj.not_web = True bt_box_task = task_obj.start_task_new def task_ExecShell(fucn_name: str, **kw): """ 仅运行 /www/server/panel/YakTask/task_script.py 下的包装函数 可通过 kw 参数扩展前置检查，例如: - kw['paths_exists']: List [str] (例如, 检查邮局插件是否存在, 任何一个检查不存在则不执行 paths_exists=['/www/server/panel/plugin/mail_sys/mail_sys_main.py', '/www/vmail']) """ if PYTHON_BIN in fucn_name: raise ValueError("valid function name required") if kw.get("paths_exists") and isinstance(kw["paths_exists"], list): for p in kw["paths_exists"]: try: if not os.path.exists(str(p)): logger.debug(f"Skip task [{fucn_name}]: path not exists") return except Exception as e: raise ValueError(f"Invalid path in paths_exists: '{p}', error: {e}") cmd = f"{PYTHON_BIN} /www/server/panel/YakTask/task_script.py {fucn_name}" _, err = exec_shell(cmd) if err: raise Exception(err) # 系统监控全局缓存 _system_task_state = { "table_ensure": False, # {"timestamp": 0, "total_up": 0, "total_down": 0, "up_packets": {}, "down_packets": {}} "last_network_io": {}, # {"timestamp": 0, "read_count": 0, "write_count": 0, "read_bytes": 0, "write_bytes": 0, "read_time": 0, "write_time": 0} "last_disk_io": {}, # {pid: (create_time, cpu_time, disk_read, disk_write, timestamp)} "last_process_cache": {}, # {pid: (inactive_count, last_check_time)} "inactive_process_cache": {}, "last_clear_time": 0, "last_cpu_times": None, } # 系统监控任务 def systemTask(): def get_cpu_percent_smooth() -> float: """窗口时间内CPU占用率""" try: current_times = psutil.cpu_times() last_times = _system_task_state.get("last_cpu_times") if not last_times: _system_task_state["last_cpu_times"] = current_times return psutil.cpu_percent(interval=0.1) all_delta = sum(current_times) - sum(last_times) if all_delta == 0.0: return 0.0 idle_delta = getattr(current_times, "idle", 0) - getattr(last_times, "idle", 0) cpu_percent = ((all_delta - idle_delta) / all_delta) * 100 _system_task_state["last_cpu_times"] = current_times return max(0.0, min(100.0, cpu_percent)) except Exception: return psutil.cpu_percent(interval=0.1) def get_mem_used_percent() -> float: """内存使用率""" try: mem = psutil.virtual_memory() total = mem.total / 1024 / 1024 free = mem.free / 1024 / 1024 buffers = getattr(mem, "buffers", 0) / 1024 / 1024 cached = getattr(mem, "cached", 0) / 1024 / 1024 used = total - free - buffers - cached return used / (total / 100.0) if total else 1.0 except Exception: return 1.0 # noinspection PyUnusedLocal,PyTypeChecker def get_swap_used_percent() -> float: """ 获取Swap内存已占用的百分比（返回0.0~100.0，异常时返回100.0） 逻辑：Swap使用率 = (已使用Swap / 总Swap容量) * 100% """ try: # 获取Swap内存信息（psutil.swap_memory()返回namedtuple） swap = psutil.swap_memory() # Swap总容量（bytes → MB，和物理内存计算单位保持一致） swap_total = swap.total / 1024 / 1024 # Swap已使用量（bytes → MB） swap_used = swap.used / 1024 / 1024 swap_free = swap.free / 1024 / 1024 # 避免除以0（无Swap分区时） if swap_total == 0: return 0.0, 0, 0, 0 # 无Swap时默认返回100%（或根据需求改0.0） # 计算使用率（保留2位小数，确保返回float类型） used_percent = round((swap_used / swap_total) * 100.0, 2) # 边界值修正（防止因系统浮点误差导致超过100%） # return min(used_percent, 100.0),swap.total/1024,swap.used/1024,swap.free/1024 return min(used_percent, 100.0), swap.total, swap.used, swap.free # 捕获所有异常，返回100%（和物理内存函数的异常返回逻辑一致） except Exception: return 100.0, 0, 0, 0 def get_load_average() -> Tuple[float, float, float, float]: """负载平均""" try: one, five, fifteen = os.getloadavg() max_v = psutil.cpu_count() * 2 lpro = round((one / max_v) * 100, 2) if max_v else 0 if lpro > 100: lpro = 100 return lpro, float(one), float(five), float(fifteen) except Exception: return 0.0, 0.0, 0.0, 0.0 def get_network_io() -> Optional[dict]: """网络IO""" try: network_io = psutil.net_io_counters(pernic=True) ret = { 'total_up': sum(v.bytes_sent for v in network_io.values()), 'total_down': sum(v.bytes_recv for v in network_io.values()), 'timestamp': time.time(), 'down_packets': {k: v.bytes_recv for k, v in network_io.items()}, 'up_packets': {k: v.bytes_sent for k, v in network_io.items()} } last_io = _system_task_state["last_network_io"] if not last_io: _system_task_state["last_network_io"] = ret return None diff_t = (ret["timestamp"] - last_io.get("timestamp", 0)) * 1024 # 转KB if diff_t <= 0: return None res = { 'up': round((ret['total_up'] - last_io.get('total_up', 0)) / diff_t, 2), 'down': round((ret['total_down'] - last_io.get('total_down', 0)) / diff_t, 2), 'total_up': ret['total_up'], 'total_down': ret['total_down'], 'down_packets': { k: round((v - last_io.get('down_packets', {}).get(k, 0)) / diff_t, 2) for k, v in ret['down_packets'].items() }, 'up_packets': { k: round((v - last_io.get('up_packets', {}).get(k, 0)) / diff_t, 2) for k, v in ret['up_packets'].items() } } _system_task_state["last_network_io"] = ret return res except Exception: return None def get_disk_io() -> Optional[dict]: """磁盘IO""" if not os.path.exists('/proc/diskstats'): return None try: disk_io = psutil.disk_io_counters() if not disk_io: return None ret = { 'read_count': disk_io.read_count, 'write_count': disk_io.write_count, 'read_bytes': disk_io.read_bytes, 'write_bytes': disk_io.write_bytes, 'read_time': disk_io.read_time, 'write_time': disk_io.write_time, 'timestamp': time.time() } last_io = _system_task_state["last_disk_io"] if not last_io: _system_task_state["last_disk_io"] = ret return None diff_t = ret["timestamp"] - last_io.get("timestamp", 0) if diff_t <= 0: return None res = { 'read_count': int((ret["read_count"] - last_io.get("read_count", 0)) / diff_t), 'write_count': int((ret["write_count"] - last_io.get("write_count", 0)) / diff_t), 'read_bytes': int((ret["read_bytes"] - last_io.get("read_bytes", 0)) / diff_t), 'write_bytes': int((ret["write_bytes"] - last_io.get("write_bytes", 0)) / diff_t), 'read_time': int((ret["read_time"] - last_io.get("read_time", 0)) / diff_t), 'write_time': int((ret["write_time"] - last_io.get("write_time", 0)) / diff_t), } _system_task_state["last_disk_io"] = ret return res except Exception: return None _XSS_TRANS = str.maketrans({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }) def xss_encode(s: str) -> str: """XSS""" if not isinstance(s, str): s = str(s) if not any(c in s for c in '&<>"\''): return s return s.translate(_XSS_TRANS) def cut_top_list(process_list, *sort_key, top_num=5) -> list: # """最小堆取前N""" # import heapq # if not process_list or not sort_key: # return [] # try: # if len(sort_key) == 1: # return heapq.nlargest( # top_num, process_list, key=lambda x: x.get(sort_key[0], 0) # ) # else: # return heapq.nlargest( # top_num, process_list, key=lambda x: tuple(x.get(k, 0) for k in sort_key) # ) # except (TypeError, ValueError): # return [] """前N个""" if not process_list or not sort_key: return [] tops = sorted( process_list, key=lambda x: tuple(x.get(k, 0) for k in sort_key), reverse=True ) return tops[:top_num] def get_process_list() -> list: """获取进程列表""" SKIP_NAMES = { 'edac-poller', 'devfreq_wq', 'watchdogd', 'kthrotld', 'acpi_thermal_pm', 'charger_manager', 'kthreadd', 'rcu_gp', 'rcu_par_gp', 'rcu_sched', 'migration/0', 'cpuhp/0', 'kdevtmpfs', 'netns', 'oom_reaper', 'writeback', 'crypto', 'kintegrityd', 'kblockd', 'ata_sff', } try: pids = psutil.pids() current_pid = os.getpid() timer = getattr(time, "monotonic", time.time) cpu_num = psutil.cpu_count() or 1 process_list = [] new_cache = {} inactive_cache = _system_task_state["inactive_process_cache"] for pid in pids: if pid == current_pid: continue if pid < 10: # 核心进程 continue if pid in inactive_cache: inactive_count, last_check = inactive_cache[pid] # 5分钟重新检查一次 if time.time() - last_check < 300 and inactive_count >= 3: inactive_cache[pid] = (inactive_count, time.time()) # 结合后续10分钟清理逻辑 continue try: p = psutil.Process(pid) if p.status() in (psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD): # 休眠/僵尸 continue process_name = None try: process_name = p.name() if process_name in SKIP_NAMES: continue except: pass with p.oneshot(): create_time = p.create_time() if not process_name: try: process_name = p.name() except: process_name = "unknown" last_cache = _system_task_state["last_process_cache"].get(pid) if last_cache and last_cache[0] != create_time: # 进程应该被重启了 last_cache = None if pid in inactive_cache: del inactive_cache[pid] p_cpu_time = p.cpu_times() cpu_time = p_cpu_time.system + p_cpu_time.user io_counters = p.io_counters() memory_info = p.memory_info() current_time = timer() # 刷新缓存 new_cache[pid] = ( create_time, cpu_time, io_counters.read_bytes, io_counters.write_bytes, current_time ) if not last_cache: # 初次处理的进程 continue diff_t = current_time - last_cache[4] if diff_t <= 0: continue cpu_percent = max(round((cpu_time - last_cache[1]) * 100 / diff_t / cpu_num, 2), 0) disk_read = max(0, int((io_counters.read_bytes - last_cache[2]) / diff_t)) disk_write = max(0, int((io_counters.write_bytes - last_cache[3]) / diff_t)) disk_total = disk_read + disk_write if cpu_percent == 0 and disk_total == 0: if pid in inactive_cache: inactive_cache[pid] = (inactive_cache[pid][0] + 1, time.time()) else: inactive_cache[pid] = (1, time.time()) continue if pid in inactive_cache: del inactive_cache[pid] # swap占用 try: swap = p.memory_full_info().swap except: swap = 0 # connect_count = len( # p.net_connections() if hasattr(p, "net_connections") else p.connections() # noqa # ) process_info = { 'pid': pid, 'name': process_name, 'username': p.username(), 'cpu_percent': cpu_percent, 'memory': memory_info.rss, 'swap': swap, 'disk_read': disk_read, 'disk_write': disk_write, 'disk_total': disk_total or 0, 'cmdline': ' '.join(filter(lambda x: x, p.cmdline()))[:500], 'create_time': create_time, 'connect_count': 0, # future 'net_total': 0, # future 'up': 0, # future 'down': 0, # future 'up_package': 0, # future 'down_package': 0, # future } # process_info["net_total"] = process_info["up"] + process_info["down"] process_list.append(process_info) except Exception: continue current_time = time.time() inactive_cache_copy = dict(inactive_cache) for pid, (count, last_check) in inactive_cache_copy.items(): remove_flag = False if current_time - last_check > 600: # 超过 10 分钟 remove_flag = True else: # 进程残留, pid被复用 try: current_create = psutil.Process(pid).create_time() exist_create = None if pid in _system_task_state["last_process_cache"]: exist_create = _system_task_state["last_process_cache"][pid][0] elif pid in new_cache: exist_create = new_cache[pid][0] if exist_create and current_create != exist_create: remove_flag = True # pid被复用 except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): remove_flag = True if remove_flag and pid in inactive_cache: del inactive_cache[pid] _system_task_state["last_process_cache"] = new_cache _system_task_state["inactive_process_cache"] = inactive_cache return process_list except Exception: return [] # 暂无用 # def start_process_net_total(self): # # 进程流量监控，如果文件：/www/server/panel/data/is_net_task.pl 或 /www/server/panel/data/control.conf不存在，则不监控进程流量 # if not (os.path.isfile(self.proc_net_service) and os.path.isfile(self.base_service)): # return # # def process_net_total(): # class_path = '{}/class'.format(BASE_PATH) # if class_path not in sys.path: # sys.path.insert(0, class_path) # import process_task # process_task.process_network_total().start() # # import threading # th = threading.Thread(target=process_net_total, daemon=True) # th.start() def ensure_table(db_file: str) -> None: """表, 字段处理""" if not os.path.isfile(db_file): os.makedirs(os.path.dirname(db_file), exist_ok=True) open(db_file, 'w').close() conn = None cursor = None init_sql = ''' CREATE TABLE IF NOT EXISTS `cpuio` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, `pro` INTEGER, `mem` INTEGER, `swap_percent` INTEGER, `swap_total` INTEGER, `swap_used` INTEGER, `swap_free` INTEGER, `addtime` INTEGER ); CREATE TABLE IF NOT EXISTS `network` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, `up` INTEGER, `down` INTEGER, `total_up` INTEGER, `total_down` INTEGER, `down_packets` INTEGER, `up_packets` INTEGER, `addtime` INTEGER ); CREATE TABLE IF NOT EXISTS `diskio` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, `read_count` INTEGER, `write_count` INTEGER,`read_bytes` INTEGER, `write_bytes` INTEGER, `read_time` INTEGER, `write_time` INTEGER, `addtime` INTEGER ); CREATE TABLE IF NOT EXISTS `load_average` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, `pro` REAL, `one` REAL, `five` REAL, `fifteen` REAL, `addtime` INTEGER ); CREATE TABLE IF NOT EXISTS `process_top_list` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, `cpu_top` REAL, `memory_top` REAL, `disk_top` REAL, `net_top` REAL, `all_top` REAL, `swap_top` REAL, `addtime` INTEGER ); -- 索引 CREATE INDEX IF NOT EXISTS idx_cpuio_addtime ON cpuio (addtime); CREATE INDEX IF NOT EXISTS idx_network_addtime ON network (addtime); CREATE INDEX IF NOT EXISTS idx_diskio_addtime ON diskio (addtime); CREATE INDEX IF NOT EXISTS idx_load_average_addtime ON load_average (addtime); CREATE INDEX IF NOT EXISTS idx_process_top_list_addtime ON process_top_list (addtime); ''' try: conn = sqlite3.connect(db_file) try: conn.executescript(init_sql) conn.commit() except Exception as e: logger.error("Failed to initialize system.db : {}".format(e)) # 检测cpuio表是否存在swap字段 cursor = conn.cursor() try: cursor.execute( "SELECT swap_total, swap_percent, swap_used, swap_free FROM cpuio ORDER BY id DESC LIMIT 1" ) except sqlite3.OperationalError: # swap字段不存在 alter_sql = ''' ALTER TABLE cpuio ADD COLUMN swap_percent INTEGER DEFAULT 0; ALTER TABLE cpuio ADD COLUMN swap_total INTEGER DEFAULT 0; ALTER TABLE cpuio ADD COLUMN swap_used INTEGER DEFAULT 0; ALTER TABLE cpuio ADD COLUMN swap_free INTEGER DEFAULT 0; ''' try: conn.executescript(alter_sql) conn.commit() except sqlite3.OperationalError: pass # 检测process_top_list表是否存在swap_top字段 try: cursor.execute("SELECT swap_top FROM process_top_list ORDER BY id DESC LIMIT 1") except sqlite3.OperationalError: try: conn.execute("ALTER TABLE process_top_list ADD COLUMN swap_top REAL DEFAULT []") conn.commit() except sqlite3.OperationalError: pass _system_task_state["table_ensure"] = True except Exception as e: logger.error("Failed to connect to system.db : {}".format(e)) _system_task_state["table_ensure"] = False finally: if cursor: cursor.close() if conn: conn.close() def clear_expire_data(conn: sqlite3.Connection) -> None: if not _system_task_state.get("last_clear_time"): return now = int(time.time()) # 检查是否需要清理（每小时一次） if now - _system_task_state.get("last_clear_time", 0) < 3600: return cur = None try: cur = conn.cursor() deltime = now - (keep_days * 86400) cur.execute("DELETE FROM cpuio WHERE addtime < ?", (deltime,)) cur.execute("DELETE FROM network WHERE addtime < ?", (deltime,)) cur.execute("DELETE FROM diskio WHERE addtime < ?", (deltime,)) cur.execute("DELETE FROM load_average WHERE addtime < ?", (deltime,)) cur.execute("DELETE FROM process_top_list WHERE addtime < ?", (deltime,)) conn.commit() _system_task_state["last_clear_time"] = now except: pass finally: if cur: cur.close() control_conf = f"{BASE_PATH}/data/control.conf" db_file = f"{BASE_PATH}/data/system.db" # 表结构检查 if not _system_task_state["table_ensure"]: ensure_table(db_file) # 是否启用监控, 不存在时为不开启监控, 存在时为监控天数 if not os.path.exists(control_conf): return try: keep_days = int(read_file(control_conf) or 30) if keep_days < 1: return except Exception: keep_days = 30 conn = None cursor = None try: conn = sqlite3.connect(db_file, timeout=3) cursor = conn.cursor() cpu_used = get_cpu_percent_smooth() mem_used = get_mem_used_percent() swap_percent, swap_total, swap_used, swap_free = get_swap_used_percent() lpro, one, five, fifteen = get_load_average() network_io = get_network_io() disk_io = get_disk_io() process_list = get_process_list() proc_net_service = f"{BASE_PATH}/data/is_net_task.pl" addtime = int(time.time()) # cpu, swap cursor.execute( "INSERT INTO cpuio (pro, mem, swap_percent, swap_total, swap_used, swap_free, addtime) VALUES (?, ?, ?, ?, ?, ?, ?)", (cpu_used, mem_used, swap_percent, swap_total, swap_used, swap_free, addtime) ) # load cursor.execute( "INSERT INTO load_average (pro, one, five, fifteen, addtime) VALUES (?, ?, ?, ?, ?)", (lpro, one, five, fifteen, addtime) ) # network io if network_io: cursor.execute( "INSERT INTO network (up, down, total_up, total_down, down_packets, up_packets, addtime) VALUES (?, ?, ?, ?, ?, ?, ?)", ( network_io['up'], network_io['down'], network_io['total_up'], network_io['total_down'], json.dumps(network_io['down_packets']), json.dumps(network_io['up_packets']), addtime ) ) # disk io if disk_io: cursor.execute( "INSERT INTO diskio (read_count, write_count, read_bytes, write_bytes, read_time, write_time, addtime) VALUES (?, ?, ?, ?, ?, ?, ?)", ( disk_io['read_count'], disk_io['write_count'], disk_io['read_bytes'], disk_io['write_bytes'], disk_io['read_time'], disk_io['write_time'], addtime ) ) # process top list if process_list: all_top_list = cut_top_list( process_list, 'cpu_percent', 'disk_total', 'memory', 'net_total', top_num=5 ) cpu_top_list = cut_top_list(process_list, 'cpu_percent', top_num=5) disk_top_list = cut_top_list(process_list, 'disk_total', top_num=5) memory_top_list = cut_top_list(process_list, 'memory', top_num=5) swap_top_list = cut_top_list(process_list, 'swap', top_num=5) # net_top_list = all_top_list if os.path.isfile(proc_net_service): # 进程流量监控, 暂无用 # net_top_list = top_lists['net_top'] pass all_top = json.dumps([( p['cpu_percent'], p['disk_read'], p['disk_write'], p['memory'], p['up'], p['down'], p['pid'], xss_encode(p['name']), xss_encode(p['cmdline']), xss_encode(p['username']), p['create_time'] ) for p in all_top_list]) cpu_top = json.dumps([( p['cpu_percent'], p['pid'], xss_encode(p['name']), xss_encode(p['cmdline']), xss_encode(p['username']), p['create_time'] ) for p in cpu_top_list]) disk_top = json.dumps([( p['disk_total'], p['disk_read'], p['disk_write'], p['pid'], xss_encode(p['name']), xss_encode(p['cmdline']), xss_encode(p['username']), p['create_time'] ) for p in disk_top_list]) # net_top = json.dumps([( # p['net_total'], p['up'], p['down'], p['connect_count'], p['up_package'] + p['down_package'], # p['pid'], xss_encode(p['name']), xss_encode(p['cmdline']), xss_encode(p['username']), # p['create_time'] # ) for p in net_top_list]) net_top = None memory_top = json.dumps([( p['memory'], p['pid'], xss_encode(p['name']), xss_encode(p['cmdline']), xss_encode(p['username']), p['create_time'] ) for p in memory_top_list]) swap_top = json.dumps([( p['swap'], p['pid'], xss_encode(p['name']), xss_encode(p['cmdline']), xss_encode(p['username']), p['create_time'] ) for p in swap_top_list]) cursor.execute( "INSERT INTO process_top_list (all_top, cpu_top, disk_top, net_top, memory_top, swap_top, addtime) VALUES (?, ?, ?, ?, ?, ?, ?)", (all_top, cpu_top, disk_top, net_top, memory_top, swap_top, addtime) ) conn.commit() # every 1h check clear old data clear_expire_data(conn) except sqlite3.OperationalError as e: logger.error(f"SQLite OperationalError in systemTask: {e}") ensure_table(db_file) _system_task_state["table_ensure"] = False except Exception: logger.error(f"systemTask error: {traceback.format_exc()}") finally: if cursor: cursor.close() if conn: conn.close() gc.collect() def check502Task(): task_ExecShell("check502task") # 服务守护 def daemon_service(): try: obj = RestartServices() obj.main() del obj finally: gc.collect() # 项目守护 def project_daemon_service(): task_ExecShell("project_daemon_service") # 重启面板服务 def restart_panel(): def service_panel(action='reload'): if not os.path.exists('{}/init.sh'.format(BASE_PATH)): os.system("curl -k https://node.yakpanel.com/install/update_7.x_en.sh|bash &") else: os.system("nohup bash /www/server/panel/init.sh {} > /dev/null 2>&1 &".format(action)) logger.info("Panel Service: {}".format(action)) rtips = '{}/data/restart.pl'.format(BASE_PATH) reload_tips = '{}/data/reload.pl'.format(BASE_PATH) if os.path.exists(rtips): os.remove(rtips) service_panel('restart') if os.path.exists(reload_tips): os.remove(reload_tips) service_panel('reload') # 定时任务去检测邮件信息 def send_mail_time(): if not os.path.exists('/www/server/panel/plugin/mail_sys/mail_sys_main.py') or not os.path.exists('/www/vmail'): return exec_shell("{} /www/server/panel/script/mail_task.py".format(PYTHON_BIN)) # 面板推送消息 def push_msg(): def _read_file(file_path: str) -> Optional[list]: if not os.path.exists(file_path): return None content = read_file(file_path) if not content: return None try: return json.loads(content) except: return [] sender_path = f"{BASE_PATH}/data/mod_push_data/sender.json" task_path = f"{BASE_PATH}/data/mod_push_data/task.json" sender_info = _read_file(sender_path) or [] work = False for s in sender_info: # default sender_type sms data is {} if s.get("sender_type") != "sms" and s.get("data"): work = True break if not work: return if not _read_file(task_path): return task_ExecShell("push_msg") # 检测面板授权 # noinspection PyUnboundLocalVariable def panel_auth(): pro_file = '/www/server/panel/data/panel_pro.pl' update_file = '/www/server/panel/data/now_update_pro.pl' if os.path.exists(pro_file): try: from YakPanel import cache except Exception as e: logger.error("Failed to import cache from YakPanel: {}".format(e)) cache = None if cache: key = 'pro_check_sdfjslk' res = cache.get(key) if os.path.exists(update_file) or res is None: os.system('nohup {} /www/server/panel/script/check_auth.py > /dev/null 2>&1 &'.format(PYTHON_BIN)) if cache: cache.set(key, 'sddsf', 3600) if os.path.exists(update_file): os.remove(update_file) def count_ssh_logs(): task_ExecShell("count_ssh_logs") # 每天提交一次昨天的邮局发送总数 def submit_email_statistics(): task_ExecShell( "submit_email_statistics", paths_exists=[ "/www/server/panel/plugin/mail_sys/mail_sys_main.py", "/www/vmail", ]) # 每天一次 提交今天之前的统计数据 def submit_module_call_statistics(): task_ExecShell("submit_module_call_statistics") def mailsys_domain_restrictions(): if not os.path.exists('/www/server/panel/plugin/mail_sys/mail_send_bulk.py'): return if not os.path.exists('/www/vmail'): return yesterday = datetime.now() - timedelta(days=1) yesterday = yesterday.strftime('%Y-%m-%d') cloud_yesterday_submit = '{}/data/{}_update_mailsys_domain_restrictions.pl'.format( BASE_PATH, yesterday ) if os.path.exists(cloud_yesterday_submit): return if os.path.exists("/www/server/panel/plugin/mail_sys"): sys.path.insert(1, "/www/server/panel/plugin/mail_sys") # 检查版本 检查是否能查询额度 剩余额度 import public.PluginLoader as plugin_loader bulk = plugin_loader.get_module('{}/plugin/mail_sys/mail_send_bulk.py'.format(BASE_PATH)) SendMailBulk = bulk.SendMailBulk try: SendMailBulk()._get_user_quota() except: logger.error(traceback.format_exc()) return # 添加标记 write_file(cloud_yesterday_submit, '1') # 删除前天标记 before_yesterday = datetime.now() - timedelta(days=2) before_yesterday = before_yesterday.strftime('%Y-%m-%d') cloud_before_yesterday_submit = '{}/data/{}_update_mailsys_domain_restrictions.pl'.format( BASE_PATH, before_yesterday ) if os.path.exists(cloud_before_yesterday_submit): os.remove(cloud_before_yesterday_submit) return def mailsys_domain_blecklisted_alarm(): task_ExecShell( "mailsys_domain_blecklisted_alarm", paths_exists=[ "/www/server/panel/plugin/mail_sys/mail_sys_main.py", "/www/vmail", ] ) def update_vulnerabilities(): task_ExecShell("update_vulnerabilities") # 邮件域名邮箱使用限额告警 def mailsys_quota_alarm(): try: if not os.path.exists('/www/server/panel/plugin/mail_sys/mail_sys_main.py') or not os.path.exists( '/www/vmail'): return script = '/www/server/panel/plugin/mail_sys/script/check_quota_alerts.py' if not os.path.exists(script): return cmd = f"btpython {script}" exec_shell(cmd) except: pass # 邮局更新域名邮箱使用量 def mailsys_update_usage(): try: if not os.path.exists( '/www/server/panel/plugin/mail_sys/mail_sys_main.py' ) or not os.path.exists('/www/vmail'): return script = '/www/server/panel/plugin/mail_sys/script/update_usage.py' if not os.path.exists(script): return cmd = f"btpython {script}" exec_shell(cmd) except: pass # 邮局自动回复 def auto_reply_tasks(): task_ExecShell( "auto_reply_tasks", paths_exists=[ "/www/server/panel/plugin/mail_sys/mail_sys_main.py", "/www/vmail", ] ) # 邮局自动扫描异常邮箱 def auto_scan_abnormal_mail(): task_ExecShell( "auto_scan_abnormal_mail", paths_exists=[ "/www/server/panel/plugin/mail_sys/mail_sys_main.py", "/www/vmail", ] ) # 每6小时aa默认ssl检查 def domain_ssl_service(): # check 6h, inside task_ExecShell("make_suer_ssl_task") # 每隔20分钟更新一次网站报表数据 def update_monitor_requests(): task_ExecShell("update_monitor_requests") # 每隔20分钟更新一次waf报表数据 def update_waf_config(): task_ExecShell("update_waf_config") # 每6小时进行恶意文件扫描 def malicious_file_scanning(): task_ExecShell("malicious_file_scanning") # 多服务守护任务，仅在多服务下执行，每5分钟 300 s 检查一次 def multi_web_server_daemon(): task_ExecShell("multi_web_server_daemon") def parse_soft_name_of_version(name): """ @name 获取软件名称和版本 @param name 软件名称 @return tuple(string, string) 返回软件名称和版本 """ if name.find('Docker') != -1: return 'docker', '1.0' return_default = ('', '') l, r = name.find('['), name.find(']') if l == -1 or r == -1 or l > r: return return_default # 去除括号只保留括号中间的软件名称和版本 if name[l + 1:r].count("-") == 0: return return_default soft_name, soft_version = name[l + 1:r].split('-')[:2] if soft_name == 'php': soft_version = soft_version.replace('.', '') return soft_name, soft_version def check_install_status(name: str): """ @name 检查软件是否安装成功 @param name 软件名称 @return tuple(bool, string) 返回是否安装成功和安装信息 """ return_default = (1, 'Installation successful') try: # 获取安装检查配置 install_config = json.loads(read_file("{}/config/install_check.json".format(BASE_PATH))) except: return return_default try: # 获取软件名称和版本 soft_name, soft_version = parse_soft_name_of_version(name) if not soft_name or not soft_version: return return_default if soft_name not in install_config: return return_default if os.path.exists("{}/install/{}_not_support.pl".format(BASE_PATH, soft_name)): return 0, 'Not compatible with this system! Please click on the details to explain!' if os.path.exists("{}/install/{}_mem_kill.pl".format(BASE_PATH, soft_name)): return 0, 'Insufficient memory installation exception! Please click on the details to explain!' soft_config = install_config[soft_name] # 取计算机名 def get_hostname(): try: import socket return socket.gethostname() except: return 'localhost.localdomain' # 替换soft_config中所有变量 def replace_all(dat: str): if not dat: return dat if dat.find('{') == -1: return dat # 替换安装路径, 替换版本号 dat = dat.replace('{SetupPath}', '/www/server').replace('{Version}', soft_version) # 替换主机名 if dat.find("{Host") != -1: host_name = get_hostname() host = host_name.split('.')[0] dat = dat.replace("{Hostname}", host_name) dat = dat.replace("{Host}", host) return dat # 检查文件是否存在 if 'files_exists' in soft_config: for f_name in soft_config['files_exists']: filename = replace_all(f_name) if not os.path.exists(filename): return 0, 'Installation failed, file does not exist:{}'.format(filename) # 检查pid文件是否有效 if 'pid' in soft_config and soft_config['pid']: pid_file = replace_all(soft_config['pid']) if not os.path.exists(pid_file): return 0, 'Startup failed, PID file does not exist:{}'.format(pid_file) pid = read_file(pid_file) if not pid: return 0, 'Startup failed, PID file does not exist:{}'.format(pid_file) proc_file = '/proc/{}/cmdline'.format(pid.strip()) if not os.path.exists(proc_file): return 0, 'Startup failed, PID file is empty: {}({}) process does not exist'.format(pid_file, pid) # 执行命令检查 if 'cmd' in soft_config: for cmd in soft_config['cmd']: p = subprocess.Popen( replace_all(cmd['exec']), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) p.wait() res = p.stdout.read() + "\n" + p.stderr.read() if res.find(replace_all(cmd['success'])) == -1: return 0, '[{}] Abnormal service startup status'.format(soft_name) except: pass return return_default def soft_task(): # 执行面板soft corn之类的安装执行任务, from task.py -> def startTask(): def ExecShell(cmdstring, cwd=None, shell=True, symbol='&>'): try: import shlex import subprocess import time sub = subprocess.Popen( cmdstring + symbol + exlogPath, cwd=cwd, stdin=subprocess.PIPE, shell=shell, bufsize=4096 ) while sub.poll() is None: time.sleep(0.1) return sub.returncode except: return None # def TaskStart(value: dict, start: int = None, time_out: int = None): # """执行安装任务, 并检测是否已经在执行或超时""" # def is_time_out(pid: int) -> bool: # res = int(time.time()) - start > time_out if time_out else False # if res: # try: # p = psutil.Process(pid) # child = p.children(recursive=True) # for c in child: # try: c.kill() # except: continue # p.kill() # except Exception: pass # return res # # start = int(time.time()) if not start else start # try: # output, _ = exec_shell(f"pgrep -f \"{value['execstr'] + '&>' + exlogPath}\"") # pids = output.strip().split() # if pids: # target_pid = int(pids[0]) # sql.table('tasks').where("id=?", (value['id'],)).save('status', (Running,)) # while not is_time_out(target_pid): # try: # os.kill(target_pid, 0) # time.sleep(1) # 残留阻塞 # except ProcessLookupError: break # except Exception: break # if is_time_out(target_pid): # raise Exception # raise Exception # except Exception: # sql.table('tasks').where("id=?", (value['id'],)).save('status,start', (Running, start)) # ExecShell(value['execstr']) tip_file = "/dev/shm/.panelTask.pl" panel_log_path = "/www/server/panel/logs/installed/" if not os.path.exists(panel_log_path): os.mkdir(panel_log_path) os.chmod(panel_log_path, 0o600) Waitting = "0" # 等待 Running = "-1" # 执行中 Finished = "1" # 完成 try: if os.path.exists(isTask): with db.Sql() as sql: # 检测task表是否存在install_status,message字段 field = 'id,type,execstr,name,install_status,message' check_result = sql.table('tasks').order("id desc").field(field).select() if type(check_result) == str: sql.table('tasks').execute("ALTER TABLE 'tasks' ADD 'install_status' INTEGER DEFAULT 1", ()) sql.table('tasks').execute("ALTER TABLE 'tasks' ADD 'message' TEXT DEFAULT ''", ()) sql.table('tasks').where("status=?", (Running,)).setField('status', Waitting) taskArr = sql.table('tasks').where("status=?", (Waitting,)).field('id,type,execstr,name').order( "id asc" ).select() for value in taskArr: if value['type'] != 'execshell': continue if not sql.table('tasks').where("id=?", (value['id'],)).count(): write_file(tip_file, str(int(time.time()))) continue start = int(time.time()) # TaskStart(value, start=start, time_out=3600) sql.table('tasks').where("id=?", (value['id'],)).save('status,start', (Running, start)) ExecShell(value['execstr']) # 保存安装日志 target_log_file = '{}/task_{}.log'.format(panel_log_path, value['id']) shutil.copy(exlogPath, target_log_file) # 检查软件是否安装成功 end = int(time.time()) try: install_status, install_msg = check_install_status(value['name']) except Exception: install_status = 1 install_msg = "" sql.table('tasks').where("id=?", (value['id'],)).save( 'status,end,install_status,message', (Finished, end, install_status, install_msg) ) if sql.table('tasks').where("status=?", (Waitting,)).count() < 1: if os.path.exists(isTask): os.remove(isTask) write_file(tip_file, str(int(time.time()))) except Exception as e: logger.error(f"start_bt_task error: {e}") # 预安装网站监控报表 def check_site_monitor(): task_ExecShell("check_site_monitor") # 节点监控 def node_monitor(): task_ExecShell("node_monitor") # 节点监控 def node_monitor_check(): task_ExecShell("node_monitor_check") # 检测防爆破计划任务 def breaking_through(): task_ExecShell("breaking_through") # 找site favicons def find_favicons(): task_ExecShell( "find_favicons", paths_exists=[ '/www/server/panel/config/auto_favicon.conf', ] ) # 邮件日志 def maillog_event(): task_ExecShell( "maillog_event", paths_exists=[ "/www/server/panel/plugin/mail_sys/mail_sys_main.py", "/www/vmail", ] ) # 邮件处理日志聚合 def aggregate_maillogs_task(): task_ExecShell( "aggregate_maillogs_task", paths_exists=[ "/www/server/panel/plugin/mail_sys/mail_sys_main.py", "/www/vmail", ] ) # 邮件自动化发件任务 def schedule_automations(): task_ExecShell( "schedule_automations", paths_exists=[ "/www/server/panel/plugin/mail_sys/mail_sys_main.py", "/www/vmail", ] ) # 刷新docker app 列表 def refresh_dockerapps(): task_ExecShell("refresh_dockerapps") # 版本更新执行一次性 def task_version_part(): task_ExecShell("task_version_part") # ================================ 这是任务分割线 =============================== TASKS = [ # <核心任务> 面板重启检查, 面板授权检查 {"func": [restart_panel, panel_auth], "interval": 2, "is_core": True}, {"func": soft_task, "interval": 2, "is_core": True}, # 原面板任务 {"func": bt_box_task, "interval": 2, "is_core": True}, # 原面板任务 # <核心任务> 服务守护 {"func": daemon_service, "interval": 10, "is_core": True}, # <核心任务> 原系统监控 {"func": systemTask, "interval": 60, "is_core": True}, # 每1分钟系统监控任务 {"func": project_daemon_service, "interval": 120, "is_core": True}, # 每120秒项目守护 # ================================ 分割线 =============================== # <普通任务> # func 函数, interval 间隔时间秒s, 排队复用线程 (打印请用logger, 日志路径 .../logs/task.log) {"func": push_msg, "interval": 60}, # 每1分钟面板推送消息 {"func": breaking_through, "interval": 60}, # 每分钟防爆破计划任务 {"func": multi_web_server_daemon, "interval": 300}, # 每5分钟多服务守护任务 {"func": check502Task, "interval": 60 * 10}, # 每10分钟 502检查(夹杂若干任务) {"func": check_site_monitor, "interval": 60 * 10}, # 每10分钟检查站点安装监控 {"func": node_monitor, "interval": 60}, # 每1分钟节点监控任务 {"func": node_monitor_check, "interval": 60 * 60 * 24 * 30}, # 每月节点监控检测任务 {"func": update_waf_config, "interval": 60 * 20}, # 每隔20分钟更新一次waf报表数据 {"func": update_monitor_requests, "interval": 60 * 20}, # 每隔20分钟更新一次网站报表数据 {"func": find_favicons, "interval": 43200}, # 每12小时找favicons {"func": domain_ssl_service, "interval": 3600}, # 每6小时进行域名SSL服务(内置时间标记, 可提前检查) {"func": malicious_file_scanning, "interval": 60 * 60 * 6}, # 每每6小时进行恶意文件扫描 {"func": count_ssh_logs, "interval": 3600 * 24}, # 每天统计SSH登录日志 {"func": submit_module_call_statistics, "interval": 3600}, # 每天一次 提交今天之前的统计数据(内置时间标记, 可提前检查) {"func": maillog_event, "interval": 60, "loop": True}, # 邮局日志事件监控 event loop事件, 每60秒一次, 起守护作用 {"func": send_mail_time, "interval": 60 * 3}, # 每3分钟检测邮件信息 {"func": auto_reply_tasks, "interval": 3600}, # 每1小时自动回复邮件 {"func": schedule_automations, "interval": 60}, # 每1分钟邮局自动化任务 {"func": aggregate_maillogs_task, "interval": 60}, # 每1分钟聚合邮局日志 {"func": mailsys_quota_alarm, "interval": 3600 * 2}, # 每2小时邮件域名邮箱使用限额告警 {"func": auto_scan_abnormal_mail, "interval": 3600 * 2}, # 每2小时自动扫描异常邮箱 {"func": mailsys_update_usage, "interval": 3600 * 12}, # 每12小时邮局更新域名邮箱使用量 {"func": submit_email_statistics, "interval": 3600 * 24}, # 每天一次 昨日邮件发送统计 {"func": mailsys_domain_blecklisted_alarm, "interval": 3600 * 24}, # # 每天一次 邮局黑名单检测 {"func": update_vulnerabilities, "interval": 3600 * 24}, # # 每天一次 更新漏洞信息 {"func": refresh_dockerapps, "interval": 3600 * 24}, # # 每天一次 更新docker app 列表 ] def thread_register(brain: SimpleBrain, is_core: bool = True): if not is_core: # delay normal tasks logger.info("Normal Task will be join active after 30s") time.sleep(30) for index, task in enumerate(TASKS): try: if task.get("is_core", False) == is_core: if isinstance(task["func"], list): task_id = "_".join([f.__name__ for f in task["func"]]) else: task_id = task.get("id", task["func"].__name__) # delay normal tasks, 削峰 if not is_core: time.sleep(10) brain.register_task( func=task["func"], task_id=task_id, interval=task.get("interval", 3600), is_core=task.get("is_core", False), loop=task.get("loop", False), ) except Exception: import traceback logger.error(f"Register task {task} failed: {traceback.format_exc()}") continue logger.info( f"All the {'[Core]' if is_core else '[Normal]'} tasks have been registered." ) def main(max_workers: int = None): main_pid = "logs/task.pid" if os.path.exists(main_pid): os.system("kill -9 $(cat {}) &> /dev/null".format(main_pid)) pid = os.fork() if pid: sys.exit(0) os.setsid() _pid = os.fork() if _pid: write_file(main_pid, str(_pid)) sys.exit(0) sys.stdout.flush() sys.stderr.flush() logger.info("Service Up") time.sleep(5) task_version_part() # =================== Start =========================== sb = SimpleBrain(cpu_max=20.0, workers=max_workers) try: # core tasks thread_register(brain=sb, is_core=True) # normal tasks will be delayed threading.Thread( target=thread_register, args=(sb, False), daemon=True ).start() sb.run() except Exception: import traceback logger.error(traceback.format_exc()) sb._shutdown() # =================== End ======================== if __name__ == "__main__": main()