#!/bin/python
# coding: utf-8
# +-------------------------------------------------------------------
# | YakPanel
# +-------------------------------------------------------------------
# | Copyright (c) 2015-2016 YakPanel(www.yakpanel.com) All rights reserved.
# +-------------------------------------------------------------------
# | Author: yakpanel
# +-------------------------------------------------------------------

# ------------------------------
# YakPanel background scheduled tasks
# ------------------------------
import gc
import os
import shutil
import sqlite3
import subprocess
import sys
import threading
import time
import traceback
from datetime import datetime, timedelta
from typing import Optional, Tuple

import psutil

try:
    import ujson as json
except ImportError:
    try:
        os.system("btpip install ujson")
        import ujson as json
    except:
        import json

sys.path.insert(0, "/www/server/panel/class/")

try:
    from public.hook_import import hook_import

    hook_import()
except:
    pass

import db
from panelTask import bt_task
from script.restart_services import RestartServices
from YakTask.brain import SimpleBrain
from YakTask.conf import (
    BASE_PATH,
    PYTHON_BIN,
    exlogPath,
    isTask,
    logger,
)


def write_file(path: str, content: str, mode='w'):
    try:
        fp = open(path, mode)
        fp.write(content)
        fp.close()
        return True
    except:
        try:
            fp = open(path, mode, encoding="utf-8")
            fp.write(content)
            fp.close()
            return True
        except:
            return False


def read_file(filename: str):
    fp = None
    try:
        fp = open(filename, "rb")
        f_body_bytes: bytes = fp.read()
        f_body = f_body_bytes.decode("utf-8", errors='ignore')
        fp.close()
        return f_body
    except Exception:
        return False
    finally:
        if fp and not fp.closed:
            fp.close()


def exec_shell(cmdstring, timeout=None, shell=True, cwd=None):
    """
        @name 执行命令
        @param cmdstring 命令 [必传]
        @param timeout 超时时间
        @param shell 是否通过shell运行
        @return 命令执行结果
    """
    try:
        result = subprocess.run(
            cmdstring,
            shell=shell,
            cwd=cwd,
            timeout=timeout,
            capture_output=True,
            text=True,  # 直接以文本模式处理输出
            encoding='utf-8',
            errors='ignore',
            env=os.environ
        )
        return result.stdout, result.stderr
    except subprocess.TimeoutExpired:
        return 'Timed out', ''
    except Exception:
        return '', traceback.format_exc()


task_obj = bt_task()
task_obj.not_web = True
bt_box_task = task_obj.start_task_new


def task_ExecShell(fucn_name: str, **kw):
    """
    仅运行 /www/server/panel/YakTask/task_script.py 下的包装函数
    可通过 kw 参数扩展前置检查，例如:
      - kw['paths_exists']: List [str]
      (例如, 检查邮局插件是否存在, 任何一个检查不存在则不执行
      paths_exists=['/www/server/panel/plugin/mail_sys/mail_sys_main.py', '/www/vmail'])

    """
    if PYTHON_BIN in fucn_name:
        raise ValueError("valid function name required")

    if kw.get("paths_exists") and isinstance(kw["paths_exists"], list):
        for p in kw["paths_exists"]:
            try:
                if not os.path.exists(str(p)):
                    logger.debug(f"Skip task [{fucn_name}]: path not exists")
                    return
            except Exception as e:
                raise ValueError(f"Invalid path in paths_exists: '{p}', error: {e}")

    cmd = f"{PYTHON_BIN} /www/server/panel/YakTask/task_script.py {fucn_name}"
    _, err = exec_shell(cmd)
    if err:
        raise Exception(err)


# 系统监控全局缓存
_system_task_state = {
    "table_ensure": False,
    # {"timestamp": 0, "total_up": 0, "total_down": 0, "up_packets": {}, "down_packets": {}}
    "last_network_io": {},
    # {"timestamp": 0, "read_count": 0, "write_count": 0, "read_bytes": 0, "write_bytes": 0, "read_time": 0, "write_time": 0}
    "last_disk_io": {},
    # {pid: (create_time, cpu_time, disk_read, disk_write, timestamp)}
    "last_process_cache": {},
    # {pid: (inactive_count, last_check_time)}
    "inactive_process_cache": {},
    "last_clear_time": 0,
    "last_cpu_times": None,
}


# 系统监控任务
def systemTask():
    def get_cpu_percent_smooth() -> float:
        """窗口时间内CPU占用率"""
        try:
            current_times = psutil.cpu_times()
            last_times = _system_task_state.get("last_cpu_times")
            if not last_times:
                _system_task_state["last_cpu_times"] = current_times
                return psutil.cpu_percent(interval=0.1)
            all_delta = sum(current_times) - sum(last_times)
            if all_delta == 0.0:
                return 0.0
            idle_delta = getattr(current_times, "idle", 0) - getattr(last_times, "idle", 0)
            cpu_percent = ((all_delta - idle_delta) / all_delta) * 100
            _system_task_state["last_cpu_times"] = current_times
            return max(0.0, min(100.0, cpu_percent))
        except Exception:
            return psutil.cpu_percent(interval=0.1)

    def get_mem_used_percent() -> float:
        """内存使用率"""
        try:
            mem = psutil.virtual_memory()
            total = mem.total / 1024 / 1024
            free = mem.free / 1024 / 1024
            buffers = getattr(mem, "buffers", 0) / 1024 / 1024
            cached = getattr(mem, "cached", 0) / 1024 / 1024
            used = total - free - buffers - cached
            return used / (total / 100.0) if total else 1.0
        except Exception:
            return 1.0

    # noinspection PyUnusedLocal,PyTypeChecker
    def get_swap_used_percent() -> float:
        """
        获取Swap内存已占用的百分比（返回0.0~100.0，异常时返回100.0）
        逻辑：Swap使用率 = (已使用Swap / 总Swap容量) * 100%
        """
        try:
            # 获取Swap内存信息（psutil.swap_memory()返回namedtuple）
            swap = psutil.swap_memory()

            # Swap总容量（bytes → MB，和物理内存计算单位保持一致）
            swap_total = swap.total / 1024 / 1024
            # Swap已使用量（bytes → MB）
            swap_used = swap.used / 1024 / 1024

            swap_free = swap.free / 1024 / 1024

            # 避免除以0（无Swap分区时）
            if swap_total == 0:
                return 0.0, 0, 0, 0  # 无Swap时默认返回100%（或根据需求改0.0）

            # 计算使用率（保留2位小数，确保返回float类型）
            used_percent = round((swap_used / swap_total) * 100.0, 2)

            # 边界值修正（防止因系统浮点误差导致超过100%）
            # return min(used_percent, 100.0),swap.total/1024,swap.used/1024,swap.free/1024
            return min(used_percent, 100.0), swap.total, swap.used, swap.free

        # 捕获所有异常，返回100%（和物理内存函数的异常返回逻辑一致）
        except Exception:
            return 100.0, 0, 0, 0

    def get_load_average() -> Tuple[float, float, float, float]:
        """负载平均"""
        try:
            one, five, fifteen = os.getloadavg()
            max_v = psutil.cpu_count() * 2
            lpro = round((one / max_v) * 100, 2) if max_v else 0
            if lpro > 100:
                lpro = 100
            return lpro, float(one), float(five), float(fifteen)
        except Exception:
            return 0.0, 0.0, 0.0, 0.0

    def get_network_io() -> Optional[dict]:
        """网络IO"""
        try:
            network_io = psutil.net_io_counters(pernic=True)
            ret = {
                'total_up': sum(v.bytes_sent for v in network_io.values()),
                'total_down': sum(v.bytes_recv for v in network_io.values()),
                'timestamp': time.time(),
                'down_packets': {k: v.bytes_recv for k, v in network_io.items()},
                'up_packets': {k: v.bytes_sent for k, v in network_io.items()}
            }

            last_io = _system_task_state["last_network_io"]
            if not last_io:
                _system_task_state["last_network_io"] = ret
                return None

            diff_t = (ret["timestamp"] - last_io.get("timestamp", 0)) * 1024  # 转KB
            if diff_t <= 0:
                return None

            res = {
                'up': round((ret['total_up'] - last_io.get('total_up', 0)) / diff_t, 2),
                'down': round((ret['total_down'] - last_io.get('total_down', 0)) / diff_t, 2),
                'total_up': ret['total_up'],
                'total_down': ret['total_down'],
                'down_packets': {
                    k: round((v - last_io.get('down_packets', {}).get(k, 0)) / diff_t, 2)
                    for k, v in ret['down_packets'].items()
                },
                'up_packets': {
                    k: round((v - last_io.get('up_packets', {}).get(k, 0)) / diff_t, 2)
                    for k, v in ret['up_packets'].items()
                }
            }
            _system_task_state["last_network_io"] = ret
            return res
        except Exception:
            return None

    def get_disk_io() -> Optional[dict]:
        """磁盘IO"""
        if not os.path.exists('/proc/diskstats'):
            return None
        try:
            disk_io = psutil.disk_io_counters()
            if not disk_io:
                return None

            ret = {
                'read_count': disk_io.read_count,
                'write_count': disk_io.write_count,
                'read_bytes': disk_io.read_bytes,
                'write_bytes': disk_io.write_bytes,
                'read_time': disk_io.read_time,
                'write_time': disk_io.write_time,
                'timestamp': time.time()
            }

            last_io = _system_task_state["last_disk_io"]
            if not last_io:
                _system_task_state["last_disk_io"] = ret
                return None

            diff_t = ret["timestamp"] - last_io.get("timestamp", 0)
            if diff_t <= 0:
                return None

            res = {
                'read_count': int((ret["read_count"] - last_io.get("read_count", 0)) / diff_t),
                'write_count': int((ret["write_count"] - last_io.get("write_count", 0)) / diff_t),
                'read_bytes': int((ret["read_bytes"] - last_io.get("read_bytes", 0)) / diff_t),
                'write_bytes': int((ret["write_bytes"] - last_io.get("write_bytes", 0)) / diff_t),
                'read_time': int((ret["read_time"] - last_io.get("read_time", 0)) / diff_t),
                'write_time': int((ret["write_time"] - last_io.get("write_time", 0)) / diff_t),
            }
            _system_task_state["last_disk_io"] = ret
            return res
        except Exception:
            return None

    _XSS_TRANS = str.maketrans({
        '&': '&amp;',
        '<': '&lt;',
        '>': '&gt;',
        '"': '&quot;',
        "'": '&#x27;'
    })

    def xss_encode(s: str) -> str:
        """XSS"""
        if not isinstance(s, str):
            s = str(s)
        if not any(c in s for c in '&<>"\''):
            return s
        return s.translate(_XSS_TRANS)

    def cut_top_list(process_list, *sort_key, top_num=5) -> list:
        # """最小堆取前N"""
        # import heapq
        # if not process_list or not sort_key:
        #     return []
        # try:
        #     if len(sort_key) == 1:
        #         return heapq.nlargest(
        #             top_num, process_list, key=lambda x: x.get(sort_key[0], 0)
        #         )
        #     else:
        #         return heapq.nlargest(
        #             top_num, process_list, key=lambda x: tuple(x.get(k, 0) for k in sort_key)
        #         )
        # except (TypeError, ValueError):
        #     return []
        """前N个"""
        if not process_list or not sort_key:
            return []
        tops = sorted(
            process_list, key=lambda x: tuple(x.get(k, 0) for k in sort_key), reverse=True
        )
        return tops[:top_num]

    def get_process_list() -> list:
        """获取进程列表"""
        SKIP_NAMES = {
            'edac-poller', 'devfreq_wq', 'watchdogd', 'kthrotld', 'acpi_thermal_pm', 'charger_manager',
            'kthreadd', 'rcu_gp', 'rcu_par_gp', 'rcu_sched', 'migration/0', 'cpuhp/0', 'kdevtmpfs',
            'netns', 'oom_reaper', 'writeback', 'crypto', 'kintegrityd', 'kblockd', 'ata_sff',
        }

        try:
            pids = psutil.pids()
            current_pid = os.getpid()
            timer = getattr(time, "monotonic", time.time)
            cpu_num = psutil.cpu_count() or 1
            process_list = []
            new_cache = {}
            inactive_cache = _system_task_state["inactive_process_cache"]
            for pid in pids:
                if pid == current_pid:
                    continue
                if pid < 10:  # 核心进程
                    continue
                if pid in inactive_cache:
                    inactive_count, last_check = inactive_cache[pid]
                    # 5分钟重新检查一次
                    if time.time() - last_check < 300 and inactive_count >= 3:
                        inactive_cache[pid] = (inactive_count, time.time())  # 结合后续10分钟清理逻辑
                        continue

                try:
                    p = psutil.Process(pid)
                    if p.status() in (psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD):  # 休眠/僵尸
                        continue
                    process_name = None
                    try:
                        process_name = p.name()
                        if process_name in SKIP_NAMES:
                            continue
                    except:
                        pass

                    with p.oneshot():
                        create_time = p.create_time()
                        if not process_name:
                            try:
                                process_name = p.name()
                            except:
                                process_name = "unknown"
                        last_cache = _system_task_state["last_process_cache"].get(pid)
                        if last_cache and last_cache[0] != create_time:  # 进程应该被重启了
                            last_cache = None
                            if pid in inactive_cache:
                                del inactive_cache[pid]

                        p_cpu_time = p.cpu_times()
                        cpu_time = p_cpu_time.system + p_cpu_time.user
                        io_counters = p.io_counters()
                        memory_info = p.memory_info()
                        current_time = timer()
                        # 刷新缓存
                        new_cache[pid] = (
                            create_time, cpu_time, io_counters.read_bytes, io_counters.write_bytes, current_time
                        )
                        if not last_cache:  # 初次处理的进程
                            continue
                        diff_t = current_time - last_cache[4]
                        if diff_t <= 0:
                            continue

                        cpu_percent = max(round((cpu_time - last_cache[1]) * 100 / diff_t / cpu_num, 2), 0)
                        disk_read = max(0, int((io_counters.read_bytes - last_cache[2]) / diff_t))
                        disk_write = max(0, int((io_counters.write_bytes - last_cache[3]) / diff_t))
                        disk_total = disk_read + disk_write

                        if cpu_percent == 0 and disk_total == 0:
                            if pid in inactive_cache:
                                inactive_cache[pid] = (inactive_cache[pid][0] + 1, time.time())
                            else:
                                inactive_cache[pid] = (1, time.time())
                            continue

                        if pid in inactive_cache:
                            del inactive_cache[pid]

                        # swap占用
                        try:
                            swap = p.memory_full_info().swap
                        except:
                            swap = 0
                        # connect_count = len(
                        #     p.net_connections() if hasattr(p, "net_connections") else p.connections()  # noqa
                        # )
                        process_info = {
                            'pid': pid,
                            'name': process_name,
                            'username': p.username(),
                            'cpu_percent': cpu_percent,
                            'memory': memory_info.rss,
                            'swap': swap,
                            'disk_read': disk_read,
                            'disk_write': disk_write,
                            'disk_total': disk_total or 0,
                            'cmdline': ' '.join(filter(lambda x: x, p.cmdline()))[:500],
                            'create_time': create_time,
                            'connect_count': 0,  # future
                            'net_total': 0,  # future
                            'up': 0,  # future
                            'down': 0,  # future
                            'up_package': 0,  # future
                            'down_package': 0,  # future
                        }
                        # process_info["net_total"] = process_info["up"] + process_info["down"]
                        process_list.append(process_info)
                except Exception:
                    continue

            current_time = time.time()
            inactive_cache_copy = dict(inactive_cache)
            for pid, (count, last_check) in inactive_cache_copy.items():
                remove_flag = False
                if current_time - last_check > 600:  # 超过 10 分钟
                    remove_flag = True
                else:  # 进程残留, pid被复用
                    try:
                        current_create = psutil.Process(pid).create_time()
                        exist_create = None
                        if pid in _system_task_state["last_process_cache"]:
                            exist_create = _system_task_state["last_process_cache"][pid][0]
                        elif pid in new_cache:
                            exist_create = new_cache[pid][0]
                        if exist_create and current_create != exist_create:
                            remove_flag = True  # pid被复用
                    except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
                        remove_flag = True

                if remove_flag and pid in inactive_cache:
                    del inactive_cache[pid]

            _system_task_state["last_process_cache"] = new_cache
            _system_task_state["inactive_process_cache"] = inactive_cache
            return process_list

        except Exception:
            return []

    # 暂无用
    # def start_process_net_total(self):
    #     # 进程流量监控，如果文件：/www/server/panel/data/is_net_task.pl 或 /www/server/panel/data/control.conf不存在，则不监控进程流量
    #     if not (os.path.isfile(self.proc_net_service) and os.path.isfile(self.base_service)):
    #         return
    #
    #     def process_net_total():
    #         class_path = '{}/class'.format(BASE_PATH)
    #         if class_path not in sys.path:
    #             sys.path.insert(0, class_path)
    #         import process_task
    #         process_task.process_network_total().start()
    #
    #     import threading
    #     th = threading.Thread(target=process_net_total, daemon=True)
    #     th.start()
    def ensure_table(db_file: str) -> None:
        """表, 字段处理"""
        if not os.path.isfile(db_file):
            os.makedirs(os.path.dirname(db_file), exist_ok=True)
            open(db_file, 'w').close()
        conn = None
        cursor = None
        init_sql = '''
CREATE TABLE IF NOT EXISTS `cpuio` (
    `id` INTEGER PRIMARY KEY AUTOINCREMENT,
    `pro` INTEGER, `mem` INTEGER,
    `swap_percent` INTEGER, `swap_total` INTEGER, `swap_used` INTEGER, `swap_free` INTEGER,
    `addtime` INTEGER
);

CREATE TABLE IF NOT EXISTS `network` (
    `id` INTEGER PRIMARY KEY AUTOINCREMENT,
    `up` INTEGER, `down` INTEGER, `total_up` INTEGER, `total_down` INTEGER,
    `down_packets` INTEGER, `up_packets` INTEGER, `addtime` INTEGER
);

CREATE TABLE IF NOT EXISTS `diskio` (
    `id` INTEGER PRIMARY KEY AUTOINCREMENT,
    `read_count` INTEGER, `write_count` INTEGER,`read_bytes` INTEGER, `write_bytes` INTEGER,
    `read_time` INTEGER, `write_time` INTEGER, `addtime` INTEGER
);

CREATE TABLE IF NOT EXISTS `load_average` (
    `id` INTEGER PRIMARY KEY AUTOINCREMENT,
    `pro` REAL, `one` REAL, `five` REAL, `fifteen` REAL, `addtime` INTEGER
);

CREATE TABLE IF NOT EXISTS `process_top_list` (
    `id` INTEGER PRIMARY KEY AUTOINCREMENT,
    `cpu_top` REAL, `memory_top` REAL, `disk_top` REAL, `net_top` REAL, `all_top` REAL,
    `swap_top` REAL,
    `addtime` INTEGER
);

-- 索引
CREATE INDEX IF NOT EXISTS idx_cpuio_addtime ON cpuio (addtime);
CREATE INDEX IF NOT EXISTS idx_network_addtime ON network (addtime);
CREATE INDEX IF NOT EXISTS idx_diskio_addtime ON diskio (addtime);
CREATE INDEX IF NOT EXISTS idx_load_average_addtime ON load_average (addtime);
CREATE INDEX IF NOT EXISTS idx_process_top_list_addtime ON process_top_list (addtime);
'''
        try:
            conn = sqlite3.connect(db_file)
            try:
                conn.executescript(init_sql)
                conn.commit()
            except Exception as e:
                logger.error("Failed to initialize system.db : {}".format(e))

            # 检测cpuio表是否存在swap字段
            cursor = conn.cursor()
            try:
                cursor.execute(
                    "SELECT swap_total, swap_percent, swap_used, swap_free FROM cpuio ORDER BY id DESC LIMIT 1"
                )
            except sqlite3.OperationalError:
                # swap字段不存在
                alter_sql = '''
    ALTER TABLE cpuio ADD COLUMN swap_percent INTEGER DEFAULT 0;
    ALTER TABLE cpuio ADD COLUMN swap_total INTEGER DEFAULT 0;
    ALTER TABLE cpuio ADD COLUMN swap_used INTEGER DEFAULT 0;
    ALTER TABLE cpuio ADD COLUMN swap_free INTEGER DEFAULT 0;
    '''
                try:
                    conn.executescript(alter_sql)
                    conn.commit()
                except sqlite3.OperationalError:
                    pass

            # 检测process_top_list表是否存在swap_top字段
            try:
                cursor.execute("SELECT swap_top FROM process_top_list ORDER BY id DESC LIMIT 1")
            except sqlite3.OperationalError:
                try:
                    conn.execute("ALTER TABLE process_top_list ADD COLUMN swap_top REAL DEFAULT []")
                    conn.commit()
                except sqlite3.OperationalError:
                    pass
            _system_task_state["table_ensure"] = True
        except Exception as e:
            logger.error("Failed to connect to system.db : {}".format(e))
            _system_task_state["table_ensure"] = False
        finally:
            if cursor: cursor.close()
            if conn: conn.close()

    def clear_expire_data(conn: sqlite3.Connection) -> None:
        if not _system_task_state.get("last_clear_time"):
            return
        now = int(time.time())
        # 检查是否需要清理（每小时一次）
        if now - _system_task_state.get("last_clear_time", 0) < 3600:
            return
        cur = None
        try:
            cur = conn.cursor()
            deltime = now - (keep_days * 86400)
            cur.execute("DELETE FROM cpuio WHERE addtime < ?", (deltime,))
            cur.execute("DELETE FROM network WHERE addtime < ?", (deltime,))
            cur.execute("DELETE FROM diskio WHERE addtime < ?", (deltime,))
            cur.execute("DELETE FROM load_average WHERE addtime < ?", (deltime,))
            cur.execute("DELETE FROM process_top_list WHERE addtime < ?", (deltime,))
            conn.commit()
            _system_task_state["last_clear_time"] = now
        except:
            pass
        finally:
            if cur: cur.close()

    control_conf = f"{BASE_PATH}/data/control.conf"
    db_file = f"{BASE_PATH}/data/system.db"

    # 表结构检查
    if not _system_task_state["table_ensure"]:
        ensure_table(db_file)

    # 是否启用监控, 不存在时为不开启监控, 存在时为监控天数
    if not os.path.exists(control_conf):
        return

    try:
        keep_days = int(read_file(control_conf) or 30)
        if keep_days < 1:
            return
    except Exception:
        keep_days = 30
    conn = None
    cursor = None

    try:
        conn = sqlite3.connect(db_file, timeout=3)
        cursor = conn.cursor()
        cpu_used = get_cpu_percent_smooth()
        mem_used = get_mem_used_percent()
        swap_percent, swap_total, swap_used, swap_free = get_swap_used_percent()
        lpro, one, five, fifteen = get_load_average()
        network_io = get_network_io()
        disk_io = get_disk_io()
        process_list = get_process_list()
        proc_net_service = f"{BASE_PATH}/data/is_net_task.pl"
        addtime = int(time.time())

        # cpu, swap
        cursor.execute(
            "INSERT INTO cpuio (pro, mem, swap_percent, swap_total, swap_used, swap_free, addtime) VALUES (?, ?, ?, ?, ?, ?, ?)",
            (cpu_used, mem_used, swap_percent, swap_total, swap_used, swap_free, addtime)
        )
        # load
        cursor.execute(
            "INSERT INTO load_average (pro, one, five, fifteen, addtime) VALUES (?, ?, ?, ?, ?)",
            (lpro, one, five, fifteen, addtime)
        )
        # network io
        if network_io:
            cursor.execute(
                "INSERT INTO network (up, down, total_up, total_down, down_packets, up_packets, addtime) VALUES (?, ?, ?, ?, ?, ?, ?)",
                (
                    network_io['up'], network_io['down'],
                    network_io['total_up'], network_io['total_down'],
                    json.dumps(network_io['down_packets']),
                    json.dumps(network_io['up_packets']),
                    addtime
                )
            )

        # disk io
        if disk_io:
            cursor.execute(
                "INSERT INTO diskio (read_count, write_count, read_bytes, write_bytes, read_time, write_time, addtime) VALUES (?, ?, ?, ?, ?, ?, ?)",
                (
                    disk_io['read_count'], disk_io['write_count'],
                    disk_io['read_bytes'], disk_io['write_bytes'],
                    disk_io['read_time'], disk_io['write_time'],
                    addtime
                )
            )

        # process top list
        if process_list:
            all_top_list = cut_top_list(
                process_list, 'cpu_percent', 'disk_total', 'memory', 'net_total', top_num=5
            )
            cpu_top_list = cut_top_list(process_list, 'cpu_percent', top_num=5)
            disk_top_list = cut_top_list(process_list, 'disk_total', top_num=5)
            memory_top_list = cut_top_list(process_list, 'memory', top_num=5)
            swap_top_list = cut_top_list(process_list, 'swap', top_num=5)
            # net_top_list = all_top_list

            if os.path.isfile(proc_net_service):  # 进程流量监控, 暂无用
                # net_top_list = top_lists['net_top']
                pass

            all_top = json.dumps([(
                p['cpu_percent'], p['disk_read'], p['disk_write'], p['memory'], p['up'], p['down'], p['pid'],
                xss_encode(p['name']), xss_encode(p['cmdline']), xss_encode(p['username']),
                p['create_time']
            ) for p in all_top_list])

            cpu_top = json.dumps([(
                p['cpu_percent'], p['pid'], xss_encode(p['name']), xss_encode(p['cmdline']),
                xss_encode(p['username']), p['create_time']
            ) for p in cpu_top_list])

            disk_top = json.dumps([(
                p['disk_total'], p['disk_read'], p['disk_write'], p['pid'], xss_encode(p['name']),
                xss_encode(p['cmdline']), xss_encode(p['username']), p['create_time']
            ) for p in disk_top_list])

            # net_top = json.dumps([(
            #     p['net_total'], p['up'], p['down'], p['connect_count'], p['up_package'] + p['down_package'],
            #     p['pid'], xss_encode(p['name']), xss_encode(p['cmdline']), xss_encode(p['username']),
            #     p['create_time']
            # ) for p in net_top_list])
            net_top = None

            memory_top = json.dumps([(
                p['memory'], p['pid'], xss_encode(p['name']), xss_encode(p['cmdline']),
                xss_encode(p['username']), p['create_time']
            ) for p in memory_top_list])

            swap_top = json.dumps([(
                p['swap'], p['pid'], xss_encode(p['name']), xss_encode(p['cmdline']),
                xss_encode(p['username']), p['create_time']
            ) for p in swap_top_list])

            cursor.execute(
                "INSERT INTO process_top_list (all_top, cpu_top, disk_top, net_top, memory_top, swap_top, addtime) VALUES (?, ?, ?, ?, ?, ?, ?)",
                (all_top, cpu_top, disk_top, net_top, memory_top, swap_top, addtime)
            )

        conn.commit()
        # every 1h check clear old data
        clear_expire_data(conn)
    except sqlite3.OperationalError as e:
        logger.error(f"SQLite OperationalError in systemTask: {e}")
        ensure_table(db_file)
        _system_task_state["table_ensure"] = False
    except Exception:
        logger.error(f"systemTask error: {traceback.format_exc()}")
    finally:
        if cursor: cursor.close()
        if conn: conn.close()
        gc.collect()


def check502Task():
    task_ExecShell("check502task")


# 服务守护
def daemon_service():
    try:
        obj = RestartServices()
        obj.main()
        del obj
    finally:
        gc.collect()


# 项目守护
def project_daemon_service():
    task_ExecShell("project_daemon_service")


# 重启面板服务
def restart_panel():
    def service_panel(action='reload'):
        if not os.path.exists('{}/init.sh'.format(BASE_PATH)):
            os.system("curl -k https://node.yakpanel.com/install/update_7.x_en.sh|bash &")
        else:
            os.system("nohup bash /www/server/panel/init.sh {} > /dev/null 2>&1 &".format(action))
        logger.info("Panel Service: {}".format(action))

    rtips = '{}/data/restart.pl'.format(BASE_PATH)
    reload_tips = '{}/data/reload.pl'.format(BASE_PATH)

    if os.path.exists(rtips):
        os.remove(rtips)
        service_panel('restart')
    if os.path.exists(reload_tips):
        os.remove(reload_tips)
        service_panel('reload')


# 定时任务去检测邮件信息
def send_mail_time():
    if not os.path.exists('/www/server/panel/plugin/mail_sys/mail_sys_main.py') or not os.path.exists('/www/vmail'):
        return
    exec_shell("{} /www/server/panel/script/mail_task.py".format(PYTHON_BIN))


# 面板推送消息
def push_msg():
    def _read_file(file_path: str) -> Optional[list]:
        if not os.path.exists(file_path):
            return None
        content = read_file(file_path)
        if not content:
            return None
        try:
            return json.loads(content)
        except:
            return []

    sender_path = f"{BASE_PATH}/data/mod_push_data/sender.json"
    task_path = f"{BASE_PATH}/data/mod_push_data/task.json"
    sender_info = _read_file(sender_path) or []
    work = False
    for s in sender_info:
        # default sender_type sms data is {}
        if s.get("sender_type") != "sms" and s.get("data"):
            work = True
            break
    if not work:
        return
    if not _read_file(task_path):
        return
    task_ExecShell("push_msg")


# 检测面板授权
# noinspection PyUnboundLocalVariable
def panel_auth():
    pro_file = '/www/server/panel/data/panel_pro.pl'
    update_file = '/www/server/panel/data/now_update_pro.pl'
    if os.path.exists(pro_file):
        try:
            from YakPanel import cache
        except Exception as e:
            logger.error("Failed to import cache from YakPanel: {}".format(e))
            cache = None
        if cache:
            key = 'pro_check_sdfjslk'
            res = cache.get(key)
        if os.path.exists(update_file) or res is None:
            os.system('nohup {} /www/server/panel/script/check_auth.py > /dev/null 2>&1 &'.format(PYTHON_BIN))
            if cache:
                cache.set(key, 'sddsf', 3600)
    if os.path.exists(update_file):
        os.remove(update_file)


def count_ssh_logs():
    task_ExecShell("count_ssh_logs")


# 每天提交一次昨天的邮局发送总数
def submit_email_statistics():
    task_ExecShell(
        "submit_email_statistics",
        paths_exists=[
            "/www/server/panel/plugin/mail_sys/mail_sys_main.py",
            "/www/vmail",
        ])


# 每天一次 提交今天之前的统计数据
def submit_module_call_statistics():
    task_ExecShell("submit_module_call_statistics")


def mailsys_domain_restrictions():
    if not os.path.exists('/www/server/panel/plugin/mail_sys/mail_send_bulk.py'):
        return

    if not os.path.exists('/www/vmail'):
        return

    yesterday = datetime.now() - timedelta(days=1)
    yesterday = yesterday.strftime('%Y-%m-%d')
    cloud_yesterday_submit = '{}/data/{}_update_mailsys_domain_restrictions.pl'.format(
        BASE_PATH, yesterday
    )
    if os.path.exists(cloud_yesterday_submit):
        return

    if os.path.exists("/www/server/panel/plugin/mail_sys"):
        sys.path.insert(1, "/www/server/panel/plugin/mail_sys")

    # 检查版本 检查是否能查询额度  剩余额度
    import public.PluginLoader as plugin_loader
    bulk = plugin_loader.get_module('{}/plugin/mail_sys/mail_send_bulk.py'.format(BASE_PATH))
    SendMailBulk = bulk.SendMailBulk
    try:
        SendMailBulk()._get_user_quota()
    except:
        logger.error(traceback.format_exc())
        return

    # 添加标记
    write_file(cloud_yesterday_submit, '1')
    # 删除前天标记
    before_yesterday = datetime.now() - timedelta(days=2)
    before_yesterday = before_yesterday.strftime('%Y-%m-%d')
    cloud_before_yesterday_submit = '{}/data/{}_update_mailsys_domain_restrictions.pl'.format(
        BASE_PATH, before_yesterday
    )
    if os.path.exists(cloud_before_yesterday_submit):
        os.remove(cloud_before_yesterday_submit)
    return


def mailsys_domain_blecklisted_alarm():
    task_ExecShell(
        "mailsys_domain_blecklisted_alarm",
        paths_exists=[
            "/www/server/panel/plugin/mail_sys/mail_sys_main.py",
            "/www/vmail",
        ]
    )


def update_vulnerabilities():
    task_ExecShell("update_vulnerabilities")


# 邮件域名邮箱使用限额告警
def mailsys_quota_alarm():
    try:
        if not os.path.exists('/www/server/panel/plugin/mail_sys/mail_sys_main.py') or not os.path.exists(
                '/www/vmail'):
            return

        script = '/www/server/panel/plugin/mail_sys/script/check_quota_alerts.py'
        if not os.path.exists(script):
            return

        cmd = f"btpython {script}"
        exec_shell(cmd)
    except:
        pass


# 邮局更新域名邮箱使用量
def mailsys_update_usage():
    try:
        if not os.path.exists(
                '/www/server/panel/plugin/mail_sys/mail_sys_main.py'
        ) or not os.path.exists('/www/vmail'):
            return
        script = '/www/server/panel/plugin/mail_sys/script/update_usage.py'
        if not os.path.exists(script):
            return
        cmd = f"btpython {script}"
        exec_shell(cmd)
    except:
        pass


# 邮局自动回复
def auto_reply_tasks():
    task_ExecShell(
        "auto_reply_tasks",
        paths_exists=[
            "/www/server/panel/plugin/mail_sys/mail_sys_main.py",
            "/www/vmail",
        ]
    )


# 邮局自动扫描异常邮箱
def auto_scan_abnormal_mail():
    task_ExecShell(
        "auto_scan_abnormal_mail",
        paths_exists=[
            "/www/server/panel/plugin/mail_sys/mail_sys_main.py",
            "/www/vmail",
        ]
    )


# 每6小时aa默认ssl检查
def domain_ssl_service():
    # check 6h, inside
    task_ExecShell("make_suer_ssl_task")


# 每隔20分钟更新一次网站报表数据
def update_monitor_requests():
    task_ExecShell("update_monitor_requests")


# 每隔20分钟更新一次waf报表数据
def update_waf_config():
    task_ExecShell("update_waf_config")


# 每6小时进行恶意文件扫描
def malicious_file_scanning():
    task_ExecShell("malicious_file_scanning")


# 多服务守护任务，仅在多服务下执行，每5分钟 300 s 检查一次
def multi_web_server_daemon():
    task_ExecShell("multi_web_server_daemon")


def parse_soft_name_of_version(name):
    """
        @name 获取软件名称和版本
        @param name<string> 软件名称
        @return tuple(string, string) 返回软件名称和版本
    """
    if name.find('Docker') != -1:
        return 'docker', '1.0'
    return_default = ('', '')
    l, r = name.find('['), name.find(']')
    if l == -1 or r == -1 or l > r:
        return return_default
    # 去除括号只保留括号中间的软件名称和版本
    if name[l + 1:r].count("-") == 0:
        return return_default
    soft_name, soft_version = name[l + 1:r].split('-')[:2]
    if soft_name == 'php':
        soft_version = soft_version.replace('.', '')
    return soft_name, soft_version


def check_install_status(name: str):
    """
    @name 检查软件是否安装成功
    @param name<string> 软件名称
    @return tuple(bool, string) 返回是否安装成功和安装信息
    """
    return_default = (1, 'Installation successful')
    try:
        # 获取安装检查配置
        install_config = json.loads(read_file("{}/config/install_check.json".format(BASE_PATH)))
    except:
        return return_default
    try:
        # 获取软件名称和版本
        soft_name, soft_version = parse_soft_name_of_version(name)
        if not soft_name or not soft_version:
            return return_default

        if soft_name not in install_config:
            return return_default

        if os.path.exists("{}/install/{}_not_support.pl".format(BASE_PATH, soft_name)):
            return 0, 'Not compatible with this system! Please click on the details to explain!'

        if os.path.exists("{}/install/{}_mem_kill.pl".format(BASE_PATH, soft_name)):
            return 0, 'Insufficient memory installation exception! Please click on the details to explain!'

        soft_config = install_config[soft_name]

        # 取计算机名
        def get_hostname():
            try:
                import socket
                return socket.gethostname()
            except:
                return 'localhost.localdomain'

        # 替换soft_config中所有变量
        def replace_all(dat: str):
            if not dat:
                return dat
            if dat.find('{') == -1:
                return dat
            # 替换安装路径, 替换版本号
            dat = dat.replace('{SetupPath}', '/www/server').replace('{Version}', soft_version)
            # 替换主机名
            if dat.find("{Host") != -1:
                host_name = get_hostname()
                host = host_name.split('.')[0]
                dat = dat.replace("{Hostname}", host_name)
                dat = dat.replace("{Host}", host)
            return dat

        # 检查文件是否存在
        if 'files_exists' in soft_config:
            for f_name in soft_config['files_exists']:
                filename = replace_all(f_name)
                if not os.path.exists(filename):
                    return 0, 'Installation failed, file does not exist:{}'.format(filename)

        # 检查pid文件是否有效
        if 'pid' in soft_config and soft_config['pid']:
            pid_file = replace_all(soft_config['pid'])
            if not os.path.exists(pid_file):
                return 0, 'Startup failed, PID file does not exist:{}'.format(pid_file)
            pid = read_file(pid_file)
            if not pid:
                return 0, 'Startup failed, PID file does not exist:{}'.format(pid_file)
            proc_file = '/proc/{}/cmdline'.format(pid.strip())
            if not os.path.exists(proc_file):
                return 0, 'Startup failed, PID file is empty: {}({}) process does not exist'.format(pid_file, pid)

        # 执行命令检查
        if 'cmd' in soft_config:
            for cmd in soft_config['cmd']:
                p = subprocess.Popen(
                    replace_all(cmd['exec']), shell=True,
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
                )
                p.wait()
                res = p.stdout.read() + "\n" + p.stderr.read()
                if res.find(replace_all(cmd['success'])) == -1:
                    return 0, '[{}] Abnormal service startup status'.format(soft_name)
    except:
        pass

    return return_default


def soft_task():
    # 执行面板soft corn之类的安装执行任务, from task.py -> def startTask():
    def ExecShell(cmdstring, cwd=None, shell=True, symbol='&>'):
        try:
            import shlex
            import subprocess
            import time
            sub = subprocess.Popen(
                cmdstring + symbol + exlogPath,
                cwd=cwd,
                stdin=subprocess.PIPE,
                shell=shell,
                bufsize=4096
            )
            while sub.poll() is None:
                time.sleep(0.1)
            return sub.returncode
        except:
            return None

    # def TaskStart(value: dict, start: int = None, time_out: int = None):
    #     """执行安装任务, 并检测是否已经在执行或超时"""
    #     def is_time_out(pid: int) -> bool:
    #         res = int(time.time()) - start > time_out if time_out else False
    #         if res:
    #             try:
    #                 p = psutil.Process(pid)
    #                 child = p.children(recursive=True)
    #                 for c in child:
    #                     try: c.kill()
    #                     except: continue
    #                 p.kill()
    #             except Exception: pass
    #         return res
    #
    #     start = int(time.time()) if not start else start
    #     try:
    #         output, _ = exec_shell(f"pgrep -f \"{value['execstr'] + '&>' + exlogPath}\"")
    #         pids = output.strip().split()
    #         if pids:
    #             target_pid = int(pids[0])
    #             sql.table('tasks').where("id=?", (value['id'],)).save('status', (Running,))
    #             while not is_time_out(target_pid):
    #                 try:
    #                     os.kill(target_pid, 0)
    #                     time.sleep(1)  # 残留阻塞
    #                 except ProcessLookupError: break
    #                 except Exception: break
    #             if is_time_out(target_pid):
    #                 raise Exception
    #         raise Exception
    #     except Exception:
    #         sql.table('tasks').where("id=?", (value['id'],)).save('status,start', (Running, start))
    #         ExecShell(value['execstr'])

    tip_file = "/dev/shm/.panelTask.pl"
    panel_log_path = "/www/server/panel/logs/installed/"
    if not os.path.exists(panel_log_path):
        os.mkdir(panel_log_path)
        os.chmod(panel_log_path, 0o600)

    Waitting = "0"  # 等待
    Running = "-1"  # 执行中
    Finished = "1"  # 完成
    try:
        if os.path.exists(isTask):
            with db.Sql() as sql:
                # 检测task表是否存在install_status,message字段
                field = 'id,type,execstr,name,install_status,message'
                check_result = sql.table('tasks').order("id desc").field(field).select()
                if type(check_result) == str:
                    sql.table('tasks').execute("ALTER TABLE 'tasks' ADD 'install_status' INTEGER DEFAULT 1", ())
                    sql.table('tasks').execute("ALTER TABLE 'tasks' ADD 'message' TEXT DEFAULT ''", ())

                sql.table('tasks').where("status=?", (Running,)).setField('status', Waitting)
                taskArr = sql.table('tasks').where("status=?", (Waitting,)).field('id,type,execstr,name').order(
                    "id asc"
                ).select()
                for value in taskArr:
                    if value['type'] != 'execshell':
                        continue
                    if not sql.table('tasks').where("id=?", (value['id'],)).count():
                        write_file(tip_file, str(int(time.time())))
                        continue

                    start = int(time.time())
                    # TaskStart(value, start=start, time_out=3600)
                    sql.table('tasks').where("id=?", (value['id'],)).save('status,start', (Running, start))
                    ExecShell(value['execstr'])

                    # 保存安装日志
                    target_log_file = '{}/task_{}.log'.format(panel_log_path, value['id'])
                    shutil.copy(exlogPath, target_log_file)
                    # 检查软件是否安装成功
                    end = int(time.time())
                    try:
                        install_status, install_msg = check_install_status(value['name'])
                    except Exception:
                        install_status = 1
                        install_msg = ""

                    sql.table('tasks').where("id=?", (value['id'],)).save(
                        'status,end,install_status,message',
                        (Finished, end, install_status, install_msg)
                    )
                    if sql.table('tasks').where("status=?", (Waitting,)).count() < 1:
                        if os.path.exists(isTask):
                            os.remove(isTask)
        write_file(tip_file, str(int(time.time())))
    except Exception as e:
        logger.error(f"start_bt_task error: {e}")


# 预安装网站监控报表
def check_site_monitor():
    task_ExecShell("check_site_monitor")


# 节点监控
def node_monitor():
    task_ExecShell("node_monitor")


# 节点监控
def node_monitor_check():
    task_ExecShell("node_monitor_check")


# 检测防爆破计划任务
def breaking_through():
    task_ExecShell("breaking_through")


# 找site favicons
def find_favicons():
    task_ExecShell(
        "find_favicons",
        paths_exists=[
            '/www/server/panel/config/auto_favicon.conf',
        ]
    )


# 邮件日志
def maillog_event():
    task_ExecShell(
        "maillog_event",
        paths_exists=[
            "/www/server/panel/plugin/mail_sys/mail_sys_main.py",
            "/www/vmail",
        ]
    )


# 邮件处理日志聚合
def aggregate_maillogs_task():
    task_ExecShell(
        "aggregate_maillogs_task",
        paths_exists=[
            "/www/server/panel/plugin/mail_sys/mail_sys_main.py",
            "/www/vmail",
        ]
    )


# 邮件自动化发件任务
def schedule_automations():
    task_ExecShell(
        "schedule_automations",
        paths_exists=[
            "/www/server/panel/plugin/mail_sys/mail_sys_main.py",
            "/www/vmail",
        ]
    )


# 刷新docker app 列表
def refresh_dockerapps():
    task_ExecShell("refresh_dockerapps")


# 版本更新执行一次性
def task_version_part():
    task_ExecShell("task_version_part")


# ================================ 这是任务分割线 ===============================


TASKS = [
    # <核心任务> 面板重启检查, 面板授权检查
    {"func": [restart_panel, panel_auth], "interval": 2, "is_core": True},
    {"func": soft_task, "interval": 2, "is_core": True},  # 原面板任务
    {"func": bt_box_task, "interval": 2, "is_core": True},  # 原面板任务
    # <核心任务> 服务守护
    {"func": daemon_service, "interval": 10, "is_core": True},
    # <核心任务> 原系统监控
    {"func": systemTask, "interval": 60, "is_core": True},  # 每1分钟系统监控任务
    {"func": project_daemon_service, "interval": 120, "is_core": True},  # 每120秒项目守护

    # ================================ 分割线 ===============================
    # <普通任务>
    # func 函数, interval 间隔时间秒s, 排队复用线程 (打印请用logger, 日志路径 .../logs/task.log)
    {"func": push_msg, "interval": 60},  # 每1分钟面板推送消息
    {"func": breaking_through, "interval": 60},  # 每分钟防爆破计划任务

    {"func": multi_web_server_daemon, "interval": 300},  # 每5分钟多服务守护任务
    {"func": check502Task, "interval": 60 * 10},  # 每10分钟 502检查(夹杂若干任务)
    {"func": check_site_monitor, "interval": 60 * 10},  # 每10分钟检查站点安装监控
    {"func": node_monitor, "interval": 60},  # 每1分钟节点监控任务
    {"func": node_monitor_check, "interval": 60 * 60 * 24 * 30},  # 每月节点监控检测任务
    {"func": update_waf_config, "interval": 60 * 20},  # 每隔20分钟更新一次waf报表数据
    {"func": update_monitor_requests, "interval": 60 * 20},  # 每隔20分钟更新一次网站报表数据

    {"func": find_favicons, "interval": 43200},  # 每12小时找favicons
    {"func": domain_ssl_service, "interval": 3600},  # 每6小时进行域名SSL服务(内置时间标记, 可提前检查)
    {"func": malicious_file_scanning, "interval": 60 * 60 * 6},  # 每每6小时进行恶意文件扫描
    {"func": count_ssh_logs, "interval": 3600 * 24},  # 每天统计SSH登录日志
    {"func": submit_module_call_statistics, "interval": 3600},  # 每天一次 提交今天之前的统计数据(内置时间标记, 可提前检查)

    {"func": maillog_event, "interval": 60, "loop": True},  # 邮局日志事件监控 event loop事件, 每60秒一次, 起守护作用
    {"func": send_mail_time, "interval": 60 * 3},  # 每3分钟检测邮件信息
    {"func": auto_reply_tasks, "interval": 3600},  # 每1小时自动回复邮件
    {"func": schedule_automations, "interval": 60},  # 每1分钟邮局自动化任务
    {"func": aggregate_maillogs_task, "interval": 60},  # 每1分钟聚合邮局日志
    {"func": mailsys_quota_alarm, "interval": 3600 * 2},  # 每2小时邮件域名邮箱使用限额告警
    {"func": auto_scan_abnormal_mail, "interval": 3600 * 2},  # 每2小时自动扫描异常邮箱
    {"func": mailsys_update_usage, "interval": 3600 * 12},  # 每12小时邮局更新域名邮箱使用量
    {"func": submit_email_statistics, "interval": 3600 * 24},  # 每天一次 昨日邮件发送统计
    {"func": mailsys_domain_blecklisted_alarm, "interval": 3600 * 24},  # # 每天一次 邮局黑名单检测

    {"func": update_vulnerabilities, "interval": 3600 * 24},  # # 每天一次 更新漏洞信息
    {"func": refresh_dockerapps, "interval": 3600 * 24},  # # 每天一次 更新docker app 列表
]


def thread_register(brain: SimpleBrain, is_core: bool = True):
    if not is_core:  # delay normal tasks
        logger.info("Normal Task will be join active after 30s")
        time.sleep(30)

    for index, task in enumerate(TASKS):
        try:
            if task.get("is_core", False) == is_core:
                if isinstance(task["func"], list):
                    task_id = "_".join([f.__name__ for f in task["func"]])
                else:
                    task_id = task.get("id", task["func"].__name__)

                # delay normal tasks, 削峰
                if not is_core:
                    time.sleep(10)

                brain.register_task(
                    func=task["func"],
                    task_id=task_id,
                    interval=task.get("interval", 3600),
                    is_core=task.get("is_core", False),
                    loop=task.get("loop", False),
                )
        except Exception:
            import traceback
            logger.error(f"Register task {task} failed: {traceback.format_exc()}")
            continue
    logger.info(
        f"All the {'[Core]' if is_core else '[Normal]'} tasks have been registered."
    )


def main(max_workers: int = None):
    main_pid = "logs/task.pid"
    if os.path.exists(main_pid):
        os.system("kill -9 $(cat {}) &> /dev/null".format(main_pid))
    pid = os.fork()
    if pid:
        sys.exit(0)

    os.setsid()
    _pid = os.fork()

    if _pid:
        write_file(main_pid, str(_pid))
        sys.exit(0)

    sys.stdout.flush()
    sys.stderr.flush()

    logger.info("Service Up")
    time.sleep(5)
    task_version_part()
    # =================== Start ===========================
    sb = SimpleBrain(cpu_max=20.0, workers=max_workers)
    try:
        # core tasks
        thread_register(brain=sb, is_core=True)
        # normal tasks will be delayed
        threading.Thread(
            target=thread_register, args=(sb, False), daemon=True
        ).start()
        sb.run()
    except Exception:
        import traceback
        logger.error(traceback.format_exc())
        sb._shutdown()
    # =================== End ========================


if __name__ == "__main__":
    main()
