Files
yakpanel-core/mod/project/python/serviceMod.py
2026-04-07 02:04:22 +05:30

789 lines
27 KiB
Python

# coding: utf-8
# -------------------------------------------------------------------
# YakPanel
# -------------------------------------------------------------------
# Copyright (c) 2014-2099 YakPanel(www.yakpanel.com) All rights reserved.
# -------------------------------------------------------------------
# Author: yakpanel
# -------------------------------------------------------------------
# ------------------------------
# py service model app
# ------------------------------
import json
import os.path
import re
import sys
import time
import psutil
from uuid import uuid4
from typing import Optional, Union, List, Dict, Tuple, Any, Set
SERVICE_PATH = "/www/server/python_project/service"
if not os.path.isdir(SERVICE_PATH):
try:
os.makedirs(SERVICE_PATH, 0o755)
except:
pass
if "/www/server/panel/class" not in sys.path:
sys.path.insert(0, "/www/server/panel/class")
if "/www/server/panel" not in sys.path:
sys.path.insert(0, "/www/server/panel")
import public
from mod.base import json_response
from mod.project.python.pyenv_tool import EnvironmentManager
from public.exceptions import HintException
class Environment(object):
def __init__(self, project_name: str, python_path: str, python_bin: str, project_path: str, user: str,
env_list: List[Dict[str, str]], env_file: str):
self.python_path = python_path
self.project_path = project_path
self.env_list = env_list
self.env_file = env_file
self.project_name = project_name
self.user = user
self._env_cache: Optional[str] = None
self.pyenv = EnvironmentManager().get_env_py_path(python_bin)
@classmethod
def form_project_conf(cls, project_config: dict) -> Union["Environment", str]:
if not isinstance(project_config, dict):
return 'Invalid project configuration file format'
python_path: str = project_config.get("vpath")
python_bin: str = project_config.get("python_bin", project_config.get("vpath"))
project_path: str = project_config.get("path")
env_list = project_config.get("env_list", [])
env_file = project_config.get("env_file", "")
project_name = project_config.get("pjname")
user = project_config.get("user", "root")
if not python_path or not project_path or not project_name:
return 'Invalid project configuration file format'
if not os.path.isdir(python_path) or not os.path.isdir(project_path):
return 'The project directory or virtual environment directory specified in the config does not exist'
python_path = python_path.rstrip("/")
project_path = project_path.rstrip("/")
if not python_path.endswith("/bin"):
python_path = python_path + "/bin"
if not os.path.isdir(python_path):
return 'The virtual environment directory specified in the config does not exist'
return cls(project_name, python_path, python_bin, project_path, user, env_list, env_file)
# 组合环境变量,用于启动服务
def shell_env(self) -> str:
if self._env_cache is not None:
return self._env_cache
# cd 到指定路径, 加载环境变量, 加载环境变量文件, 设置Python环境到首位
res_env_list = ["cd {}".format(self.project_path)]
if isinstance(self.env_list, list):
for i in self.env_list:
if not isinstance(i, dict):
continue
if 'k' in i and 'v' in i:
res_env_list.append("export {}={}".format(i['k'], i['v']))
if self.env_file and os.path.isfile(self.env_file):
res_env_list.append("source {}".format(self.env_file))
res_env_list.append(self.pyenv.activate_shell())
self._env_cache = "\n".join(res_env_list)
return self._env_cache
class PythonService(object):
def __init__(self, sid: str, name: str, command: str, level: Optional[int], log_type: Optional[str]):
self.sid = sid
self.name = name
self.command = command
self.level = level
self.log_type = log_type
self.env: Optional[Environment] = None
def set_env(self, env: Environment):
self.env = env
def write_pid(self, pid: int):
if not self.env:
raise RuntimeError('Env not set')
pid_file = os.path.join(SERVICE_PATH, '{}/{}.pid'.format(self.env.project_name, self.name))
if not os.path.isdir(os.path.dirname(pid_file)):
os.makedirs(os.path.dirname(pid_file), 0o755)
public.writeFile(pid_file, str(pid))
def read_pid(self) -> Optional[int]:
if not self.env:
raise RuntimeError('Env not set')
pid_file = os.path.join(SERVICE_PATH, '{}/{}.pid'.format(self.env.project_name, self.name))
if not os.path.isfile(pid_file):
return None
res = None
try:
res = int(public.readFile(pid_file))
except:
pass
if isinstance(res, int) and res > 0:
return res
return None
@classmethod
def from_config(cls, config: dict,
env: Optional[Environment]) -> Union['PythonService', "MainPythonService", 'CeleryService', str]:
sid = config.get('sid', None)
if sid == 'main':
return MainPythonService(env)
name: str = config.get('name', "")
command: str = config.get('command', "")
if not sid or not name or not command:
return 'Missing required parameters'
if not isinstance(command, str) or not isinstance(name, str) or not isinstance(sid, str):
return 'Invalid parameter type'
level = config.get('level', 11)
log_type = config.get('log_type', "append")
if command.split()[0].endswith("celery"):
res = CeleryService(sid, name, command, level, log_type)
else:
res = cls(sid, name, command, level, log_type)
if env:
res.set_env(env)
return res
# 执行启动服务并返回PID信息或错误信息
def start(self) -> Optional[int]:
if not self.env:
raise RuntimeError('Env not set')
log_file = os.path.join(SERVICE_PATH, '{}/{}.log'.format(self.env.project_name, self.name))
pid_file = os.path.join(SERVICE_PATH, '{}/{}.pid'.format(self.env.project_name, self.name))
if not os.path.exists(os.path.dirname(pid_file)):
os.makedirs(os.path.dirname(pid_file), 0o755)
if os.path.exists(pid_file):
os.remove(pid_file)
prep_sh = self.env.shell_env()
prep_sh += "\nexport BT_PYTHON_SERVICE_SID={}".format(self.sid)
if not os.path.isfile(log_file):
public.writeFile(log_file, '')
public.set_own(log_file, self.env.user)
public.set_mode(log_file, "755")
if self.log_type == "append":
prep_sh += "\nnohup {} &>> {} &".format(self.command, log_file)
else:
prep_sh += "\nnohup {} &> {} &".format(self.command, log_file)
public.ExecShell(prep_sh, user=self.env.user)
time.sleep(0.5)
return self.get_service_pid()
def get_service_pid(self, only_service: bool = False) -> Optional[int]:
pid = self.read_pid()
if pid and psutil.pid_exists(pid):
return pid
if not pid:
pid = self.get_pid_by_env_key()
if not pid and not only_service:
pid = self.get_pid_by_command()
if pid:
self.write_pid(pid)
return pid
return None
def get_pid_by_env_key(self) -> Optional[int]:
env_key = "BT_PYTHON_SERVICE_SID={}".format(self.sid)
target = []
for p in psutil.pids():
try:
data: str = public.readFile("/proc/{}/environ".format(p))
if data.rfind(env_key) != -1:
target.append(p)
except:
continue
for i in target:
try:
p = psutil.Process(i)
if p.ppid() not in target:
return i
except:
continue
return None
def get_pid_by_command(self) -> Optional[int]:
cmd_list = self.split_command()
target = []
for p in psutil.process_iter(["cmdline", "pid", "exe"]):
try:
real_cmd = p.cmdline()
if cmd_list == real_cmd:
target.append(p)
if real_cmd[2:] == cmd_list[1:] and real_cmd[0].startswith(self.env.python_path):
target.append(p)
except:
continue
for p in target:
try:
if p.ppid() not in target:
return p.pid
except:
continue
return None
def split_command(self) -> List[str]:
res = []
tmp = ""
in_quot = False
for i in self.command:
if i in (' ', '\t', '\r'):
if tmp and not in_quot:
res.append(tmp)
tmp = ""
if in_quot:
tmp += ' '
elif i in ("'", '"'):
if in_quot:
in_quot = False
else:
in_quot = True
else:
tmp += i
if tmp:
res.append(tmp)
return res
def stop(self) -> None:
pid = self.get_service_pid()
if not pid:
return
try:
p = psutil.Process(pid)
p.kill()
except:
pass
def get_log(self) -> str:
if not self.env:
raise RuntimeError('env not set')
log_file = os.path.join(SERVICE_PATH, '{}/{}.log'.format(self.env.project_name, self.name))
if not os.path.isfile(log_file):
return 'No logs available'
data = public.GetNumLines(log_file, 1000)
if not data:
return 'No logs available'
return data
@staticmethod
def _get_ports_by_pid(pid: int) -> List[int]:
try:
res = set()
for con in psutil.Process(pid).connections(): # NOQA
if con.status == 'LISTEN':
res.add(con.laddr.port)
return list(res)
except:
return []
def get_info(self) -> Dict[str, Any]:
if not self.env:
raise RuntimeError('env not set')
pid = self.get_service_pid()
if isinstance(pid, int) and psutil.pid_exists(pid):
ports = self._get_ports_by_pid(pid)
return {
'pid': pid,
'ports': ports
}
return {"pid": None, "ports": []}
class MainPythonService(PythonService):
from projectModelV2.pythonModel import main as py_project_main
_py_main_class = py_project_main
def __init__(self, env: Environment):
super().__init__('main', 'main', 'main', 10, 'append')
self.set_env(env)
@property
def py_main(self):
return self._py_main_class()
def start(self) -> Optional[int]:
if not self.env:
raise RuntimeError('Env not set')
self.py_main.only_start_main_project(self.env.project_name)
return self.get_service_pid()
def get_service_pid(self, only_service: bool = False) -> Optional[int]:
if not self.env:
raise RuntimeError('env not set')
pids: List[int] = self.py_main.get_project_run_state(self.env.project_name)
if not pids:
return None
pids.sort()
return pids[0]
def stop(self) -> None:
if not self.env:
raise RuntimeError('env not set')
self.py_main.only_stop_main_project(self.env.project_name)
def get_log(self) -> str:
if not self.env:
raise RuntimeError('env not set')
get_obj = public.dict_obj()
get_obj.name = self.env.project_name
res = self.py_main.GetProjectLog(get_obj)
data = None
if res.get("status"):
data = res.get("message", {}).get("data", "")
if not data:
return 'no log found'
return data
def get_info(self):
res = super().get_info()
res['name'] = "Main project service"
return res
class CeleryService(PythonService):
def get_celery_env(self) -> Tuple[str, str]:
celery = "{}/celery".format(self.env.python_path)
if not os.path.isfile(celery):
return '', ''
celery_data = public.readFile(celery)
if not isinstance(celery_data, str):
return '', ''
celery_python = celery_data.split("\n", 1)[0]
if celery_python.startswith("#!"):
celery_python = celery_python[2:].strip()
return celery_python, celery
def get_pid_by_command(self) -> Optional[int]:
celery_env = self.get_celery_env()
if not celery_env[0] or not celery_env[1]:
return super().get_pid_by_command()
target = []
cmd_list = list(celery_env) + self.split_command()[1:]
for p in psutil.process_iter(["cmdline", "pid"]):
try:
if cmd_list == p.cmdline():
target.append(p)
except:
continue
for p in target:
try:
if p.ppid() not in target:
return p.pid
except:
continue
return None
# 协同服务管理类, 包括主服务和其他服务
class ServiceManager:
MAIN_SERVICE_CONF = {
"sid": "main",
"name": "main",
"command": "main",
"level": 10,
"log_type": "append",
}
def __init__(self, project_name: str, project_config: dict):
self.project_name = project_name
self.project_config = project_config
self._other_services: Optional[List[Dict]] = None
self._env: Optional[Environment] = None
@classmethod
def new_mgr(cls, project_name: str) -> Union["ServiceManager", str]:
data = public.M("sites").where(
'project_type=? AND name=? ', ('Python', project_name)
).field('id,project_config').find()
if not data:
raise HintException("Project [{}] Not Found!".format(project_name))
try:
project_config = json.loads(data['project_config'])
except json.JSONDecodeError:
raise HintException("Project [{}] db's Project Config Error!".format(project_name))
return cls(project_name, project_config)
@property
def service_list(self) -> List[Dict]:
res = [self.MAIN_SERVICE_CONF]
res.extend(self.other_services)
res.sort(key=lambda x: x['level'])
return res
@property
def other_services(self) -> List[Dict]:
if self._other_services is None:
services = []
for service in self.project_config.get('services', []):
if service.get('sid') == 'main':
continue
services.append(service)
self._other_services = services
return self._other_services
@staticmethod
def new_id() -> str:
return uuid4().hex[::3]
def save_service_conf(self) -> Optional[str]:
data = public.M("sites").where(
'project_type=? AND name=? ', ('Python', self.project_name)
).field('id,project_config').find()
if not data:
return "Website information not found"
data['project_config'] = json.loads(data['project_config'])
data['project_config']['services'] = self.other_services
public.M("sites").where('id=?', (data['id'],)).update({'project_config': json.dumps(data['project_config'])})
return None
def add_service(self, service_conf: dict) -> Optional[str]:
try:
conf = {
"name": service_conf.get("name", "").strip(),
"command": service_conf.get("command", "").strip(),
"level": int(service_conf.get("level", 11)),
"log_type": service_conf.get("log_type", "append"),
}
except:
return "Parameter error"
if re.search(r"[\s$^`]+", conf['name']):
return "Service name cannot contain spaces or special characters"
for i in self.other_services:
if i['name'] == conf['name']:
return "Service name must be unique"
if i['command'] == conf['command']:
return "This start command already exists; service name: {}".format(i['name'])
if not (conf['name'] and conf['command']):
return "Service name and start command cannot be empty"
conf["sid"] = self.new_id()
self.other_services.append(conf)
self.save_service_conf()
return None
def modify_service(self, sid: str, service_conf: dict) -> Optional[str]:
target_data = None
for i in self.other_services:
if i["sid"] == sid:
target_data = i
break
if target_data is None:
return "Service not found"
name = target_data["name"]
if "name" in service_conf and service_conf["name"] != target_data["name"]:
name = service_conf["name"].strip()
if re.search(r"[\s$^`]+", name):
return "Service name cannot contain spaces or special characters"
command = target_data["command"]
if "command" in service_conf and service_conf["command"] != target_data["command"]:
command = service_conf["command"].strip()
for i in self.other_services:
if i["sid"] == sid:
continue
if i["name"] == name:
return "Service name must be unique"
if i["command"] == command:
return "This start command already exists; service name: {}".format(i["name"])
if name != target_data["name"]:
log_file = os.path.join(SERVICE_PATH, '{}/{}.log'.format(self.project_name, target_data["name"]))
pid_file = os.path.join(SERVICE_PATH, '{}/{}.pid'.format(self.project_name, target_data["name"]))
if os.path.exists(log_file):
os.rename(log_file, os.path.join(SERVICE_PATH, '{}/{}.log'.format(self.project_name, name)))
if os.path.exists(pid_file):
os.rename(pid_file, os.path.join(SERVICE_PATH, '{}/{}.pid'.format(self.project_name, name)))
target_data["name"] = name
target_data["command"] = command
target_data["level"] = int(service_conf.get("level", int(target_data.get("level", 11))))
target_data["log_type"] = service_conf.get("log_type", target_data["log_type"])
self.save_service_conf()
return None
def remove_service(self, sid: str) -> Optional[str]:
del_idx = None
for idx, i in enumerate(self.other_services):
if i["sid"] == sid:
del_idx = idx
break
if del_idx is None:
return "Service not found"
del_conf = self.other_services.pop(del_idx)
self.save_service_conf()
log_file = os.path.join(SERVICE_PATH, '{}/{}.log'.format(self.project_name, del_conf["name"]))
pid_file = os.path.join(SERVICE_PATH, '{}/{}.pid'.format(self.project_name, del_conf["name"]))
if os.path.exists(log_file):
os.remove(log_file)
if os.path.exists(pid_file):
os.remove(pid_file)
return None
def _get_service_conf_by_sid(self, sid: str) -> Optional[Dict]:
for i in self.service_list:
if i["sid"] == sid:
return i
return None
def _build_service_by_conf(self, conf: dict) -> Union[PythonService, str]:
if not self._env:
self._env = Environment.form_project_conf(self.project_config)
if isinstance(self._env, str):
return self._env
return PythonService.from_config(conf, env=self._env)
def handle_service(self, sid: str, action: str = "start") -> Optional[str]:
conf = self._get_service_conf_by_sid(sid)
if conf is None:
return "Service not found"
service = self._build_service_by_conf(conf)
if isinstance(service, str):
return service
pid = service.get_service_pid()
if not pid:
pid = -1
if action == "start":
if not psutil.pid_exists(pid):
service.start()
elif action == "stop":
service.stop()
elif action == "restart":
if psutil.pid_exists(pid):
service.stop()
for i in range(50):
if not psutil.pid_exists(pid):
break
time.sleep(0.1)
else:
service.stop()
time.sleep(1)
service.start()
else:
return "Unknown action"
return None
def get_service_log(self, sid: str) -> Tuple[bool, str]:
conf = self._get_service_conf_by_sid(sid)
if conf is None:
return False, "Service Not Found"
service = self._build_service_by_conf(conf)
if isinstance(service, str):
return False, service
return True, service.get_log()
def get_services_info(self) -> List[dict]:
res = []
for i in self.service_list:
service = self._build_service_by_conf(i)
if isinstance(service, str):
i["error"] = service
res.append(i)
else:
i.update(service.get_info())
res.append(i)
return res
def start_project(self):
services = [
self._build_service_by_conf(i) for i in self.service_list
]
for i in services:
try:
if isinstance(i, str):
continue
pid = i.get_service_pid()
if isinstance(pid, int) and pid > 0 and psutil.pid_exists(pid):
continue
i.start()
time.sleep(0.5)
except Exception:
import traceback
public.print_log("start project service error: {}".format(traceback.format_exc()))
continue
return True
def stop_project(self):
services = [self._build_service_by_conf(i) for i in self.service_list]
for i in services[::-1]:
if isinstance(i, str):
continue
i.stop()
return "Stop command executed"
def other_service_pids(self) -> Set[int]:
res_pid = []
for i in self.other_services:
service = self._build_service_by_conf(i)
if isinstance(service, str):
continue
pid = service.get_service_pid()
if isinstance(pid, int) and pid > 0 and psutil.pid_exists(pid):
res_pid.append(pid)
sub_pid = []
def get_sub_pid(pro: psutil.Process) -> List[int]:
tmp_res = []
if pro.status() != psutil.STATUS_ZOMBIE and pro.children():
for sub_pro in pro.children():
tmp_res.append(sub_pro.pid)
tmp_res.extend(get_sub_pid(sub_pro))
return tmp_res
for i in res_pid:
try:
p = psutil.Process(i)
sub_pid.extend(get_sub_pid(p))
except:
pass
return set(res_pid + sub_pid)
# 协同服务api
class main:
def __init__(self):
pass
@staticmethod
def get_services_info(get):
try:
project_name = get.project_name.strip()
except:
return json_response(False, 'Parameter error')
s_mgr = ServiceManager.new_mgr(project_name)
if isinstance(s_mgr, str):
return json_response(False, s_mgr)
return json_response(True, data=s_mgr.get_services_info())
@staticmethod
def add_service(get):
try:
project_name = get.project_name.strip()
service_conf = get.service_conf
if isinstance(service_conf, str):
service_conf = json.loads(service_conf)
if not isinstance(service_conf, dict):
return json_response(False, 'Invalid service configuration parameters')
except:
return json_response(False, 'Parameter error')
s_mgr = ServiceManager.new_mgr(project_name)
if isinstance(s_mgr, str):
return json_response(False, s_mgr)
res = s_mgr.add_service(service_conf)
if isinstance(res, str):
return json_response(False, res)
public.set_module_logs('python_project', 'add_service', 1)
return json_response(True, msg="Added successfully")
@staticmethod
def modify_service(get):
try:
project_name = get.project_name.strip()
sid = get.sid.strip()
service_conf = get.service_conf
if isinstance(service_conf, str):
service_conf = json.loads(service_conf)
if not isinstance(service_conf, dict):
return json_response(False, 'Invalid service configuration parameters')
except:
return json_response(False, 'Parameter error')
s_mgr = ServiceManager.new_mgr(project_name)
if isinstance(s_mgr, str):
return json_response(False, s_mgr)
res = s_mgr.modify_service(sid, service_conf)
if isinstance(res, str):
return json_response(False, res)
return json_response(True, msg="Updated successfully")
@staticmethod
def remove_service(get):
try:
project_name = get.project_name.strip()
sid = get.sid.strip()
except:
return json_response(False, 'Parameter error')
s_mgr = ServiceManager.new_mgr(project_name)
if isinstance(s_mgr, str):
return json_response(False, s_mgr)
res = s_mgr.remove_service(sid)
if isinstance(res, str):
return json_response(False, res)
return json_response(True, msg="Deleted successfully")
@staticmethod
def handle_service(get):
try:
project_name = get.project_name.strip()
sid = get.sid.strip()
action = get.option.strip()
except:
return json_response(False, 'Parameter error')
s_mgr = ServiceManager.new_mgr(project_name)
if isinstance(s_mgr, str):
return json_response(False, s_mgr)
res = s_mgr.handle_service(sid, action)
if isinstance(res, str):
return json_response(False, res)
return json_response(True, msg="Operation successful")
@staticmethod
def get_service_log(get):
try:
project_name = get.project_name.strip()
sid = get.sid.strip()
except:
return json_response(False, 'Parameter error')
s_mgr = ServiceManager.new_mgr(project_name)
if isinstance(s_mgr, str):
return json_response(False, s_mgr)
res, log = s_mgr.get_service_log(sid)
if not res:
return json_response(False, log)
return json_response(True, data=log)