mirror of https://github.com/lework/script
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
151 lines
5.1 KiB
151 lines
5.1 KiB
#!/usr/bin/python |
|
# -*- coding: utf-8 -*- |
|
# |
|
# @Time : 2019-10-16 |
|
# @Author : lework |
|
# @Desc : 一个事件监听器,订阅PROCESS_STATE_CHANGE事件。当supervisor管理的进程意外过渡到EXITED状态时,它将发送邮件。 |
|
|
|
# [eventlistener:supervisor_event_exited] |
|
# process_name=%(program_name)s |
|
# command=/usr/bin/python /data/scripts/supervisor_event_exited.py |
|
# autostart=true |
|
# autorestart=true |
|
# events=PROCESS_STATE |
|
# log_stdout=true |
|
# log_stderr=true |
|
# stdout_logfile=/var/log/supervisor/supervisor_event_exited-stdout.log |
|
# stdout_logfile_maxbytes=50MB |
|
# stdout_logfile_backups=3 |
|
# buffer_size=10 |
|
# stderr_logfile=/var/log/supervisor/supervisor_event_exited-stderr.log |
|
# stderr_logfile_maxbytes=50MB |
|
# stderr_logfile_backups=3 |
|
|
|
|
|
import os |
|
import smtplib |
|
import socket |
|
import sys |
|
from supervisor import childutils |
|
from email.header import Header |
|
from email.mime.text import MIMEText |
|
|
|
|
|
class CrashMail: |
|
def __init__(self, mail_config, programs): |
|
self.mail_config = mail_config |
|
self.programs = programs |
|
self.stdin = sys.stdin |
|
self.stdout = sys.stdout |
|
self.stderr = sys.stderr |
|
self.time = '' |
|
|
|
def write_stderr(self, s): |
|
s = s+'\n' |
|
if self.time: |
|
s = '[%s] %s' % (self.time, s) |
|
self.stderr.write(s) |
|
self.stderr.flush() |
|
|
|
def runforever(self): |
|
# 死循环, 处理完 event 不退出继续处理下一个 |
|
while 1: |
|
# 使用 self.stdin, self.stdout, self.stderr 代替 sys.* |
|
headers, payload = childutils.listener.wait(self.stdin, self.stdout) |
|
|
|
self.time = childutils.get_asctime() |
|
self.write_stderr('[headers] %s' % str(headers)) |
|
self.write_stderr('[payload] %s' % str(payload)) |
|
|
|
# 不处理不是 PROCESS_STATE_EXITED 类型的 event, 直接向 stdout 写入"RESULT\nOK" |
|
if headers['eventname'] != 'PROCESS_STATE_EXITED': |
|
childutils.listener.ok(self.stdout) |
|
continue |
|
|
|
# 解析 payload, 这里我们只用这个 pheaders. |
|
# pdata 在 PROCESS_LOG_STDERR 和 PROCESS_COMMUNICATION_STDOUT 等类型的 event 中才有 |
|
pheaders, pdata = childutils.eventdata(payload + '\n') |
|
|
|
# 如果在programs中设置,就只处理programs中的,否则全部处理. |
|
if len(self.programs) !=0 and pheaders['groupname'] not in self.programs: |
|
childutils.listener.ok(self.stdout) |
|
continue |
|
|
|
# 过滤掉 expected 的 event, 仅处理 unexpected 的 |
|
# 当 program 的退出码为对应配置中的 exitcodes 值时, expected=1; 否则为0 |
|
if int(pheaders['expected']): |
|
childutils.listener.ok(self.stdout) |
|
continue |
|
|
|
# 获取系统主机名和ip地址 |
|
hostname = socket.gethostname() |
|
ip = socket.gethostbyname(hostname) |
|
|
|
# 构造报警内容 |
|
msg = "Host: %s(%s)\nProcess: %s\nPID: %s\nEXITED unexpectedly from state: %s" % \ |
|
(hostname, ip, pheaders['processname'], pheaders['pid'], pheaders['from_state']) |
|
|
|
subject = '[Supervistord] %s crashed at %s' % (pheaders['processname'], self.time) |
|
|
|
self.write_stderr('[INFO] unexpected exit, mailing') |
|
|
|
# 发送邮件 |
|
self.send_mail(subject, msg) |
|
|
|
# 向 stdout 写入"RESULT\nOK",并进入下一次循环 |
|
childutils.listener.ok(self.stdout) |
|
|
|
def send_mail(self, subject, content): |
|
""" |
|
:param subject: str |
|
:param content: str |
|
:return: bool |
|
""" |
|
|
|
mail_port = self.mail_config.get('mail_port', '') |
|
mail_host = self.mail_config.get('mail_host', '') |
|
mail_user = self.mail_config.get('mail_user', '') |
|
mail_pass = self.mail_config.get('mail_pass', '') |
|
to_list = self.mail_config.get('to_list', []) |
|
|
|
msg = MIMEText(content, _subtype='plain', _charset='utf-8') |
|
msg['Subject'] = Header(subject, 'utf-8') |
|
msg['From'] = mail_user |
|
msg['to'] = ",".join(to_list) |
|
try: |
|
s = smtplib.SMTP_SSL(mail_host, mail_port) |
|
s.login(mail_user, mail_pass) |
|
s.sendmail(mail_user, to_list, msg.as_string()) |
|
s.quit() |
|
self.write_stderr('[mail] ok') |
|
return True |
|
except Exception as e: |
|
self.write_stderr('[mail] error\n\n%s\n' % e) |
|
return False |
|
|
|
|
|
def main(): |
|
# listener 必须交由 supervisor 管理, 直接运行是不行的 |
|
if not 'SUPERVISOR_SERVER_URL' in os.environ: |
|
sys.stderr.write('crashmail must be run as a supervisor event ' |
|
'listener\n') |
|
sys.stderr.flush() |
|
return |
|
|
|
# 设置smtp信息 |
|
mail_config = { |
|
'mail_host': 'smtp.lework.com', |
|
'mail_port': '465', |
|
'mail_user': 'ops@lework.com', |
|
'mail_pass': '123123123', |
|
'to_list': ['lework@lework.com'] |
|
} |
|
|
|
# 设置要检测的program,不设置则检测全部 |
|
programs = [] |
|
prog = CrashMail(mail_config, programs) |
|
prog.runforever() |
|
|
|
|
|
if __name__ == '__main__': |
|
main()
|
|
|