script/python/supervisor_event_exited.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# @Time    : 2019-10-16
# @Author  : lework
# @Desc    : 一个事件监听器，订阅PROCESS_STATE_CHANGE事件。当supervisor管理的进程意外过渡到EXITED状态时，它将发送邮件。

# [eventlistener:supervisor_event_exited]
# process_name=%(program_name)s
# command=/usr/bin/python /data/scripts/supervisor_event_exited.py
# autostart=true
# autorestart=true
# events=PROCESS_STATE
# log_stdout=true
# log_stderr=true
# stdout_logfile=/var/log/supervisor/supervisor_event_exited-stdout.log
# stdout_logfile_maxbytes=50MB
# stdout_logfile_backups=3
# buffer_size=10
# stderr_logfile=/var/log/supervisor/supervisor_event_exited-stderr.log
# stderr_logfile_maxbytes=50MB
# stderr_logfile_backups=3


import os
import smtplib
import socket
import sys
from supervisor import childutils
from email.header import Header
from email.mime.text import MIMEText


class CrashMail:
    def __init__(self, mail_config, programs):
        self.mail_config = mail_config
        self.programs = programs
        self.stdin = sys.stdin
        self.stdout = sys.stdout
        self.stderr = sys.stderr
        self.time = ''

    def write_stderr(self, s):
        s = s+'\n'
	if self.time:
           s = '[%s] %s' % (self.time, s)
        self.stderr.write(s)
        self.stderr.flush()

    def runforever(self):
        # 死循环, 处理完 event 不退出继续处理下一个
        while 1:
            # 使用 self.stdin, self.stdout, self.stderr 代替 sys.*
            headers, payload = childutils.listener.wait(self.stdin, self.stdout)

            self.time = childutils.get_asctime()
            self.write_stderr('[headers] %s' % str(headers))
            self.write_stderr('[payload] %s' % str(payload))

            # 不处理不是 PROCESS_STATE_EXITED 类型的 event, 直接向 stdout 写入"RESULT\nOK"
            if headers['eventname'] != 'PROCESS_STATE_EXITED':
                childutils.listener.ok(self.stdout)
                continue

            # 解析 payload, 这里我们只用这个 pheaders.
            # pdata 在 PROCESS_LOG_STDERR 和 PROCESS_COMMUNICATION_STDOUT 等类型的 event 中才有
            pheaders, pdata = childutils.eventdata(payload + '\n')

            # 如果在programs中设置，就只处理programs中的，否则全部处理.
            if len(self.programs) !=0 and pheaders['groupname'] not in self.programs:
                childutils.listener.ok(self.stdout)
	        continue

            # 过滤掉 expected 的 event, 仅处理 unexpected 的
            # 当 program 的退出码为对应配置中的 exitcodes 值时, expected=1; 否则为0
            if int(pheaders['expected']):
                childutils.listener.ok(self.stdout)
                continue

            # 获取系统主机名和ip地址
            hostname = socket.gethostname()
            ip = socket.gethostbyname(hostname)

            # 构造报警内容
            msg = "Host: %s(%s)\nProcess: %s\nPID: %s\nEXITED unexpectedly from state: %s" % \
                  (hostname, ip, pheaders['processname'], pheaders['pid'], pheaders['from_state'])

            subject = '[Supervistord] %s crashed at %s' % (pheaders['processname'], self.time)

            self.write_stderr('[INFO] unexpected exit, mailing')

            # 发送邮件
            self.send_mail(subject, msg)

            # 向 stdout 写入"RESULT\nOK"，并进入下一次循环
            childutils.listener.ok(self.stdout)

    def send_mail(self, subject, content):
        """
        :param subject: str
        :param content: str
        :return: bool
        """

        mail_port = self.mail_config.get('mail_port', '')
        mail_host = self.mail_config.get('mail_host', '')
        mail_user = self.mail_config.get('mail_user', '')
        mail_pass = self.mail_config.get('mail_pass', '')
        to_list = self.mail_config.get('to_list', [])

        msg = MIMEText(content, _subtype='plain', _charset='utf-8')
        msg['Subject'] = Header(subject, 'utf-8')
        msg['From'] = mail_user
        msg['to'] = ",".join(to_list)
        try:
            s = smtplib.SMTP_SSL(mail_host, mail_port)
            s.login(mail_user, mail_pass)
            s.sendmail(mail_user, to_list, msg.as_string())
            s.quit()
            self.write_stderr('[mail] ok')
            return True
        except Exception as e:
            self.write_stderr('[mail] error\n\n%s\n' % e)
            return False


def main():
    # listener 必须交由 supervisor 管理, 直接运行是不行的
    if not 'SUPERVISOR_SERVER_URL' in os.environ:
        sys.stderr.write('crashmail must be run as a supervisor event '
                         'listener\n')
        sys.stderr.flush()
        return

    # 设置smtp信息
    mail_config = {
        'mail_host': 'smtp.lework.com',
        'mail_port': '465',
        'mail_user': 'ops@lework.com',
        'mail_pass': '123123123',
        'to_list': ['lework@lework.com']
    }

    # 设置要检测的program,不设置则检测全部
    programs = []
    prog = CrashMail(mail_config, programs)
    prog.runforever()


if __name__ == '__main__':
    main()