From ccde433e9bf606da0f1df93d66710b7477766327 Mon Sep 17 00:00:00 2001 From: lework Date: Sat, 12 Oct 2019 18:15:47 +0800 Subject: [PATCH] add os_check --- python/facts_os_check/ansible.py | 383 + python/facts_os_check/run.sh | 9 + python/facts_os_check/templates/report.html | 14963 ++++++++++++++++ .../templates/report_cssinline.html | 7793 ++++++++ 4 files changed, 23148 insertions(+) create mode 100644 python/facts_os_check/ansible.py create mode 100644 python/facts_os_check/run.sh create mode 100644 python/facts_os_check/templates/report.html create mode 100644 python/facts_os_check/templates/report_cssinline.html diff --git a/python/facts_os_check/ansible.py b/python/facts_os_check/ansible.py new file mode 100644 index 0000000..81beea9 --- /dev/null +++ b/python/facts_os_check/ansible.py @@ -0,0 +1,383 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# @Time : 2019-10-09 +# @Author : lework + + +import datetime +import smtplib +import os +import json +import codecs +import copy +from email.header import Header +from email.mime.text import MIMEText +import urllib.request +from jinja2 import FileSystemLoader, Environment + + +def deepupdate(target, src, overwrite=True): + """Deep update target list, dict or set or other iterable with src + For each k,v in src: if k doesn't exist in target, it is deep copied from + src to target. Otherwise, if v is a list, target[k] is extended with + src[k]. If v is a set, target[k] is updated with v, If v is a dict, + recursively deep-update it. If `overwrite` is False, existing values in + target will not be overwritten. + Examples: + \>>> t = {'name': 'Ferry', 'hobbies': ['programming', 'sci-fi']} + \>>> deepupdate(t, {'hobbies': ['gaming']}) + \>>> print t + {'name': 'Ferry', 'hobbies': ['programming', 'sci-fi', 'gaming']} + """ + for k, v in src.items(): + if type(v) == list: + if not k in target: + target[k] = copy.deepcopy(v) + elif overwrite is True: + target[k].extend(v) + elif type(v) == dict: + if not k in target: + target[k] = copy.deepcopy(v) + else: + deepupdate(target[k], v, overwrite=overwrite) + elif type(v) == set: + if not k in target: + target[k] = v.copy() + elif overwrite is True: + if type(target[k]) == list: + target[k].extend(v) + elif type(target[k]) == set: + target[k].update(v) + else: + raise TypeError("Cannot update {} with {}".format(type(target[k]), type(v))) + else: + if k not in target or overwrite is True: + target[k] = copy.copy(v) + + +def send_mail(mail_config, to_list, subject, content): + """ + 发送HTML类型的邮件 + :param mail_config: dict + :param to_list: list + :param subject: str + :param content: str + :return: + """ + + mail_port = mail_config.get('mail_port', '') + mail_host = mail_config.get('mail_host', '') + mail_user = mail_config.get('mail_user', '') + mail_pass = mail_config.get('mail_pass', '') + + me = mail_user + msg = MIMEText(content, _subtype='html', _charset='utf-8') + msg['Subject'] = Header(subject, 'utf-8') + msg['From'] = me + msg['to'] = ",".join(to_list) + try: + s = smtplib.SMTP_SSL(mail_host, mail_port) + s.login(mail_user, mail_pass) + s.sendmail(me, to_list, msg.as_string()) + s.quit() + print("[Send mail] success.") + return True + except Exception as e: + print("[Send mail] error. %s" % e) + return False + + +class Ansible(object): + """ + 生成主机信息 + """ + + def __init__(self, fact_dirs, fact_cache=False): + + self.fact_dirs = fact_dirs + self.fact_cache = fact_cache + self.host_data = {} + self.remote_timestamp = 0 + + # 条件 + self.bad_threshold = 80 + self.critical_threshold = 90 + self.time_threshold = 10 * 60 + + self.default_host_info = [ + 'ansible_hostname', + 'ansible_default_ipv4', + 'ansible_distribution', + 'ansible_distribution_version', + 'ansible_kernel', + 'ansible_dns', + 'ansible_uptime_seconds', + 'ansible_date_time', + 'ansible_memory_mb', + 'ansible_memfree_mb', + 'ansible_memtotal_mb', + 'ansible_mounts', + 'ansible_swaptotal_mb', + 'ansible_swapfree_mb' + ] + + self.check_result = { + 'time': '', + 'summary': { + 'ok': 0, + 'bad': 0, + 'critical': 0, + 'total': 0, + 'error': 0 + }, + 'ok': [], + 'bad': [], + 'critical': [], + 'ok_item': {}, + 'bad_item': {}, + 'critical_item': {}, + 'error_item': {} + } + + for fact_dir in self.fact_dirs: + self._parse_fact_dir(fact_dir, self.fact_cache) + + self._set_remote_timestamp() + + def _parse_fact_dir(self, fact_dir, fact_cache=False): + if not os.path.isdir(fact_dir): + raise IOError("Not a directory: '{0}'".format(fact_dir)) + + flist = [] + for (dirpath, dirnames, filenames) in os.walk(fact_dir): + flist.extend(filenames) + break + + for fname in flist: + if fname.startswith('.'): + continue + hostname = fname + + fd = codecs.open(os.path.join(fact_dir, fname), 'r', encoding='utf8') + s = fd.readlines() + fd.close() + try: + x = json.loads(''.join(s)) + # for compatibility with fact_caching=jsonfile + # which omits the "ansible_facts" parent key added by the setup module + if self.fact_cache: + x = json.loads('{ "ansible_facts": ' + ''.join(s) + ' }') + self.update_host(hostname, x) + self.update_host(hostname, {'name': hostname}) + except ValueError as e: + # Ignore non-JSON files (and bonus errors) + print("Error parsing: %s: %s" % (fname, e)) + + def _set_remote_timestamp(self): + time_api = "http://api.m.taobao.com/rest/api3.do?api=mtop.common.getTimestamp" + try: + response = urllib.request.urlopen(time_api) + result = json.loads(response.read().decode('utf-8')) + self.remote_timestamp = int(result['data'].get('t', '0')) + except Exception as e: + self.remote_timestamp = datetime.datetime.timestamp(datetime.datetime.now()) + + def check_time(self, host, item, now): + now_timestamp = datetime.datetime.timestamp(datetime.datetime.strptime(now, "%Y-%m-%dT%H:%M:%SZ")) + time_zone = 8 * 60 * 60 + + if abs(self.remote_timestamp - now_timestamp - time_zone) >= self.time_threshold: + self.check_result['critical'].append(host) + if host not in self.check_result['critical_item']: + self.check_result['critical_item'][host] = {'critical': [], 'bad': []} + self.check_result['critical_item'][host]['critical'].append(item) + + def check_usedutilization(self, host, item, now): + if now >= self.critical_threshold: + self.check_result['critical'].append(host) + if host not in self.check_result['critical_item']: + self.check_result['critical_item'][host] = {'critical': [], 'bad': []} + self.check_result['critical_item'][host]['critical'].append(item) + + elif now >= self.bad_threshold: + if host in self.check_result['critical']: + self.check_result['critical_item'][host]['bad'].append(item) + return + self.check_result['bad'].append(host) + if host not in self.check_result['bad_item']: + self.check_result['bad_item'][host] = {'bad': []} + self.check_result['bad_item'][host]['bad'].append(item) + else: + if host in self.check_result['critical'] or host in self.check_result['bad']: + return + self.check_result['ok'].append(host) + + def update_host(self, hostname, key_values, overwrite=True): + """ + Update a hosts information. This is called by various collectors such + as the ansible setup module output and the hosts parser to add + informatio to a host. It does some deep inspection to make sure nested + information can be updated. + """ + default_empty_host = { + 'name': hostname, + } + host_info = self.host_data.get(hostname, default_empty_host) + deepupdate(host_info, key_values, overwrite=overwrite) + self.host_data[hostname] = host_info + + def get_check_result(self): + for key, host in self.host_data.items(): + print('[Check] %s' % key) + if 'ansible_facts' not in host: + self.check_result['summary']['error'] += 1 + self.check_result['error_item'][key] = {'msg': host['msg']} + continue + + usedutilization = { + 'os_time': '', + 'mem': '', + 'swap': '', + 'disk': [] + } + + iso8601 = host['ansible_facts']['ansible_date_time'].get('iso8601', None) + ansible_memtotal_mb = host['ansible_facts'].get('ansible_memtotal_mb', 0) + ansible_memfree_mb = host['ansible_facts'].get('ansible_memfree_mb', 0) + ansible_swaptotal_mb = host['ansible_facts'].get('ansible_swaptotal_mb', 0) + ansible_swapfree_mb = host['ansible_facts'].get('ansible_swapfree_mb', 0) + + if ansible_memtotal_mb != 0: + usedutilization['mem'] = int( + (ansible_memtotal_mb - ansible_memfree_mb) / ansible_memtotal_mb * 10000) / 100 + else: + usedutilization['mem'] = 0 + + if ansible_swaptotal_mb != 0: + usedutilization['swap'] = int( + (ansible_swaptotal_mb - ansible_swapfree_mb) / ansible_swaptotal_mb * 10000) / 100 + else: + usedutilization['swap'] = 0 + + usedutilization['os_time'] = iso8601 + + for disk in host['ansible_facts'].get('ansible_mounts', []): + mount = disk.get('mount', '') + fstype = disk.get('fstype', '') + if 'containers' in mount or 'iso9660' in fstype: + continue + size_total = disk.get('size_total', 0) + size_available = disk.get('size_available', 0) + block_used = disk.get('block_used', 0) + block_total = disk.get('block_total', 0) + inode_total = disk.get('inode_total', 0) + inode_used = disk.get('inode_used', 0) + + size_usedutilization = 0 + block_usedutilization = 0 + inode_usedutilization = 0 + + if size_total != 0: + size_usedutilization = int((size_total - size_available) / size_total * 10000) / 100 + + if block_total != 0: + block_usedutilization = int(block_used / block_total * 10000) / 100 + + if inode_total != 0: + inode_usedutilization = int(inode_used / inode_total * 10000) / 100 + + usedutilization['disk'].append( + {'mount': mount, 'size': size_usedutilization, 'block': block_usedutilization, + 'inode': inode_usedutilization}) + + self.check_usedutilization(key, 'mem', usedutilization['mem']) + self.check_usedutilization(key, 'swap', usedutilization['swap']) + + for du in usedutilization['disk']: + self.check_usedutilization(key, 'mount_size_' + mount, du['size']) + self.check_usedutilization(key, 'mount_block_' + mount, du['block']) + self.check_usedutilization(key, 'mount_inode_' + mount, du['inode']) + + # self.check_time(key, 'time', iso8601) + self.host_data[key]['usedutilization'] = usedutilization + + self.check_result['ok'] = sorted( + list(set(self.check_result['ok']).difference(set(self.check_result['critical'])))) + self.check_result['ok'] = sorted(list(set(self.check_result['ok']).difference(set(self.check_result['bad'])))) + self.check_result['bad'] = sorted( + list(set(self.check_result['bad']).difference(set(self.check_result['critical'])))) + self.check_result['critical'] = sorted(set(self.check_result['critical'])) + self.check_result['total'] = len(self.host_data.values()) + + self.check_result['summary']['ok'] = len(self.check_result['ok']) + self.check_result['summary']['bad'] = len(self.check_result['bad']) + self.check_result['summary']['critical'] = len(self.check_result['critical']) + self.check_result['summary']['total'] = len(self.host_data.values()) + self.check_result['summary']['error'] + + self.check_result['time'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + + for ok_item in self.check_result['ok']: + self.check_result['ok_item'][ok_item] = {} + for info in self.default_host_info: + self.check_result['ok_item'][ok_item][info] = self.host_data[ok_item]['ansible_facts'][info] + self.check_result['ok_item'][ok_item]['usedutilization'] = self.host_data[ok_item]['usedutilization'] + + for bad_item in self.check_result['bad']: + for info in self.default_host_info: + self.check_result['bad_item'][bad_item][info] = self.host_data[bad_item]['ansible_facts'][info] + self.check_result['bad_item'][bad_item]['usedutilization'] = self.host_data[bad_item]['usedutilization'] + + for critical_item in self.check_result['critical']: + for info in self.default_host_info: + self.check_result['critical_item'][critical_item][info] = \ + self.host_data[critical_item]['ansible_facts'][ + info] + self.check_result['critical_item'][critical_item]['usedutilization'] = self.host_data[critical_item][ + 'usedutilization'] + + return self.check_result + + +if __name__ == '__main__': + # 定义基础数据 + print('[Init] Set configuration') + current_path = os.path.dirname(os.path.abspath(__file__)) + now_date = datetime.datetime.now().strftime('%Y-%m-%d') + report_path = os.path.join(current_path, 'report', 'report-%s.html' % now_date) + template_path = os.path.join(current_path, 'templates') + # template_file = 'report.html' + template_file = 'report_cssinline.html' + + # 设置fact目录 + fact_dirs = [os.path.join(current_path, 'facts')] + + # 获取检查结果 + print('[Check] Get Result') + ansible = Ansible(fact_dirs=fact_dirs) + check_result = ansible.get_check_result() + + # 生成报告 + print('[Check] Generate report') + TemplateLoader = FileSystemLoader(searchpath=template_path) + TemplateEnv = Environment(loader=TemplateLoader) + template = TemplateEnv.get_template(template_file) + html = template.render(data=check_result) + + # 存储报告 + print('[Check] Save report') + with open(report_path, 'w', encoding='utf-8') as f: + f.write(html) + + # 发送邮件 + subject = 'System Check Report [%s]' % now_date + to_list = ['lework@ops.com'] + + mail_config = { + 'mail_host': 'smtp.lework.com', + 'mail_port': '465', + 'mail_user': 'ops@lework.com', + 'mail_pass': '123123' + } + print(json.dumps(check_result)) + send_mail(mail_config, to_list, subject, html) diff --git a/python/facts_os_check/run.sh b/python/facts_os_check/run.sh new file mode 100644 index 0000000..059b157 --- /dev/null +++ b/python/facts_os_check/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +[ ! -d ./report ] && mkdir ./report +[ ! -d ./facts ] && mkdir ./facts +rm -rf ./facts + +ansible all -m setup --tree ./facts + +python3 ansible.py diff --git a/python/facts_os_check/templates/report.html b/python/facts_os_check/templates/report.html new file mode 100644 index 0000000..bdcbd53 --- /dev/null +++ b/python/facts_os_check/templates/report.html @@ -0,0 +1,14963 @@ + + + + + + + + 巡检报告 + + + + +
+
+
+
+ +
+
+
+

巡检报告

+
+ 生成时间：
+ {{ data.time }} +
+
+
+
+ +
+
+
+

Error Total

+
+

{{ data.summary.error}}

+
+
+
+
+ +
+
+
+

Critical Total

+
+

{{ data.summary.critical}}

+
+
+
+
+ +
+
+
+

Bad Total

+
+

{{ data.summary.bad}}

+
+
+
+
+ +
+
+
+

OK Total

+
+

{{ data.summary.ok}}

+
+
+
+
+
+
+
+

Total

+
+

{{ data.summary.total}}

+
+
+
+
+
+
Bad评判条件: 80 <= 使用率 < 90, Critical评判条件: 使用率 >= 90
+
+
+ {% if data.summary.error != 0 %} +
+
+
+
+

Error List

+
+
+ + + + + + + + + {% for key, value in data.error_item.items() %} + + + + + {% endfor %} + +
Hostmsg
{{ key }}{{ value.msg }}
+
+
+
+
+
+
+ {% endif %} + {% if data.summary.critical != 0 %} +
+
+
+
+

Critical List

+
+
+ + + + + + + + + + + + + + + {% for key, value in data.critical_item.items() %} + + + + + + + + + + + {% endfor %} + +
HostnameMain IPOSMem UsedSwap UsedDisk Size UsedDisk Inode UsedTimestamp
{{ value.ansible_hostname }}{{ value.ansible_default_ipv4.address | default(key)}}{{ value.ansible_distribution }} {{ value.ansible_distribution_version }}{% if 'mem' in value.critical %}{% elif 'mem' in value.bad %}{%else%}{% endif %}{{ value.usedutilization.mem }}% {% if 'swap' in value.critical %}{% elif 'swap' in value.bad %}{%else%}{% endif %}{{ value.usedutilization.swap }}% + {% for d in value.usedutilization.disk %} + {% if 'mount_size_' + d.mount in value.critical %}{% elif 'mount_size_' +d.mount in value.bad %}{%else%}{% endif %} + {{ d.mount }} [{{ d.size }}%] + + {% if not loop.last %}
{% endif %} + {% endfor%} + 		+ {% for d in value.usedutilization.disk %} + {% if 'mount_inode_' + d.mount in value.critical %}{% elif 'mount_inode_' + d.mount in value.bad %} {% else %}{% endif %} + {{ d.mount }} [{{ d.inode }}%] + + {% if not loop.last %}
{% endif %} + {% endfor%} + 		{% if 'time' in value.critical %}{% elif 'time' in value.bad %}{% else %}{% endif %}{{ value.usedutilization.os_time }}
+
+
+
+
+
+
+ {% endif %} + {% if data.summary.bad != 0 %} +
+
+
+
+

Bad List

+
+
+ + + + + + + + + + + + + + + {% for key, value in data.bad_item.items() %} + + + + + + + + + + + {% endfor %} + +
HostnameMain IPOSMem UsedSwap UsedDisk Size UsedDisk Inode UsedTimestamp
{{ value.ansible_hostname }}{{ value.ansible_default_ipv4.address | default(key) }}{{ value.ansible_distribution }} {{ value.ansible_distribution_version }}{% if 'mem' in value.bad %}{% else %}{% endif %}{{ value.usedutilization.mem }}%{% if 'swap' in value.bad %}{% else %}{% endif %}{{ value.usedutilization.swap }}% + {% for d in value.usedutilization.disk %} + {% if 'mount_size_' + d.mount in value.bad %}{% else %} {% endif %} + {{ d.mount }} [{{ d.size }}%] + + {% if not loop.last %}
{% endif %} + {% endfor%} + 		+ {% for d in value.usedutilization.disk %} + {% if 'mount_inode_' + d.mount in value.bad %} {% else %} {% endif %} > + {{ d.mount }} [{{ d.inode }}%] + + {% if not loop.last %}
{% endif %} + {% endfor%} + 		{% if 'time' in value.bad %}{% else %}{% endif %}{{ value.usedutilization.os_time }}
+
+
+
+
+
+
+ {% endif %} + {% if data.summary.ok != 0 %} +
+
+
+
+

OK List

+
+
+ + + + + + + + + + + + + + + {% for key, value in data.ok_item.items() %} + + + + + + + + + + + {% endfor %} + +
HostnameMain IPOSMem UsedSwap UsedDisk Size UsedDisk Inode UsedTimestamp
{{ value.ansible_hostname }}{{ value.ansible_default_ipv4.address | default(key) }}{{ value.ansible_distribution }} {{ value.ansible_distribution_version }}{{ value.usedutilization.mem }}%{{ value.usedutilization.swap }}%{% for d in value.usedutilization.disk %} + {{ d.mount }} [{{ d.size }}%] {% if not loop.last %}
{% endif %} + {% endfor%}		{% for d in value.usedutilization.disk %} + {{ d.mount }} [{{ d.inode }}%] {% if not loop.last %}
{% endif %} + {% endfor%}		{{ value.usedutilization.os_time }}
+
+
+
+
+
+
+ {% endif %} +
+
+ +
+
+

Copyright ©2019 Developed by lework

+
+
+
+ + + \ No newline at end of file diff --git a/python/facts_os_check/templates/report_cssinline.html b/python/facts_os_check/templates/report_cssinline.html new file mode 100644 index 0000000..c23775b --- /dev/null +++ b/python/facts_os_check/templates/report_cssinline.html @@ -0,0 +1,7793 @@ + + + + + + + + 巡检报告 + + + + +
+
+
+
+ +
+
+
+

巡检报告

+
+ 生成时间：
+ {{ data.time }} +
+
+
+
+ +
+
+
+

Error Total

+
+

{{ data.summary.error}}

+
+
+
+
+ +
+
+
+

Critical Total

+
+

{{ data.summary.critical}}

+
+
+
+
+ +
+
+
+

Bad Total

+
+

{{ data.summary.bad}}

+
+
+
+
+ +
+
+
+

OK Total

+
+

{{ data.summary.ok}}

+
+
+
+
+
+
+
+

Total

+
+

{{ data.summary.total}}

+
+
+
+
+
+
Bad评判条件: 80 <= 使用率 < 90, Critical评判条件: 使用率 >= 90
+
+
+ {% if data.summary.error != 0 %} +
+
+
+
+

Error List

+
+
+ + + + + + + + + {% for key, value in data.error_item.items() %} + + + + + {% endfor %} + +
Hostmsg
{{ key }}{{ value.msg }}
+
+
+
+
+
+
+ {% endif %} + {% if data.summary.critical != 0 %} +
+
+
+
+

Critical List

+
+
+ + + + + + + + + + + + + + + {% for key, value in data.critical_item.items() %} + + + + + + + + + + + {% endfor %} + +
HostnameMain IPOSMem UsedSwap UsedDisk Size UsedDisk Inode UsedTimestamp
{{ value.ansible_hostname }}{{ value.ansible_default_ipv4.address | default(key)}}{{ value.ansible_distribution }} {{ value.ansible_distribution_version }}{% if 'mem' in value.critical %}{% elif 'mem' in value.bad %}{%else%}{% endif %}{{ value.usedutilization.mem }}% {% if 'swap' in value.critical %}{% elif 'swap' in value.bad %}{%else%}{% endif %}{{ value.usedutilization.swap }}% + {% for d in value.usedutilization.disk %} + {% if 'mount_size_' + d.mount in value.critical %}{% elif 'mount_size_' +d.mount in value.bad %}{%else%}{% endif %} + {{ d.mount }} [{{ d.size }}%] + + {% if not loop.last %}
{% endif %} + {% endfor%} + 		+ {% for d in value.usedutilization.disk %} + {% if 'mount_inode_' + d.mount in value.critical %}{% elif 'mount_inode_' + d.mount in value.bad %} {% else %}{% endif %} + {{ d.mount }} [{{ d.inode }}%] + + {% if not loop.last %}
{% endif %} + {% endfor%} + 		{% if 'time' in value.critical %}{% elif 'time' in value.bad %}{% else %}{% endif %}{{ value.usedutilization.os_time }}
+
+
+
+
+
+
+ {% endif %} + {% if data.summary.bad != 0 %} +
+
+
+
+

Bad List

+
+
+ + + + + + + + + + + + + + + {% for key, value in data.bad_item.items() %} + + + + + + + + + + + {% endfor %} + +
HostnameMain IPOSMem UsedSwap UsedDisk Size UsedDisk Inode UsedTimestamp
{{ value.ansible_hostname }}{{ value.ansible_default_ipv4.address | default(key) }}{{ value.ansible_distribution }} {{ value.ansible_distribution_version }}{% if 'mem' in value.bad %}{% else %}{% endif %}{{ value.usedutilization.mem }}%{% if 'swap' in value.bad %}{% else %}{% endif %}{{ value.usedutilization.swap }}% + {% for d in value.usedutilization.disk %} + {% if 'mount_size_' + d.mount in value.bad %}{% else %} {% endif %} + {{ d.mount }} [{{ d.size }}%] + + {% if not loop.last %}
{% endif %} + {% endfor%} + 		+ {% for d in value.usedutilization.disk %} + {% if 'mount_inode_' + d.mount in value.bad %} {% else %} {% endif %} > + {{ d.mount }} [{{ d.inode }}%] + + {% if not loop.last %}
{% endif %} + {% endfor%} + 		{% if 'time' in value.bad %}{% else %}{% endif %}{{ value.usedutilization.os_time }}
+
+
+
+
+
+
+ {% endif %} + {% if data.summary.ok != 0 %} +
+
+
+
+

OK List

+
+
+ + + + + + + + + + + + + + + {% for key, value in data.ok_item.items() %} + + + + + + + + + + + {% endfor %} + +
HostnameMain IPOSMem UsedSwap UsedDisk Size UsedDisk Inode UsedTimestamp
{{ value.ansible_hostname }}{{ value.ansible_default_ipv4.address | default(key) }}{{ value.ansible_distribution }} {{ value.ansible_distribution_version }}{{ value.usedutilization.mem }}%{{ value.usedutilization.swap }}%{% for d in value.usedutilization.disk %} + {{ d.mount }} [{{ d.size }}%] {% if not loop.last %}
{% endif %} + {% endfor%}		{% for d in value.usedutilization.disk %} + {{ d.mount }} [{{ d.inode }}%] {% if not loop.last %}
{% endif %} + {% endfor%}		{{ value.usedutilization.os_time }}
+
+
+
+
+
+
+ {% endif %} +
+
+ +
+
+

Copyright ©2019 Developed by lework

+
+
+
+ + + \ No newline at end of file