#!/usr/bin/python import argparse import sys import json import time import re import requests from requests.auth import HTTPBasicAuth import urllib3 import subprocess from subprocess import call, check_output import hashlib import base64 import string import smtplib import random import os import getpass import keyring urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) log_directory = "/home/nagios/logs" alert_log_file = log_directory+"/log_alerts" nodes_log_file = log_directory+"/log_nodes" mail_log_file = log_directory+"/log_mail" certificate_file = log_directory+"/certificate.pem" class node: def __init__(self, uuid, state, name, IP): self.uuid = uuid self.alertsUUID = [] self.alertsMessage = [] self.state = state self.name = name self.IP = IP def add_alert(self, alertUUID, alertMessage): self.alertsUUID.append(alertUUID) self.alertsMessage.append(alertMessage) def get_node_status(self): IP = "" alertNumber = 0 if "Available" in self.IP: IP = self.IP else: for i in self.IP: IP += i + ", " IP = IP[:-2] if self.alertsMessage: message = "Node: " + self.name + "; with IP: "+ IP + "; is in " + self.state + " state; with the following alerts; \n" for i in self.alertsMessage: alertNumber += 1 message += str(alertNumber) + ") " + i + "\n" else: message = "Node: " + self.name + "; with IP: "+ IP + "; is in " + self.state + " state; with no alerts\n" return message def is_json(myjson): try: json_object = json.loads(myjson) except ValueError, e: return "Not a json" return "It is a json" def api_call(url, lxca_instance): storedUsername = keyring.get_password(lxca_instance, "username") storedPassword = keyring.get_password(lxca_instance, "password") certificate = keyring.get_password(lxca_instance, "certificate") with open(certificate_file, 'w+') as f: f.write(certificate) response = requests.get(url, verify=certificate_file, auth=HTTPBasicAuth(storedUsername, storedPassword)) os.remove(certificate_file) response.raise_for_status() return response.json() def api_call_certificate(url, username, password): response = requests.get(url, verify=False, auth=HTTPBasicAuth(username, password)) response.raise_for_status() return response.content def get_node_info(lxca_instance, uuid): storedUrl = keyring.get_password(lxca_instance, "url") tempUrl = storedUrl+"/nodes/"+uuid+"?includeAttributes=name,overallHealthState,ipv4Addresses,ipv6Addresses,errorFields" return api_call(tempUrl.format(url), lxca_instance) def get_devices(lxca_instance): storedUrl = keyring.get_password(lxca_instance, "url") return api_call("{0}/nodes?includeAttributes=name,overallHealthState,ipv4Addresses&status=managed".format(storedUrl), lxca_instance) def get_storages(lxca_instance): storedUrl = keyring.get_password(lxca_instance, "url") return api_call("{0}/storage?includeAttributes=name,overallHealthState,ipv4Addresses&status=managed".format(storedUrl), lxca_instance) def get_switches(lxca_instance): storedUrl = keyring.get_password(lxca_instance, "url") return api_call("{0}/switches?includeAttributes=name,overallHealthState,ipv4Addresses&status=managed".format(storedUrl), lxca_instance) def get_cmms(lxca_instance): storedUrl = keyring.get_password(lxca_instance, "url") return api_call("{0}/cmms?includeAttributes=name,overallHealthState,ipv4Addresses&status=managed".format(storedUrl), lxca_instance) def get_chassis(lxca_instance): storedUrl = keyring.get_password(lxca_instance, "url") return api_call("{0}/chassis?includeAttributes=name,overallHealthState,ipv4Addresses&status=managed".format(storedUrl), lxca_instance) def get_certificate(url, username, password): return api_call_certificate("{0}/signingCertificate/pem".format(url), username, password) def get_node_alerts(lxca_instance, uuid): storedUrl = keyring.get_password(lxca_instance, "url") return api_call("{0}/events/activeAlerts/?uidList=[\""+uuid+"\"]".format(storedUrl), lxca_instance) def get_alerts(lxca_instance, uuid): storedUrl = keyring.get_password(lxca_instance, "url") return api_call("{0}/events/activeAlerts/?uidList=[\""+uuid+"\"]".format(storedUrl), lxca_instance) def get_all_alerts(lxca_instance): storedUrl = keyring.get_password(lxca_instance, "url") return api_call("{0}/events/activeAlerts/?includeAttributes=name,overallHealthState,ipv4Addresses,ipv6Addresses,UUID".format(storedUrl), lxca_instance) def print_help(parser): parser.print_help() exit(1) def write_to_log(message, location): f = open(location,"w+") f.write(message) f.close() def send_mail(_args, alert): SERVER = "localhost" FROM = "nagios@localhost.cezar" TO = [_args.contactemail] SUBJECT = '** '+_args.notificationtype+' Service Alert: '+_args.hostalias+'/'+_args.servicedesc+' is ' +_args.servicestate+' **' TEXT = '***** Nagios *****\n\nNotification Type: '+_args.notificationtype+'\n\nService: '+_args.servicedesc+'\nHost: '+_args.hostalias+'\nAddress: '+_args.hostaddress+'\nState: '+_args.servicestate+'\n\nDate/Time: '+_args.longdatetime+'\n\nAdditional Info:\n\n'+alert+'\n' message = string.join(( "From: %s" % FROM, "To: %s" % ', '.join(TO), "Subject: %s" % SUBJECT , "", TEXT ), "\r\n") server = smtplib.SMTP(SERVER) server.set_debuglevel(3) server.sendmail(FROM, TO, message) server.quit() return "OK" def clear_log_file(logToDelete): logFileTemp = open(logToDelete, "w+") logFileTemp.close() def get_encryption_key(): uuid = '' test = subprocess.Popen(['cat', '/etc/fstab'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout,stderr = test.communicate() for i in stdout.split("\n"): if "boot" in i: for j in i.split(" "): if "UUID" in j: uuid = j ips = check_output(['hostname', '--all-ip-addresses']) uuid += ips.split(" ")[0] return hashlib.sha224(uuid).hexdigest() def encode(key, clear): enc = [] for i in range(len(clear)): key_c = key[i % len(key)] enc_c = chr((ord(clear[i]) + ord(key_c)) % 256) enc.append(enc_c) return base64.urlsafe_b64encode("".join(enc)) def decode(key, enc): dec = [] enc = base64.urlsafe_b64decode(enc) for i in range(len(enc)): key_c = key[i % len(key)] dec_c = chr((256 + ord(enc[i]) - ord(key_c)) % 256) dec.append(dec_c) return "".join(dec) def generate_hash_credentials(): username = raw_input('Enter username: ') password = getpass.getpass('Enter password: ') print 'Enter URL for LXCA instance (example: https://10.252.48.38)' url = raw_input('URL: ') keyring.set_password("system", "username", username) keyring.set_password("system", "password", password) keyring.set_password("system", "url", url) certificate = get_certificate(url, username, password) keyring.set_password("system", "certificate", certificate) def get_bad_devices(lxca_instance): bad_devices = [] good_devices = [] devices = get_devices(lxca_instance).get("nodeList") storages = get_storages(lxca_instance).get("storageList") switches = get_switches(lxca_instance).get("switchList") cmms = get_cmms(lxca_instance).get("cmmList") ok_counter = 0 warning_counter = 0 error_counter = 0 exit_status = 0 ok_state = ["Normal"] warning_state = ["Non-Critical", "Warning"] error_state = ["Minor-Failure", "Major-Failure", "Non-Recoverable", "Critical", "Unknown"] for device in devices: state = device["overallHealthState"] uuid = device["uuid"] name = device["name"] if device["ipv4Addresses"]: ip = device["ipv4Addresses"] else: ip = "Not Available" if state != "Normal": if state in warning_state: bad_device = node(uuid, "Warning", name, ip) else: bad_device = node(uuid, "Critical", name, ip) bad_devices.append(bad_device) else: good_device = node(uuid, state, name, ip) good_devices.append(good_device) if state in ok_state: ok_counter += 1 elif state in warning_state: warning_counter += 1 elif state in error_state: error_counter += 1 if exit_status == 0 and device["overallHealthState"] in warning_state: exit_status = 1 elif exit_status == 0 and device["overallHealthState"] in error_state: exit_status = 2 elif exit_status == 1 and device["overallHealthState"] in error_state: exit_status = 2 for device in switches: state = device["overallHealthState"] uuid = device["uuid"] name = device["name"] try: if device["ipv4Addresses"]: ip = device["ipv4Addresses"] else: ip = "Not Available" except KeyError: ip = "Not Available" if state != "Normal": if state in warning_state: bad_device = node(uuid, "Warning", name, ip) else: bad_device = node(uuid, "Critical", name, ip) bad_devices.append(bad_device) else: good_device = node(uuid, state, name, ip) good_devices.append(good_device) if state in ok_state: ok_counter += 1 elif state in warning_state: warning_counter += 1 elif state in error_state: error_counter += 1 if exit_status == 0 and device["overallHealthState"] in warning_state: exit_status = 1 elif exit_status == 0 and device["overallHealthState"] in error_state: exit_status = 2 elif exit_status == 1 and device["overallHealthState"] in error_state: exit_status = 2 for device in storages: state = device["overallHealthState"] uuid = device["uuid"] name = device["name"] try: if device["ipv4Addresses"]: ip = device["ipv4Addresses"] else: ip = "Not Available" except KeyError: ip = "Not Available" if state != "Normal": if state in warning_state: bad_device = node(uuid, "Warning", name, ip) else: bad_device = node(uuid, "Critical", name, ip) bad_devices.append(bad_device) else: good_device = node(uuid, state, name, ip) good_devices.append(good_device) if state in ok_state: ok_counter += 1 elif state in warning_state: warning_counter += 1 elif state in error_state: error_counter += 1 if exit_status == 0 and device["overallHealthState"] in warning_state: exit_status = 1 elif exit_status == 0 and device["overallHealthState"] in error_state: exit_status = 2 elif exit_status == 1 and device["overallHealthState"] in error_state: exit_status = 2 for device in cmms: state = device["overallHealthState"] uuid = device["uuid"] name = device["name"] try: if device["ipv4Addresses"]: ip = device["ipv4Addresses"] else: ip = "Not Available" except KeyError: ip = "Not Available" if state != "Normal": if state in warning_state: bad_device = node(uuid, "Warning", name, ip) else: bad_device = node(uuid, "Critical", name, ip) bad_devices.append(bad_device) else: good_device = node(uuid, state, name, ip) good_devices.append(good_device) if state in ok_state: ok_counter += 1 elif state in warning_state: warning_counter += 1 elif state in error_state: error_counter += 1 if exit_status == 0 and device["overallHealthState"] in warning_state: exit_status = 1 elif exit_status == 0 and device["overallHealthState"] in error_state: exit_status = 2 elif exit_status == 1 and device["overallHealthState"] in error_state: exit_status = 2 return (bad_devices, good_devices, exit_status) def sort_alerts(devices, alerts, write_to_log_f): write_alert_log = "" for i in alerts: componentID = i["componentID"] write_alert_log += i["alertID"] + "\n" for j in devices: deviceUUID = j.uuid if componentID == deviceUUID: alertID = i["alertID"] alertMsg = i["msg"] j.add_alert(alertID, alertMsg) if write_to_log_f: write_to_log(write_alert_log, alert_log_file) write_alert_log = write_alert_log.split("\n") return devices def changed_alerts(_args): alerts_log = "" new_alerts = "" removed_alerts = "" all_alerts = get_all_alerts(_args.lxca) all_alerts_dumps = json.dumps(all_alerts) all_alerts_present = json.loads(all_alerts_dumps) if os.path.exists(alert_log_file): with open(alert_log_file, 'r+') as f: alerts_log = f.read() else: with open(alert_log_file, 'w+') as f: alerts_log = f.read() all_alerts = "" for i in all_alerts_present: present_uuid = i["alertID"] all_alerts += i["alertID"] if present_uuid not in alerts_log: new_alerts += "Node: " + i["componentID"] + " generated alert: " + i["alertID"] + " with the following message: " + i["msg"] + "\n" alerts_log = alerts_log.split("\n") for i in alerts_log: if i not in all_alerts: temp_alert_string = "" temp_alert = i.split(":") temp_node = str(temp_alert[-1]) temp_alert = temp_alert[:-1] for j in temp_alert: temp_alert_string += j removed_alerts += "Node: " + temp_node + " no longer has the alert with ID: " + temp_alert_string + "\n" return new_alerts def send_changes_mail(new_alerts, changed_status, _args): status = "" if new_alerts: status += send_mail(_args, new_alerts) if changed_status: status += send_mail(_args, changed_status) return status def check_node_state(longServiceOutput): message = "" logFile = "" logFileTemp = "" readLog = "" readLogTemp = "" if os.path.exists(nodes_log_file): with open(nodes_log_file, 'r+') as origFile: logFileTemp = origFile.read() else: with open(nodes_log_file, 'w+') as origFile: logFileTemp = origFile.read() readLogTemp = str(longServiceOutput) logFileTemp = logFileTemp.split("\n") readLogTemp = readLogTemp.split("\n") for i in logFileTemp: if("LONGSERVICEOUTPUT" in i): continue if("Devices:" in i): continue if("Issues:" in i): continue if not i: continue logFile += i logFile += "\n" for q in readLogTemp: if("LONGSERVICEOUTPUT" in q): continue if("Devices:" in q): continue if("Issues:" in q): continue if not q: continue readLog += q readLog += "\n" logFile = logFile.split("\n") readLog = readLog.split("\n") logFile.pop() readLog.pop() for i in logFile: if not i.startswith("Node:"): continue logInfo = i.split(";") hostnameLog = logInfo[0] stateLog = logInfo[2] for j in readLog: if not j.startswith("Node:"): continue readInfo = j.split(";") hostnameRead = readInfo[0] stateRead = readInfo[2] if hostnameLog.replace(" ", "") == hostnameRead.replace(" ", ""): if stateLog.replace(" ", "").replace("\n","") != stateRead.replace(" ", "").replace("\n",""): stateLog = stateLog[6:] stateLog = stateLog[:-5] stateRead = stateRead[6:] stateRead = stateRead[:-5] message += hostnameLog + " changed from \"" + stateLog + "\" to \"" + stateRead + "\" state\n" return message def check_lxca(_args, write_to_log_f): bad_devices, good_devices, exit_status = get_bad_devices(_args.lxca) all_alerts = get_all_alerts(_args.lxca) all_alerts_dumps = json.dumps(all_alerts) all_alerts_json = json.loads(all_alerts_dumps) alerts_per_system = sort_alerts(bad_devices, all_alerts_json, write_to_log_f) return (alerts_per_system, good_devices, exit_status) def get_lxca_status(good_nodes, bad_nodes, write): message = "To see whole error log please switch from SERVICEOUTPUT to LONGSERVICEOUTPUT\n" for i in good_nodes: message += i.get_node_status() for j in bad_nodes: message += j.get_node_status() message += "Issues: " if write: write_to_log(message, nodes_log_file) return message if __name__ == '__main__': exit_status = 0 if not os.path.exists(log_directory): os.makedirs(log_directory) if (len(sys.argv) == 17): parser = argparse.ArgumentParser( description="Check Lenovo xClarity (LXCA) for SystemStatus on all managed devices") parser.add_argument('--notificationtype', type=str) parser.add_argument('--servicedesc', type=str) parser.add_argument('--hostaddress', type=str) parser.add_argument('--servicestate', type=str) parser.add_argument('--longdatetime', type=str) parser.add_argument('--longserviceoutput', type=str) parser.add_argument('--hostalias', type=str) parser.add_argument('--contactemail', type=str) _args = parser.parse_args() mail_file_content = "" if os.path.exists(mail_log_file): with open(mail_log_file, 'r+') as mail_file: mail_file_content = mail_file.read() else: with open(mail_log_file, 'w+') as mail_file: mail_file_content = mail_file.read() mail_file_content = mail_file_content.split("-----") changed_status = mail_file_content[0] new_alerts = mail_file_content[1] print "Changed status: " + changed_status print "New_alerts: " + new_alerts if "alert" in new_alerts: send_changes_mail(new_alerts, changed_status, _args) else: send_changes_mail("", changed_status, _args) exit(0) if (len(sys.argv) == 2): generate_hash_credentials() exit(0) if (len(sys.argv) == 3): parser = argparse.ArgumentParser( description="Check Lenovo xClarity (LXCA) for SystemStatus on all managed devices") parser.add_argument('--lxca', type=str) _args = parser.parse_args() write_to_log_flag = 0 new_alerts = "" changed_status = "" new_alerts = "" status_lxca_bad, status_lxca_good, exit_status = check_lxca(_args, write_to_log_flag) all_nodes = get_lxca_status(status_lxca_good, status_lxca_bad, write_to_log_flag) changed_status = check_node_state(all_nodes) new_alerts = changed_alerts(_args) to_send_mail = changed_status + "-----\n" + new_alerts write_to_log(to_send_mail, mail_log_file) write_to_log_flag = 1 check_lxca(_args, write_to_log_flag) print get_lxca_status(status_lxca_good, status_lxca_bad, write_to_log_flag) exit(exit_status)