This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository mum. See http://git.chorem.org/mum.git commit b47efaed91b459bd51ec86eff159cf44aaac7b6d Author: Alexis Guilbaud <guilbaud@codelutin.com> Date: Mon Mar 9 12:10:52 2015 +0100 les modules de connection sont chargés comme ceux de monitoring --- app/app.py | 5 +- app/module_loader.py | 184 ++++++++++-------------- app/modules/connection_modules/__init__.py | 3 +- app/modules/connection_modules/nagios.py | 1 - app/modules/connection_modules/snmp.py | 1 - app/modules/connection_modules/ssh.py | 25 +++- app/modules/detection_modules/nmap_detection.py | 5 +- app/modules/monitoring_modules/__init__.py | 3 +- app/modules/monitoring_modules/disk.py | 7 +- app/modules/monitoring_modules/memory.py | 8 +- app/modules/storage_modules/shelve_db.py | 55 ++++--- 11 files changed, 145 insertions(+), 152 deletions(-) diff --git a/app/app.py b/app/app.py index 81a94f3..2d6d636 100755 --- a/app/app.py +++ b/app/app.py @@ -168,8 +168,9 @@ if __name__ == '__main__': global ml global wsc ml = ModuleLoader() - ml.get_all_monitoring_modules() + ml.load_all_monitoring_modules() + ml.load_all_connection_modules() wsc = WebSocketContainer(ml.get_db()) - process_monitoring.init(ml, wsc) + #process_monitoring.init(ml, wsc) port = int(os.environ.get('PORT', 1337)) run(host='0.0.0.0', port=port, debug=True, server=GeventWebSocketServer) \ No newline at end of file diff --git a/app/module_loader.py b/app/module_loader.py index 4abb83e..92bf62b 100644 --- a/app/module_loader.py +++ b/app/module_loader.py @@ -14,20 +14,24 @@ import re import pkgutil import sys + class ModuleLoader: """ - Loads dynamically modules from packages connection_modules, detection_modules, monitoring_modules, storage_modules. + Loads dynamically modules from packages connection_modules, detection_modules, monitoring_modules, storage_modules + and contains several methods in order to call the methods once these modules loaded. """ def __init__(self): - fconf = open('conf.txt') - dict_conf = {} + dict_conf = {} # See conf.txt + fconf = open('conf.txt', 'r') for line in fconf.read().splitlines(): fields = line.split('=') dict_conf[fields[0]] = fields[1] + fconf.close() self.db_loc = dict_conf['db_location'] self.external_mod_loc = dict_conf['external_modules_location'] self.db = self.load_db() - self.loaded_mod_infos = {} + self.loaded_mod_moni = {} # See load_all_monitoring_modules + self.loaded_mod_conn = {} # See load_all_connection_modules def load_db(self): """ @@ -42,7 +46,8 @@ class ModuleLoader: def get_db(self): return self.db - def run_nmap_detection(self, param, opt, db, ws, list_mod_conn, dict_mod_monitoring): + @staticmethod + def run_nmap_detection(param, opt, db, ws, list_mod_conn, dict_mod_monitoring): """ Instanciates the nmap_detection module from detection_modules, and runs the detection. :param param: parameter to put in nmap command. can be a hostname, a ip address or a ip range @@ -64,18 +69,6 @@ class ModuleLoader: ws.send(json.dumps({"40": hnfe.__str__()})) return None - def load_conn(self, conn_name, addr_host, username, key_location): # /home/aguilbaud/.ssh/id_rsa - """ - Instanciates and creates a connection with a connection module. - :param conn_name: the name of the detection module - :param addr_host: the IP adress of the host we want to create a connection - :param key_location: the location of the public key - :return: the instance of connection module created - """ - conn = __import__("modules.connection_modules." + conn_name, fromlist=modules.connection_modules) - conn_instance = getattr(conn, conn_name)(addr_host, username, key_location, modules.CommandNotFoundException) - return conn_instance - def run_all_detection_modules(self, os, conn, db, ws): """ Instanciates and runs every detection_modules listed in the __init__.py file of the package corresponding to @@ -97,64 +90,50 @@ class ModuleLoader: if ws is not None: ws.send(json.dumps({"40": cnfe.__str__()})) - def get_all_monitoring_modules(self): + def load_all_monitoring_modules(self): """ Instanciates and stores the informations about each monitoring modules avaliable (internal and loaded - externally) + externally) on the loaded_mod_moni attribute """ for importer, mod_name, ispkg in pkgutil.iter_modules(["app/modules/monitoring_modules/"]): if mod_name not in sys.modules: - loaded_mod = __import__("modules.monitoring_modules." + mod_name, fromlist=[mod_name]) - class_name = getattr(loaded_mod, "get_class_name")() - if mod_name == 'ping': - mod_inst = getattr(loaded_mod, class_name)(None, None) - else: - mod_inst = getattr(loaded_mod, class_name)(None, None, None) - infos_mod = {} - infos_mod['imported'] = loaded_mod - infos_mod['class_name'] = getattr(mod_inst, 'get_name')() - infos_mod['compatible_os'] = getattr(mod_inst, 'get_compatible_os')() - infos_mod['block'] = getattr(mod_inst, 'get_block')() - infos_mod['unit'] = getattr(mod_inst, 'get_unit')() - infos_mod['external'] = False - self.loaded_mod_infos[mod_name] = infos_mod - - # Now for external modules - # Adding the external diretory into pythonpath : - sys.path.insert(0, self.external_mod_loc) + try: + loaded_mod = __import__("modules.monitoring_modules." + mod_name, fromlist=[mod_name]) + class_name = getattr(loaded_mod, "get_class_name")() + if mod_name == 'ping': + mod_inst = getattr(loaded_mod, class_name)(None, None) + else: + mod_inst = getattr(loaded_mod, class_name)(None, None, None) + infos_mod = {} + infos_mod['imported'] = loaded_mod + infos_mod['class_name'] = getattr(mod_inst, 'get_name')() + infos_mod['compatible_os'] = getattr(mod_inst, 'get_compatible_os')() + infos_mod['block'] = getattr(mod_inst, 'get_block')() + infos_mod['unit'] = getattr(mod_inst, 'get_unit')() + infos_mod['external'] = False + self.loaded_mod_moni[mod_name] = infos_mod + except AttributeError: + print "Error : internal monitoring module " + mod_name + " could not have been loaded. " + + # Now for external modules: + sys.path.insert(0, self.external_mod_loc) # Adding the external diretory into pythonpath for importer, mod_name, ispkg in pkgutil.iter_modules([self.external_mod_loc]): if mod_name not in sys.modules: - loaded_mod = __import__(mod_name, fromlist=[mod_name]) - class_name = getattr(loaded_mod, "get_class_name")() - mod_inst = getattr(loaded_mod, class_name)(None, None, None) - infos_mod = {} - infos_mod['imported'] = loaded_mod - infos_mod['class_name'] = getattr(mod_inst, 'get_name')() - infos_mod['compatible_os'] = getattr(mod_inst, 'get_compatible_os')() - infos_mod['block'] = getattr(mod_inst, 'get_block')() - infos_mod['unit'] = getattr(mod_inst, 'get_unit')() - infos_mod['external'] = True - self.loaded_mod_infos[mod_name] = infos_mod - - - - """ - __import__("modules.monitoring_modules." + os) - pack_mod_os = __import__("modules.monitoring_modules." + os, fromlist=modules.monitoring_modules.__all__) - for mod_name in pack_mod_os.__all__: - mod = __import__("modules.monitoring_modules." + os + "." + mod_name, fromlist=modules.monitoring_modules.unix.__all__) # on charge le module - mod_instance = getattr(mod, mod_name)(conn, db, modules.ModuleNotCompatibleException) # on appelle le constructeur - try: - mod_instance.check() - except modules.ModuleNotCompatibleException.ModuleNotCompatibleException as mnce: - print mnce.__str__() - if ws is not None: - ws.send(json.dumps({"40": mnce.__str__()})) - except modules.CommandNotFoundException.CommandNotFoundException as cnfe: - print cnfe.__str__() - if ws is not None: - ws.send(json.dumps({"40": cnfe.__str__()})) - """ + try: + loaded_mod = __import__(mod_name, fromlist=[mod_name]) + class_name = getattr(loaded_mod, "get_class_name")() + mod_inst = getattr(loaded_mod, class_name)(None, None, None) + infos_mod = {} + infos_mod['imported'] = loaded_mod + infos_mod['class_name'] = getattr(mod_inst, 'get_name')() + infos_mod['compatible_os'] = getattr(mod_inst, 'get_compatible_os')() + infos_mod['block'] = getattr(mod_inst, 'get_block')() + infos_mod['unit'] = getattr(mod_inst, 'get_unit')() + infos_mod['external'] = True + self.loaded_mod_moni[mod_name] = infos_mod + except AttributeError: + print "Error : external monitoring module " + mod_name + " could not have been loaded. " + print "Please verify that every necessary methods have been implemented." def run_one_monitoring_module(self, mod_name, addr_host, conn, db): """ @@ -163,42 +142,23 @@ class ModuleLoader: :param mod_name: the name of the monitoring_module to run :param conn: an instance of a connection module :param db: an instance of a storage module - :param ws: a websocket connection if the function have been called from a client. Is None otherwise """ if db is None: db = self.get_db() if mod_name == 'ping': - mod_inst = getattr(self.loaded_mod_infos[mod_name]['imported'], - self.loaded_mod_infos[mod_name]['class_name'])(db, addr_host) + mod_inst = getattr(self.loaded_mod_moni[mod_name]['imported'], + self.loaded_mod_moni[mod_name]['class_name'])(db, addr_host) mod_inst.check() else: - mod_inst = getattr(self.loaded_mod_infos[mod_name]['imported'], - self.loaded_mod_infos[mod_name]['class_name'])(conn, db, + mod_inst = getattr(self.loaded_mod_moni[mod_name]['imported'], + self.loaded_mod_moni[mod_name]['class_name'])(conn, db, modules.ModuleNotCompatibleException) - mod_inst.check() - """ - if conn is None: - if mod_name == 'ping': - # in this case, the connection have not been configurated yet, so we just launch a ping check - ping_mod = __import__("modules.monitoring_modules.ping", fromlist=modules.monitoring_modules.special_modules) - ping_mod_inst = getattr(ping_mod, "ping")(db, addr_host) - ping_mod_inst.check() - else: - conn = load_conn("ssh", addr_host, "aguilbaud", "/home/aguilbaud/.ssh/id_rsa") - __import__("modules.monitoring_modules." + os) - mod = __import__("modules.monitoring_modules." + os + "." + mod_name, fromlist=modules.monitoring_modules.unix.__all__) - mod_instance = getattr(mod, mod_name)(conn, db, modules.ModuleNotCompatibleException) # on appelle le constructeur - try: - mod_instance.check() - except modules.ModuleNotCompatibleException.ModuleNotCompatibleException as mnce: - print mnce.__str__() - if ws is not None: - ws.send(json.dumps({"40": mnce.__str__()})) - except modules.CommandNotFoundException.CommandNotFoundException as cnfe: - print cnfe.__str__() - if ws is not None: - ws.send(json.dumps({"40": cnfe.__str__()})) - """ + try: + mod_inst.check() + except modules.ModuleNotCompatibleException.ModuleNotCompatibleException as mnce: + print mnce.__str__() + except modules.CommandNotFoundException.CommandNotFoundException as cnfe: + print cnfe.__str__() def get_info_mod_monitoring(self): """ @@ -217,7 +177,23 @@ class ModuleLoader: } } """ - return self.loaded_mod_infos + return self.loaded_mod_moni + + def load_all_connection_modules(self): + """ + Instanciates and stores the informations about each connection modules avaliable + on the loaded_mod_conn attribute + """ + for importer, mod_name, ispkg in pkgutil.iter_modules(["app/modules/connection_modules/"]): + if mod_name not in sys.modules: + loaded_mod = __import__("modules.connection_modules." + mod_name, fromlist=[mod_name]) + class_name = getattr(loaded_mod, "get_class_name")() + mod_inst = getattr(loaded_mod, class_name)(None, None, None) + infos_mod = {} + infos_mod['imported'] = loaded_mod + infos_mod['class_name'] = getattr(mod_inst, 'get_name')() + infos_mod['params'] = getattr(mod_inst, 'get_parameters')() + self.loaded_mod_conn[mod_name] = infos_mod def get_conection_modules_list(self): """ @@ -226,25 +202,23 @@ class ModuleLoader: :return: a list containing the names of the different connection modules declared on the __init__.py file of the connection_modules package. """ - pack_conn_os = __import__("modules.connection_modules", fromlist=modules.connection_modules.__all__) - return pack_conn_os.__all__ + return self.loaded_mod_conn def update_global_conf(self): """ Asks the database to update the configuration if new monitoring modules are added in function of the modules informations stored. """ - self.db.create_global_conf(self.loaded_mod_infos) + self.db.create_global_conf(self.loaded_mod_moni) - def get_all_monitoring_instructions(self, db): + def get_all_monitoring_instructions(self): """ runs on the database the function get_monitoring_instructions for all hosts under monitoring - :param db: the database to perform operation :return:the monitoring instructions for all hosts """ res = [] - for addr_host in db.get_list_addr_hosts(): - for instr in db.get_monitoring_instructions(addr_host): + for addr_host in self.db.get_list_addr_hosts(): + for instr in self.db.get_monitoring_instructions(addr_host): res.append(instr) return res diff --git a/app/modules/connection_modules/__init__.py b/app/modules/connection_modules/__init__.py index ef5647c..6ed406e 100644 --- a/app/modules/connection_modules/__init__.py +++ b/app/modules/connection_modules/__init__.py @@ -1,2 +1 @@ -__author__ = 'aguilbaud' -__all__ = ['ssh'] \ No newline at end of file +__author__ = 'aguilbaud' \ No newline at end of file diff --git a/app/modules/connection_modules/nagios.py b/app/modules/connection_modules/nagios.py deleted file mode 100644 index fcb43f2..0000000 --- a/app/modules/connection_modules/nagios.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'aguilbaud' diff --git a/app/modules/connection_modules/snmp.py b/app/modules/connection_modules/snmp.py deleted file mode 100644 index fcb43f2..0000000 --- a/app/modules/connection_modules/snmp.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'aguilbaud' diff --git a/app/modules/connection_modules/ssh.py b/app/modules/connection_modules/ssh.py index 74786cb..b407dc8 100644 --- a/app/modules/connection_modules/ssh.py +++ b/app/modules/connection_modules/ssh.py @@ -1,19 +1,32 @@ import paramiko -class ssh: +def get_class_name(): + return "SSH" - def __init__(self, addr_host, usrname, key_location, cnfe): - key = paramiko.RSAKey.from_private_key_file(key_location) # "/home/aguilbaud/.ssh/id_rsa" - self.ssh = paramiko.SSHClient() - self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - self.ssh.connect(addr_host, username=usrname, pkey=key) + +class SSH: + def __init__(self, addr_host, params, cnfe): + self.parameters = {"username": "string", "public_key": "file", "port": "int"} + self.name = get_class_name() self.addr_host = addr_host self.CommandNotFoundException = cnfe + if params is not None: + key = paramiko.RSAKey.from_private_key_file(params['public_key']) + self.ssh = paramiko.SSHClient() + self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.ssh.connect(addr_host, username=params['username'], pkey=key, port=params['port']) + + def get_name(self): + return self.name def get_addr_host(self): + # Called by monitoring modules return self.addr_host + def get_parameters(self): + return self.parameters + def exec_command(self, cmd): stdin, stdout, stderr = self.ssh.exec_command(cmd) out = stdout.read() diff --git a/app/modules/detection_modules/nmap_detection.py b/app/modules/detection_modules/nmap_detection.py index 4e78d7e..80216c9 100644 --- a/app/modules/detection_modules/nmap_detection.py +++ b/app/modules/detection_modules/nmap_detection.py @@ -158,10 +158,7 @@ class nmap_detection: state = port.getElementsByTagName('state')[0] service = port.getElementsByTagName('service')[0] if service.hasAttribute("ostype"): - if service.getAttribute("ostype").lower() == "linux": - dict_host['os'] = 'unix' - else: - dict_host['os'] = service.getAttribute("ostype").lower() + dict_host['os'] = service.getAttribute("ostype").lower() if state.getAttribute('state') == 'open': dict_port['portid'] = port.getAttribute('portid') dict_port['portname'] = service.getAttribute('name') diff --git a/app/modules/monitoring_modules/__init__.py b/app/modules/monitoring_modules/__init__.py index 30c04b0..6ed406e 100644 --- a/app/modules/monitoring_modules/__init__.py +++ b/app/modules/monitoring_modules/__init__.py @@ -1,2 +1 @@ -__author__ = 'aguilbaud' -special_modules = ['ping'] \ No newline at end of file +__author__ = 'aguilbaud' \ No newline at end of file diff --git a/app/modules/monitoring_modules/disk.py b/app/modules/monitoring_modules/disk.py index 57b3ba7..33ceec1 100644 --- a/app/modules/monitoring_modules/disk.py +++ b/app/modules/monitoring_modules/disk.py @@ -2,16 +2,15 @@ __author__ = 'aguilbaud' import re -""" -Check and returns the percentage of disk used ( = percentage of use of the partition mounted on /) -""" - def get_class_name(): return "Disk" class Disk: + """ + Check and returns the percentage of disk used ( = percentage of use of the partition mounted on /) + """ def __init__(self, conn, db, mnce): self.conn = conn self.db = db diff --git a/app/modules/monitoring_modules/memory.py b/app/modules/monitoring_modules/memory.py index 5c7f4cf..521fcf8 100644 --- a/app/modules/monitoring_modules/memory.py +++ b/app/modules/monitoring_modules/memory.py @@ -3,16 +3,14 @@ __author__ = 'aguilbaud' import re -""" -Check and returns the percentage of total memory used of the machine -""" - - def get_class_name(): return "Memory" class Memory: + """ + Check and returns the percentage of total memory used of the machine + """ def __init__(self, conn, db, mnce): self.conn = conn self.db = db diff --git a/app/modules/storage_modules/shelve_db.py b/app/modules/storage_modules/shelve_db.py index e8adf8e..1f739b6 100644 --- a/app/modules/storage_modules/shelve_db.py +++ b/app/modules/storage_modules/shelve_db.py @@ -62,7 +62,7 @@ class shelve_db: self.db["hosts"][addr_host]["detected"]["nmap"] = nmap_res # Preconfiguration self.db["hosts"][addr_host]["conf"] = {} - self.db["hosts"][addr_host]["conf"]["connections"] = {} #self.init_conn(json.loads(nmap_res), list_mod_conn) + self.db["hosts"][addr_host]["conf"]["connections"] = self.init_conn(json.loads(nmap_res), list_mod_conn) os_host = json.loads(nmap_res)['os'] self.db["hosts"][addr_host]["conf"]["monitoring"] = self.generate_global_conf(dict_mod_info, addr_host, os_host) @@ -79,6 +79,32 @@ class shelve_db: finally: self.close_db() + @staticmethod + def init_conn(dict_nmap_res, conn_list): + """ + Returns an initialization for the connection configuration on a host. + :param dict_nmap_res: The result of the nmap detection formatted to datastructures. + :param conn_list: A list of all connection modules avaliable + :return: + """ + dict_conn = {} + cpt = 1 + for port in dict_nmap_res['openports']: + if port["portname"] in conn_list: + dict_conn[port["portname"]] = { + "priority": cpt, + "portid": int(port["portid"]) + } + cpt += 1 + else: + dict_conn[port["portname"]] = { + "priority": 0, + "portid": None + } + dict_conn[port["portname"]]["login"] = None + dict_conn[port["portname"]]["key_couple"] = None + return dict_conn + def generate_global_conf(self, dict_mod_info, addr_host, os_host): """ Configures automatically the monitoring for a host for each of the monitoring modules avaliable, in @@ -141,23 +167,14 @@ class shelve_db: res['ping'] = ping_conf return res - def init_conn(self, dict_nmap_res, conn_list): - """ - Returns an initialization for the connection configuration on a host. - :param dict_nmap_res: The result of the nmap detection formatted to datastructures. - :param conn_list: A list of all connection modules avaliable - :return: - """ - dict_conn = {} - cpt = 1 - for port in dict_nmap_res['openports']: - if port["portname"] in conn_list: - dict_conn[port["portname"]] = { - "priority": cpt, - "portid": int(port["portid"]), - } - cpt += 1 - return dict_conn + def get_conn_param(self, addr_host): + self.open_db() + res = None + try: + res = self.db['hosts'][addr_host]['conf']['connections'] + finally: + self.close_db() + return res def get_monitoring_instructions(self, addr_host): """ @@ -620,8 +637,6 @@ class shelve_db: os = detected['os'].lower() if os == "Unknown": raise Exception('Os not detected') - if os == "linux": - os = "unix" return os finally: self.close_db() \ No newline at end of file -- To stop receiving notification emails like this one, please contact chorem.org SCM administrator <admin+scm@chorem.org>.