This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository mum. See http://git.chorem.org/mum.git commit 440088c864dcce0638ac1e0801a8b087781aca7b Author: Alexis Guilbaud <guilbaud@codelutin.com> Date: Thu Feb 26 11:06:38 2015 +0100 monitoring automatique fonctionnel --- app/app.py | 15 +++++-- app/module_loader.py | 6 ++- app/modules/storage_modules/shelve_db.py | 69 ++++++++++++++++++++++++++++++++ app/process_monitoring.py | 63 +++++++++++++++++++---------- static/js/controllers/mainCtrl.js | 10 +++++ views/dashboard.html | 2 +- views/hostpage.html | 1 + 7 files changed, 139 insertions(+), 27 deletions(-) diff --git a/app/app.py b/app/app.py index 8fcddfe..9c22377 100755 --- a/app/app.py +++ b/app/app.py @@ -8,6 +8,7 @@ from bottle_websocket import websocket import json import threading import module_loader +import process_monitoring NMAP_SCAN_DEMAND = "10" DETECTION_DEMAND = "11" @@ -45,9 +46,11 @@ class ThreadDetect(threading.Thread): for ip in json.loads(scanned_ip): conn = module_loader.load_conn("ssh", ip, "aguilbaud", "/home/aguilbaud/.ssh/id_rsa") module_loader.run_all_detection_modules(db.get_host_os(ip), conn, db, self.ws) - module_loader.run_all_monitoring_modules("unix", conn, db, self.ws) - print "first monitoring done" - + monitoring_intructions = db.get_monitoring_instructions(ip) + for instr in monitoring_intructions: + process_monitoring.add_to_waiting_list(instr) + #module_loader.run_all_monitoring_modules("unix", conn, db, self.ws) + # adding entries on process monitoring @route('/') def index(section='home'): @@ -132,6 +135,9 @@ def receive(ws): db = module_loader.load_db() ws.send(json.dumps({GET_HOSTS_RESPONSE: db.get_hosts()})) del db + elif code == HOST_INFO_DEMAND: + db = module_loader.load_db() + ws.send({INFO_HOST: db.get_host_inoformations(msg[HOST_INFO_DEMAND])}) else: break except: # Should be WebSocketError when closing the connection @@ -140,5 +146,6 @@ def receive(ws): # Lancement du serveur a l'adresse 0.0.0.0:1337 if __name__ == '__main__': + process_monitoring.init() port = int(os.environ.get('PORT', 1337)) - run(host='0.0.0.0', port=port, debug=True, reloader=True, server=GeventWebSocketServer) + run(host='0.0.0.0', port=port, debug=True, reloader=True, server=GeventWebSocketServer) \ No newline at end of file diff --git a/app/module_loader.py b/app/module_loader.py index d79199f..f1ded30 100644 --- a/app/module_loader.py +++ b/app/module_loader.py @@ -110,7 +110,7 @@ def run_all_monitoring_modules(os, conn, db, ws): ws.send(json.dumps({"40": cnfe.__str__()})) -def run_one_monitoring_module(mod_name, os, conn, db, ws): +def run_one_monitoring_module(mod_name, addr_host, os, conn, db, ws): """ Instanciates and runs one monitoring_module of the package corresponding to the operating system entered in parameters. @@ -120,6 +120,10 @@ def run_one_monitoring_module(mod_name, os, conn, db, ws): :param db: an instance of a storage module :param ws: a websocket connection if the function have been called from a client. Is None otherwise """ + if conn is None: + conn = load_conn("ssh", addr_host, "aguilbaud", "/home/aguilbaud/.ssh/id_rsa") + if db is None: + db = load_db() __import__("modules.monitoring_modules." + os) mod = __import__("modules.monitoring_modules." + os + "." + mod_name, fromlist=modules.monitoring_modules.unix.__all__) mod_instance = getattr(mod, mod_name)(conn, db, modules.ModuleNotCompatibleException) # on appelle le constructeur diff --git a/app/modules/storage_modules/shelve_db.py b/app/modules/storage_modules/shelve_db.py index ef26d36..aab9e92 100644 --- a/app/modules/storage_modules/shelve_db.py +++ b/app/modules/storage_modules/shelve_db.py @@ -140,6 +140,35 @@ class shelve_db: cpt += 1 return dict_conn + def get_monitoring_instructions(self, addr_host): + """ + Necessary to launch for the first time all activated monitoring modules for a given host + :param addr_host: the address IP of the host + :return: structured informations about monitoring in form : + { + 'addr' : val, => the IP address of the host + 'os', val, => the operating system of the host + 'mod_name', val, => the name of the monitoring module + 'time', val, => the time at when to launch the monitoring module + 'freq', val => the frequency check (in seconds) + } + """ + self.open_db() + res = [] + try: + for mod in self.db['hosts'][addr_host]['conf']['monitoring']: + if self.db['hosts'][addr_host]['conf']['monitoring'][mod]['activated']: + dict = {} + dict['addr'] = addr_host + dict['os'] = json.loads(self.db['hosts'][addr_host]['detected']['nmap'])['os'] + dict['mod_name'] = mod + dict['time'] = datetime.now() + dict['freq'] = self.db['hosts'][addr_host]['conf']['monitoring'][mod]['check_frequency'] + res.append(dict) + finally: + self.close_db() + return res + def get_hosts(self): """ Returns the essential data about all hosts under monitoring @@ -194,6 +223,46 @@ class shelve_db: self.close_db() return json.dumps(res) + def get_host_inoformations(self, addr_host): + """ + Get every informations necessary for the hostpage summary page. + :param addr_host: the IP address of the host + :return: informations concerning this host on the form : + { + addr_host:{ + 'monitoring':{ + mod_name:{ + 'state':val, + 'value':val, + 'unit':val, + 'last_check':date + } + } + 'detection':{ + mod_name:res + } + 'custom_info';val + 'interventions':list + } + } + """ + self.open_db() + res = {} + try: + res['monitoring'] = {} + for mod in self.db['hosts'][addr_host]['monitoring']: + print mod + res['monitoring'][mod] = self.db['hosts'][addr_host]['monitoring'][mod] + res['detected'] = {} + for mod in self.db['hosts'][addr_host]['detected']: + res['detected'][mod] = json.loads(self.db['hosts'][addr_host]['detected'][mod]) + res['custom_info'] = self.db['hosts'][addr_host]['conf']['custom_info'] + res['interventions'] = self.db['hosts'][addr_host]['conf']['interventions'] + finally: + self.close_db() + return json.dumps({addr_host: res}) + + def remove_host(self, addr_host): """ Removes from the database the host at the adress on parameter. If the host is part of a group, it will be diff --git a/app/process_monitoring.py b/app/process_monitoring.py index 7c76a41..c9f03b8 100644 --- a/app/process_monitoring.py +++ b/app/process_monitoring.py @@ -3,7 +3,15 @@ __author__ = 'aguilbaud' import threading from datetime import datetime from datetime import timedelta +from module_loader import run_one_monitoring_module import time +import sys + +waiting_list = [] + +def init(): + pm = ProcessMonitoring() + pm.start() class ProcessMonitoring(threading.Thread): @@ -21,38 +29,51 @@ class ProcessMonitoring(threading.Thread): The data are ordored by crescent time. This is for optimize the complexity when poping the most recent time. We could use here the deque structure from Python in order to use optimized poping left, but this structure don't have implemented the insert() function, which is used for adding new data and keep the queue ordored. - - buffer_launcher : a list containing structured data of the same type than waiting_queue, - but elements here have to be executed for check. """ def __init__(self): threading.Thread.__init__(self) - self.waiting_queue = [] - self.buffer_launcher = [] def run(self): + """ + Each second, the process will check if the last element of the list have to be lauch. + If it is the case, it will add this element to the launching list, as well with the previous elements + having the same launching date. + """ + global waiting_list while True: - if not self.waiting_queue == []: - while self.waiting_queue[len(self.waiting_queue) - 1]['time'] <= datetime.now(): - dict_mod = self.waiting_queue.pop(len(self.waiting_queue) - 1) - self.buffer_launcher.append(dict_mod) + if not waiting_list == []: + modules_to_run = [] + while waiting_list[len(waiting_list) - 1]['time'] <= datetime.now(): + dict_mod = waiting_list.pop(len(waiting_list) - 1) + modules_to_run.append(dict_mod) dict_mod['time'] = dict_mod['time'] + timedelta(seconds=dict_mod['freq']) - self.add_to_waiting_queue(dict_mod) + add_to_waiting_list(dict_mod) + rm = RunMonitoring(modules_to_run) + rm.start() time.sleep(1) - def add_to_waiting_queue(self, dict_mod): - if self.waiting_queue == []: - self.waiting_queue.append(dict_mod) - else: - pos_queue = 0 - while pos_queue < len(self.waiting_queue) and dict_mod['time'] <= self.waiting_queue[pos_queue]['time']: - pos_queue += 1 - self.waiting_queue.insert(pos_queue, dict_mod) -""" +def add_to_waiting_list(dict_mod): + """ + Adds an element to the waiting queue and keep it ordered by crescent launching times. + """ + global waiting_list + if waiting_list == []: + waiting_list.append(dict_mod) + else: + pos_queue = 0 + while pos_queue < len(waiting_list) and dict_mod['time'] <= waiting_list[pos_queue]['time']: + pos_queue += 1 + waiting_list.insert(pos_queue, dict_mod) + + class RunMonitoring(threading.Thread): - def __init__(self, list_of_dict_mod): + def __init__(self, list_dict_mod): threading.Thread.__init__(self) - self.list_of_dict_mod = list_of_dict_mod + self.list_dict_mod = list_dict_mod def run(self): - """ \ No newline at end of file + for dict_mod in self.list_dict_mod: + print "Launching " + str(dict_mod['os']) + "." + str(dict_mod['mod_name']) + " on " + str(dict_mod['addr']) + sys.stdout.flush() + run_one_monitoring_module(dict_mod['mod_name'], dict_mod['addr'], dict_mod['os'], None, None, None) \ No newline at end of file diff --git a/static/js/controllers/mainCtrl.js b/static/js/controllers/mainCtrl.js index c71643f..452e6c3 100644 --- a/static/js/controllers/mainCtrl.js +++ b/static/js/controllers/mainCtrl.js @@ -101,6 +101,13 @@ mumApp.controller('mainController', ['$scope', 'toastr', '$interval', '$filter', $scope.scan_is_over = false; // pour afficher ou non certaines parties de la page $scope.ip_scanned = {}; + /* + // Concerning the hostpage + $scope.host_informations = null; + $scope.get_host_informations = function(addr_host){ + var request = '{"13" : "'+ addr_host + '"}'; + ws.send(request); + }*/ // Concerning WebSocket var ws = new WebSocket("ws://0.0.0.0:1337/websocket"); @@ -121,6 +128,9 @@ mumApp.controller('mainController', ['$scope', 'toastr', '$interval', '$filter', }); toastr.success(value, "Success on module execution"); case 21: // Informations concerning one host + /*$scope.$apply(function(){ + $scope.host_informations = JSON.parse(value); + });*/ break; case 22: // List of hosts under monitoring $scope.$apply(function(){ diff --git a/views/dashboard.html b/views/dashboard.html index d405482..e621f0d 100644 --- a/views/dashboard.html +++ b/views/dashboard.html @@ -68,7 +68,7 @@ orderBy:sort.sortingOrder:sort.reverse | filter:{addr:addr_filter, name:name_filter, status:status_filter, group:{name:group_filter}}" class={{item.status}}> - <td><a href="#/hostpage/{{item.addr}}">{{item.addr}}</a></td> + <td><a href="#/hostpage/{{item.addr}}" ng-click="get_host_informations({{item.addr}})">{{item.addr}}</a></td> <td>{{item.name}}</td> <td>{{item.status}}</td> <td>{{getGroupsByAddr(item.addr)}}</td> diff --git a/views/hostpage.html b/views/hostpage.html index fd9bd28..f25aec8 100644 --- a/views/hostpage.html +++ b/views/hostpage.html @@ -5,6 +5,7 @@ <button type="button" class="btn btn-primary btn-xs" data-toggle="modal" data-target="#modal_conn">Connection settings</button> <button type="button" class="btn btn-info btn-xs">Launch a new detection</button> <button type="button" class="btn btn-danger btn-xs">Remove this host</button> + {{host_informations}} <table class="table table-condensed table-hover"> <thead> -- To stop receiving notification emails like this one, please contact chorem.org SCM administrator <admin+scm@chorem.org>.