This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository mum. See http://git.chorem.org/mum.git commit ad00240170f3e0e9bd249b8908f6adb6b1e9de55 Author: Alexis Guilbaud <guilbaud@codelutin.com> Date: Fri Apr 3 12:00:26 2015 +0200 lock ajouté dans la classe shelve + refctoring des méthodes appellant process_monitoring afin de ne plus ajour de dépendances shelve/process_monitoring (tout centraliser dans module_loader) --- app/module_loader.py | 43 ++++++++++++++++++++------- app/modules/storage_modules/shelve_db.py | 50 +++++++++++++++++--------------- app/mum.py | 21 ++++++-------- static/js/controllers/headCtrl.js | 3 ++ static/js/controllers/hostPageCtrl.js | 15 +++++----- 5 files changed, 79 insertions(+), 53 deletions(-) diff --git a/app/module_loader.py b/app/module_loader.py index 1328a59..451aa08 100644 --- a/app/module_loader.py +++ b/app/module_loader.py @@ -8,6 +8,8 @@ import modules.storage_modules import modules.ModuleNotCompatibleException import modules.CommandNotFoundException import modules.HostNotFoundException +from modules.notification_modules.websocket_container import WebSocketContainer +import process_monitoring import json import re @@ -21,28 +23,25 @@ class ModuleLoader: Loads dynamically modules from packages connection_modules, detection_modules, monitoring_modules, storage_modules and contains several methods in order to call the methods once these modules loaded. """ - def __init__(self, add_func, rem_func, conf): + def __init__(self, conf): self.conf = conf - ''' - self.db_loc = dict_conf['db_location'] - self.external_mod_loc = dict_conf['external_modules_location'] - self.public_keys_loc = dict_conf['keys_location'] - ''' - self.db = self.load_db(add_func, rem_func, self.conf['keys_location']) + self.db = self.load_db(self.conf['keys_location']) self.loaded_mod_moni = {} # See load_all_monitoring_modules self.loaded_mod_detect = {} # See load_all_detection_modules self.loaded_mod_conn = {} # See load_all_connection_modules self.loaded_mod_notif = {} # See load_all_notification_modules self.compatible_os_list = ['other'] # Will contain the list of os compatibles for every monitoring module loaded + self.wsc = WebSocketContainer(self.db) + process_monitoring.init(self, self.wsc) - def load_db(self, add_func, rem_func, key_loc): + def load_db(self, key_loc): """ Creates an instance of the class shelve_db from storage_modules. :return: an instance of the shelve_db class """ db_name = "shelve_db" db = __import__("modules.storage_modules." + db_name, fromlist=modules.storage_modules) - db_instance = getattr(db, db_name)(self.conf['db_location'], add_func, rem_func, key_loc) + db_instance = getattr(db, db_name)(self.conf['db_location'], key_loc) return db_instance def get_db(self): @@ -51,6 +50,17 @@ class ModuleLoader: def get_public_keys_loc(self): return self.conf['keys_location'] + def get_websocket_container(self): + return self.wsc + + @staticmethod + def add_to_waiting_list(instr): + process_monitoring.add_to_waiting_list(instr) + + @staticmethod + def stop_monitoring(): + process_monitoring.end = True + @staticmethod def run_nmap_detection(param, opt, db, ws, list_mod_conn, dict_mod_monitoring): """ @@ -397,11 +407,24 @@ class ModuleLoader: return getattr(self.db, dict_instr['func'])(dict_instr['args']) def update_activated_modules(self, args): - self.db.config_mod_activation(args, self.get_monitoring_modules_list()) + dict_instr = self.db.config_mod_activation(args) + for add_instr in dict_instr['add']: + process_monitoring.add_to_waiting_list(add_instr) + for rem_instr in dict_instr['rem']: + process_monitoring.remove_to_waiting_list(rem_instr['addr'], rem_instr['mod_name']) + + def set_conf_moni_mod(self, args): + self.db.set_conf_mod(args) + process_monitoring.update_mod_on_waiting_list(args) def get_public_keys_list(self): return os.listdir(self.conf['keys_location']) + def remove_host(self, args): + self.db.remove_host(args) + # removing monitoring instructions for this host + process_monitoring.remove_to_waiting_list(args['addr_host'], None) + def get_host_info(self, addr_host): host_info = self.db.get_host_informations(addr_host) host_info['compatible_os_list'] = self.compatible_os_list diff --git a/app/modules/storage_modules/shelve_db.py b/app/modules/storage_modules/shelve_db.py index 36bae67..5aaa385 100644 --- a/app/modules/storage_modules/shelve_db.py +++ b/app/modules/storage_modules/shelve_db.py @@ -4,6 +4,7 @@ from datetime import datetime import json import shelve import traceback +import threading import os.path @@ -14,18 +15,19 @@ class shelve_db: Storage module for the persistant objects in Python : Shelve. Every function in need to access the database have to be moved in this class. """ - def __init__(self, db_loc, add_func, rem_func, key_loc): + def __init__(self, db_loc, key_loc): self.db = None + self.lock = threading.Lock() self.db_loc = db_loc - self.add_to_monitoring_list = add_func - self.rem_to_monitoring_list = rem_func self.key_loc = key_loc def open_db(self): """ Open the shelve database from the file specified in conf.txt. If the file donesn't exists, it will be created and the first structure will also be initialized. + Moreover, a lock is activated during all the transaction, because Shelve is not thread safe. """ + self.lock.acquire() if not os.path.isfile(self.db_loc): # init of the database at the first opening self.db = shelve.open(self.db_loc, writeback=True) try: @@ -40,10 +42,11 @@ class shelve_db: def close_db(self): """ - Closes the database + Closes the database and release the lock. """ self.db.close() self.db = None + self.lock.release() def init_global_conf(self, loaded_mod_moni): """ @@ -397,8 +400,6 @@ class shelve_db: addr_host = args['addr_host'] self.open_db() try: - # removing monitoring instructions for this host - self.rem_to_monitoring_list(addr_host, None) # removing monitoring entries for this host if addr_host in self.db['hosts']: del self.db['hosts'][addr_host] @@ -489,7 +490,7 @@ class shelve_db: finally: self.close_db() - def config_mod_activation(self, args, dict_mod_info): + def config_mod_activation(self, args): """ Activates or desactivates monitoring modules for a given host :param args: a dictionary containing : @@ -499,19 +500,16 @@ class shelve_db: mod_name: bool } } - :param dict_mod_info: A dictionary containing informations about all notification modules, in the form: + :return: a dictionary containing the instructions to send to the process monitoring in the form: { - mod_name: - { - 'class_name': string, => the name of the class to instanciate - 'compatible_os': [string1, string2, ...], => a list containing the compatibles os - 'unit': string, => the unit type of return ('%', 'bool' or other) - 'block': string, => the monitoring block of the module - 'external': bool => indicates if this modules comes from external directory - } + 'add': [{'addr': str, 'os': str, 'mod_name': str, 'time': datetime, 'freq': int}, ...] + 'rem': [{'addr': str, 'mod_name': str}, ... ] } """ addr_host = args['addr_host'] + dict_instr = {} + dict_instr['add'] = [] + dict_instr['rem'] = [] self.open_db() try: for mod_name in args['activated']: @@ -526,21 +524,25 @@ class shelve_db: if args['activated'][mod_name]: # if it is now activated, we have to add it to the monitoring list self.db["hosts"][addr_host]["conf"]["monitoring"][mod_name]["activated"] = True - dict_instr = {} - dict_instr['addr'] = addr_host - dict_instr['os'] = json.loads(self.db['hosts'][addr_host]['detected']['nmap'])['os'] - dict_instr['mod_name'] = mod_name - dict_instr['time'] = datetime.now() - dict_instr['freq'] = self.db['hosts'][addr_host]['conf']['monitoring'][mod_name]['check_frequency'] - self.add_to_monitoring_list(dict_instr) + add_instr = {} + add_instr['addr'] = addr_host + add_instr['os'] = json.loads(self.db['hosts'][addr_host]['detected']['nmap'])['os'] + add_instr['mod_name'] = mod_name + add_instr['time'] = datetime.now() + add_instr['freq'] = self.db['hosts'][addr_host]['conf']['monitoring'][mod_name]['check_frequency'] + dict_instr['add'].append(add_instr) else: # if not, we have to remove it from the monitoring list self.db["hosts"][addr_host]["conf"]["monitoring"][mod_name]["activated"] = False - self.rem_to_monitoring_list(args['addr_host'], mod_name) + rem_instr = {} + rem_instr['addr'] = args['addr_host'] + rem_instr['mod_name'] = mod_name + dict_instr['rem'].append(rem_instr) except Exception: print traceback.format_exc() finally: self.close_db() + return dict_instr def get_conf_mod(self, args): """ diff --git a/app/mum.py b/app/mum.py index fee7382..e84e63d 100755 --- a/app/mum.py +++ b/app/mum.py @@ -9,8 +9,6 @@ from geventwebsocket import WebSocketError import json import threading from module_loader import ModuleLoader -import process_monitoring -from modules.notification_modules.websocket_container import WebSocketContainer import argparse # Pour lancer la detection nmap avec un nouveau thread @@ -32,7 +30,7 @@ class ThreadDetect(threading.Thread): for ip in json.loads(scanned_ip): monitoring_intructions = db.get_monitoring_instructions(ip) for instr in monitoring_intructions: - process_monitoring.add_to_waiting_list(instr) + ml.add_to_waiting_list(instr) """ # now launching full detection for ip in json.loads(scanned_ip): @@ -117,7 +115,7 @@ def bower_files(filepath): @get('/websocket', apply=[websocket]) def receive(ws): global ml - wsc.add_websocket(ws) + ml.get_websocket_container().add_websocket(ws) while True: try: response = ws.receive() @@ -134,6 +132,9 @@ def receive(ws): ws.send(json.dumps({"RES_GET_HOSTS": db.get_hosts()})) elif code == "GET_HOST_INFO": # asked from hostpage ws.send(json.dumps({"RES_INFO_HOST": ml.get_host_info(msg["GET_HOST_INFO"])})) + elif code == "REM_HOST": + ml.remove_host(msg["REM_HOST"]) + ws.send(json.dumps({"RES_REM_HOST": True})) elif code == "CALL_FUNC_DB": # asked when the request can directly pass by database res = ml.launch_db_function(msg["CALL_FUNC_DB"]) msg = json.dumps({"RES_CALL_FUNC_DB": { @@ -159,8 +160,7 @@ def receive(ws): except Exception: ws.send(json_dumps({"RES_TEST_CONN": False})) elif code == "SET_CONF_MOD": # asked from hostpage, at the monitoring module configuration - ml.launch_db_function({'func': 'set_conf_mod', 'args': msg["SET_CONF_MOD"]}) - process_monitoring.update_mod_on_waiting_list(msg["SET_CONF_MOD"]) + ml.set_conf_moni_mod(msg["SET_CONF_MOD"]) elif code == "CHECK_NOW": # asekd from hostpage args = msg["CHECK_NOW"] ml.run_one_monitoring_module(args['mod_name'], args['addr_host'], None, None) @@ -169,7 +169,7 @@ def receive(ws): else: break except WebSocketError: # Should be WebSocketError when closing the connection - wsc.remove_websocket(ws) + ml.get_websocket_container().remove_websocket(ws) break @route('/upload', method='POST') @@ -186,7 +186,6 @@ def do_upload(): # Lancement du serveur a l'adresse 0.0.0.0:1337 if __name__ == '__main__': global ml - global wsc # creating the parser for the command line arguments parser = argparse.ArgumentParser(description="Mum (Machines Under Monotoring) is a web-oriented and modulable " @@ -219,17 +218,15 @@ if __name__ == '__main__': if dict_args[arg] is not None: conf[arg] = dict_args[arg] - ml = ModuleLoader(process_monitoring.add_to_waiting_list, process_monitoring.remove_to_waiting_list, conf) + ml = ModuleLoader(conf) ml.load_all_monitoring_modules() ml.load_all_connection_modules() ml.load_all_detection_modules() ml.load_all_notification_modules() ml.get_db().init_global_conf(ml.get_monitoring_modules_list()) - wsc = WebSocketContainer(ml.get_db()) #dict_notif = ml.db.add_check('127.0.0.1', "ping", False) #ml.run_notification_modules(dict_notif) - process_monitoring.init(ml, wsc) port = int(os.environ.get('PORT', int(conf['server_port']))) run(host=conf['server_addr'], port=port, debug=True, server=GeventWebSocketServer) # after ending - process_monitoring.end = True \ No newline at end of file + ml.stop_monitoring() \ No newline at end of file diff --git a/static/js/controllers/headCtrl.js b/static/js/controllers/headCtrl.js index eea029c..d57023e 100644 --- a/static/js/controllers/headCtrl.js +++ b/static/js/controllers/headCtrl.js @@ -41,6 +41,9 @@ mumApp.controller('headCtrl', function($scope, $rootScope, toastr, $interval, $r $scope.items = DataHosts.Items; }); break; + case "RES_REM_HOST": + $rootScope.$broadcast("remHost"); + break; case "RES_GET_LOADED_CONN_MOD": $rootScope.$broadcast("resGetLoadedConnMod", obj[key]); break; diff --git a/static/js/controllers/hostPageCtrl.js b/static/js/controllers/hostPageCtrl.js index 23f562e..a35f434 100644 --- a/static/js/controllers/hostPageCtrl.js +++ b/static/js/controllers/hostPageCtrl.js @@ -49,12 +49,6 @@ mumApp.controller('hostPageCtrl', function($scope, $rootScope, $route, $routePar if(args.func == 'update_custom_informations'){ $route.reload(); } - if(args.func == 'remove_host'){ - $scope.$apply(function(){ - $rootScope.$broadcast("sendViaWs", JSON.stringify({"GET_HOSTS": ""})); - $location.path('/'); - }); - } }); $scope.model = {custom_infos : '', @@ -100,6 +94,13 @@ mumApp.controller('hostPageCtrl', function($scope, $rootScope, $route, $routePar $rootScope.$broadcast("sendViaWs", JSON.stringify({"GET_HOST_INFO": $scope.addr_host})); }); + $scope.$on("remHost", function (event, args) { + $scope.$apply(function(){ + $rootScope.$broadcast("sendViaWs", JSON.stringify({"GET_HOSTS": ""})); + $location.path('/'); + }); + }); + // save custom informations $scope.save_custom_infos = function(){ var args = {}; @@ -111,7 +112,7 @@ mumApp.controller('hostPageCtrl', function($scope, $rootScope, $route, $routePar $scope.remove_host = function(){ var args = {}; args['addr_host'] = $scope.addr_host; - $rootScope.$broadcast("sendViaWs", JSON.stringify({"CALL_FUNC_DB": {'func': 'remove_host', 'args': args}})); + $rootScope.$broadcast("sendViaWs", JSON.stringify({"REM_HOST": args})); }; // creation of modals -- To stop receiving notification emails like this one, please contact chorem.org SCM administrator <admin+scm@chorem.org>.