This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository mum. See http://git.chorem.org/mum.git commit e5713afaaea50b13a5518085a1427609a0436dbd Author: Alexis Guilbaud <guilbaud@codelutin.com> Date: Mon May 4 15:38:45 2015 +0200 archiving + stats calculation after a check (except for dict values) + angular-charts bower dependency added as well a sample chart on statistics page --- app/modules/storage_modules/shelve_db.py | 119 +++++++++++++++++++++---------- app/process_monitoring.py | 12 ++-- bower.json | 3 +- static/css/dashboard.css | 6 ++ static/js/controllers/statsCtrl.js | 33 +++++++++ static/js/mumApp.js | 6 +- views/index.html | 3 + views/stats.html | 11 ++- 8 files changed, 145 insertions(+), 48 deletions(-) diff --git a/app/modules/storage_modules/shelve_db.py b/app/modules/storage_modules/shelve_db.py index 5dc918a..de44699 100644 --- a/app/modules/storage_modules/shelve_db.py +++ b/app/modules/storage_modules/shelve_db.py @@ -2,6 +2,7 @@ __author__ = 'aguilbaud' from datetime import datetime from datetime import timedelta +from sys import maxint import json import shelve import traceback @@ -90,14 +91,12 @@ class shelve_db: # once the check, if it exists a connection that can lauch the module mod_conf['activated'] = True mod_conf['check_frequency'] = 60 - """ - mod_conf['nb_minute'] = 30 + mod_conf['nb_min'] = 30 mod_conf['nb_hour'] = 12 mod_conf['nb_day'] = 15 mod_conf['nb_week'] = 2 mod_conf['nb_month'] = 6 mod_conf['nb_year'] = None - """ mod_conf['subparts'] = [] # a list that can contain for example the names of the disk partitions unit = loaded_mod_moni[mod]['unit'] mod_conf['unit'] = unit @@ -240,6 +239,7 @@ class shelve_db: self.db["hosts"][addr_host]["status"]["state"] = "" # Create structure for archiving data self.db["hosts"][addr_host]["archive"] = {} + self.db["hosts"][addr_host]["stats"] = {} except Exception: print traceback.format_exc() finally: @@ -827,48 +827,59 @@ class shelve_db: """ dict_notif = {} self.open_db() - new_val = {"date": str(datetime.now()), "value": val} + dict_new_val = {"date": str(datetime.now()), "value": val} try: if mod_name not in self.db['hosts'][addr_host]["monitoring"]: # creating the monitoring structure for this module if not exists self.db['hosts'][addr_host]["monitoring"][mod_name] = {} + self.db['hosts'][addr_host]['stats'][mod_name] = {} + self.db['hosts'][addr_host]['stats'][mod_name]['nb_check'] = 0 + self.db['hosts'][addr_host]['stats'][mod_name]['delta'] = 0 + self.db['hosts'][addr_host]['stats'][mod_name]['mean'] = 0 + self.db['hosts'][addr_host]['stats'][mod_name]['M2'] = 0 + self.db['hosts'][addr_host]['stats'][mod_name]['lin_reg'] = 0 + self.db['hosts'][addr_host]['stats'][mod_name]['min'] = maxint + self.db['hosts'][addr_host]['stats'][mod_name]['max'] = 0 + self.db['hosts'][addr_host]['stats'][mod_name]['total'] = 0 + self.db['hosts'][addr_host]['archive'][mod_name] = {} if isinstance(val, type({})): # if dictionary - for subval in val: - new_val = self.set_new_val(val[subval], - new_val, + for key in val: + dict_new_val = self.set_new_val(val[key], + dict_new_val, self.db['hosts'][addr_host]['conf']['monitoring'][mod_name]['minor_limit'], self.db['hosts'][addr_host]['conf']['monitoring'][mod_name]['major_limit']) else: - new_val = self.set_new_val(val, - new_val, + dict_new_val = self.set_new_val(val, + dict_new_val, self.db['hosts'][addr_host]['conf']['monitoring'][mod_name]['minor_limit'], self.db['hosts'][addr_host]['conf']['monitoring'][mod_name]['major_limit']) previous_val = self.db['hosts'][addr_host]["monitoring"][mod_name] - self.db['hosts'][addr_host]['monitoring'][mod_name] = new_val + self.db['hosts'][addr_host]['monitoring'][mod_name] = dict_new_val # updating the global state of the host - self.db['hosts'][addr_host]['status']['date'] = new_val['date'] + self.db['hosts'][addr_host]['status']['date'] = dict_new_val['date'] if 'state' not in self.db['hosts'][addr_host]['status']: - self.db['hosts'][addr_host]['status']['state'] = new_val['state'] + self.db['hosts'][addr_host]['status']['state'] = dict_new_val['state'] else: # now updating the global state of the host (most recent check and worst state for all modules) state = 'success' - for mod_name in self.db['hosts'][addr_host]['monitoring']: - if self.db['hosts'][addr_host]['monitoring'][mod_name]['state'] == 'danger': + for mod_name_stored in self.db['hosts'][addr_host]['monitoring']: + if self.db['hosts'][addr_host]['monitoring'][mod_name_stored]['state'] == 'danger': state = 'danger' - elif self.db['hosts'][addr_host]['monitoring'][mod_name]['state'] == 'warning' \ + elif self.db['hosts'][addr_host]['monitoring'][mod_name_stored]['state'] == 'warning' \ and not state == 'danger': state = 'warning' self.db['hosts'][addr_host]['status']['state'] = state # create a notification structure if the state is not a success - if not new_val['state'] == 'success': - dict_notif = self.create_notif_structure(addr_host, mod_name, new_val['state']) - # now performing archiving - """ - if mod_name in self.db['hosts'][addr_host]['archive']: - self.db['hosts'][addr_host]['archive'][mod_name] = \ - self.update_stats(self.db['hosts'][addr_host]['archive'][mod_name], val) - """ + if not dict_new_val['state'] == 'success': + dict_notif = self.create_notif_structure(addr_host, mod_name, dict_new_val['state']) + # updating and saving statistics + self.db['hosts'][addr_host]['stats'][mod_name] = \ + self.update_stats(self.db['hosts'][addr_host]['stats'][mod_name], val) + self.db['hosts'][addr_host]['archive'][mod_name] = \ + self.update_archive(self.db['hosts'][addr_host]['archive'][mod_name], + dict_new_val, + self.db['hosts'][addr_host]['conf']['monitoring'][mod_name]) except Exception: print traceback.format_exc() finally: @@ -910,27 +921,63 @@ class shelve_db: dict_val['state'] = new_status return dict_val - def update_stats(self, stats, val): + @staticmethod + def update_stats(stats, val): """ Updates calulated statistics once a new value is received. :param stats: a dictionary taken from the database and corresponding to the statistics stored :param val: the new value :return: the statistics dictionary updated """ - stats['nb_check'] += 1 - stats['total'] += val - if stats['min'] > val: - stats['min'] = val - if stats['max'] < val: - stats['max'] = val - # Compute linear regression - stats['lr'] += val * stats['nb_check'] - # Compute variance - stats['delta'] = val - stats['mean'] - stats['mean'] += stats['delta'] / stats['nb_check'] - stats['M2'] += stats['delta'] * (val - stats['mean']) + if isinstance(val, type(True)): + if val: + val = 1 + else: + val = 0 + if not isinstance(val, type({})): + stats['nb_check'] += 1 + stats['total'] += val + if stats['min'] > val: + stats['min'] = val + if stats['max'] < val: + stats['max'] = val + # Compute linear regression + stats['lin_reg'] += val * stats['nb_check'] + # Compute variance + stats['delta'] = val - stats['mean'] + stats['mean'] += stats['delta'] / stats['nb_check'] + stats['M2'] += stats['delta'] * (val - stats['mean']) return stats + @staticmethod + def update_archive(dict_archive, dict_check, conf_mod): + if dict_archive == {}: + for period in ['min', 'hour', 'day', 'week', 'month', 'year']: + dict_archive[period] = [] + dict_archive[period].append(dict_check) + else: + date_last_check = datetime.strptime(dict_check['date'],'%Y-%m-%d %H:%M:%S.%f') + for period in ['min', 'hour', 'day', 'week', 'month', 'year']: + date_period_str = dict_archive[period][len(dict_archive[period]) - 1]['date'] + date_period = datetime.strptime(date_period_str,'%Y-%m-%d %H:%M:%S.%f') + time_diff = date_last_check - date_period + print time_diff + if period == 'min' and time_diff.seconds >= 60: + dict_archive[period].append(dict_check) + if period == 'hour' and time_diff.seconds >= 3600: + dict_archive[period].append(dict_check) + if period == 'day' and time_diff.days >= 1: + dict_archive[period].append(dict_check) + if period == 'week' and time_diff.days >= 7: + dict_archive[period].append(dict_check) + if period == 'month' and time_diff.days >= 30: + dict_archive[period].append(dict_check) + if period == 'year' and time_diff.days >= 365: + dict_archive[period].append(dict_check) + if conf_mod['nb_' + period] is not None and len(dict_archive[period]) > conf_mod['nb_' + period]: + dict_archive[period].pop(0) + return dict_archive + def create_notif_structure(self, addr_host, moni_mod, status): """ Creates and returns a data structure that will be used by the notification modules. diff --git a/app/process_monitoring.py b/app/process_monitoring.py index f9c7982..126aebc 100644 --- a/app/process_monitoring.py +++ b/app/process_monitoring.py @@ -30,7 +30,7 @@ class ProcessMonitoring(threading.Thread): } The data are ordored by crescent time. This is for optimize the complexity when poping the most recent time. We could use here the deque structure from Python in order to use optimized poping left, but this structure - don't have implemented the insert() function, which is used for adding new data and keep the queue ordored. + don't have implemented the insert() function, which is used for adding new data and keep the list ordored. """ def __init__(self, ml, wsc): threading.Thread.__init__(self) @@ -64,7 +64,7 @@ class ProcessMonitoring(threading.Thread): def add_to_waiting_list(dict_mod): """ - Adds an element to the waiting queue and keep it ordered by crescent launching times. + Adds an element to the waiting list and keep it ordered by crescent launching times. :param dict_mod: a dictionary containing: {'addr': str, 'mod_name': str, 'time': datetime, 'freq': int} """ @@ -72,10 +72,10 @@ def add_to_waiting_list(dict_mod): if waiting_list == []: waiting_list.append(dict_mod) else: - pos_queue = 0 - while pos_queue < len(waiting_list) and dict_mod['time'] <= waiting_list[pos_queue]['time']: - pos_queue += 1 - waiting_list.insert(pos_queue, dict_mod) + pos_list = 0 + while pos_list < len(waiting_list) and dict_mod['time'] <= waiting_list[pos_list]['time']: + pos_list += 1 + waiting_list.insert(pos_list, dict_mod) def remove_to_waiting_list(addr_host, modname): diff --git a/bower.json b/bower.json index 0e80c92..de2f50b 100644 --- a/bower.json +++ b/bower.json @@ -7,7 +7,8 @@ "angular-toastr": "0.4.2", "angular-route": "~1.2.28", "angular-bootstrap": "~0.12.1", - "angular-file-upload": "~1.1.5" + "angular-file-upload": "~1.1.5", + "angular-charts": "~0.2.7" }, "resolutions": { "angular": "~1.2.28", diff --git a/static/css/dashboard.css b/static/css/dashboard.css index c85aa71..e639ce7 100644 --- a/static/css/dashboard.css +++ b/static/css/dashboard.css @@ -121,3 +121,9 @@ body { padding-left:15px; } + +/* For angular charts */ +.chart { + width: 500px; + height: 300px; +} \ No newline at end of file diff --git a/static/js/controllers/statsCtrl.js b/static/js/controllers/statsCtrl.js new file mode 100644 index 0000000..6499b78 --- /dev/null +++ b/static/js/controllers/statsCtrl.js @@ -0,0 +1,33 @@ +mumApp.controller('statsCtrl', function($scope, $rootScope, $timeout) { + $scope.config = { + title: 'Products', + tooltips: true, + labels: false, + mouseover: function() {}, + mouseout: function() {}, + click: function() {}, + legend: { + display: true, + //could be 'left, right' + position: 'right' + } + }; + + $scope.data = { + series: ['Sales', 'Income', 'Expense', 'Laptops', 'Keyboards'], + data: [{ + x: "Laptops", + y: [100, 500, 0], + tooltip: "this is tooltip" + }, { + x: "Desktops", + y: [300, 100, 100] + }, { + x: "Mobiles", + y: [351] + }, { + x: "Tablets", + y: [54, 0, 879] + }] + }; +}); \ No newline at end of file diff --git a/static/js/mumApp.js b/static/js/mumApp.js index c2e3afa..91e22dd 100644 --- a/static/js/mumApp.js +++ b/static/js/mumApp.js @@ -1,4 +1,4 @@ -var mumApp = angular.module('mumApp', ['angularFileUpload', 'ngRoute', 'ui.bootstrap', 'toastr']); +var mumApp = angular.module('mumApp', ['angularFileUpload', 'angularCharts', 'ngRoute', 'ui.bootstrap', 'toastr']); mumApp.factory('DataHosts', function(){ return {Items: []}; @@ -47,8 +47,8 @@ mumApp.config(function($routeProvider){ //controller : 'mainController' }) .when('/stats/',{ - templateUrl : 'stats.html' - //controller : 'mainController' + templateUrl : 'stats.html', + controller : 'statsCtrl' }) .when('/users/',{ templateUrl : 'users.html', diff --git a/views/index.html b/views/index.html index 1025258..08cebaf 100644 --- a/views/index.html +++ b/views/index.html @@ -24,6 +24,8 @@ <script src="bower_components/angular-route/angular-route.min.js"></script> <script src="bower_components/angular-toastr/dist/angular-toastr.min.js"></script> <script src="bower_components/angular-bootstrap/ui-bootstrap-tpls.min.js"></script> + <script src="bower_components/d3/d3.min.js"></script> + <script src="bower_components/angular-charts/dist/angular-charts.min.js"></script> <script src="static/js/mumApp.js"></script> @@ -35,6 +37,7 @@ <script src="static/js/controllers/profileCtrl.js"></script> <script src="static/js/controllers/scanCtrl.js"></script> <script src="static/js/controllers/settingsCtrl.js"></script> + <script src="static/js/controllers/statsCtrl.js"></script> <script src="static/js/controllers/usersCtrl.js"></script> diff --git a/views/stats.html b/views/stats.html index 276f7df..9120281 100644 --- a/views/stats.html +++ b/views/stats.html @@ -2,7 +2,14 @@ <div class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main"> <h1 class="page-header">Statistics</h1> - <div class="row placeholders"> + <div + data-ac-chart="'bar'" + data-ac-data="data" + data-ac-config="config" + class="chart"> + </div> + + <!--<div class="row placeholders"> <div class="col-xs-6 col-sm-3 placeholder"> <img src="data/index.svg" class="img-responsive" alt="Generic placeholder thumbnail"> <h4>Label</h4> @@ -23,5 +30,5 @@ <h4>Label</h4> <span class="text-muted">Something else</span> </div> - </div> + </div>--> </div> -- To stop receiving notification emails like this one, please contact chorem.org SCM administrator <admin+scm@chorem.org>.