From dab1d03d81c538966d03fb9318a4588a9e803b44 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 24 Mar 2012 11:20:07 -0500 Subject: Allow to run directly from a git checkout (#1037) Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Reports/__init__.py | 1 + src/lib/Bcfg2/Server/Reports/backends.py | 34 ++ src/lib/Bcfg2/Server/Reports/importscript.py | 310 +++++++++++++++ src/lib/Bcfg2/Server/Reports/manage.py | 11 + src/lib/Bcfg2/Server/Reports/nisauth.py | 44 +++ src/lib/Bcfg2/Server/Reports/reports/__init__.py | 1 + .../Reports/reports/fixtures/initial_version.xml | 39 ++ src/lib/Bcfg2/Server/Reports/reports/models.py | 343 +++++++++++++++++ .../Bcfg2/Server/Reports/reports/sql/client.sql | 9 + .../Server/Reports/reports/templates/404.html | 8 + .../Reports/reports/templates/base-timeview.html | 25 ++ .../Server/Reports/reports/templates/base.html | 95 +++++ .../Reports/reports/templates/clients/detail.html | 127 +++++++ .../reports/templates/clients/detailed-list.html | 46 +++ .../Reports/reports/templates/clients/history.html | 20 + .../Reports/reports/templates/clients/index.html | 34 ++ .../Reports/reports/templates/clients/manage.html | 45 +++ .../reports/templates/config_items/item.html | 115 ++++++ .../reports/templates/config_items/listing.html | 33 ++ .../reports/templates/displays/summary.html | 42 +++ .../Reports/reports/templates/displays/timing.html | 38 ++ .../reports/templates/widgets/filter_bar.html | 13 + .../reports/templates/widgets/interaction_list.inc | 38 ++ .../reports/templates/widgets/page_bar.html | 23 ++ .../Reports/reports/templatetags/__init__.py | 0 .../Reports/reports/templatetags/bcfg2_tags.py | 276 ++++++++++++++ .../reports/templatetags/syntax_coloring.py | 49 +++ src/lib/Bcfg2/Server/Reports/reports/urls.py | 55 +++ src/lib/Bcfg2/Server/Reports/reports/views.py | 415 +++++++++++++++++++++ src/lib/Bcfg2/Server/Reports/settings.py | 161 ++++++++ src/lib/Bcfg2/Server/Reports/updatefix.py | 190 ++++++++++ src/lib/Bcfg2/Server/Reports/urls.py | 14 + src/lib/Bcfg2/Server/Reports/utils.py | 124 ++++++ 33 files changed, 2778 insertions(+) create mode 100644 src/lib/Bcfg2/Server/Reports/__init__.py create mode 100644 src/lib/Bcfg2/Server/Reports/backends.py create mode 100755 src/lib/Bcfg2/Server/Reports/importscript.py create mode 100755 src/lib/Bcfg2/Server/Reports/manage.py create mode 100644 src/lib/Bcfg2/Server/Reports/nisauth.py create mode 100644 src/lib/Bcfg2/Server/Reports/reports/__init__.py create mode 100644 src/lib/Bcfg2/Server/Reports/reports/fixtures/initial_version.xml create mode 100644 src/lib/Bcfg2/Server/Reports/reports/models.py create mode 100644 src/lib/Bcfg2/Server/Reports/reports/sql/client.sql create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/404.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/base.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py create mode 100644 src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py create mode 100644 src/lib/Bcfg2/Server/Reports/reports/urls.py create mode 100644 src/lib/Bcfg2/Server/Reports/reports/views.py create mode 100644 src/lib/Bcfg2/Server/Reports/settings.py create mode 100644 src/lib/Bcfg2/Server/Reports/updatefix.py create mode 100644 src/lib/Bcfg2/Server/Reports/urls.py create mode 100755 src/lib/Bcfg2/Server/Reports/utils.py (limited to 'src/lib/Bcfg2/Server/Reports') diff --git a/src/lib/Bcfg2/Server/Reports/__init__.py b/src/lib/Bcfg2/Server/Reports/__init__.py new file mode 100644 index 000000000..bdf908f4a --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/__init__.py @@ -0,0 +1 @@ +__all__ = ['manage', 'nisauth', 'reports', 'settings', 'backends', 'urls', 'importscript'] diff --git a/src/lib/Bcfg2/Server/Reports/backends.py b/src/lib/Bcfg2/Server/Reports/backends.py new file mode 100644 index 000000000..85241932f --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/backends.py @@ -0,0 +1,34 @@ +from django.contrib.auth.models import User +from nisauth import * + + +class NISBackend(object): + + def authenticate(self, username=None, password=None): + try: + print("start nis authenticate") + n = nisauth(username, password) + temp_pass = User.objects.make_random_password(100) + nis_user = dict(username=username, + ) + + user_session_obj = dict(email=username, + first_name=None, + last_name=None, + uid=n.uid) + user, created = User.objects.get_or_create(username=username) + + return user + + except NISAUTHError: + e = sys.exc_info()[1] + print(e) + return None + + def get_user(self, user_id): + try: + return User.objects.get(pk=user_id) + except User.DoesNotExist: + e = sys.exc_info()[1] + print(e) + return None diff --git a/src/lib/Bcfg2/Server/Reports/importscript.py b/src/lib/Bcfg2/Server/Reports/importscript.py new file mode 100755 index 000000000..cbdf019f5 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/importscript.py @@ -0,0 +1,310 @@ +#! /usr/bin/env python +""" +Imports statistics.xml and clients.xml files in to database backend for +new statistics engine +""" + +import binascii +import os +import sys +try: + import Bcfg2.Server.Reports.settings +except Exception: + e = sys.exc_info()[1] + sys.stderr.write("Failed to load configuration settings. %s\n" % e) + sys.exit(1) + +project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__) +project_name = os.path.basename(project_directory) +sys.path.append(os.path.join(project_directory, '..')) +project_module = __import__(project_name, '', '', ['']) +sys.path.pop() +# Set DJANGO_SETTINGS_MODULE appropriately. +os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name + +from Bcfg2.Server.Reports.reports.models import * +from lxml.etree import XML, XMLSyntaxError +from getopt import getopt, GetoptError +from datetime import datetime +from time import strptime +from django.db import connection +from Bcfg2.Server.Reports.updatefix import update_database +import logging +import Bcfg2.Logger +import platform + +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser + + +def build_reason_kwargs(r_ent, encoding, logger): + binary_file = False + sensitive_file = False + if r_ent.get('sensitive') in ['true', 'True']: + sensitive_file = True + rc_diff = '' + elif r_ent.get('current_bfile', False): + binary_file = True + rc_diff = r_ent.get('current_bfile') + if len(rc_diff) > 1024 * 1024: + rc_diff = '' + elif len(rc_diff) == 0: + # No point in flagging binary if we have no data + binary_file = False + elif r_ent.get('current_bdiff', False): + rc_diff = binascii.a2b_base64(r_ent.get('current_bdiff')) + elif r_ent.get('current_diff', False): + rc_diff = r_ent.get('current_diff') + else: + rc_diff = '' + if not binary_file: + try: + rc_diff = rc_diff.decode(encoding) + except: + logger.error("Reason isn't %s encoded, cannot decode it" % encoding) + rc_diff = '' + return dict(owner=r_ent.get('owner', default=""), + current_owner=r_ent.get('current_owner', default=""), + group=r_ent.get('group', default=""), + current_group=r_ent.get('current_group', default=""), + perms=r_ent.get('perms', default=""), + current_perms=r_ent.get('current_perms', default=""), + status=r_ent.get('status', default=""), + current_status=r_ent.get('current_status', default=""), + to=r_ent.get('to', default=""), + current_to=r_ent.get('current_to', default=""), + version=r_ent.get('version', default=""), + current_version=r_ent.get('current_version', default=""), + current_exists=r_ent.get('current_exists', default="True").capitalize() == "True", + current_diff=rc_diff, + is_binary=binary_file, + is_sensitive=sensitive_file) + + +def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''): + clients = {} + [clients.__setitem__(c.name, c) \ + for c in Client.objects.all()] + + pingability = {} + [pingability.__setitem__(n.get('name'), n.get('pingable', default='N')) \ + for n in cdata.findall('Client')] + + for node in sdata.findall('Node'): + name = node.get('name') + c_inst, created = Client.objects.get_or_create(name=name) + if vlevel > 0: + logger.info("Client %s added to db" % name) + clients[name] = c_inst + try: + pingability[name] + except KeyError: + pingability[name] = 'N' + for statistics in node.findall('Statistics'): + timestamp = datetime(*strptime(statistics.get('time'))[0:6]) + ilist = Interaction.objects.filter(client=c_inst, + timestamp=timestamp) + if ilist: + current_interaction = ilist[0] + if vlevel > 0: + logger.info("Interaction for %s at %s with id %s already exists" % \ + (c_inst.id, timestamp, current_interaction.id)) + continue + else: + newint = Interaction(client=c_inst, + timestamp=timestamp, + state=statistics.get('state', + default="unknown"), + repo_rev_code=statistics.get('revision', + default="unknown"), + client_version=statistics.get('client_version', + default="unknown"), + goodcount=statistics.get('good', + default="0"), + totalcount=statistics.get('total', + default="0"), + server=location) + newint.save() + current_interaction = newint + if vlevel > 0: + logger.info("Interaction for %s at %s with id %s INSERTED in to db" % (c_inst.id, + timestamp, current_interaction.id)) + + counter_fields = {TYPE_CHOICES[0]: 0, + TYPE_CHOICES[1]: 0, + TYPE_CHOICES[2]: 0} + pattern = [('Bad/*', TYPE_CHOICES[0]), + ('Extra/*', TYPE_CHOICES[2]), + ('Modified/*', TYPE_CHOICES[1])] + for (xpath, type) in pattern: + for x in statistics.findall(xpath): + counter_fields[type] = counter_fields[type] + 1 + kargs = build_reason_kwargs(x, encoding, logger) + + try: + rr = None + try: + rr = Reason.objects.filter(**kargs)[0] + except IndexError: + rr = Reason(**kargs) + rr.save() + if vlevel > 0: + logger.info("Created reason: %s" % rr.id) + except Exception: + ex = sys.exc_info()[1] + logger.error("Failed to create reason for %s: %s" % (x.get('name'), ex)) + rr = Reason(current_exists=x.get('current_exists', + default="True").capitalize() == "True") + rr.save() + + entry, created = Entries.objects.get_or_create(\ + name=x.get('name'), kind=x.tag) + + Entries_interactions(entry=entry, reason=rr, + interaction=current_interaction, + type=type[0]).save() + if vlevel > 0: + logger.info("%s interaction created with reason id %s and entry %s" % (xpath, rr.id, entry.id)) + + # Update interaction counters + current_interaction.bad_entries = counter_fields[TYPE_CHOICES[0]] + current_interaction.modified_entries = counter_fields[TYPE_CHOICES[1]] + current_interaction.extra_entries = counter_fields[TYPE_CHOICES[2]] + current_interaction.save() + + mperfs = [] + for times in statistics.findall('OpStamps'): + for metric, value in list(times.items()): + mmatch = [] + if not quick: + mmatch = Performance.objects.filter(metric=metric, value=value) + + if mmatch: + mperf = mmatch[0] + else: + mperf = Performance(metric=metric, value=value) + mperf.save() + mperfs.append(mperf) + current_interaction.performance_items.add(*mperfs) + + for key in list(pingability.keys()): + if key not in clients: + continue + try: + pmatch = Ping.objects.filter(client=clients[key]).order_by('-endtime')[0] + if pmatch.status == pingability[key]: + pmatch.endtime = datetime.now() + pmatch.save() + continue + except IndexError: + pass + Ping(client=clients[key], status=pingability[key], + starttime=datetime.now(), + endtime=datetime.now()).save() + + if vlevel > 1: + logger.info("---------------PINGDATA SYNCED---------------------") + + #Clients are consistent + +if __name__ == '__main__': + from sys import argv + verb = 0 + cpath = "/etc/bcfg2.conf" + clientpath = False + statpath = False + syslog = False + + try: + opts, args = getopt(argv[1:], "hvudc:s:CS", ["help", + "verbose", + "updates", + "debug", + "clients=", + "stats=", + "config=", + "syslog"]) + except GetoptError: + mesg = sys.exc_info()[1] + # print help information and exit: + print("%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-S] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg)) + raise SystemExit(2) + + for o, a in opts: + if o in ("-h", "--help"): + print("Usage:\nimportscript.py [-h] [-v] -c -s \n") + print("h : help; this message") + print("v : verbose; print messages on record insertion/skip") + print("u : updates; print status messages as items inserted semi-verbose") + print("d : debug; print most SQL used to manipulate database") + print("C : path to bcfg2.conf config file.") + print("c : clients.xml file") + print("s : statistics.xml file") + print("S : syslog; output to syslog") + raise SystemExit + if o in ["-C", "--config"]: + cpath = a + + if o in ("-v", "--verbose"): + verb = 1 + if o in ("-u", "--updates"): + verb = 2 + if o in ("-d", "--debug"): + verb = 3 + if o in ("-c", "--clients"): + clientspath = a + + if o in ("-s", "--stats"): + statpath = a + if o in ("-S", "--syslog"): + syslog = True + + logger = logging.getLogger('importscript.py') + logging.getLogger().setLevel(logging.INFO) + Bcfg2.Logger.setup_logging('importscript.py', + True, + syslog) + + cf = ConfigParser.ConfigParser() + cf.read([cpath]) + + if not statpath: + try: + statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + print("Could not read bcfg2.conf; exiting") + raise SystemExit(1) + try: + statsdata = XML(open(statpath).read()) + except (IOError, XMLSyntaxError): + print("StatReports: Failed to parse %s" % (statpath)) + raise SystemExit(1) + + try: + encoding = cf.get('components', 'encoding') + except: + encoding = 'UTF-8' + + if not clientpath: + try: + clientspath = "%s/Metadata/clients.xml" % \ + cf.get('server', 'repository') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + print("Could not read bcfg2.conf; exiting") + raise SystemExit(1) + try: + clientsdata = XML(open(clientspath).read()) + except (IOError, XMLSyntaxError): + print("StatReports: Failed to parse %s" % (clientspath)) + raise SystemExit(1) + + q = '-O3' in sys.argv + # Be sure the database is ready for new schema + update_database() + load_stats(clientsdata, + statsdata, + encoding, + verb, + logger, + quick=q, + location=platform.node()) diff --git a/src/lib/Bcfg2/Server/Reports/manage.py b/src/lib/Bcfg2/Server/Reports/manage.py new file mode 100755 index 000000000..858bddeca --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/manage.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +from django.core.management import execute_manager +try: + import settings # Assumed to be in the same directory. +except ImportError: + import sys + sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) + sys.exit(1) + +if __name__ == "__main__": + execute_manager(settings) diff --git a/src/lib/Bcfg2/Server/Reports/nisauth.py b/src/lib/Bcfg2/Server/Reports/nisauth.py new file mode 100644 index 000000000..b3e37113b --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/nisauth.py @@ -0,0 +1,44 @@ +import crypt +import nis +from Bcfg2.Server.Reports.settings import AUTHORIZED_GROUP + +"""Checks with NIS to see if the current user is in the support group""" + + +class NISAUTHError(Exception): + """NISAUTHError is raised when somehting goes boom.""" + pass + + +class nisauth(object): + group_test = False + samAcctName = None + distinguishedName = None + sAMAccountName = None + telephoneNumber = None + title = None + memberOf = None + department = None # this will be a list + mail = None + extensionAttribute1 = None # badgenumber + badge_no = None + uid = None + + def __init__(self, login, passwd=None): + """get user profile from NIS""" + try: + p = nis.match(login, 'passwd.byname').split(":") + print(p) + except: + raise NISAUTHError('username') + # check user password using crypt and 2 character salt from passwd file + if p[1] == crypt.crypt(passwd, p[1][:2]): + # check to see if user is in valid support groups + # will have to include these groups in a settings file eventually + if not login in nis.match(AUTHORIZED_GROUP, + 'group.byname').split(':')[-1].split(','): + raise NISAUTHError('group') + self.uid = p[2] + print(self.uid) + else: + raise NISAUTHError('password') diff --git a/src/lib/Bcfg2/Server/Reports/reports/__init__.py b/src/lib/Bcfg2/Server/Reports/reports/__init__.py new file mode 100644 index 000000000..ccdce8943 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/__init__.py @@ -0,0 +1 @@ +__all__ = ['templatetags'] diff --git a/src/lib/Bcfg2/Server/Reports/reports/fixtures/initial_version.xml b/src/lib/Bcfg2/Server/Reports/reports/fixtures/initial_version.xml new file mode 100644 index 000000000..919265d48 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/fixtures/initial_version.xml @@ -0,0 +1,39 @@ + + + + 0 + 2008-08-05 11:03:50 + + + 1 + 2008-08-05 11:04:10 + + + 2 + 2008-08-05 13:37:19 + + + 3 + 2008-08-11 08:44:36 + + + 10 + 2008-08-22 11:28:50 + + + 11 + 2009-01-13 12:26:10 + + + 16 + 2010-06-01 12:26:10 + + + 17 + 2010-07-02 00:00:00 + + + 18 + 2011-06-30 00:00:00 + + diff --git a/src/lib/Bcfg2/Server/Reports/reports/models.py b/src/lib/Bcfg2/Server/Reports/reports/models.py new file mode 100644 index 000000000..870239641 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/models.py @@ -0,0 +1,343 @@ +"""Django models for Bcfg2 reports.""" +from django.db import models +from django.db import connection, transaction +from django.db.models import Q +from datetime import datetime, timedelta +from time import strptime + +KIND_CHOICES = ( + #These are the kinds of config elements + ('Package', 'Package'), + ('Path', 'directory'), + ('Path', 'file'), + ('Path', 'permissions'), + ('Path', 'symlink'), + ('Service', 'Service'), +) +PING_CHOICES = ( + #These are possible ping states + ('Up (Y)', 'Y'), + ('Down (N)', 'N') +) +TYPE_BAD = 1 +TYPE_MODIFIED = 2 +TYPE_EXTRA = 3 + +TYPE_CHOICES = ( + (TYPE_BAD, 'Bad'), + (TYPE_MODIFIED, 'Modified'), + (TYPE_EXTRA, 'Extra'), +) + + +def convert_entry_type_to_id(type_name): + """Convert a entry type to its entry id""" + for e_id, e_name in TYPE_CHOICES: + if e_name.lower() == type_name.lower(): + return e_id + return -1 + + +class ClientManager(models.Manager): + """Extended client manager functions.""" + def active(self, timestamp=None): + """returns a set of clients that have been created and have not + yet been expired as of optional timestmamp argument. Timestamp + should be a datetime object.""" + + if timestamp == None: + timestamp = datetime.now() + elif not isinstance(timestamp, datetime): + raise ValueError('Expected a datetime object') + else: + try: + timestamp = datetime(*strptime(timestamp, + "%Y-%m-%d %H:%M:%S")[0:6]) + except ValueError: + return self.none() + + return self.filter(Q(expiration__gt=timestamp) | Q(expiration__isnull=True), + creation__lt=timestamp) + + +class Client(models.Model): + """Object representing every client we have seen stats for.""" + creation = models.DateTimeField(auto_now_add=True) + name = models.CharField(max_length=128,) + current_interaction = models.ForeignKey('Interaction', + null=True, blank=True, + related_name="parent_client") + expiration = models.DateTimeField(blank=True, null=True) + + def __str__(self): + return self.name + + objects = ClientManager() + + class Admin: + pass + + +class Ping(models.Model): + """Represents a ping of a client (sparsely).""" + client = models.ForeignKey(Client, related_name="pings") + starttime = models.DateTimeField() + endtime = models.DateTimeField() + status = models.CharField(max_length=4, choices=PING_CHOICES) # up/down + + class Meta: + get_latest_by = 'endtime' + + +class InteractiveManager(models.Manager): + """Manages interactions objects.""" + + def recent_interactions_dict(self, maxdate=None, active_only=True): + """ + Return the most recent interactions for clients as of a date. + + This method uses aggregated queries to return a ValuesQueryDict object. + Faster then raw sql since this is executed as a single query. + """ + + return list(self.values('client').annotate(max_timestamp=Max('timestamp')).values()) + + def interaction_per_client(self, maxdate=None, active_only=True): + """ + Returns the most recent interactions for clients as of a date + + Arguments: + maxdate -- datetime object. Most recent date to pull. (dafault None) + active_only -- Include only active clients (default True) + + """ + + if maxdate and not isinstance(maxdate, datetime): + raise ValueError('Expected a datetime object') + return self.filter(id__in=self.get_interaction_per_client_ids(maxdate, active_only)) + + def get_interaction_per_client_ids(self, maxdate=None, active_only=True): + """ + Returns the ids of most recent interactions for clients as of a date. + + Arguments: + maxdate -- datetime object. Most recent date to pull. (dafault None) + active_only -- Include only active clients (default True) + + """ + from django.db import connection + cursor = connection.cursor() + cfilter = "expiration is null" + + sql = 'select reports_interaction.id, x.client_id from (select client_id, MAX(timestamp) ' + \ + 'as timer from reports_interaction' + if maxdate: + if not isinstance(maxdate, datetime): + raise ValueError('Expected a datetime object') + sql = sql + " where timestamp <= '%s' " % maxdate + cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate) + sql = sql + ' GROUP BY client_id) x, reports_interaction where ' + \ + 'reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer' + if active_only: + sql = sql + " and x.client_id in (select id from reports_client where %s)" % \ + cfilter + try: + cursor.execute(sql) + return [item[0] for item in cursor.fetchall()] + except: + '''FIXME - really need some error hadling''' + pass + return [] + + +class Interaction(models.Model): + """Models each reconfiguration operation interaction between client and server.""" + client = models.ForeignKey(Client, related_name="interactions",) + timestamp = models.DateTimeField() # Timestamp for this record + state = models.CharField(max_length=32) # good/bad/modified/etc + repo_rev_code = models.CharField(max_length=64) # repo revision at time of interaction + client_version = models.CharField(max_length=32) # Client Version + goodcount = models.IntegerField() # of good config-items + totalcount = models.IntegerField() # of total config-items + server = models.CharField(max_length=256) # Name of the server used for the interaction + bad_entries = models.IntegerField(default=-1) + modified_entries = models.IntegerField(default=-1) + extra_entries = models.IntegerField(default=-1) + + def __str__(self): + return "With " + self.client.name + " @ " + self.timestamp.isoformat() + + def percentgood(self): + if not self.totalcount == 0: + return (self.goodcount / float(self.totalcount)) * 100 + else: + return 0 + + def percentbad(self): + if not self.totalcount == 0: + return ((self.totalcount - self.goodcount) / (float(self.totalcount))) * 100 + else: + return 0 + + def isclean(self): + if (self.bad_entry_count() == 0 and self.goodcount == self.totalcount): + return True + else: + return False + + def isstale(self): + if (self == self.client.current_interaction): # Is Mostrecent + if(datetime.now() - self.timestamp > timedelta(hours=25)): + return True + else: + return False + else: + #Search for subsequent Interaction for this client + #Check if it happened more than 25 hrs ago. + if (self.client.interactions.filter(timestamp__gt=self.timestamp) + .order_by('timestamp')[0].timestamp - + self.timestamp > timedelta(hours=25)): + return True + else: + return False + + def save(self): + super(Interaction, self).save() # call the real save... + self.client.current_interaction = self.client.interactions.latest() + self.client.save() # save again post update + + def delete(self): + '''Override the default delete. Allows us to remove Performance items''' + pitems = list(self.performance_items.all()) + super(Interaction, self).delete() + for perf in pitems: + if perf.interaction.count() == 0: + perf.delete() + + def badcount(self): + return self.totalcount - self.goodcount + + def bad(self): + return Entries_interactions.objects.select_related().filter(interaction=self, type=TYPE_BAD) + + def bad_entry_count(self): + """Number of bad entries. Store the count in the interation field to save db queries.""" + if self.bad_entries < 0: + self.bad_entries = Entries_interactions.objects.filter(interaction=self, type=TYPE_BAD).count() + self.save() + return self.bad_entries + + def modified(self): + return Entries_interactions.objects.select_related().filter(interaction=self, type=TYPE_MODIFIED) + + def modified_entry_count(self): + """Number of modified entries. Store the count in the interation field to save db queries.""" + if self.modified_entries < 0: + self.modified_entries = Entries_interactions.objects.filter(interaction=self, type=TYPE_MODIFIED).count() + self.save() + return self.modified_entries + + def extra(self): + return Entries_interactions.objects.select_related().filter(interaction=self, type=TYPE_EXTRA) + + def extra_entry_count(self): + """Number of extra entries. Store the count in the interation field to save db queries.""" + if self.extra_entries < 0: + self.extra_entries = Entries_interactions.objects.filter(interaction=self, type=TYPE_EXTRA).count() + self.save() + return self.extra_entries + + objects = InteractiveManager() + + class Admin: + list_display = ('client', 'timestamp', 'state') + list_filter = ['client', 'timestamp'] + pass + + class Meta: + get_latest_by = 'timestamp' + ordering = ['-timestamp'] + unique_together = ("client", "timestamp") + + +class Reason(models.Model): + """reason why modified or bad entry did not verify, or changed.""" + owner = models.TextField(max_length=128, blank=True) + current_owner = models.TextField(max_length=128, blank=True) + group = models.TextField(max_length=128, blank=True) + current_group = models.TextField(max_length=128, blank=True) + perms = models.TextField(max_length=4, blank=True) # txt fixes typing issue + current_perms = models.TextField(max_length=4, blank=True) + status = models.TextField(max_length=3, blank=True) # on/off/(None) + current_status = models.TextField(max_length=1, blank=True) # on/off/(None) + to = models.TextField(max_length=256, blank=True) + current_to = models.TextField(max_length=256, blank=True) + version = models.TextField(max_length=128, blank=True) + current_version = models.TextField(max_length=128, blank=True) + current_exists = models.BooleanField() # False means its missing. Default True + current_diff = models.TextField(max_length=1280, blank=True) + is_binary = models.BooleanField(default=False) + is_sensitive = models.BooleanField(default=False) + + def _str_(self): + return "Reason" + + @staticmethod + @transaction.commit_on_success + def prune_orphans(): + '''Prune oprhaned rows... no good way to use the ORM''' + cursor = connection.cursor() + cursor.execute('delete from reports_reason where not exists (select rei.id from reports_entries_interactions rei where rei.reason_id = reports_reason.id)') + transaction.set_dirty() + + +class Entries(models.Model): + """Contains all the entries feed by the client.""" + name = models.CharField(max_length=128, db_index=True) + kind = models.CharField(max_length=16, choices=KIND_CHOICES, db_index=True) + + def __str__(self): + return self.name + + @staticmethod + @transaction.commit_on_success + def prune_orphans(): + '''Prune oprhaned rows... no good way to use the ORM''' + cursor = connection.cursor() + cursor.execute('delete from reports_entries where not exists (select rei.id from reports_entries_interactions rei where rei.entry_id = reports_entries.id)') + transaction.set_dirty() + + +class Entries_interactions(models.Model): + """Define the relation between the reason, the interaction and the entry.""" + entry = models.ForeignKey(Entries) + reason = models.ForeignKey(Reason) + interaction = models.ForeignKey(Interaction) + type = models.IntegerField(choices=TYPE_CHOICES) + + +class Performance(models.Model): + """Object representing performance data for any interaction.""" + interaction = models.ManyToManyField(Interaction, related_name="performance_items") + metric = models.CharField(max_length=128) + value = models.DecimalField(max_digits=32, decimal_places=16) + + def __str__(self): + return self.metric + + @staticmethod + @transaction.commit_on_success + def prune_orphans(): + '''Prune oprhaned rows... no good way to use the ORM''' + cursor = connection.cursor() + cursor.execute('delete from reports_performance where not exists (select ri.id from reports_performance_interaction ri where ri.performance_id = reports_performance.id)') + transaction.set_dirty() + + +class InternalDatabaseVersion(models.Model): + """Object that tell us to witch version is the database.""" + version = models.IntegerField() + updated = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return "version %d updated the %s" % (self.version, self.updated.isoformat()) diff --git a/src/lib/Bcfg2/Server/Reports/reports/sql/client.sql b/src/lib/Bcfg2/Server/Reports/reports/sql/client.sql new file mode 100644 index 000000000..8c63754c9 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/sql/client.sql @@ -0,0 +1,9 @@ +CREATE VIEW reports_current_interactions AS SELECT x.client_id AS client_id, reports_interaction.id AS interaction_id FROM (select client_id, MAX(timestamp) as timer FROM reports_interaction GROUP BY client_id) x, reports_interaction WHERE reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer; + +create index reports_interaction_client_id on reports_interaction (client_id); +create index reports_extra_interactions_client_id on reports_extra_interactions(interaction_id); +create index reports_modified_interactions_client_id on reports_modified_interactions(interaction_id); +create index reports_client_current_interaction_id on reports_client (current_interaction_id); +create index reports_performance_interaction_performance_id on reports_performance_interaction (performance_id); +create index reports_interaction_timestamp on reports_interaction (timestamp); +create index reports_performance_interation_interaction_id on reports_performance_interaction (interaction_id); \ No newline at end of file diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/404.html b/src/lib/Bcfg2/Server/Reports/reports/templates/404.html new file mode 100644 index 000000000..168bd9fec --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/404.html @@ -0,0 +1,8 @@ +{% extends 'base.html' %} +{% block title %}Bcfg2 - Page not found{% endblock %} +{% block fullcontent %} +

Page not found

+

+The page or object requested could not be found. +

+{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html b/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html new file mode 100644 index 000000000..842de36f0 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html @@ -0,0 +1,25 @@ +{% extends "base.html" %} + +{% block timepiece %} + +{% if not timestamp %}Rendered at {% now "Y-m-d H:i" %} | {% else %}View as of {{ timestamp|date:"Y-m-d H:i" }} | {% endif %}{% spaceless %} + [change] +
+{% endspaceless %} +{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/base.html b/src/lib/Bcfg2/Server/Reports/reports/templates/base.html new file mode 100644 index 000000000..f541c0d2b --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/base.html @@ -0,0 +1,95 @@ +{% load bcfg2_tags %} + + + + + +{% block title %}Bcfg2 Reporting System{% endblock %} + + + + + + + + + + + + + +{% block extra_header_info %}{% endblock %} + + + + + + +
+
+ {% block fullcontent %} +
+

{% block pagebanner %}Page Banner{% endblock %}

+
{% block timepiece %}Rendered at {% now "Y-m-d H:i" %}{% endblock %}
+
+
+ {% block content %}{% endblock %} +
+ {% endblock %} +
+
+ {% block sidemenu %} +
    +
  • Overview
    • +
+ +
    +
  • Clients
    • +
+ +
    +
  • Entries Configured
    • +
+ +{% comment %} + TODO +
    +
  • Entry Types
    • +
+ +{% endcomment %} + + {% endblock %} +
+
+
+ + + + + diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html new file mode 100644 index 000000000..dd4295f21 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html @@ -0,0 +1,127 @@ +{% extends "base.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Client {{client.name}}{% endblock %} + +{% block extra_header_info %} + +{% endblock %} + +{% block body_onload %}javascript:clientdetailload(){% endblock %} + +{% block pagebanner %}Client Details{% endblock %} + +{% block content %} +
+

{{client.name}}

+ [manage] + View History | Jump to  + +
+ + {% if interaction.isstale %} +
+ This node did not run within the last 24 hours — it may be out of date. +
+ {% endif %} + + + {% if interaction.server %} + + {% endif %} + {% if interaction.repo_rev_code %} + + {% endif %} + + + {% if not interaction.isclean %} + + {% endif %} +
Timestamp{{interaction.timestamp}}
Served by{{interaction.server}}
Revision{{interaction.repo_rev_code}}
State{{interaction.state|capfirst}}
Managed entries{{interaction.totalcount}}
Deviation{{interaction.percentbad|floatformat:"3"}}%
+ + {% if interaction.bad_entry_count %} +
+
+

Bad Entries — {{ interaction.bad_entry_count }}

+
[+]
+
+ + {% for e in interaction.bad|sortwell %} + + + + + {% endfor %} +
{{e.entry.kind}}: + {{e.entry.name}}
+
+ {% endif %} + + {% if interaction.modified_entry_count %} +
+
+

Modified Entries — {{ interaction.modified_entry_count }}

+
[+]
+
+ + {% for e in interaction.modified|sortwell %} + + + + + {% endfor %} +
{{e.entry.kind}}: + {{e.entry.name}}
+
+ {% endif %} + + {% if interaction.extra_entry_count %} +
+
+

Extra Entries — {{ interaction.extra_entry_count }}

+
[+]
+
+ + {% for e in interaction.extra|sortwell %} + + + + + {% endfor %} +
{{e.entry.kind}}:{{e.entry.name}}
+
+ {% endif %} + + {% if entry_list %} +
+
+

Recent Interactions

+
+
+ {% include "widgets/interaction_list.inc" %} + +
+
+ {% endif %} +{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html new file mode 100644 index 000000000..0c1fae8d5 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html @@ -0,0 +1,46 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Detailed Client Listing{% endblock %} +{% block pagebanner %}Clients - Detailed View{% endblock %} + +{% block content %} +
+{% if entry_list %} + {% filter_navigator %} + + + + + + + + + + + + {% for entry in entry_list %} + + + + + + + + + + + {% endfor %} +
NodeStateGoodBadModifiedExtraLast RunServer
{{ entry.client.name }}{{ entry.state }}{{ entry.goodcount }}{{ entry.bad_entry_count }}{{ entry.modified_entry_count }}{{ entry.extra_entry_count }}{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }} + {% if entry.server %} + {{ entry.server }} + {% else %} +   + {% endif %} +
+{% else %} +

No client records are available.

+{% endif %} +
+{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html new file mode 100644 index 000000000..01d4ec2f4 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Interaction History{% endblock %} +{% block pagebanner %}Interaction history{% if client %} for {{ client.name }}{% endif %}{% endblock %} + +{% block extra_header_info %} +{% endblock %} + +{% block content %} +
+{% if entry_list %} + {% filter_navigator %} + {% include "widgets/interaction_list.inc" %} +{% else %} +

No client records are available.

+{% endif %} +
+{% page_navigator %} +{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html new file mode 100644 index 000000000..e0c0d2d7a --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html @@ -0,0 +1,34 @@ +{% extends "base-timeview.html" %} + +{% block extra_header_info %} +{% endblock%} + +{% block title %}Bcfg2 - Client Grid View{% endblock %} + +{% block pagebanner %}Clients - Grid View{% endblock %} + +{% block content %} + +{% if inter_list %} + + {% for inter in inter_list %} + {% if forloop.first %}{% endif %} + + {% if forloop.last %} + + {% else %} + {% if forloop.counter|divisibleby:"4" %}{% endif %} + {% endif %} + {% endfor %} +
+ {{ inter.client.name }} +
+{% else %} +

No client records are available.

+{% endif %} +{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html new file mode 100644 index 000000000..5725ae577 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html @@ -0,0 +1,45 @@ +{% extends "base.html" %} + +{% block extra_header_info %} +{% endblock%} + +{% block title %}Bcfg2 - Manage Clients{% endblock %} + +{% block pagebanner %}Clients - Manage{% endblock %} + +{% block content %} +
+ {% if message %} +
{{ message }}
+ {% endif %} +{% if clients %} + + + + + + + {% for client in clients %} + + + + + + {% endfor %} +
NodeExpirationManage
+ + + {{ client.name }}{% firstof client.expiration 'Active' %} +
+
{# here for no reason other then to validate #} + + + +
+
+
+
+{% else %} +

No client records are available.

+{% endif %} +{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html new file mode 100644 index 000000000..cc99ef503 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html @@ -0,0 +1,115 @@ +{% extends "base.html" %} +{% load syntax_coloring %} + + +{% block title %}Bcfg2 - Element Details{% endblock %} + + +{% block extra_header_info %} + +{% endblock%} + +{% block pagebanner %}Element Details{% endblock %} + +{% block content %} +
+

{{mod_or_bad|capfirst}} {{item.entry.kind}}: {{item.entry.name}}

+
+ +
+ + {% if isextra %} +

This item exists on the host but is not defined in the configuration.

+ {% endif %} + + {% if not item.reason.current_exists %} +
This item does not currently exist on the host but is specified to exist in the configuration.
+ {% endif %} + + {% if item.reason.current_owner or item.reason.current_group or item.reason.current_perms or item.reason.current_status or item.reason.current_status or item.reason.current_to or item.reason.current_version %} + + + + {% if item.reason.current_owner %} + + + {% endif %} + {% if item.reason.current_group %} + + + {% endif %} + {% if item.reason.current_perms %} + + + {% endif %} + {% if item.reason.current_status %} + + + {% endif %} + {% if item.reason.current_to %} + + + {% endif %} + {% if item.reason.current_version %} + + + {% endif %} +
Problem TypeExpectedFound
Owner{{item.reason.owner}}{{item.reason.current_owner}}
Group{{item.reason.group}}{{item.reason.current_group}}
Permissions{{item.reason.perms}}{{item.reason.current_perms}}
Status{{item.reason.status}}{{item.reason.current_status}}
Symlink Target{{item.reason.to}}{{item.reason.current_to}}
Package Version{{item.reason.version|cut:"("|cut:")"}}{{item.reason.current_version|cut:"("|cut:")"}}
+ {% endif %} + + {% if item.reason.current_diff or item.reason.is_sensitive %} +
+
+ {% if item.reason.is_sensitive %} +

File contents unavailable, as they might contain sensitive data.

+ {% else %} +

Incorrect file contents

+ {% endif %} +
+ {% if not item.reason.is_sensitive %} +
+ {{ item.reason.current_diff|syntaxhilight }} +
+ {% endif %} +
+ {% endif %} + + +
+
+

Occurences on {{ timestamp|date:"Y-m-d" }}

+
+ {% if associated_list %} + + {% for inter in associated_list %} + + + + {% endfor %} +
{{inter.client.name}}{{inter.timestamp}}
+ {% else %} +

Missing client list

+ {% endif %} +
+ +
+{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html new file mode 100644 index 000000000..9b1026a08 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html @@ -0,0 +1,33 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Element Listing{% endblock %} + +{% block extra_header_info %} +{% endblock%} + +{% block pagebanner %}{{mod_or_bad|capfirst}} Element Listing{% endblock %} + +{% block content %} +{% if item_list_dict %} + {% for kind, entries in item_list_dict.items %} + +
+
+

{{ kind }} — {{ entries|length }}

+
[–]
+
+ + + {% for e in entries %} + + + + {% endfor %} +
{{e.entry.name}}
+
+ {% endfor %} +{% else %} +

There are currently no inconsistent configuration entries.

+{% endif %} +{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html new file mode 100644 index 000000000..b9847cf96 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html @@ -0,0 +1,42 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Client Summary{% endblock %} +{% block pagebanner %}Clients - Summary{% endblock %} + +{% block body_onload %}javascript:hide_table_array(hide_tables){% endblock %} + +{% block extra_header_info %} + +{% endblock%} + +{% block content %} +
+

{{ node_count }} nodes reporting in

+
+{% if summary_data %} + {% for summary in summary_data %} +
+
+

{{ summary.nodes|length }} {{ summary.label }}

+
[+]
+
+ + + {% for node in summary.nodes|sort_interactions_by_name %} + + + + {% endfor %} +
{{ node.client.name }}
+
+ {% endfor %} +{% else %} +

No data to report on

+{% endif %} +{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html new file mode 100644 index 000000000..ff775ded5 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html @@ -0,0 +1,38 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Performance Metrics{% endblock %} +{% block pagebanner %}Performance Metrics{% endblock %} + + +{% block extra_header_info %} +{% endblock%} + +{% block content %} +
+ {% if metrics %} + + + + + + + + + + + {% for metric in metrics|dictsort:"name" %} + + + {% for mitem in metric|build_metric_list %} + + {% endfor %} + + {% endfor %} +
NameParseProbeInventoryInstallConfigTotal
{{ metric.name }}{{ mitem }}
+ {% else %} +

No metric data available

+ {% endif %} +
+{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html new file mode 100644 index 000000000..6fbe585ab --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html @@ -0,0 +1,13 @@ +{% spaceless %} +{% if filters %} +{% for filter, filter_url in filters %} + {% if forloop.first %} +
Active filters (click to remove): + {% endif %} + {{ filter|capfirst }}{% if not forloop.last %}, {% endif %} + {% if forloop.last %} +
+ {% endif %} +{% endfor %} +{% endif %} +{% endspaceless %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc new file mode 100644 index 000000000..8f2dec1dc --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc @@ -0,0 +1,38 @@ +{% load bcfg2_tags %} +
+ + + + {% if not client %} + + {% endif %} + + + + + + + + {% for entry in entry_list %} + + + {% if not client %} + + {% endif %} + + + + + + + + {% endfor %} +
TimestampClientStateGoodBadModifiedExtraServer
{{ entry.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}{{ entry.client.name }}{{ entry.state }}{{ entry.goodcount }}{{ entry.bad_entry_count }}{{ entry.modified_entry_count }}{{ entry.extra_entry_count }} + {% if entry.server %} + {{ entry.server }} + {% else %} +   + {% endif %} +
+
diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html new file mode 100644 index 000000000..aa0def83e --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html @@ -0,0 +1,23 @@ +{% spaceless %} +{% for page, page_url in pager %} + {% if forloop.first %} +
+ {% if prev_page %}< Prev {% endif %} + {% if first_page %}1 ... {% endif %} + {% endif %} + {% ifequal page current_page %} + {{ page }} + {% else %} + {{ page }} + {% endifequal %} + {% if forloop.last %} + {% if last_page %} ... {{ total_pages }} {% endif %} + {% if next_page %}Next > {% endif %} + |{% for limit, limit_url in page_limits %} {{ limit }}{% endfor %} +
+ {% else %} +   + {% endif %} +{% endfor %} +{% endspaceless %} + diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py new file mode 100644 index 000000000..f738f7bdd --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py @@ -0,0 +1,276 @@ +from django import template +from django.core.urlresolvers import resolve, reverse, Resolver404, NoReverseMatch +from django.utils.encoding import smart_unicode, smart_str +from datetime import datetime, timedelta +from Bcfg2.Server.Reports.utils import filter_list + +register = template.Library() + +__PAGE_NAV_LIMITS__ = (10, 25, 50, 100) + +@register.inclusion_tag('widgets/page_bar.html', takes_context=True) +def page_navigator(context): + """ + Creates paginated links. + + Expects the context to be a RequestContext and views.prepare_paginated_list() + to have populated page information. + """ + fragment = dict() + try: + path = context['request'].META['PATH_INFO'] + total_pages = int(context['total_pages']) + records_per_page = int(context['records_per_page']) + except KeyError: + return fragment + except ValueError: + return fragment + + if total_pages < 2: + return {} + + try: + view, args, kwargs = resolve(path) + current_page = int(kwargs.get('page_number',1)) + fragment['current_page'] = current_page + fragment['page_number'] = current_page + fragment['total_pages'] = total_pages + fragment['records_per_page'] = records_per_page + if current_page > 1: + kwargs['page_number'] = current_page - 1 + fragment['prev_page'] = reverse(view, args=args, kwargs=kwargs) + if current_page < total_pages: + kwargs['page_number'] = current_page + 1 + fragment['next_page'] = reverse(view, args=args, kwargs=kwargs) + + view_range = 5 + if total_pages > view_range: + pager_start = current_page - 2 + pager_end = current_page + 2 + if pager_start < 1: + pager_end += (1 - pager_start) + pager_start = 1 + if pager_end > total_pages: + pager_start -= (pager_end - total_pages) + pager_end = total_pages + else: + pager_start = 1 + pager_end = total_pages + + if pager_start > 1: + kwargs['page_number'] = 1 + fragment['first_page'] = reverse(view, args=args, kwargs=kwargs) + if pager_end < total_pages: + kwargs['page_number'] = total_pages + fragment['last_page'] = reverse(view, args=args, kwargs=kwargs) + + pager = [] + for page in range(pager_start, int(pager_end) + 1): + kwargs['page_number'] = page + pager.append( (page, reverse(view, args=args, kwargs=kwargs)) ) + + kwargs['page_number'] = 1 + page_limits = [] + for limit in __PAGE_NAV_LIMITS__: + kwargs['page_limit'] = limit + page_limits.append( (limit, reverse(view, args=args, kwargs=kwargs)) ) + # resolver doesn't like this + del kwargs['page_number'] + del kwargs['page_limit'] + page_limits.append( ('all', reverse(view, args=args, kwargs=kwargs) + "|all") ) + + fragment['pager'] = pager + fragment['page_limits'] = page_limits + + except Resolver404: + path = "404" + except NoReverseMatch: + nr = sys.exc_info()[1] + path = "NoReverseMatch: %s" % nr + except ValueError: + path = "ValueError" + #FIXME - Handle these + + fragment['path'] = path + return fragment + +@register.inclusion_tag('widgets/filter_bar.html', takes_context=True) +def filter_navigator(context): + try: + path = context['request'].META['PATH_INFO'] + view, args, kwargs = resolve(path) + + # Strip any page limits and numbers + if 'page_number' in kwargs: + del kwargs['page_number'] + if 'page_limit' in kwargs: + del kwargs['page_limit'] + + filters = [] + for filter in filter_list: + if filter in kwargs: + myargs = kwargs.copy() + del myargs[filter] + filters.append( (filter, reverse(view, args=args, kwargs=myargs) ) ) + filters.sort(lambda x,y: cmp(x[0], y[0])) + return { 'filters': filters } + except (Resolver404, NoReverseMatch, ValueError, KeyError): + pass + return dict() + +def _subtract_or_na(mdict, x, y): + """ + Shortcut for build_metric_list + """ + try: + return round(mdict[x] - mdict[y], 4) + except: + return "n/a" + +@register.filter +def build_metric_list(mdict): + """ + Create a list of metric table entries + + Moving this here it simplify the view. Should really handle the case where these + are missing... + """ + td_list = [] + # parse + td_list.append( _subtract_or_na(mdict, 'config_parse', 'config_download')) + #probe + td_list.append( _subtract_or_na(mdict, 'probe_upload', 'start')) + #inventory + td_list.append( _subtract_or_na(mdict, 'inventory', 'initialization')) + #install + td_list.append( _subtract_or_na(mdict, 'install', 'inventory')) + #cfg download & parse + td_list.append( _subtract_or_na(mdict, 'config_parse', 'probe_upload')) + #total + td_list.append( _subtract_or_na(mdict, 'finished', 'start')) + return td_list + +@register.filter +def isstale(timestamp, entry_max=None): + """ + Check for a stale timestamp + + Compares two timestamps and returns True if the + difference is greater then 24 hours. + """ + if not entry_max: + entry_max = datetime.now() + return entry_max - timestamp > timedelta(hours=24) + +@register.filter +def sort_interactions_by_name(value): + """ + Sort an interaction list by client name + """ + inters = list(value) + inters.sort(lambda a,b: cmp(a.client.name, b.client.name)) + return inters + +class AddUrlFilter(template.Node): + def __init__(self, filter_name, filter_value): + self.filter_name = filter_name + self.filter_value = filter_value + self.fallback_view = 'Bcfg2.Server.Reports.reports.views.render_history_view' + + def render(self, context): + link = '#' + try: + path = context['request'].META['PATH_INFO'] + view, args, kwargs = resolve(path) + filter_value = self.filter_value.resolve(context, True) + if filter_value: + filter_name = smart_str(self.filter_name) + filter_value = smart_unicode(filter_value) + kwargs[filter_name] = filter_value + # These two don't make sense + if filter_name == 'server' and 'hostname' in kwargs: + del kwargs['hostname'] + elif filter_name == 'hostname' and 'server' in kwargs: + del kwargs['server'] + try: + link = reverse(view, args=args, kwargs=kwargs) + except NoReverseMatch: + link = reverse(self.fallback_view, args=None, + kwargs={ filter_name: filter_value }) + except NoReverseMatch: + rm = sys.exc_info()[1] + raise rm + except (Resolver404, ValueError): + pass + return link + +@register.tag +def add_url_filter(parser, token): + """ + Return a url with the filter added to the current view. + + Takes a new filter and resolves the current view with the new filter + applied. Resolves to Bcfg2.Server.Reports.reports.views.client_history + by default. + + {% add_url_filter server=interaction.server %} + """ + try: + tag_name, filter_pair = token.split_contents() + filter_name, filter_value = filter_pair.split('=', 1) + filter_name = filter_name.strip() + filter_value = parser.compile_filter(filter_value) + except ValueError: + raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) + if not filter_name or not filter_value: + raise template.TemplateSyntaxError("argument should be a filter=value pair") + + return AddUrlFilter(filter_name, filter_value) + +@register.filter +def sortwell(value): + """ + Sorts a list(or evaluates queryset to list) of bad, extra, or modified items in the best + way for presentation + """ + + configItems = list(value) + configItems.sort(lambda x,y: cmp(x.entry.name, y.entry.name)) + configItems.sort(lambda x,y: cmp(x.entry.kind, y.entry.kind)) + return configItems + +class MediaTag(template.Node): + def __init__(self, filter_value): + self.filter_value = filter_value + + def render(self, context): + base = context['MEDIA_URL'] + try: + request = context['request'] + try: + base = request.environ['bcfg2.media_url'] + except: + if request.path != request.META['PATH_INFO']: + offset = request.path.find(request.META['PATH_INFO']) + if offset > 0: + base = "%s/%s" % (request.path[:offset], \ + context['MEDIA_URL'].strip('/')) + except: + pass + return "%s/%s" % (base, self.filter_value) + +@register.tag +def to_media_url(parser, token): + """ + Return a url relative to the media_url. + + {% to_media_url /bcfg2.css %} + """ + try: + tag_name, filter_value = token.split_contents() + filter_value = parser.compile_filter(filter_value) + except ValueError: + raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) + + return MediaTag(filter_value) + diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py new file mode 100644 index 000000000..2e30125f9 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py @@ -0,0 +1,49 @@ +import sys +from django import template +from django.utils.encoding import smart_unicode, smart_str +from django.utils.html import conditional_escape +from django.utils.safestring import mark_safe + +register = template.Library() + +try: + from pygments import highlight + from pygments.lexers import get_lexer_by_name + from pygments.formatters import HtmlFormatter + colorize = True + +except: + colorize = False + +# py3k compatibility +def u_str(string): + if sys.hexversion >= 0x03000000: + return string + else: + return unicode(string) + +@register.filter +def syntaxhilight(value, arg="diff", autoescape=None): + """ + Returns a syntax-hilighted version of Code; requires code/language arguments + """ + + if autoescape: + value = conditional_escape(value) + arg = conditional_escape(arg) + + if colorize: + try: + output = u_str('') + + lexer = get_lexer_by_name(arg) + output += highlight(value, lexer, HtmlFormatter()) + return mark_safe(output) + except: + return value + else: + return mark_safe(u_str('
Tip: Install pygments for highlighting
%s
') % value) +syntaxhilight.needs_autoescape = True + diff --git a/src/lib/Bcfg2/Server/Reports/reports/urls.py b/src/lib/Bcfg2/Server/Reports/reports/urls.py new file mode 100644 index 000000000..434ce07b7 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/urls.py @@ -0,0 +1,55 @@ +from django.conf.urls.defaults import * +from django.core.urlresolvers import reverse, NoReverseMatch +from django.http import HttpResponsePermanentRedirect +from Bcfg2.Server.Reports.utils import filteredUrls, paginatedUrls, timeviewUrls + +def newRoot(request): + try: + grid_view = reverse('reports_grid_view') + except NoReverseMatch: + grid_view = '/grid' + return HttpResponsePermanentRedirect(grid_view) + +urlpatterns = patterns('Bcfg2.Server.Reports.reports', + (r'^$', newRoot), + + url(r'^manage/?$', 'views.client_manage', name='reports_client_manage'), + url(r'^client/(?P[^/]+)/(?P\d+)/?$', 'views.client_detail', name='reports_client_detail_pk'), + url(r'^client/(?P[^/]+)/?$', 'views.client_detail', name='reports_client_detail'), + url(r'^elements/(?P\w+)/(?P\d+)/?$', 'views.config_item', name='reports_item'), +) + +urlpatterns += patterns('Bcfg2.Server.Reports.reports', + *timeviewUrls( + (r'^grid/?$', 'views.client_index', None, 'reports_grid_view'), + (r'^summary/?$', 'views.display_summary', None, 'reports_summary'), + (r'^timing/?$', 'views.display_timing', None, 'reports_timing'), + (r'^elements/(?P\w+)/?$', 'views.config_item_list', None, 'reports_item_list'), +)) + +urlpatterns += patterns('Bcfg2.Server.Reports.reports', + *filteredUrls(*timeviewUrls( + (r'^detailed/?$', + 'views.client_detailed_list', None, 'reports_detailed_list') +))) + +urlpatterns += patterns('Bcfg2.Server.Reports.reports', + *paginatedUrls( *filteredUrls( + (r'^history/?$', + 'views.render_history_view', None, 'reports_history'), + (r'^history/(?P[^/|]+)/?$', + 'views.render_history_view', None, 'reports_client_history'), +))) + + # Uncomment this for admin: + #(r'^admin/', include('django.contrib.admin.urls')), + + +## Uncomment this section if using authentication +#urlpatterns += patterns('', +# (r'^login/$', 'django.contrib.auth.views.login', +# {'template_name': 'auth/login.html'}), +# (r'^logout/$', 'django.contrib.auth.views.logout', +# {'template_name': 'auth/logout.html'}) +# ) + diff --git a/src/lib/Bcfg2/Server/Reports/reports/views.py b/src/lib/Bcfg2/Server/Reports/reports/views.py new file mode 100644 index 000000000..ccd71a60e --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/views.py @@ -0,0 +1,415 @@ +""" +Report views + +Functions to handle all of the reporting views. +""" +from datetime import datetime, timedelta +import sys +from time import strptime + +from django.template import Context, RequestContext +from django.http import \ + HttpResponse, HttpResponseRedirect, HttpResponseServerError, Http404 +from django.shortcuts import render_to_response, get_object_or_404 +from django.core.urlresolvers import \ + resolve, reverse, Resolver404, NoReverseMatch +from django.db import connection + +from Bcfg2.Server.Reports.reports.models import * + + +class PaginationError(Exception): + """This error is raised when pagination cannot be completed.""" + pass + + +def server_error(request): + """ + 500 error handler. + + For now always return the debug response. Mailing isn't appropriate here. + + """ + from django.views import debug + return debug.technical_500_response(request, *sys.exc_info()) + + +def timeview(fn): + """ + Setup a timeview view + + Handles backend posts from the calendar and converts date pieces + into a 'timestamp' parameter + + """ + def _handle_timeview(request, **kwargs): + """Send any posts back.""" + if request.method == 'POST': + cal_date = request.POST['cal_date'] + try: + fmt = "%Y/%m/%d" + if cal_date.find(' ') > -1: + fmt += " %H:%M" + timestamp = datetime(*strptime(cal_date, fmt)[0:6]) + view, args, kw = resolve(request.META['PATH_INFO']) + kw['year'] = "%0.4d" % timestamp.year + kw['month'] = "%02.d" % timestamp.month + kw['day'] = "%02.d" % timestamp.day + if cal_date.find(' ') > -1: + kw['hour'] = timestamp.hour + kw['minute'] = timestamp.minute + return HttpResponseRedirect(reverse(view, + args=args, + kwargs=kw)) + except KeyError: + pass + except: + pass + # FIXME - Handle this + + """Extract timestamp from args.""" + timestamp = None + try: + timestamp = datetime(int(kwargs.pop('year')), + int(kwargs.pop('month')), + int(kwargs.pop('day')), int(kwargs.pop('hour', 0)), + int(kwargs.pop('minute', 0)), 0) + kwargs['timestamp'] = timestamp + except KeyError: + pass + except: + raise + return fn(request, **kwargs) + + return _handle_timeview + + +def config_item(request, pk, type="bad"): + """ + Display a single entry. + + Dispalys information about a single entry. + + """ + item = get_object_or_404(Entries_interactions, id=pk) + timestamp = item.interaction.timestamp + time_start = item.interaction.timestamp.replace(hour=0, + minute=0, + second=0, + microsecond=0) + time_end = time_start + timedelta(days=1) + + todays_data = Interaction.objects.filter(timestamp__gte=time_start, + timestamp__lt=time_end) + shared_entries = Entries_interactions.objects.filter(entry=item.entry, + reason=item.reason, + type=item.type, + interaction__in=[x['id']\ + for x in todays_data.values('id')]) + + associated_list = Interaction.objects.filter(id__in=[x['interaction']\ + for x in shared_entries.values('interaction')])\ + .order_by('client__name', 'timestamp').select_related().all() + + return render_to_response('config_items/item.html', + {'item': item, + 'isextra': item.type == TYPE_EXTRA, + 'mod_or_bad': type, + 'associated_list': associated_list, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +@timeview +def config_item_list(request, type, timestamp=None): + """Render a listing of affected elements""" + mod_or_bad = type.lower() + type = convert_entry_type_to_id(type) + if type < 0: + raise Http404 + + current_clients = Interaction.objects.get_interaction_per_client_ids(timestamp) + item_list_dict = {} + seen = dict() + for x in Entries_interactions.objects.filter(interaction__in=current_clients, + type=type).select_related(): + if (x.entry, x.reason) in seen: + continue + seen[(x.entry, x.reason)] = 1 + if item_list_dict.get(x.entry.kind, None): + item_list_dict[x.entry.kind].append(x) + else: + item_list_dict[x.entry.kind] = [x] + + for kind in item_list_dict: + item_list_dict[kind].sort(lambda a, b: cmp(a.entry.name, b.entry.name)) + + return render_to_response('config_items/listing.html', + {'item_list_dict': item_list_dict, + 'mod_or_bad': mod_or_bad, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +@timeview +def client_index(request, timestamp=None): + """ + Render a grid view of active clients. + + Keyword parameters: + timestamp -- datetime objectto render from + + """ + list = Interaction.objects.interaction_per_client(timestamp).select_related()\ + .order_by("client__name").all() + + return render_to_response('clients/index.html', + {'inter_list': list, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +@timeview +def client_detailed_list(request, timestamp=None, **kwargs): + """ + Provides a more detailed list view of the clients. Allows for extra + filters to be passed in. + + """ + + kwargs['interaction_base'] = Interaction.objects.interaction_per_client(timestamp).select_related() + kwargs['orderby'] = "client__name" + kwargs['page_limit'] = 0 + return render_history_view(request, 'clients/detailed-list.html', **kwargs) + + +def client_detail(request, hostname=None, pk=None): + context = dict() + client = get_object_or_404(Client, name=hostname) + if(pk == None): + context['interaction'] = client.current_interaction + return render_history_view(request, 'clients/detail.html', page_limit=5, + client=client, context=context) + else: + context['interaction'] = client.interactions.get(pk=pk) + return render_history_view(request, 'clients/detail.html', page_limit=5, + client=client, maxdate=context['interaction'].timestamp, context=context) + + +def client_manage(request): + """Manage client expiration""" + message = '' + if request.method == 'POST': + try: + client_name = request.POST.get('client_name', None) + client_action = request.POST.get('client_action', None) + client = Client.objects.get(name=client_name) + if client_action == 'expire': + client.expiration = datetime.now() + client.save() + message = "Expiration for %s set to %s." % \ + (client_name, client.expiration.strftime("%Y-%m-%d %H:%M:%S")) + elif client_action == 'unexpire': + client.expiration = None + client.save() + message = "%s is now active." % client_name + else: + message = "Missing action" + except Client.DoesNotExist: + if not client_name: + client_name = "" + message = "Couldn't find client \"%s\"" % client_name + + return render_to_response('clients/manage.html', + {'clients': Client.objects.order_by('name').all(), 'message': message}, + context_instance=RequestContext(request)) + + +@timeview +def display_summary(request, timestamp=None): + """ + Display a summary of the bcfg2 world + """ + query = Interaction.objects.interaction_per_client(timestamp).select_related() + node_count = query.count() + recent_data = query.all() + if not timestamp: + timestamp = datetime.now() + + collected_data = dict(clean=[], + bad=[], + modified=[], + extra=[], + stale=[], + pings=[]) + for node in recent_data: + if timestamp - node.timestamp > timedelta(hours=24): + collected_data['stale'].append(node) + # If stale check for uptime + try: + if node.client.pings.latest().status == 'N': + collected_data['pings'].append(node) + except Ping.DoesNotExist: + collected_data['pings'].append(node) + continue + if node.bad_entry_count() > 0: + collected_data['bad'].append(node) + else: + collected_data['clean'].append(node) + if node.modified_entry_count() > 0: + collected_data['modified'].append(node) + if node.extra_entry_count() > 0: + collected_data['extra'].append(node) + + # label, header_text, node_list + summary_data = [] + get_dict = lambda name, label: {'name': name, + 'nodes': collected_data[name], + 'label': label} + if len(collected_data['clean']) > 0: + summary_data.append(get_dict('clean', + 'nodes are clean.')) + if len(collected_data['bad']) > 0: + summary_data.append(get_dict('bad', + 'nodes are bad.')) + if len(collected_data['modified']) > 0: + summary_data.append(get_dict('modified', + 'nodes were modified.')) + if len(collected_data['extra']) > 0: + summary_data.append(get_dict('extra', + 'nodes have extra configurations.')) + if len(collected_data['stale']) > 0: + summary_data.append(get_dict('stale', + 'nodes did not run within the last 24 hours.')) + if len(collected_data['pings']) > 0: + summary_data.append(get_dict('pings', + 'are down.')) + + return render_to_response('displays/summary.html', + {'summary_data': summary_data, 'node_count': node_count, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +@timeview +def display_timing(request, timestamp=None): + mdict = dict() + inters = Interaction.objects.interaction_per_client(timestamp).select_related().all() + [mdict.__setitem__(inter, {'name': inter.client.name}) \ + for inter in inters] + for metric in Performance.objects.filter(interaction__in=list(mdict.keys())).all(): + for i in metric.interaction.all(): + mdict[i][metric.metric] = metric.value + return render_to_response('displays/timing.html', + {'metrics': list(mdict.values()), + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +def render_history_view(request, template='clients/history.html', **kwargs): + """ + Provides a detailed history of a clients interactions. + + Renders a detailed history of a clients interactions. Allows for various + filters and settings. Automatically sets pagination data into the context. + + Keyword arguments: + interaction_base -- Interaction QuerySet to build on + (default Interaction.objects) + context -- Additional context data to render with + page_number -- Page to display (default 1) + page_limit -- Number of results per page, if 0 show all (default 25) + client -- Client object to render + hostname -- Client hostname to lookup and render. Returns a 404 if + not found + server -- Filter interactions by server + state -- Filter interactions by state + entry_max -- Most recent interaction to display + orderby -- Sort results using this field + + """ + + context = kwargs.get('context', dict()) + max_results = int(kwargs.get('page_limit', 25)) + page = int(kwargs.get('page_number', 1)) + + client = kwargs.get('client', None) + if not client and 'hostname' in kwargs: + client = get_object_or_404(Client, name=kwargs['hostname']) + if client: + context['client'] = client + + entry_max = kwargs.get('maxdate', None) + context['entry_max'] = entry_max + + # Either filter by client or limit by clients + iquery = kwargs.get('interaction_base', Interaction.objects) + if client: + iquery = iquery.filter(client__exact=client).select_related() + + if 'orderby' in kwargs and kwargs['orderby']: + iquery = iquery.order_by(kwargs['orderby']) + + if 'state' in kwargs and kwargs['state']: + iquery = iquery.filter(state__exact=kwargs['state']) + if 'server' in kwargs and kwargs['server']: + iquery = iquery.filter(server__exact=kwargs['server']) + + if entry_max: + iquery = iquery.filter(timestamp__lte=entry_max) + + if max_results < 0: + max_results = 1 + entry_list = [] + if max_results > 0: + try: + rec_start, rec_end = prepare_paginated_list(request, + context, + iquery, + page, + max_results) + except PaginationError: + page_error = sys.exc_info()[1] + if isinstance(page_error[0], HttpResponse): + return page_error[0] + return HttpResponseServerError(page_error) + context['entry_list'] = iquery.all()[rec_start:rec_end] + else: + context['entry_list'] = iquery.all() + + return render_to_response(template, context, + context_instance=RequestContext(request)) + + +def prepare_paginated_list(request, context, paged_list, page=1, max_results=25): + """ + Prepare context and slice an object for pagination. + """ + if max_results < 1: + raise PaginationError("Max results less then 1") + if paged_list == None: + raise PaginationError("Invalid object") + + try: + nitems = paged_list.count() + except TypeError: + nitems = len(paged_list) + + rec_start = (page - 1) * int(max_results) + try: + total_pages = (nitems / int(max_results)) + 1 + except: + total_pages = 1 + if page > total_pages: + # If we passed beyond the end send back + try: + view, args, kwargs = resolve(request.META['PATH_INFO']) + kwargs['page_number'] = total_pages + raise PaginationError(HttpResponseRedirect(reverse(view, + kwards=kwargs))) + except (Resolver404, NoReverseMatch, ValueError): + raise "Accessing beyond last page. Unable to resolve redirect." + + context['total_pages'] = total_pages + context['records_per_page'] = max_results + return (rec_start, rec_start + int(max_results)) diff --git a/src/lib/Bcfg2/Server/Reports/settings.py b/src/lib/Bcfg2/Server/Reports/settings.py new file mode 100644 index 000000000..c8ceb5d88 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/settings.py @@ -0,0 +1,161 @@ +import django +import sys + +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser +# Django settings for bcfg2 reports project. +c = ConfigParser.ConfigParser() +if len(c.read(['/etc/bcfg2.conf', '/etc/bcfg2-web.conf'])) == 0: + raise ImportError("Please check that bcfg2.conf or bcfg2-web.conf exists " + "and is readable by your web server.") + +try: + DEBUG = c.getboolean('statistics', 'web_debug') +except: + DEBUG = False + +if DEBUG: + print("Warning: Setting web_debug to True causes extraordinary memory " + "leaks. Only use this setting if you know what you're doing.") + +TEMPLATE_DEBUG = DEBUG + +ADMINS = ( + ('Root', 'root'), +) + +MANAGERS = ADMINS +try: + db_engine = c.get('statistics', 'database_engine') +except ConfigParser.NoSectionError: + e = sys.exc_info()[1] + raise ImportError("Failed to determine database engine: %s" % e) +db_name = '' +if c.has_option('statistics', 'database_name'): + db_name = c.get('statistics', 'database_name') +if db_engine == 'sqlite3' and db_name == '': + db_name = "%s/etc/brpt.sqlite" % c.get('server', 'repository') + +DATABASES = { + 'default': { + 'ENGINE': "django.db.backends.%s" % db_engine, + 'NAME': db_name + } +} + +if db_engine != 'sqlite3': + DATABASES['default']['USER'] = c.get('statistics', 'database_user') + DATABASES['default']['PASSWORD'] = c.get('statistics', 'database_password') + DATABASES['default']['HOST'] = c.get('statistics', 'database_host') + try: + DATABASES['default']['PORT'] = c.get('statistics', 'database_port') + except: # An empty string tells Django to use the default port. + DATABASES['default']['PORT'] = '' + +if django.VERSION[0] == 1 and django.VERSION[1] < 2: + DATABASE_ENGINE = db_engine + DATABASE_NAME = DATABASES['default']['NAME'] + if DATABASE_ENGINE != 'sqlite3': + DATABASE_USER = DATABASES['default']['USER'] + DATABASE_PASSWORD = DATABASES['default']['PASSWORD'] + DATABASE_HOST = DATABASES['default']['HOST'] + DATABASE_PORT = DATABASES['default']['PORT'] + + +# Local time zone for this installation. All choices can be found here: +# http://docs.djangoproject.com/en/dev/ref/settings/#time-zone +try: + TIME_ZONE = c.get('statistics', 'time_zone') +except: + if django.VERSION[0] == 1 and django.VERSION[1] > 2: + TIME_ZONE = None + +# Language code for this installation. All choices can be found here: +# http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes +# http://blogs.law.harvard.edu/tech/stories/storyReader$15 +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# Absolute path to the directory that holds media. +# Example: "/home/media/media.lawrence.com/" +MEDIA_ROOT = '' + +# URL that handles the media served from MEDIA_ROOT. +# Example: "http://media.lawrence.com" +MEDIA_URL = '/site_media' +if c.has_option('statistics', 'web_prefix'): + MEDIA_URL = c.get('statistics', 'web_prefix').rstrip('/') + MEDIA_URL + +# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a +# trailing slash. +# Examples: "http://foo.com/media/", "/media/". +ADMIN_MEDIA_PREFIX = '/media/' + +# Make this unique, and don't share it with anybody. +SECRET_KEY = 'eb5+y%oy-qx*2+62vv=gtnnxg1yig_odu0se5$h0hh#pc*lmo7' + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.load_template_source', + 'django.template.loaders.app_directories.load_template_source', +) + +MIDDLEWARE_CLASSES = ( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.middleware.doc.XViewMiddleware', +) + +ROOT_URLCONF = 'Bcfg2.Server.Reports.urls' + +# Authentication Settings +# Use NIS authentication backend defined in backends.py +AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend', + 'Bcfg2.Server.Reports.backends.NISBackend') +# The NIS group authorized to login to BCFG2's reportinvg system +AUTHORIZED_GROUP = '' +#create login url area: +try: + import django.contrib.auth +except ImportError: + raise ImportError('Import of Django module failed. Is Django installed?') +django.contrib.auth.LOGIN_URL = '/login' + +SESSION_EXPIRE_AT_BROWSER_CLOSE = True + + + +TEMPLATE_DIRS = ( + # Put strings here, like "/home/html/django_templates". + # Always use forward slashes, even on Windows. + '/usr/share/python-support/python-django/django/contrib/admin/templates/', + 'Bcfg2.Server.Reports.reports' +) + +if django.VERSION[0] == 1 and django.VERSION[1] < 2: + TEMPLATE_CONTEXT_PROCESSORS = ( + 'django.core.context_processors.auth', + 'django.core.context_processors.debug', + 'django.core.context_processors.i18n', + 'django.core.context_processors.media', + 'django.core.context_processors.request' + ) +else: + TEMPLATE_CONTEXT_PROCESSORS = ( + 'django.contrib.auth.context_processors.auth', + 'django.core.context_processors.debug', + 'django.core.context_processors.i18n', + 'django.core.context_processors.media', + 'django.core.context_processors.request' + ) + +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.admin', + 'Bcfg2.Server.Reports.reports' +) diff --git a/src/lib/Bcfg2/Server/Reports/updatefix.py b/src/lib/Bcfg2/Server/Reports/updatefix.py new file mode 100644 index 000000000..c6593fb9c --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/updatefix.py @@ -0,0 +1,190 @@ +import Bcfg2.Server.Reports.settings + +from django.db import connection +import django.core.management +import logging +import traceback +from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion, \ + TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA +logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix') + + +# all update function should go here +def _merge_database_table_entries(): + cursor = connection.cursor() + insert_cursor = connection.cursor() + find_cursor = connection.cursor() + cursor.execute(""" + Select name, kind from reports_bad + union + select name, kind from reports_modified + union + select name, kind from reports_extra + """) + # this fetch could be better done + entries_map = {} + for row in cursor.fetchall(): + insert_cursor.execute("insert into reports_entries (name, kind) \ + values (%s, %s)", (row[0], row[1])) + entries_map[(row[0], row[1])] = insert_cursor.lastrowid + + cursor.execute(""" + Select name, kind, reason_id, interaction_id, 1 from reports_bad + inner join reports_bad_interactions on reports_bad.id=reports_bad_interactions.bad_id + union + Select name, kind, reason_id, interaction_id, 2 from reports_modified + inner join reports_modified_interactions on reports_modified.id=reports_modified_interactions.modified_id + union + Select name, kind, reason_id, interaction_id, 3 from reports_extra + inner join reports_extra_interactions on reports_extra.id=reports_extra_interactions.extra_id + """) + for row in cursor.fetchall(): + key = (row[0], row[1]) + if entries_map.get(key, None): + entry_id = entries_map[key] + else: + find_cursor.execute("Select id from reports_entries where name=%s and kind=%s", key) + rowe = find_cursor.fetchone() + entry_id = rowe[0] + insert_cursor.execute("insert into reports_entries_interactions \ + (entry_id, interaction_id, reason_id, type) values (%s, %s, %s, %s)", (entry_id, row[3], row[2], row[4])) + + +def _interactions_constraint_or_idx(): + '''sqlite doesn't support alter tables.. or constraints''' + cursor = connection.cursor() + try: + cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') + except: + cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') + + +def _populate_interaction_entry_counts(): + '''Populate up the type totals for the interaction table''' + cursor = connection.cursor() + count_field = {TYPE_BAD: 'bad_entries', + TYPE_MODIFIED: 'modified_entries', + TYPE_EXTRA: 'extra_entries'} + + for type in list(count_field.keys()): + cursor.execute("select count(type), interaction_id " + + "from reports_entries_interactions where type = %s group by interaction_id" % type) + updates = [] + for row in cursor.fetchall(): + updates.append(row) + try: + cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) + except Exception: + e = sys.exc_info()[1] + print(e) + cursor.close() + + +# be sure to test your upgrade query before reflecting the change in the models +# the list of function and sql command to do should go here +_fixes = [_merge_database_table_entries, + # this will remove unused tables + "drop table reports_bad;", + "drop table reports_bad_interactions;", + "drop table reports_extra;", + "drop table reports_extra_interactions;", + "drop table reports_modified;", + "drop table reports_modified_interactions;", + "drop table reports_repository;", + "drop table reports_metadata;", + "alter table reports_interaction add server varchar(256) not null default 'N/A';", + # fix revision data type to support $VCS hashes + "alter table reports_interaction add repo_rev_code varchar(64) default '';", + # Performance enhancements for large sites + 'alter table reports_interaction add column bad_entries integer not null default -1;', + 'alter table reports_interaction add column modified_entries integer not null default -1;', + 'alter table reports_interaction add column extra_entries integer not null default -1;', + _populate_interaction_entry_counts, + _interactions_constraint_or_idx, + 'alter table reports_reason add is_binary bool NOT NULL default False;', + 'alter table reports_reason add is_sensitive bool NOT NULL default False;', +] + +# this will calculate the last possible version of the database +lastversion = len(_fixes) + + +def rollupdate(current_version): + """ function responsible to coordinates all the updates + need current_version as integer + """ + ret = None + if current_version < lastversion: + for i in range(current_version, lastversion): + try: + if type(_fixes[i]) == str: + connection.cursor().execute(_fixes[i]) + else: + _fixes[i]() + except: + logger.error("Failed to perform db update %s" % (_fixes[i]), exc_info=1) + # since array start at 0 but version start at 1 we add 1 to the normal count + ret = InternalDatabaseVersion.objects.create(version=i + 1) + return ret + else: + return None + + +def dosync(): + """Function to do the syncronisation for the models""" + # try to detect if it's a fresh new database + try: + cursor = connection.cursor() + # If this table goes missing then don't forget to change it to the new one + cursor.execute("Select * from reports_client") + # if we get here with no error then the database has existing tables + fresh = False + except: + logger.debug("there was an error while detecting the freshness of the database") + #we should get here if the database is new + fresh = True + + # ensure database connection are close, so that the management can do it's job right + try: + cursor.close() + connection.close() + except: + # ignore any errors from missing/invalid dbs + pass + # Do the syncdb according to the django version + if "call_command" in dir(django.core.management): + # this is available since django 1.0 alpha. + # not yet tested for full functionnality + django.core.management.call_command("syncdb", interactive=False, verbosity=0) + if fresh: + django.core.management.call_command("loaddata", 'initial_version.xml', verbosity=0) + elif "syncdb" in dir(django.core.management): + # this exist only for django 0.96.* + django.core.management.syncdb(interactive=False, verbosity=0) + if fresh: + logger.debug("loading the initial_version fixtures") + django.core.management.load_data(fixture_labels=['initial_version'], verbosity=0) + else: + logger.warning("Don't forget to run syncdb") + + +def update_database(): + ''' methode to search where we are in the revision of the database models and update them ''' + try: + logger.debug("Running upgrade of models to the new one") + dosync() + know_version = InternalDatabaseVersion.objects.order_by('-version') + if not know_version: + logger.debug("No version, creating initial version") + know_version = InternalDatabaseVersion.objects.create(version=0) + else: + know_version = know_version[0] + logger.debug("Presently at %s" % know_version) + if know_version.version < lastversion: + new_version = rollupdate(know_version.version) + if new_version: + logger.debug("upgraded to %s" % new_version) + except: + logger.error("Error while updating the database") + for x in traceback.format_exc().splitlines(): + logger.error(x) diff --git a/src/lib/Bcfg2/Server/Reports/urls.py b/src/lib/Bcfg2/Server/Reports/urls.py new file mode 100644 index 000000000..d7ff1eee5 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/urls.py @@ -0,0 +1,14 @@ +from django.conf.urls.defaults import * +from django.http import HttpResponsePermanentRedirect + +handler500 = 'Bcfg2.Server.Reports.reports.views.server_error' + +urlpatterns = patterns('', + (r'^', include('Bcfg2.Server.Reports.reports.urls')) +) + +#urlpatterns += patterns("django.views", +# url(r"media/(?P.*)$", "static.serve", { +# "document_root": '/Users/tlaszlo/svn/bcfg2/reports/site_media/', +# }) +#) diff --git a/src/lib/Bcfg2/Server/Reports/utils.py b/src/lib/Bcfg2/Server/Reports/utils.py new file mode 100755 index 000000000..e0b6ead59 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/utils.py @@ -0,0 +1,124 @@ +"""Helper functions for reports""" +from django.conf.urls.defaults import * +import re + +"""List of filters provided by filteredUrls""" +filter_list = ('server', 'state') + + +class BatchFetch(object): + """Fetch Django objects in smaller batches to save memory""" + + def __init__(self, obj, step=10000): + self.count = 0 + self.block_count = 0 + self.obj = obj + self.data = None + self.step = step + self.max = obj.count() + + def __iter__(self): + return self + + def next(self): + """Provide compatibility with python < 3.0""" + return self.__next__() + + def __next__(self): + """Return the next object from our array and fetch from the + database when needed""" + if self.block_count + self.count - self.step == self.max: + raise StopIteration + if self.block_count == 0 or self.count == self.step: + # Without list() this turns into LIMIT 1 OFFSET x queries + self.data = list(self.obj.all()[self.block_count: \ + (self.block_count + self.step)]) + self.block_count += self.step + self.count = 0 + self.count += 1 + return self.data[self.count - 1] + + +def generateUrls(fn): + """ + Parse url tuples and send to functions. + + Decorator for url generators. Handles url tuple parsing + before the actual function is called. + """ + def url_gen(*urls): + results = [] + for url_tuple in urls: + if isinstance(url_tuple, (list, tuple)): + results += fn(*url_tuple) + else: + raise ValueError("Unable to handle compiled urls") + return results + return url_gen + + +@generateUrls +def paginatedUrls(pattern, view, kwargs=None, name=None): + """ + Takes a group of url tuples and adds paginated urls. + + Extends a url tuple to include paginated urls. + Currently doesn't handle url() compiled patterns. + + """ + results = [(pattern, view, kwargs, name)] + tail = '' + mtail = re.search('(/+\+?\\*?\??\$?)$', pattern) + if mtail: + tail = mtail.group(1) + pattern = pattern[:len(pattern) - len(tail)] + results += [(pattern + "/(?P\d+)" + tail, view, kwargs)] + results += [(pattern + "/(?P\d+)\|(?P\d+)" + + tail, view, kwargs)] + if not kwargs: + kwargs = dict() + kwargs['page_limit'] = 0 + results += [(pattern + "/?\|(?Pall)" + tail, view, kwargs)] + return results + + +@generateUrls +def filteredUrls(pattern, view, kwargs=None, name=None): + """ + Takes a url and adds filtered urls. + + Extends a url tuple to include filtered view urls. Currently doesn't + handle url() compiled patterns. + """ + results = [(pattern, view, kwargs, name)] + tail = '' + mtail = re.search('(/+\+?\\*?\??\$?)$', pattern) + if mtail: + tail = mtail.group(1) + pattern = pattern[:len(pattern) - len(tail)] + for filter in ('/state/(?P\w+)', + '/server/(?P[\w\-\.]+)', + '/server/(?P[\w\-\.]+)/(?P[A-Za-z]+)'): + results += [(pattern + filter + tail, view, kwargs)] + return results + + +@generateUrls +def timeviewUrls(pattern, view, kwargs=None, name=None): + """ + Takes a url and adds timeview urls + + Extends a url tuple to include filtered view urls. Currently doesn't + handle url() compiled patterns. + """ + results = [(pattern, view, kwargs, name)] + tail = '' + mtail = re.search('(/+\+?\\*?\??\$?)$', pattern) + if mtail: + tail = mtail.group(1) + pattern = pattern[:len(pattern) - len(tail)] + for filter in ('/(?P\d{4})-(?P\d{2})-(?P\d{2})/' + \ + '(?P\d\d)-(?P\d\d)', + '/(?P\d{4})-(?P\d{2})-(?P\d{2})'): + results += [(pattern + filter + tail, view, kwargs)] + return results -- cgit v1.2.3-1-g7c22