Initial version of admin tool for reports

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5886 ce84e21b-d406-0410-9b95-82705330c041

2 files changed, 190 insertions, 1 deletions

diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py
new file mode 100755
index 000000000..e4cbae8e1
--- /dev/null
+++ b/src/lib/Server/Admin/Reports.py
@@ -0,0 +1,189 @@
+'''Admin interface for dynamic reports'''
+import Bcfg2.Logger
+import Bcfg2.Server.Admin
+import ConfigParser
+import os
+import logging
+import pickle
+import sys
+from Bcfg2.Server.Reports.importscript import load_stats
+from Bcfg2.Server.Reports.updatefix import update_database
+from lxml.etree import XML, XMLSyntaxError
+
+try:
+    from hashlib import md5
+except ImportError:
+    from md5 import md5
+
+# Load django
+import django.core.management
+
+# FIXME - settings file uses a hardcoded path for /etc/bcfg2.conf
+try:
+    import Bcfg2.Server.Reports.settings
+except Exception, e:
+    sys.stderr.write("Failed to load configuration settings. %s\n" % e)
+    sys.exit(1)
+
+project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__)
+project_name = os.path.basename(project_directory)
+sys.path.append(os.path.join(project_directory, '..'))
+project_module = __import__(project_name, '', '', [''])
+sys.path.pop()
+
+# Set DJANGO_SETTINGS_MODULE appropriately.
+os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name
+from django.db import connection
+
+from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, \
+				Entries_interactions, Performance, \
+				Reason, Ping, TYPE_CHOICES, InternalDatabaseVersion
+
+class Reports(Bcfg2.Server.Admin.Mode):
+    '''Admin interface for dynamic reports'''
+    __shorthelp__ = "Manage dynamic reports"
+    __longhelp__ = (__shorthelp__)
+    __usage__ = ("bcfg2-admin reports [command] [options]\n"
+                 "    -v|--verbose         Be verbose\n"
+                 "    -q|--quiet           Print only errors\n"
+                 "\n"
+                 "  Commands:\n"
+                 "    load_stats           Load statistics data\n"
+                 "      -s|--stats         Path to statistics.xml file\n"
+                 "      -c|--clients-file  Path to clients.xml file\n"
+                 "      -O3                Fast mode.  Duplicates data!\n"
+                 "    scrub                Scrub the database for duplicate reasons\n"
+                 "    update               Apply any updates to the reporting database\n"
+                 "\n")
+
+    def __init__(self, cfile):
+        Bcfg2.Server.Admin.Mode.__init__(self, cfile)
+        self.log.setLevel(logging.INFO)
+        self.django_commands = [ 'syncdb', 'sqlall', 'validate' ]
+        self.__usage__ = self.__usage__ + "  Django commands:\n    " + \
+             "\n    ".join(self.django_commands)
+
+    def __call__(self, args):
+        Bcfg2.Server.Admin.Mode.__call__(self, args)
+        if len(args) == 0 or args[0] == '-h':
+            print(self.__usage__)
+            raise SystemExit(0)
+
+        verb = 0
+
+        if '-v' in args or '--verbose' in args:
+            self.log.setLevel(logging.DEBUG)
+            verb = 1
+        if '-q' in args or '--quiet' in args:
+            self.log.setLevel(logging.WARNING)
+
+        # FIXME - dry run
+
+        if args[0] in self.django_commands:
+            self.django_command_proxy(args[0])
+        elif args[0] == 'scrub':
+            self.scrub()
+        elif args[0] == 'update':
+            update_database()
+        elif args[0] == 'load_stats':
+            quick = '-O3' in args
+            stats_file=None
+            clients_file=None
+            i=1
+            while i < len(args):
+                if args[i] == '-s' or args[i] == '--stats':
+                    stats_file = args[i+1]
+                    if stats_file[0] == '-':
+                        self.errExit("Invalid statistics file: %s" % stats_file)
+                elif args[i] == '-c' or args[i] == '--clients-file':
+                    clients_file = args[i+1]
+                    if clients_file[0] == '-':
+                        self.errExit("Invalid clients file: %s" % clients_file)
+                i = i + 1
+            self.load_stats(stats_file, clients_file, verb, quick)
+        else:
+            print "Unknown command: %s" % args[0]
+
+    def scrub(self):
+        ''' Perform a thorough scrub and cleanup of the database '''
+
+        # Currently only reasons are a problem
+        try:
+            start_count = Reason.objects.count()
+        except Exception, e:
+            self.log.error("Failed to load reason objects: %s" % e)
+            return
+        dup_reasons = []
+    
+        cmp_reasons = dict()
+        for reason in Reason.objects.all():
+            ''' Loop through each reason and create a key out of the data. \
+                This lets us take advantage of a fast hash lookup for \
+                comparisons '''
+            id = reason.id
+            reason.id = None
+            key=md5(pickle.dumps(reason)).hexdigest()
+            reason.id = id
+
+            if key in cmp_reasons:
+                self.log.debug("Update interactions from %d to %d" \
+                                    % (reason.id, cmp_reasons[key]))
+                try:
+                    Entries_interactions.objects.filter(reason=reason).\
+                                    update(reason=cmp_reasons[key])
+                    dup_reasons.append(reason.id)
+                except Exception, ex:
+                    self.log.error("Failed to update interactions for %d : %s" \
+                                    % (reason.id, ex))
+            else:
+                cmp_reasons[key] = reason.id
+            self.log.debug("key %d" % reason.id)
+    
+        self.log.debug("Done with updates, deleting dupes")
+        for dup in dup_reasons:
+            self.log.debug("Deleting %d" % dup)
+            try:
+                Reason.objects.get(id=dup).delete()
+            except Exception, ex:
+                self.log.error("Failed to delete reason %d: %s" % (dup, ex))
+
+        self.log.info("Found %d dupes out of %d" % (len(dup_reasons), start_count))
+
+    def django_command_proxy(self, command):
+        '''Call a django command'''
+        if command == 'sqlall':
+            django.core.management.call_command(command, 'reports')
+        else:
+            django.core.management.call_command(command)
+
+    def load_stats(self, stats_file=None, clientspath=None, verb=0, quick=False):
+        '''Load statistics data into the database'''
+        location = ''
+
+        if not stats_file:
+            try:
+                stats_file = "%s/etc/statistics.xml" % self.cfp.get('server', 'repository')
+            except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+                self.errExit("Could not read bcfg2.conf; exiting")
+        try:
+            statsdata = XML(open(stats_file).read())
+        except (IOError, XMLSyntaxError):
+            self.errExit("StatReports: Failed to parse %s"%(stats_file))
+
+        if not clientspath:
+            try:
+                clientspath = "%s/Metadata/clients.xml" % \
+                          self.cfp.get('server', 'repository')
+            except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+                self.errExit("Could not read bcfg2.conf; exiting")
+        try:
+            clientsdata = XML(open(clientspath).read())
+        except (IOError, XMLSyntaxError):
+            self.errExit("StatReports: Failed to parse %s"%(clientspath))
+
+        try:
+            location = self.cfp.get('components', 'bcfg2')
+        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+            self.errExit("Could not read bcfg2 from bcfg2.conf; exiting")
+        load_stats(clientsdata, statsdata, verb, self.log, quick=quick, location=location)
+
diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py
index 89415cf9f..99f52bda4 100644
--- a/src/lib/Server/Admin/__init__.py
+++ b/src/lib/Server/Admin/__init__.py
@@ -1,7 +1,7 @@
 __revision__ = '$Revision$'
 
 __all__ = ['Mode', 'Client', 'Compare', 'Init', 'Minestruct', 'Perf',
-           'Pull', 'Query', 'Snapshots', 'Tidy', 'Viz', 'Xcmd']
+           'Pull', 'Query', 'Reports', 'Snapshots', 'Tidy', 'Viz', 'Xcmd']
 
 import ConfigParser
 import logging