Get importscript properly function-ified

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@4819 ce84e21b-d406-0410-9b95-82705330c041

1 files changed, 87 insertions, 93 deletions

diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py
index a37f64214..fbc3c81f9 100755
--- a/src/lib/Server/Reports/importscript.py
+++ b/src/lib/Server/Reports/importscript.py
@@ -19,7 +19,6 @@ os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name
 
 from Bcfg2.Server.Reports.reports.models import Client, Interaction, Bad, Modified, Extra, Performance, Reason, Ping
 from lxml.etree import XML, XMLSyntaxError
-from sys import argv
 from getopt import getopt, GetoptError
 from datetime import datetime
 from time import strptime
@@ -46,100 +45,31 @@ def build_reason_kwargs(r_ent):
                 current_exists=r_ent.get('current_exists', default="True").capitalize()=="True",
                 current_diff=rc_diff)
 
-if __name__ == '__main__':
-    somewhatverbose = False
-    verbose = False
-    veryverbose = False
-    cpath = "/etc/bcfg2.conf"
-    clientpath = False
-    statpath = False
-    
-    try:
-        opts, args = getopt(argv[1:], "hvudc:s:", ["help", "verbose", "updates" ,
-                                                   "debug", "clients=", "stats=",
-                                                   "config="])
-    except GetoptError, mesg:
-        # print help information and exit:
-        print "%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg) 
-        raise SystemExit, 2
-
-    for o, a in opts:
-        if o in ("-h", "--help"):
-            print "Usage:\nimportscript.py [-h] [-v] -c <clients-file> -s <statistics-file> \n"
-            print "h : help; this message"
-            print "v : verbose; print messages on record insertion/skip"
-            print "u : updates; print status messages as items inserted semi-verbose"
-            print "d : debug; print most SQL used to manipulate database"
-            print "C : path to bcfg2.conf config file."
-            print "c : clients.xml file"
-            print "s : statistics.xml file"
-            raise SystemExit
-        if o in ["-C", "--config"]:
-            cpath = a
-
-        if o in ("-v", "--verbose"):
-            verbose = True
-        if o in ("-u", "--updates"):
-            somewhatverbose = True
-        if o in ("-d", "--debug"):
-            veryverbose = True
-        if o in ("-c", "--clients"):
-            clientspath = a
-
-        if o in ("-s", "--stats"):
-            statpath = a
-
-    cf = ConfigParser.ConfigParser()
-    cf.read([cpath])
-
-    if not statpath:
-        try:
-            statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository')
-        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
-            print "Could not read bcfg2.conf; exiting"
-            raise SystemExit, 1
-    try:
-        statsdata = XML(open(statpath).read())
-    except (IOError, XMLSyntaxError):
-        print("StatReports: Failed to parse %s"%(statpath))
-        raise SystemExit, 1
-
-    if not clientpath:
-        try:
-            clientspath = "%s/Metadata/clients.xml" % \
-                          cf.get('server', 'repository')
-        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
-            print "Could not read bcfg2.conf; exiting"
-            raise SystemExit, 1
-    try:
-        clientsdata = XML(open(clientspath).read())
-    except (IOError, XMLSyntaxError):
-        print("StatReports: Failed to parse %s"%(clientspath))
-        raise SystemExit, 1
 
+def load_stats(cdata, sdata, vlevel):
     cursor = connection.cursor()
     clients = {}
     cursor.execute("SELECT name, id from reports_client;")
     [clients.__setitem__(a, b) for a, b in cursor.fetchall()]
     
-    for node in statsdata.findall('Node'):
+    for node in sdata.findall('Node'):
         name = node.get('name')
         if not clients.has_key(name):
             cursor.execute(\
                 "INSERT INTO reports_client VALUES (NULL, %s, %s, NULL, NULL)",
                 [datetime.now(), name])
             clients[name] = cursor.lastrowid
-            if verbose:
+            if vlevel > 0:
                 print("Client %s added to db" % name)
         else:
-            if verbose:
+            if vlevel > 0:
                 print("Client %s already exists in db" % name)
 
     pingability = {}
     [pingability.__setitem__(n.get('name'), n.get('pingable', default='N')) \
-     for n in clientsdata.findall('Client')]
+     for n in cdata.findall('Client')]
 
-    for node in statsdata.findall('Node'):
+    for node in sdata.findall('Node'):
         name = node.get('name')
         c_inst = Client.objects.filter(id=clients[name])[0]
         try:
@@ -155,7 +85,7 @@ if __name__ == '__main__':
                                                timestamp=timestamp)
             if ilist:
                 current_interaction_id = ilist[0].id
-                if verbose:
+                if vlevel > 0:
                     print("Interaction for %s at %s with id %s already exists"%(clients[name],
                         datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction_id))
                 continue
@@ -169,7 +99,7 @@ if __name__ == '__main__':
                                      totalcount=statistics.get('total',default="0"))
                 newint.save()
                 current_interaction_id = newint.id
-                if verbose:
+                if vlevel > 0:
                     print("Interaction for %s at %s with id %s INSERTED in to db"%(clients[name],
                         timestamp, current_interaction_id))
 
@@ -187,12 +117,12 @@ if __name__ == '__main__':
 
                     if rls:
                         rr = rls[0]
-                        if verbose:
+                        if vlevel > 0:
                             print "Reason exists: %s"% (rr.id)
                     else:
                         rr = Reason(**kargs)
                         rr.save()
-                        if verbose:
+                        if vlevel > 0:
                             print "Created reason: %s" % rr.id
                     if '-O3' not in sys.argv:
                         links = obj.objects.filter(name=x.get('name'),
@@ -203,7 +133,7 @@ if __name__ == '__main__':
                         
                     if links:
                         item_id = links[0].id
-                        if verbose:
+                        if vlevel > 0:
                             print "%s item exists, has reason id %s and ID %s" % (xpath, rr.id, item_id)
                     else:
                         newitem = obj(name=x.get('name'),
@@ -211,7 +141,7 @@ if __name__ == '__main__':
                                       reason=rr)
                         newitem.save()
                         item_id = newitem.id
-                        if verbose:
+                        if vlevel > 0:
                             print "%s item INSERTED having reason id %s and ID %s" % (xpath, rr.id, item_id)
                     try:
                         cursor.execute("INSERT INTO "+tablename+"_interactions VALUES (NULL, %s, %s);",
@@ -238,13 +168,13 @@ if __name__ == '__main__':
                     except:
                         pass
 
-    if (somewhatverbose or verbose):
+    if vlevel > 1:
         print("----------------INTERACTIONS SYNCED----------------")
     cursor.execute("select reports_interaction.id, x.client_id from (select client_id, MAX(timestamp) as timer from reports_interaction Group BY client_id) x, reports_interaction where reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer")
     for row in cursor.fetchall():
         cursor.execute("UPDATE reports_client SET current_interaction_id = %s where reports_client.id = %s",
                        [row[0],row[1]])
-    if (somewhatverbose or verbose):
+    if vlevel > 1:
         print("------------LATEST INTERACTION SET----------------")
 
     for key in pingability.keys():
@@ -267,18 +197,82 @@ if __name__ == '__main__':
                         starttime=datetime.now(), endtime=datetime.now())
             newp.save()
 
-    if (somewhatverbose or verbose):
+    if vlevel > 1:
         print "---------------PINGDATA SYNCED---------------------"
 
     connection._commit()
     #Clients are consistent
-    if veryverbose:
-        for q in connection.queries:
-            if not (q['sql'].startswith('INSERT INTO reports_bad_interactions')|
-                    q['sql'].startswith('INSERT INTO reports_extra_interactions')|
-                    q['sql'].startswith('INSERT INTO reports_performance_interaction')|
-                    q['sql'].startswith('INSERT INTO reports_modified_interactions')|
-                    q['sql'].startswith('UPDATE reports_client SET current_interaction_id ')):
-                print q
 
     raise SystemExit, 0
+
+if __name__ == '__main__':
+    from sys import argv
+    verb = 0
+    cpath = "/etc/bcfg2.conf"
+    clientpath = False
+    statpath = False
+    
+    try:
+        opts, args = getopt(argv[1:], "hvudc:s:", ["help", "verbose", "updates" ,
+                                                   "debug", "clients=", "stats=",
+                                                   "config="])
+    except GetoptError, mesg:
+        # print help information and exit:
+        print "%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg) 
+        raise SystemExit, 2
+
+    for o, a in opts:
+        if o in ("-h", "--help"):
+            print "Usage:\nimportscript.py [-h] [-v] -c <clients-file> -s <statistics-file> \n"
+            print "h : help; this message"
+            print "v : verbose; print messages on record insertion/skip"
+            print "u : updates; print status messages as items inserted semi-verbose"
+            print "d : debug; print most SQL used to manipulate database"
+            print "C : path to bcfg2.conf config file."
+            print "c : clients.xml file"
+            print "s : statistics.xml file"
+            raise SystemExit
+        if o in ["-C", "--config"]:
+            cpath = a
+
+        if o in ("-v", "--verbose"):
+            verb = 1
+        if o in ("-u", "--updates"):
+            verb = 2
+        if o in ("-d", "--debug"):
+            verb = 3
+        if o in ("-c", "--clients"):
+            clientspath = a
+
+        if o in ("-s", "--stats"):
+            statpath = a
+
+    cf = ConfigParser.ConfigParser()
+    cf.read([cpath])
+
+    if not statpath:
+        try:
+            statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository')
+        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+            print "Could not read bcfg2.conf; exiting"
+            raise SystemExit, 1
+    try:
+        statsdata = XML(open(statpath).read())
+    except (IOError, XMLSyntaxError):
+        print("StatReports: Failed to parse %s"%(statpath))
+        raise SystemExit, 1
+
+    if not clientpath:
+        try:
+            clientspath = "%s/Metadata/clients.xml" % \
+                          cf.get('server', 'repository')
+        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+            print "Could not read bcfg2.conf; exiting"
+            raise SystemExit, 1
+    try:
+        clientsdata = XML(open(clientspath).read())
+    except (IOError, XMLSyntaxError):
+        print("StatReports: Failed to parse %s"%(clientspath))
+        raise SystemExit, 1
+
+    load_stats(clientsdata, statsdata, verb)