DBStats: Improved handling of binary objects in the database

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5962 ce84e21b-d406-0410-9b95-82705330c041

5 files changed, 25 insertions, 6 deletions

diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py
index 1b8052c2d..d6d6d7819 100644
--- a/src/lib/Server/Plugins/DBStats.py
+++ b/src/lib/Server/Plugins/DBStats.py
@@ -1,3 +1,4 @@
+import binascii
 import difflib
 import logging
 import lxml.etree
@@ -83,8 +84,14 @@ class DBStats(Bcfg2.Server.Plugin.Plugin,
                 ret.append(getattr(entry.reason, "current_%s" % t))
 
         if entry.reason.current_diff != '':
-            ret.append('\n'.join(difflib.restore(\
-                entry.reason.current_diff.split('\n'), 1)))
+            if entry.reason.is_binary:
+                ret.append(binascii.a2b_base64(entry.reason.current_diff))
+            else:
+                ret.append('\n'.join(difflib.restore(\
+                    entry.reason.current_diff.split('\n'), 1)))
+        elif entry.reason.is_binary:
+            # If len is zero the object was too large to store
+                raise Bcfg2.Server.Plugin.PluginExecutionError
         else:
             ret.append(None)
         return ret
diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py
index 69147abbb..0c061d65b 100755
--- a/src/lib/Server/Reports/importscript.py
+++ b/src/lib/Server/Reports/importscript.py
@@ -25,15 +25,20 @@ from time import strptime
 from django.db import connection
 from Bcfg2.Server.Reports.updatefix import update_database
 import ConfigParser
-import difflib
 import logging
 import Bcfg2.Logger
 import platform
 
 def build_reason_kwargs(r_ent):
+    binary_file=False
     if r_ent.get('current_bfile', False):
-        contents = binascii.a2b_base64(r_ent.get('current_bfile'))
-        rc_diff = '\n'.join(difflib.ndiff([], contents.split('\n')))
+        binary_file=True
+        rc_diff = r_ent.get('current_bfile')
+        if len(rc_diff) > 1024*1024:
+            rc_diff = ''
+        elif len(rc_diff) == 0:
+            # No point in flagging binary if we have no data
+            binary_file=False
     elif r_ent.get('current_bdiff', False):
         rc_diff = binascii.a2b_base64(r_ent.get('current_bdiff'))
     elif r_ent.get('current_diff', False):
@@ -53,7 +58,8 @@ def build_reason_kwargs(r_ent):
                 version=r_ent.get('version', default=""),
                 current_version=r_ent.get('current_version', default=""),
                 current_exists=r_ent.get('current_exists', default="True").capitalize()=="True",
-                current_diff=rc_diff)
+                current_diff=rc_diff,
+                is_binary=binary_file)
 
 
 def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''):
diff --git a/src/lib/Server/Reports/reports/fixtures/initial_version.xml b/src/lib/Server/Reports/reports/fixtures/initial_version.xml
index 5c9ca2de5..1b1359eed 100644
--- a/src/lib/Server/Reports/reports/fixtures/initial_version.xml
+++ b/src/lib/Server/Reports/reports/fixtures/initial_version.xml
@@ -28,4 +28,8 @@
 	  <field type='IntegerField' name='version'>16</field>
 	  <field type='DateTimeField' name='updated'>2010-06-01 12:26:10</field>
 	</object>
+	<object pk="7" model="reports.internaldatabaseversion">
+	  <field type='IntegerField' name='version'>17</field>
+	  <field type='DateTimeField' name='updated'>2010-07-02 00:00:00</field>
+	</object>
 </django-objects>
diff --git a/src/lib/Server/Reports/reports/models.py b/src/lib/Server/Reports/reports/models.py
index 5410ce37b..52073c153 100644
--- a/src/lib/Server/Reports/reports/models.py
+++ b/src/lib/Server/Reports/reports/models.py
@@ -222,6 +222,7 @@ class Reason(models.Model):
     current_version = models.TextField(max_length=128, blank=True)
     current_exists = models.BooleanField()#False means its missing. Default True
     current_diff = models.TextField(max_length=1280, blank=True)
+    is_binary = models.BooleanField(default=False)
     def _str_(self):
         return "Reason"
 
diff --git a/src/lib/Server/Reports/updatefix.py b/src/lib/Server/Reports/updatefix.py
index df2e812df..6d9b5e952 100644
--- a/src/lib/Server/Reports/updatefix.py
+++ b/src/lib/Server/Reports/updatefix.py
@@ -98,6 +98,7 @@ _fixes = [_merge_database_table_entries,
           'alter table reports_interaction add column extra_entries integer not null default -1;',
           _populate_interaction_entry_counts,
           _interactions_constraint_or_idx,
+          'alter table reports_reason add is_binary bool NOT NULL default False;',
 ]
 
 # this will calculate the last possible version of the database