Accept non-ASCII diffs

Currently, client reports don't include diffs of files which aren't
US-ASCII encoded.  The client transmits such files as Base64 blobs.  As
we'd like to change that, this commit teaches the server to properly
handle non-ASCII diffs.

5 files changed, 31 insertions, 6 deletions

diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py
index 942477a49..c9f3d3f58 100644
--- a/src/lib/Server/Admin/Reports.py
+++ b/src/lib/Server/Admin/Reports.py
@@ -257,6 +257,11 @@ class Reports(Bcfg2.Server.Admin.Mode):
         except (IOError, XMLSyntaxError):
             self.errExit("StatReports: Failed to parse %s" % (stats_file))
 
+        try:
+            encoding = self.cfp.get('components', 'encoding')
+        except:
+            encoding = 'UTF-8'
+
         if not clientspath:
             try:
                 clientspath = "%s/Metadata/clients.xml" % \
@@ -271,6 +276,7 @@ class Reports(Bcfg2.Server.Admin.Mode):
         try:
             load_stats(clientsdata,
                        statsdata,
+                       encoding,
                        verb,
                        self.log,
                        quick=quick,
diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py
index 8915492a3..b34d7108c 100644
--- a/src/lib/Server/Admin/__init__.py
+++ b/src/lib/Server/Admin/__init__.py
@@ -113,7 +113,8 @@ class MetadataCore(Mode):
     def __init__(self, configfile, usage, pwhitelist=None, pblacklist=None):
         Mode.__init__(self, configfile)
         options = {'plugins': Bcfg2.Options.SERVER_PLUGINS,
-                   'configfile': Bcfg2.Options.CFILE}
+                   'configfile': Bcfg2.Options.CFILE,
+                   'encoding': Bcfg2.Options.ENCODING}
         setup = Bcfg2.Options.OptionParser(options)
         setup.hm = usage
         setup.parse(sys.argv[1:])
@@ -126,7 +127,7 @@ class MetadataCore(Mode):
         try:
             self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(),
                                                 setup['plugins'],
-                                                'foo', 'UTF-8')
+                                                'foo', setup['encoding'])
         except Bcfg2.Server.Core.CoreInitError:
             msg = sys.exc_info()[1]
             self.errExit("Core load failed because %s" % msg)
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
index 5e3cca847..c08e8c4b6 100644
--- a/src/lib/Server/Plugins/Cfg.py
+++ b/src/lib/Server/Plugins/Cfg.py
@@ -197,7 +197,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
             if os.path.exists("%s.genshi" % name):
                 logger.error("Cfg: Unable to pull data for genshi types")
                 raise Bcfg2.Server.Plugin.PluginExecutionError
-            open(name, 'w').write(new_entry['text'])
+            try:
+                etext = new_entry['text'].encode(self.encoding)
+            except:
+                logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding))
+                raise Bcfg2.Server.Plugin.PluginExecutionError
+            open(name, 'w').write(etext)
             if log:
                 logger.info("Wrote file %s" % name)
         badattr = [attr for attr in ['owner', 'group', 'perms']
diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py
index 5ef1920e1..103fb7353 100644
--- a/src/lib/Server/Plugins/DBStats.py
+++ b/src/lib/Server/Plugins/DBStats.py
@@ -55,6 +55,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin,
             try:
                 Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata,
                                                              container,
+                                                             self.core.encoding,
                                                              0,
                                                              logger,
                                                              True,
diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py
index b6a3c2599..68774cec6 100755
--- a/src/lib/Server/Reports/importscript.py
+++ b/src/lib/Server/Reports/importscript.py
@@ -38,7 +38,7 @@ import platform
 from Bcfg2.Bcfg2Py3k import ConfigParser
 
 
-def build_reason_kwargs(r_ent):
+def build_reason_kwargs(r_ent, encoding, logger):
     binary_file = False
     if r_ent.get('current_bfile', False):
         binary_file = True
@@ -54,6 +54,12 @@ def build_reason_kwargs(r_ent):
         rc_diff = r_ent.get('current_diff')
     else:
         rc_diff = ''
+    if not binary_file:
+        try:
+            rc_diff = rc_diff.decode(encoding)
+        except:
+            logger.error("Reason isn't %s encoded, cannot decode it" % encoding)
+            rc_diff = ''
     return dict(owner=r_ent.get('owner', default=""),
                 current_owner=r_ent.get('current_owner', default=""),
                 group=r_ent.get('group', default=""),
@@ -71,7 +77,7 @@ def build_reason_kwargs(r_ent):
                 is_binary=binary_file)
 
 
-def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''):
+def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''):
     clients = {}
     [clients.__setitem__(c.name, c) \
         for c in Client.objects.all()]
@@ -129,7 +135,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''):
             for (xpath, type) in pattern:
                 for x in statistics.findall(xpath):
                     counter_fields[type] = counter_fields[type] + 1
-                    kargs = build_reason_kwargs(x)
+                    kargs = build_reason_kwargs(x, encoding, logger)
 
                     try:
                         rr = None
@@ -270,6 +276,11 @@ if __name__ == '__main__':
         print("StatReports: Failed to parse %s" % (statpath))
         raise SystemExit(1)
 
+    try:
+        encoding = cf.get('components', 'encoding')
+    except:
+        encoding = 'UTF-8'
+
     if not clientpath:
         try:
             clientspath = "%s/Metadata/clients.xml" % \
@@ -288,6 +299,7 @@ if __name__ == '__main__':
     update_database()
     load_stats(clientsdata,
                statsdata,
+               encoding,
                verb,
                logger,
                quick=q,