Unicode support (from stousignant) (Resolves Ticket #549)

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@4731 ce84e21b-d406-0410-9b95-82705330c041

15 files changed, 69 insertions, 33 deletions

diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py
index ec7d3383d..df8558e1d 100644
--- a/src/lib/Client/Tools/POSIX.py
+++ b/src/lib/Client/Tools/POSIX.py
@@ -320,11 +320,15 @@ class POSIX(Bcfg2.Client.Tools.Tool):
                 self.logger.error("Cannot verify incomplete ConfigFile %s" % (entry.get('name')))
                 return False
             tempdata = entry.text
+            if type(tempdata) == unicode:
+                tempdata = tempdata.encode(self.setup['encoding'])
         try:
             content = open(entry.get('name')).read()
         except IOError, error:
             self.logger.error("Failed to read %s: %s" % (error.filename, error.strerror))
             return False
+        # comparaison should be done with figerprints or md5sum so it would be faster
+        # for big binary files
         contentStatus = content == tempdata
         if not contentStatus:
             if tbin or not isString(content):
@@ -348,7 +352,8 @@ class POSIX(Bcfg2.Client.Tools.Tool):
                         break
                 if do_diff:
                     diff = '\n'.join(rawdiff)
-                    entry.set("current_bdiff", binascii.b2a_base64(diff))
+#                    entry.set("current_bdiff", binascii.b2a_base64(diff))
+#                    entry.set("current_diff", diff)
                     udiff = '\n'.join([x for x in \
                                        difflib.unified_diff(content.split('\n'), \
                                                             tempdata.split('\n'))])
@@ -356,6 +361,7 @@ class POSIX(Bcfg2.Client.Tools.Tool):
                         eudiff = udiff.encode('ascii')
                     except:
                         eudiff = "Binary file: no diff printed"
+ 
                     nqtext = entry.get('qtext', '')
 
                     if nqtext:
@@ -417,7 +423,10 @@ class POSIX(Bcfg2.Client.Tools.Tool):
             elif entry.get('empty', 'false') == 'true':
                 filedata = ''
             else:
-                filedata = entry.text
+                if type(entry.text) == unicode:
+                    filedata = entry.text.encode(self.setup['encoding'])
+                else:
+                    filedata = entry.text
             newfile.write(filedata)
             newfile.close()
             try:
diff --git a/src/lib/Options.py b/src/lib/Options.py
index 6075d1ced..2d37d1cab 100644
--- a/src/lib/Options.py
+++ b/src/lib/Options.py
@@ -247,6 +247,9 @@ AGENT_PORT = Option('Agent port', default=6789, cmd='-p', odesc='<port>',
                     cf=('communication', 'agent-port'))
 AGENT_HOST = Option('Remote host', default=False, cmd='-H', odesc='<hostname>')
 
+ENCODING = Option('Encoding of cfg files', default=sys.getdefaultencoding(), cmd='-E', odesc='<encoding>',
+                  cf=('components', 'encoding'))
+
 class OptionParser(OptionSet):
     '''OptionParser bootstraps option parsing, getting the value of the config file'''
     def __init__(self, args):
diff --git a/src/lib/Server/Admin/Pull.py b/src/lib/Server/Admin/Pull.py
index ede4271fb..71bfdefc2 100644
--- a/src/lib/Server/Admin/Pull.py
+++ b/src/lib/Server/Admin/Pull.py
@@ -70,7 +70,7 @@ class Pull(Bcfg2.Server.Admin.Mode):
             new_entry['text'] = '\n'.join(difflib.restore(diff.split('\n'), 1))
         else:
             print "found no data::"
-            print lxml.etree.tostring(cfentry)
+            print lxml.etree.tostring(cfentry, encoding='UTF-8', xml_declaration=True)
             raise SystemExit(1)
         return new_entry
 
diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py
index e7e22f3c3..b3742e8d3 100644
--- a/src/lib/Server/Core.py
+++ b/src/lib/Server/Core.py
@@ -200,7 +200,7 @@ except ImportError:
 class Core(object):
     '''The Core object is the container for all Bcfg2 Server logic, and modules'''
 
-    def __init__(self, repo, structures, generators, password, svn):
+    def __init__(self, repo, structures, generators, password, svn, encoding):
         object.__init__(self)
         self.datastore = repo
         try:
@@ -215,6 +215,7 @@ class Core(object):
         self.revision = '-1'
         self.password = password
         self.svn = svn
+        self.encoding = encoding
         try:
             if self.svn:
                 self.read_svn_revision()
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index 6b308d1f7..61e7021c6 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -444,7 +444,7 @@ class Specificity:
         return False
 
 class SpecificData(object):
-    def __init__(self, name, _, specific):
+    def __init__(self, name, _, specific, encoding):
         self.name = name
         self.specific = specific
 
@@ -459,13 +459,14 @@ class SpecificData(object):
 class EntrySet:
     '''Entry sets deal with the host- and group-specific entries'''
     ignore = re.compile("^(.*~|\\..*\\.(tmp|sw[px]))$")
-    def __init__(self, basename, path, props, entry_type):
+    def __init__(self, basename, path, props, entry_type, encoding):
         self.path = path
         self.entry_type = entry_type
         self.entries = {}
         self.properties = props
         self.metadata = default_file_metadata.copy()
         self.infoxml = None
+        self.encoding = encoding
         pattern = '(.*/)?%s(\.((H_(?P<hostname>\S+))|' % basename
         pattern += '(G(?P<prio>\d+)_(?P<group>\S+))))?$'
         self.specific = re.compile(pattern)
@@ -509,7 +510,7 @@ class EntrySet:
                 return
             self.entries[event.filename] = self.entry_type(fpath,
                                                            self.properties,
-                                                           spec)
+                                                           spec, self.encoding)
         self.entries[event.filename].handle_event(event)
 
     def specificity_from_filename(self, fname):
@@ -631,6 +632,7 @@ class GroupSpool(GeneratorPlugin):
         self.entries = {}
         self.handles = {}
         self.AddDirectoryMonitor('')
+        self.encoding = core.encoding
         if self.use_props:
             try:
                 self.properties = TemplateProperties( \
@@ -661,7 +663,8 @@ class GroupSpool(GeneratorPlugin):
                 self.entries[ident] = self.es_cls(self.filename_pattern,
                                                   dirpath,
                                                   self.properties,
-                                                  self.es_child_cls)
+                                                  self.es_child_cls,
+                                                  self.encoding)
                 self.Entries['ConfigFile'][ident] =  self.entries[ident].bind_entry
             if not posixpath.isdir(epath):
                 # do not pass through directory events
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
index 80c9d7467..5f3f1688b 100644
--- a/src/lib/Server/Plugins/Cfg.py
+++ b/src/lib/Server/Plugins/Cfg.py
@@ -49,8 +49,8 @@ class CfgMatcher:
         return self.basefile_reg.match(fname)
 
 class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
-    def __init__(self, basename, path, props, entry_type):
-        Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, props, entry_type)
+    def __init__(self, basename, path, props, entry_type, encoding):
+        Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, props, entry_type, encoding)
         self.specific = CfgMatcher(path.split('/')[-1])
 
     def sort_by_specific(self, one, other):
@@ -79,7 +79,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
         if entry.get('encoding') == 'base64':
             entry.text = binascii.b2a_base64(data)
         else:
-            entry.text = data            
+            entry.text = unicode(data, self.encoding)
         if entry.text in ['', None]:
             entry.set('empty', 'true')
 
diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py
index 084873471..b78d05ada 100644
--- a/src/lib/Server/Plugins/Metadata.py
+++ b/src/lib/Server/Plugins/Metadata.py
@@ -296,7 +296,7 @@ class Metadata(Bcfg2.Server.Plugin.MetadataPlugin,
                                           value=self.probedata[client][probe])
                 for group in self.cgroups[client]:
                     lxml.etree.SubElement(cx, "Group", name=group)
-            data = lxml.etree.tostring(top)
+            data = lxml.etree.tostring(top, encoding='UTF-8', xml_declaration=True)
             try:
                 datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w')
             except IOError:
diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py
index ec82f5753..e2b4d370d 100644
--- a/src/lib/Server/Plugins/TCheetah.py
+++ b/src/lib/Server/Plugins/TCheetah.py
@@ -9,10 +9,11 @@ logger = logging.getLogger('Bcfg2.Plugins.TCheetah')
 
 class TemplateFile:
     '''Template file creates Cheetah template structures for the loaded file'''
-    def __init__(self, name, properties, specific):
+    def __init__(self, name, properties, specific, encoding):
         self.name = name
         self.properties = properties
         self.specific = specific
+        self.encoding = encoding
         self.template = None
     
     def handle_event(self, event):
@@ -34,7 +35,11 @@ class TemplateFile:
         self.template.path = entry.get('realname', entry.get('name'))
         
         try:
-            entry.text = str(self.template)
+            if type(self.template) == unicode:
+                entry.text = self.template
+            else :
+                logger.debug("Override encoding of template to %s" % self.encoding)
+                entry.text = unicode(str(self.template), self.encoding)
         except:
             (a, b, c) = sys.exc_info()
             msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1]
diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py
index e58fb8017..16470a231 100644
--- a/src/lib/Server/Plugins/TGenshi.py
+++ b/src/lib/Server/Plugins/TGenshi.py
@@ -17,10 +17,11 @@ def removecomment(stream):
 
 class TemplateFile:
     '''Template file creates Genshi template structures for the loaded file'''
-    def __init__(self, name, properties, specific):
+    def __init__(self, name, properties, specific, encoding):
         self.name = name
         self.properties = properties
         self.specific = specific
+        self.encoding = encoding
         if self.specific.all:
             matchname = self.name
         elif self.specific.group:
@@ -52,9 +53,19 @@ class TemplateFile:
                 name=fname, metadata=metadata, path=self.name,
                 properties=self.properties).filter(removecomment)
             if isinstance(self.template, TextTemplate):
-                entry.text = stream.render('text')
+                textdata = stream.render('text')
+                if type(textdata) == unicode:
+                    entry.text = textdata
+                else:
+                    logger.debug("Override encoding of template to %s" % self.encoding)
+                    entry.text = unicode(textdata, self.encoding)
             else:
-                entry.text = stream.render('xml')
+                xmldata = stream.render('xml')
+                if type(xmldata) == unicode:
+                    entry.text = xmldata
+                else:
+                    logger.debug("Override encoding of template to %s" % self.encoding)
+                    entry.text = unicode(xmldata, self.encoding)
         except TemplateError, terror:
             logger.error('Genshi template error: %s' % terror)
             raise Bcfg2.Server.Plugin.PluginExecutionError
diff --git a/src/lib/Server/Statistics.py b/src/lib/Server/Statistics.py
index bbfc684bd..7111113aa 100644
--- a/src/lib/Server/Statistics.py
+++ b/src/lib/Server/Statistics.py
@@ -28,7 +28,7 @@ class Statistics(object):
             except IOError, ioerr:
                 self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr))
             else:
-                fout.write(lxml.etree.tostring(self.element))
+                fout.write(lxml.etree.tostring(self.element, encoding='UTF-8', xml_declaration=True))
                 fout.close()
                 os.rename(self.filename + '.new', self.filename)
                 self.dirty = 0
diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2
index 472eab795..5ac1e3aac 100755
--- a/src/sbin/bcfg2
+++ b/src/sbin/bcfg2
@@ -116,6 +116,7 @@ class Client:
             'agent-background': Bcfg2.Options.CLIENT_BACKGROUND,
             'key': Bcfg2.Options.SERVER_KEY,
             'decision-list': DECISION_LIST,
+            'encoding': Bcfg2.Options.ENCODING,
             }
 
         self.setup = Bcfg2.Options.OptionParser(optinfo)
@@ -244,7 +245,7 @@ class Client:
             if len(probes.findall(".//probe")) > 0:
                 try:
                     # upload probe responses
-                    proxy.RecvProbeData(Bcfg2.Client.XML.tostring(probedata))
+                    proxy.RecvProbeData(Bcfg2.Client.XML.tostring(probedata, encoding='UTF-8', xml_declaration=True))
                 except:
                     self.logger.error("Failed to upload probe data", exc_info=1)
                     raise SystemExit(1)
@@ -293,7 +294,7 @@ class Client:
             feedback = self.tools.GenerateStats()
 
             try:
-                proxy.RecvStats(Bcfg2.Client.XML.tostring(feedback))
+                proxy.RecvStats(Bcfg2.Client.XML.tostring(feedback, encoding='UTF-8', xml_declaration=True))
             except xmlrpclib.Fault:
                 self.logger.error("Failed to upload configuration statistics")
                 raise SystemExit(2)
diff --git a/src/sbin/bcfg2-build-reports b/src/sbin/bcfg2-build-reports
index 534cbdd91..8ae8475a3 100755
--- a/src/sbin/bcfg2-build-reports
+++ b/src/sbin/bcfg2-build-reports
@@ -100,7 +100,7 @@ def rss(reportxml, delivery, report):
             for item in items:
                 channel.append(item)
 
-        tree = "<?xml version=\"1.0\"?>" + tostring(rssdata)
+        tree = tostring(rssdata, encoding='UTF-8', xml_declaration=True)
         fil.write(tree)
         fil.close()
 
@@ -249,7 +249,7 @@ if __name__ == '__main__':
 
             #apply XSLT, different ones based on report type, and options
             if deliverymechanism == 'null-operator': #Special Cases
-                fileout(tostring(ElementTree(procnodereport).getroot()), deliv)
+                fileout(tostring(ElementTree(procnodereport).getroot(), encoding='UTF-8', xml_declaration=True), deliv)
                 break
             transform = delivtype + '-' + deliverymechanism + '.xsl'
 
@@ -301,7 +301,7 @@ if __name__ == '__main__':
                                            (toastring, socket.getfqdn(), outputstring)
                             mail(outputstring, c) #call function to send
             else:
-                outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot())
+                outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot(), encoding='UTF-8', xml_declaration=True)
                 if deliverymechanism == 'rss':
                     rss(outputstring, deliv, reprt)
                 else: # must be deliverymechanism == 'www':
diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info
index dac7fa6bf..9cffc8719 100755
--- a/src/sbin/bcfg2-info
+++ b/src/sbin/bcfg2-info
@@ -23,10 +23,10 @@ def printTabular(rows):
         print fstring % row
 
 class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core):
-    def __init__(self, repo, struct, gens, passwd, svn):
+    def __init__(self, repo, struct, gens, passwd, svn, encoding):
         cmd.Cmd.__init__(self)
         try:
-            Bcfg2.Server.Core.Core.__init__(self, repo, struct, gens, passwd, svn)
+            Bcfg2.Server.Core.Core.__init__(self, repo, struct, gens, passwd, svn, encoding)
         except Bcfg2.Server.Core.CoreInitError, msg:
             print "Core load failed because %s" % msg
             raise SystemExit(1)
@@ -97,7 +97,7 @@ Usage: [quit|exit]"""
         if len(args.split()) == 2:
             client, ofile = args.split()
             output = open(ofile, 'w')
-            data = lxml.etree.tostring(self.BuildConfiguration(client))
+            data = lxml.etree.tostring(self.BuildConfiguration(client), encoding='UTF-8', xml_declaration=True)
             output.write(data)
             output.close()
         else:
@@ -121,7 +121,7 @@ Usage: [quit|exit]"""
             entry = lxml.etree.Element('ConfigFile', name=fname)
             metadata = self.metadata.get_metadata(client)
             self.Bind(entry, metadata)
-            print lxml.etree.tostring(entry)
+            print lxml.etree.tostring(entry, encoding="UTF-8", xml_declaration=True)
         else:
             print 'Usage: buildfile filename hostname'
 
@@ -271,12 +271,13 @@ if __name__ == '__main__':
                     'svn': Bcfg2.Options.SERVER_SVN,
                     'structures': Bcfg2.Options.SERVER_STRUCTURES,
                     'generators': Bcfg2.Options.SERVER_GENERATORS,
-                    'password': Bcfg2.Options.SERVER_PASSWORD})
+                    'password': Bcfg2.Options.SERVER_PASSWORD,
+                    'encoding': Bcfg2.Options.ENCODING})
     setup = Bcfg2.Options.OptionParser(optinfo)
     setup.parse(sys.argv[1:])
 
     loop = infoCore(setup['repo'], setup['structures'], setup['generators'],
-                    setup['password'], setup['svn'])
+                    setup['password'], setup['svn'], setup['encoding'])
     loop.plugins['Metadata']
     if "args" in setup and setup['args']:
         loop.onecmd(" ".join(setup['args']))
diff --git a/src/sbin/bcfg2-ping-sweep b/src/sbin/bcfg2-ping-sweep
index 54c645277..88d8bd782 100755
--- a/src/sbin/bcfg2-ping-sweep
+++ b/src/sbin/bcfg2-ping-sweep
@@ -63,6 +63,6 @@ if __name__ == '__main__':
                 elm.set("pingable",'N')
 
     fout = open(clientdatapath, 'w')
-    fout.write(lxml.etree.tostring(clientElement.getroot()))
+    fout.write(lxml.etree.tostring(clientElement.getroot(), encoding='UTF-8', xml_declaration=True))
     fout.close()
                         
diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server
index a5c5310ba..d26fce9bf 100755
--- a/src/sbin/bcfg2-server
+++ b/src/sbin/bcfg2-server
@@ -41,7 +41,8 @@ class Bcfg2Serv(Bcfg2.Component.Component):
 
         try:
             self.Core = Core(setup['repo'], setup['structures'],
-                             setup['generators'], setup['password'], setup['svn'])
+                             setup['generators'], setup['password'], 
+                             setup['svn'], setup['encoding'])
         except CoreInitError, msg:
             logger.critical("Fatal error: %s" % (msg))
             raise SystemExit, 1
@@ -104,7 +105,7 @@ class Bcfg2Serv(Bcfg2.Component.Component):
                            if isinstance(p, Bcfg2.Server.Plugin.ProbingPlugin)]:
                 for probe in plugin.GetProbes(meta):
                     resp.append(probe)
-            return tostring(resp)
+            return tostring(resp, encoding='UTF-8', xml_declaration=True)
         except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError:
             warning = 'Client metadata resolution error for %s; check server log' % address[0]
             self.logger.warning(warning)
@@ -158,7 +159,7 @@ class Bcfg2Serv(Bcfg2.Component.Component):
         '''Build config for a client'''
         try:
             client = self.Core.metadata.resolve_client(address)
-            return tostring(self.Core.BuildConfiguration(client))
+            return tostring(self.Core.BuildConfiguration(client), encoding='UTF-8', xml_declaration=True)
         except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError:
             self.logger.warning("Metadata consistency failure for %s" % (address))
             raise Fault, (6, "Metadata consistency failure")
@@ -201,6 +202,7 @@ if __name__ == '__main__':
                     'location' : Bcfg2.Options.SERVER_LOCATION,
                     'passwd'   : Bcfg2.Options.SERVER_PASSWORD,
                     'static'   : Bcfg2.Options.SERVER_STATIC,
+                    'encoding' : Bcfg2.Options.ENCODING,
                     })