Unicode support (from stousignant) (Resolves Ticket #549)

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@4731 ce84e21b-d406-0410-9b95-82705330c041

8 files changed, 36 insertions, 16 deletions

diff --git a/src/lib/Server/Admin/Pull.py b/src/lib/Server/Admin/Pull.py
index ede4271fb..71bfdefc2 100644
--- a/src/lib/Server/Admin/Pull.py
+++ b/src/lib/Server/Admin/Pull.py
@@ -70,7 +70,7 @@ class Pull(Bcfg2.Server.Admin.Mode):
             new_entry['text'] = '\n'.join(difflib.restore(diff.split('\n'), 1))
         else:
             print "found no data::"
-            print lxml.etree.tostring(cfentry)
+            print lxml.etree.tostring(cfentry, encoding='UTF-8', xml_declaration=True)
             raise SystemExit(1)
         return new_entry
 
diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py
index e7e22f3c3..b3742e8d3 100644
--- a/src/lib/Server/Core.py
+++ b/src/lib/Server/Core.py
@@ -200,7 +200,7 @@ except ImportError:
 class Core(object):
     '''The Core object is the container for all Bcfg2 Server logic, and modules'''
 
-    def __init__(self, repo, structures, generators, password, svn):
+    def __init__(self, repo, structures, generators, password, svn, encoding):
         object.__init__(self)
         self.datastore = repo
         try:
@@ -215,6 +215,7 @@ class Core(object):
         self.revision = '-1'
         self.password = password
         self.svn = svn
+        self.encoding = encoding
         try:
             if self.svn:
                 self.read_svn_revision()
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index 6b308d1f7..61e7021c6 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -444,7 +444,7 @@ class Specificity:
         return False
 
 class SpecificData(object):
-    def __init__(self, name, _, specific):
+    def __init__(self, name, _, specific, encoding):
         self.name = name
         self.specific = specific
 
@@ -459,13 +459,14 @@ class SpecificData(object):
 class EntrySet:
     '''Entry sets deal with the host- and group-specific entries'''
     ignore = re.compile("^(.*~|\\..*\\.(tmp|sw[px]))$")
-    def __init__(self, basename, path, props, entry_type):
+    def __init__(self, basename, path, props, entry_type, encoding):
         self.path = path
         self.entry_type = entry_type
         self.entries = {}
         self.properties = props
         self.metadata = default_file_metadata.copy()
         self.infoxml = None
+        self.encoding = encoding
         pattern = '(.*/)?%s(\.((H_(?P<hostname>\S+))|' % basename
         pattern += '(G(?P<prio>\d+)_(?P<group>\S+))))?$'
         self.specific = re.compile(pattern)
@@ -509,7 +510,7 @@ class EntrySet:
                 return
             self.entries[event.filename] = self.entry_type(fpath,
                                                            self.properties,
-                                                           spec)
+                                                           spec, self.encoding)
         self.entries[event.filename].handle_event(event)
 
     def specificity_from_filename(self, fname):
@@ -631,6 +632,7 @@ class GroupSpool(GeneratorPlugin):
         self.entries = {}
         self.handles = {}
         self.AddDirectoryMonitor('')
+        self.encoding = core.encoding
         if self.use_props:
             try:
                 self.properties = TemplateProperties( \
@@ -661,7 +663,8 @@ class GroupSpool(GeneratorPlugin):
                 self.entries[ident] = self.es_cls(self.filename_pattern,
                                                   dirpath,
                                                   self.properties,
-                                                  self.es_child_cls)
+                                                  self.es_child_cls,
+                                                  self.encoding)
                 self.Entries['ConfigFile'][ident] =  self.entries[ident].bind_entry
             if not posixpath.isdir(epath):
                 # do not pass through directory events
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
index 80c9d7467..5f3f1688b 100644
--- a/src/lib/Server/Plugins/Cfg.py
+++ b/src/lib/Server/Plugins/Cfg.py
@@ -49,8 +49,8 @@ class CfgMatcher:
         return self.basefile_reg.match(fname)
 
 class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
-    def __init__(self, basename, path, props, entry_type):
-        Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, props, entry_type)
+    def __init__(self, basename, path, props, entry_type, encoding):
+        Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, props, entry_type, encoding)
         self.specific = CfgMatcher(path.split('/')[-1])
 
     def sort_by_specific(self, one, other):
@@ -79,7 +79,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
         if entry.get('encoding') == 'base64':
             entry.text = binascii.b2a_base64(data)
         else:
-            entry.text = data            
+            entry.text = unicode(data, self.encoding)
         if entry.text in ['', None]:
             entry.set('empty', 'true')
 
diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py
index 084873471..b78d05ada 100644
--- a/src/lib/Server/Plugins/Metadata.py
+++ b/src/lib/Server/Plugins/Metadata.py
@@ -296,7 +296,7 @@ class Metadata(Bcfg2.Server.Plugin.MetadataPlugin,
                                           value=self.probedata[client][probe])
                 for group in self.cgroups[client]:
                     lxml.etree.SubElement(cx, "Group", name=group)
-            data = lxml.etree.tostring(top)
+            data = lxml.etree.tostring(top, encoding='UTF-8', xml_declaration=True)
             try:
                 datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w')
             except IOError:
diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py
index ec82f5753..e2b4d370d 100644
--- a/src/lib/Server/Plugins/TCheetah.py
+++ b/src/lib/Server/Plugins/TCheetah.py
@@ -9,10 +9,11 @@ logger = logging.getLogger('Bcfg2.Plugins.TCheetah')
 
 class TemplateFile:
     '''Template file creates Cheetah template structures for the loaded file'''
-    def __init__(self, name, properties, specific):
+    def __init__(self, name, properties, specific, encoding):
         self.name = name
         self.properties = properties
         self.specific = specific
+        self.encoding = encoding
         self.template = None
     
     def handle_event(self, event):
@@ -34,7 +35,11 @@ class TemplateFile:
         self.template.path = entry.get('realname', entry.get('name'))
         
         try:
-            entry.text = str(self.template)
+            if type(self.template) == unicode:
+                entry.text = self.template
+            else :
+                logger.debug("Override encoding of template to %s" % self.encoding)
+                entry.text = unicode(str(self.template), self.encoding)
         except:
             (a, b, c) = sys.exc_info()
             msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1]
diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py
index e58fb8017..16470a231 100644
--- a/src/lib/Server/Plugins/TGenshi.py
+++ b/src/lib/Server/Plugins/TGenshi.py
@@ -17,10 +17,11 @@ def removecomment(stream):
 
 class TemplateFile:
     '''Template file creates Genshi template structures for the loaded file'''
-    def __init__(self, name, properties, specific):
+    def __init__(self, name, properties, specific, encoding):
         self.name = name
         self.properties = properties
         self.specific = specific
+        self.encoding = encoding
         if self.specific.all:
             matchname = self.name
         elif self.specific.group:
@@ -52,9 +53,19 @@ class TemplateFile:
                 name=fname, metadata=metadata, path=self.name,
                 properties=self.properties).filter(removecomment)
             if isinstance(self.template, TextTemplate):
-                entry.text = stream.render('text')
+                textdata = stream.render('text')
+                if type(textdata) == unicode:
+                    entry.text = textdata
+                else:
+                    logger.debug("Override encoding of template to %s" % self.encoding)
+                    entry.text = unicode(textdata, self.encoding)
             else:
-                entry.text = stream.render('xml')
+                xmldata = stream.render('xml')
+                if type(xmldata) == unicode:
+                    entry.text = xmldata
+                else:
+                    logger.debug("Override encoding of template to %s" % self.encoding)
+                    entry.text = unicode(xmldata, self.encoding)
         except TemplateError, terror:
             logger.error('Genshi template error: %s' % terror)
             raise Bcfg2.Server.Plugin.PluginExecutionError
diff --git a/src/lib/Server/Statistics.py b/src/lib/Server/Statistics.py
index bbfc684bd..7111113aa 100644
--- a/src/lib/Server/Statistics.py
+++ b/src/lib/Server/Statistics.py
@@ -28,7 +28,7 @@ class Statistics(object):
             except IOError, ioerr:
                 self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr))
             else:
-                fout.write(lxml.etree.tostring(self.element))
+                fout.write(lxml.etree.tostring(self.element, encoding='UTF-8', xml_declaration=True))
                 fout.close()
                 os.rename(self.filename + '.new', self.filename)
                 self.dirty = 0