summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorNarayan Desai <desai@mcs.anl.gov>2008-06-30 15:59:01 +0000
committerNarayan Desai <desai@mcs.anl.gov>2008-06-30 15:59:01 +0000
commite32bbfbca5233d4ad7a5bee74698d614ff0b1f24 (patch)
tree5f6b08da35732532f2b3d8761bd7f26a61ce8aab /src
parent477603f8f0680f40781559ec1cee8f85c2d76913 (diff)
downloadbcfg2-e32bbfbca5233d4ad7a5bee74698d614ff0b1f24.tar.gz
bcfg2-e32bbfbca5233d4ad7a5bee74698d614ff0b1f24.tar.bz2
bcfg2-e32bbfbca5233d4ad7a5bee74698d614ff0b1f24.zip
Unicode support (from stousignant) (Resolves Ticket #549)
git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@4731 ce84e21b-d406-0410-9b95-82705330c041
Diffstat (limited to 'src')
-rw-r--r--src/lib/Client/Tools/POSIX.py13
-rw-r--r--src/lib/Options.py3
-rw-r--r--src/lib/Server/Admin/Pull.py2
-rw-r--r--src/lib/Server/Core.py3
-rw-r--r--src/lib/Server/Plugin.py11
-rw-r--r--src/lib/Server/Plugins/Cfg.py6
-rw-r--r--src/lib/Server/Plugins/Metadata.py2
-rw-r--r--src/lib/Server/Plugins/TCheetah.py9
-rw-r--r--src/lib/Server/Plugins/TGenshi.py17
-rw-r--r--src/lib/Server/Statistics.py2
-rwxr-xr-xsrc/sbin/bcfg25
-rwxr-xr-xsrc/sbin/bcfg2-build-reports6
-rwxr-xr-xsrc/sbin/bcfg2-info13
-rwxr-xr-xsrc/sbin/bcfg2-ping-sweep2
-rwxr-xr-xsrc/sbin/bcfg2-server8
15 files changed, 69 insertions, 33 deletions
diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py
index ec7d3383d..df8558e1d 100644
--- a/src/lib/Client/Tools/POSIX.py
+++ b/src/lib/Client/Tools/POSIX.py
@@ -320,11 +320,15 @@ class POSIX(Bcfg2.Client.Tools.Tool):
self.logger.error("Cannot verify incomplete ConfigFile %s" % (entry.get('name')))
return False
tempdata = entry.text
+ if type(tempdata) == unicode:
+ tempdata = tempdata.encode(self.setup['encoding'])
try:
content = open(entry.get('name')).read()
except IOError, error:
self.logger.error("Failed to read %s: %s" % (error.filename, error.strerror))
return False
+ # comparaison should be done with figerprints or md5sum so it would be faster
+ # for big binary files
contentStatus = content == tempdata
if not contentStatus:
if tbin or not isString(content):
@@ -348,7 +352,8 @@ class POSIX(Bcfg2.Client.Tools.Tool):
break
if do_diff:
diff = '\n'.join(rawdiff)
- entry.set("current_bdiff", binascii.b2a_base64(diff))
+# entry.set("current_bdiff", binascii.b2a_base64(diff))
+# entry.set("current_diff", diff)
udiff = '\n'.join([x for x in \
difflib.unified_diff(content.split('\n'), \
tempdata.split('\n'))])
@@ -356,6 +361,7 @@ class POSIX(Bcfg2.Client.Tools.Tool):
eudiff = udiff.encode('ascii')
except:
eudiff = "Binary file: no diff printed"
+
nqtext = entry.get('qtext', '')
if nqtext:
@@ -417,7 +423,10 @@ class POSIX(Bcfg2.Client.Tools.Tool):
elif entry.get('empty', 'false') == 'true':
filedata = ''
else:
- filedata = entry.text
+ if type(entry.text) == unicode:
+ filedata = entry.text.encode(self.setup['encoding'])
+ else:
+ filedata = entry.text
newfile.write(filedata)
newfile.close()
try:
diff --git a/src/lib/Options.py b/src/lib/Options.py
index 6075d1ced..2d37d1cab 100644
--- a/src/lib/Options.py
+++ b/src/lib/Options.py
@@ -247,6 +247,9 @@ AGENT_PORT = Option('Agent port', default=6789, cmd='-p', odesc='<port>',
cf=('communication', 'agent-port'))
AGENT_HOST = Option('Remote host', default=False, cmd='-H', odesc='<hostname>')
+ENCODING = Option('Encoding of cfg files', default=sys.getdefaultencoding(), cmd='-E', odesc='<encoding>',
+ cf=('components', 'encoding'))
+
class OptionParser(OptionSet):
'''OptionParser bootstraps option parsing, getting the value of the config file'''
def __init__(self, args):
diff --git a/src/lib/Server/Admin/Pull.py b/src/lib/Server/Admin/Pull.py
index ede4271fb..71bfdefc2 100644
--- a/src/lib/Server/Admin/Pull.py
+++ b/src/lib/Server/Admin/Pull.py
@@ -70,7 +70,7 @@ class Pull(Bcfg2.Server.Admin.Mode):
new_entry['text'] = '\n'.join(difflib.restore(diff.split('\n'), 1))
else:
print "found no data::"
- print lxml.etree.tostring(cfentry)
+ print lxml.etree.tostring(cfentry, encoding='UTF-8', xml_declaration=True)
raise SystemExit(1)
return new_entry
diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py
index e7e22f3c3..b3742e8d3 100644
--- a/src/lib/Server/Core.py
+++ b/src/lib/Server/Core.py
@@ -200,7 +200,7 @@ except ImportError:
class Core(object):
'''The Core object is the container for all Bcfg2 Server logic, and modules'''
- def __init__(self, repo, structures, generators, password, svn):
+ def __init__(self, repo, structures, generators, password, svn, encoding):
object.__init__(self)
self.datastore = repo
try:
@@ -215,6 +215,7 @@ class Core(object):
self.revision = '-1'
self.password = password
self.svn = svn
+ self.encoding = encoding
try:
if self.svn:
self.read_svn_revision()
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index 6b308d1f7..61e7021c6 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -444,7 +444,7 @@ class Specificity:
return False
class SpecificData(object):
- def __init__(self, name, _, specific):
+ def __init__(self, name, _, specific, encoding):
self.name = name
self.specific = specific
@@ -459,13 +459,14 @@ class SpecificData(object):
class EntrySet:
'''Entry sets deal with the host- and group-specific entries'''
ignore = re.compile("^(.*~|\\..*\\.(tmp|sw[px]))$")
- def __init__(self, basename, path, props, entry_type):
+ def __init__(self, basename, path, props, entry_type, encoding):
self.path = path
self.entry_type = entry_type
self.entries = {}
self.properties = props
self.metadata = default_file_metadata.copy()
self.infoxml = None
+ self.encoding = encoding
pattern = '(.*/)?%s(\.((H_(?P<hostname>\S+))|' % basename
pattern += '(G(?P<prio>\d+)_(?P<group>\S+))))?$'
self.specific = re.compile(pattern)
@@ -509,7 +510,7 @@ class EntrySet:
return
self.entries[event.filename] = self.entry_type(fpath,
self.properties,
- spec)
+ spec, self.encoding)
self.entries[event.filename].handle_event(event)
def specificity_from_filename(self, fname):
@@ -631,6 +632,7 @@ class GroupSpool(GeneratorPlugin):
self.entries = {}
self.handles = {}
self.AddDirectoryMonitor('')
+ self.encoding = core.encoding
if self.use_props:
try:
self.properties = TemplateProperties( \
@@ -661,7 +663,8 @@ class GroupSpool(GeneratorPlugin):
self.entries[ident] = self.es_cls(self.filename_pattern,
dirpath,
self.properties,
- self.es_child_cls)
+ self.es_child_cls,
+ self.encoding)
self.Entries['ConfigFile'][ident] = self.entries[ident].bind_entry
if not posixpath.isdir(epath):
# do not pass through directory events
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
index 80c9d7467..5f3f1688b 100644
--- a/src/lib/Server/Plugins/Cfg.py
+++ b/src/lib/Server/Plugins/Cfg.py
@@ -49,8 +49,8 @@ class CfgMatcher:
return self.basefile_reg.match(fname)
class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
- def __init__(self, basename, path, props, entry_type):
- Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, props, entry_type)
+ def __init__(self, basename, path, props, entry_type, encoding):
+ Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, props, entry_type, encoding)
self.specific = CfgMatcher(path.split('/')[-1])
def sort_by_specific(self, one, other):
@@ -79,7 +79,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
if entry.get('encoding') == 'base64':
entry.text = binascii.b2a_base64(data)
else:
- entry.text = data
+ entry.text = unicode(data, self.encoding)
if entry.text in ['', None]:
entry.set('empty', 'true')
diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py
index 084873471..b78d05ada 100644
--- a/src/lib/Server/Plugins/Metadata.py
+++ b/src/lib/Server/Plugins/Metadata.py
@@ -296,7 +296,7 @@ class Metadata(Bcfg2.Server.Plugin.MetadataPlugin,
value=self.probedata[client][probe])
for group in self.cgroups[client]:
lxml.etree.SubElement(cx, "Group", name=group)
- data = lxml.etree.tostring(top)
+ data = lxml.etree.tostring(top, encoding='UTF-8', xml_declaration=True)
try:
datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w')
except IOError:
diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py
index ec82f5753..e2b4d370d 100644
--- a/src/lib/Server/Plugins/TCheetah.py
+++ b/src/lib/Server/Plugins/TCheetah.py
@@ -9,10 +9,11 @@ logger = logging.getLogger('Bcfg2.Plugins.TCheetah')
class TemplateFile:
'''Template file creates Cheetah template structures for the loaded file'''
- def __init__(self, name, properties, specific):
+ def __init__(self, name, properties, specific, encoding):
self.name = name
self.properties = properties
self.specific = specific
+ self.encoding = encoding
self.template = None
def handle_event(self, event):
@@ -34,7 +35,11 @@ class TemplateFile:
self.template.path = entry.get('realname', entry.get('name'))
try:
- entry.text = str(self.template)
+ if type(self.template) == unicode:
+ entry.text = self.template
+ else :
+ logger.debug("Override encoding of template to %s" % self.encoding)
+ entry.text = unicode(str(self.template), self.encoding)
except:
(a, b, c) = sys.exc_info()
msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1]
diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py
index e58fb8017..16470a231 100644
--- a/src/lib/Server/Plugins/TGenshi.py
+++ b/src/lib/Server/Plugins/TGenshi.py
@@ -17,10 +17,11 @@ def removecomment(stream):
class TemplateFile:
'''Template file creates Genshi template structures for the loaded file'''
- def __init__(self, name, properties, specific):
+ def __init__(self, name, properties, specific, encoding):
self.name = name
self.properties = properties
self.specific = specific
+ self.encoding = encoding
if self.specific.all:
matchname = self.name
elif self.specific.group:
@@ -52,9 +53,19 @@ class TemplateFile:
name=fname, metadata=metadata, path=self.name,
properties=self.properties).filter(removecomment)
if isinstance(self.template, TextTemplate):
- entry.text = stream.render('text')
+ textdata = stream.render('text')
+ if type(textdata) == unicode:
+ entry.text = textdata
+ else:
+ logger.debug("Override encoding of template to %s" % self.encoding)
+ entry.text = unicode(textdata, self.encoding)
else:
- entry.text = stream.render('xml')
+ xmldata = stream.render('xml')
+ if type(xmldata) == unicode:
+ entry.text = xmldata
+ else:
+ logger.debug("Override encoding of template to %s" % self.encoding)
+ entry.text = unicode(xmldata, self.encoding)
except TemplateError, terror:
logger.error('Genshi template error: %s' % terror)
raise Bcfg2.Server.Plugin.PluginExecutionError
diff --git a/src/lib/Server/Statistics.py b/src/lib/Server/Statistics.py
index bbfc684bd..7111113aa 100644
--- a/src/lib/Server/Statistics.py
+++ b/src/lib/Server/Statistics.py
@@ -28,7 +28,7 @@ class Statistics(object):
except IOError, ioerr:
self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr))
else:
- fout.write(lxml.etree.tostring(self.element))
+ fout.write(lxml.etree.tostring(self.element, encoding='UTF-8', xml_declaration=True))
fout.close()
os.rename(self.filename + '.new', self.filename)
self.dirty = 0
diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2
index 472eab795..5ac1e3aac 100755
--- a/src/sbin/bcfg2
+++ b/src/sbin/bcfg2
@@ -116,6 +116,7 @@ class Client:
'agent-background': Bcfg2.Options.CLIENT_BACKGROUND,
'key': Bcfg2.Options.SERVER_KEY,
'decision-list': DECISION_LIST,
+ 'encoding': Bcfg2.Options.ENCODING,
}
self.setup = Bcfg2.Options.OptionParser(optinfo)
@@ -244,7 +245,7 @@ class Client:
if len(probes.findall(".//probe")) > 0:
try:
# upload probe responses
- proxy.RecvProbeData(Bcfg2.Client.XML.tostring(probedata))
+ proxy.RecvProbeData(Bcfg2.Client.XML.tostring(probedata, encoding='UTF-8', xml_declaration=True))
except:
self.logger.error("Failed to upload probe data", exc_info=1)
raise SystemExit(1)
@@ -293,7 +294,7 @@ class Client:
feedback = self.tools.GenerateStats()
try:
- proxy.RecvStats(Bcfg2.Client.XML.tostring(feedback))
+ proxy.RecvStats(Bcfg2.Client.XML.tostring(feedback, encoding='UTF-8', xml_declaration=True))
except xmlrpclib.Fault:
self.logger.error("Failed to upload configuration statistics")
raise SystemExit(2)
diff --git a/src/sbin/bcfg2-build-reports b/src/sbin/bcfg2-build-reports
index 534cbdd91..8ae8475a3 100755
--- a/src/sbin/bcfg2-build-reports
+++ b/src/sbin/bcfg2-build-reports
@@ -100,7 +100,7 @@ def rss(reportxml, delivery, report):
for item in items:
channel.append(item)
- tree = "<?xml version=\"1.0\"?>" + tostring(rssdata)
+ tree = tostring(rssdata, encoding='UTF-8', xml_declaration=True)
fil.write(tree)
fil.close()
@@ -249,7 +249,7 @@ if __name__ == '__main__':
#apply XSLT, different ones based on report type, and options
if deliverymechanism == 'null-operator': #Special Cases
- fileout(tostring(ElementTree(procnodereport).getroot()), deliv)
+ fileout(tostring(ElementTree(procnodereport).getroot(), encoding='UTF-8', xml_declaration=True), deliv)
break
transform = delivtype + '-' + deliverymechanism + '.xsl'
@@ -301,7 +301,7 @@ if __name__ == '__main__':
(toastring, socket.getfqdn(), outputstring)
mail(outputstring, c) #call function to send
else:
- outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot())
+ outputstring = tostring(stylesheet.apply(ElementTree(procnodereport)).getroot(), encoding='UTF-8', xml_declaration=True)
if deliverymechanism == 'rss':
rss(outputstring, deliv, reprt)
else: # must be deliverymechanism == 'www':
diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info
index dac7fa6bf..9cffc8719 100755
--- a/src/sbin/bcfg2-info
+++ b/src/sbin/bcfg2-info
@@ -23,10 +23,10 @@ def printTabular(rows):
print fstring % row
class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core):
- def __init__(self, repo, struct, gens, passwd, svn):
+ def __init__(self, repo, struct, gens, passwd, svn, encoding):
cmd.Cmd.__init__(self)
try:
- Bcfg2.Server.Core.Core.__init__(self, repo, struct, gens, passwd, svn)
+ Bcfg2.Server.Core.Core.__init__(self, repo, struct, gens, passwd, svn, encoding)
except Bcfg2.Server.Core.CoreInitError, msg:
print "Core load failed because %s" % msg
raise SystemExit(1)
@@ -97,7 +97,7 @@ Usage: [quit|exit]"""
if len(args.split()) == 2:
client, ofile = args.split()
output = open(ofile, 'w')
- data = lxml.etree.tostring(self.BuildConfiguration(client))
+ data = lxml.etree.tostring(self.BuildConfiguration(client), encoding='UTF-8', xml_declaration=True)
output.write(data)
output.close()
else:
@@ -121,7 +121,7 @@ Usage: [quit|exit]"""
entry = lxml.etree.Element('ConfigFile', name=fname)
metadata = self.metadata.get_metadata(client)
self.Bind(entry, metadata)
- print lxml.etree.tostring(entry)
+ print lxml.etree.tostring(entry, encoding="UTF-8", xml_declaration=True)
else:
print 'Usage: buildfile filename hostname'
@@ -271,12 +271,13 @@ if __name__ == '__main__':
'svn': Bcfg2.Options.SERVER_SVN,
'structures': Bcfg2.Options.SERVER_STRUCTURES,
'generators': Bcfg2.Options.SERVER_GENERATORS,
- 'password': Bcfg2.Options.SERVER_PASSWORD})
+ 'password': Bcfg2.Options.SERVER_PASSWORD,
+ 'encoding': Bcfg2.Options.ENCODING})
setup = Bcfg2.Options.OptionParser(optinfo)
setup.parse(sys.argv[1:])
loop = infoCore(setup['repo'], setup['structures'], setup['generators'],
- setup['password'], setup['svn'])
+ setup['password'], setup['svn'], setup['encoding'])
loop.plugins['Metadata']
if "args" in setup and setup['args']:
loop.onecmd(" ".join(setup['args']))
diff --git a/src/sbin/bcfg2-ping-sweep b/src/sbin/bcfg2-ping-sweep
index 54c645277..88d8bd782 100755
--- a/src/sbin/bcfg2-ping-sweep
+++ b/src/sbin/bcfg2-ping-sweep
@@ -63,6 +63,6 @@ if __name__ == '__main__':
elm.set("pingable",'N')
fout = open(clientdatapath, 'w')
- fout.write(lxml.etree.tostring(clientElement.getroot()))
+ fout.write(lxml.etree.tostring(clientElement.getroot(), encoding='UTF-8', xml_declaration=True))
fout.close()
diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server
index a5c5310ba..d26fce9bf 100755
--- a/src/sbin/bcfg2-server
+++ b/src/sbin/bcfg2-server
@@ -41,7 +41,8 @@ class Bcfg2Serv(Bcfg2.Component.Component):
try:
self.Core = Core(setup['repo'], setup['structures'],
- setup['generators'], setup['password'], setup['svn'])
+ setup['generators'], setup['password'],
+ setup['svn'], setup['encoding'])
except CoreInitError, msg:
logger.critical("Fatal error: %s" % (msg))
raise SystemExit, 1
@@ -104,7 +105,7 @@ class Bcfg2Serv(Bcfg2.Component.Component):
if isinstance(p, Bcfg2.Server.Plugin.ProbingPlugin)]:
for probe in plugin.GetProbes(meta):
resp.append(probe)
- return tostring(resp)
+ return tostring(resp, encoding='UTF-8', xml_declaration=True)
except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError:
warning = 'Client metadata resolution error for %s; check server log' % address[0]
self.logger.warning(warning)
@@ -158,7 +159,7 @@ class Bcfg2Serv(Bcfg2.Component.Component):
'''Build config for a client'''
try:
client = self.Core.metadata.resolve_client(address)
- return tostring(self.Core.BuildConfiguration(client))
+ return tostring(self.Core.BuildConfiguration(client), encoding='UTF-8', xml_declaration=True)
except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError:
self.logger.warning("Metadata consistency failure for %s" % (address))
raise Fault, (6, "Metadata consistency failure")
@@ -201,6 +202,7 @@ if __name__ == '__main__':
'location' : Bcfg2.Options.SERVER_LOCATION,
'passwd' : Bcfg2.Options.SERVER_PASSWORD,
'static' : Bcfg2.Options.SERVER_STATIC,
+ 'encoding' : Bcfg2.Options.ENCODING,
})