From f26780538a46beac2655230a93bd2134af7fc5bc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 4 Mar 2011 15:56:09 -0500 Subject: fixed handling of group- and host-specific .genshi templates in Cfg (non-greedy matching) --- src/lib/Server/Plugins/Cfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index f851b7914..42cfb288d 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -66,7 +66,7 @@ def process_delta(data, delta): class CfgMatcher: def __init__(self, fname): name = re.escape(fname) - self.basefile_reg = re.compile('^(?P%s)(|\\.H_(?P\S+)|.G(?P\d+)_(?P\S+))(?P\\.genshi)?$' % name) + self.basefile_reg = re.compile('^(?P%s)(|\\.H_(?P\S+?)|.G(?P\d+)_(?P\S+?))(?P\\.genshi)?$' % name) self.delta_reg = re.compile('^(?P%s)(|\\.H_(?P\S+)|\\.G(?P\d+)_(?P\S+))\\.(?P(cat|diff))$' % name) self.cat_count = fname.count(".cat") self.diff_count = fname.count(".diff") -- cgit v1.2.3-1-g7c22 From 3c290811195e23c9a948ac15c304b34ea16fbe9b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 7 Mar 2011 10:11:55 -0600 Subject: Cfg: Fix unicode traceback (Resolves #993) If the Cfg plugin handled a file containing a character which isn't contained in the encoding specified, it resulted in a traceback. This now fails gracefully and suggests use of an alternate encoding. Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Cfg.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 42cfb288d..585f4f6ac 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -22,6 +22,7 @@ except: logger = logging.getLogger('Bcfg2.Plugins.Cfg') + # snipped from TGenshi def removecomment(stream): """A genshi filter that removes comments from the stream.""" @@ -30,6 +31,7 @@ def removecomment(stream): continue yield kind, data, pos + def process_delta(data, delta): if not delta.specific.delta: return data @@ -63,7 +65,9 @@ def process_delta(data, delta): raise Bcfg2.Server.Plugin.PluginExecutionError, ('delta', delta) return output + class CfgMatcher: + def __init__(self, fname): name = re.escape(fname) self.basefile_reg = re.compile('^(?P%s)(|\\.H_(?P\S+?)|.G(?P\d+)_(?P\S+?))(?P\\.genshi)?$' % name) @@ -77,7 +81,9 @@ class CfgMatcher: return self.delta_reg.match(fname) return self.basefile_reg.match(fname) + class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): + def __init__(self, basename, path, entry_type, encoding): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) @@ -87,15 +93,18 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): return cmp(one.specific, other.specific) def get_pertinent_entries(self, metadata): - '''return a list of all entries pertinent to a client => [base, delta1, delta2]''' + """return a list of all entries pertinent + to a client => [base, delta1, delta2] + """ matching = [ent for ent in self.entries.values() if \ ent.specific.matches(metadata)] matching.sort(self.sort_by_specific) - non_delta = [matching.index(m) for m in matching if not m.specific.delta] + non_delta = [matching.index(m) for m in matching + if not m.specific.delta] if not non_delta: raise Bcfg2.Server.Plugin.PluginExecutionError base = min(non_delta) - used = matching[:base+1] + used = matching[:base + 1] used.reverse() return used @@ -136,7 +145,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if entry.get('encoding') == 'base64': entry.text = binascii.b2a_base64(data) else: - entry.text = unicode(data, self.encoding) + try: + entry.text = unicode(data, self.encoding) + except UnicodeDecodeError, e: + logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) + logger.error("Please verify you are using the proper encoding.") + raise Bcfg2.Server.Plugin.PluginExecutionError if entry.text in ['', None]: entry.set('empty', 'true') @@ -168,7 +182,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): open(name, 'w').write(new_entry['text']) if log: logger.info("Wrote file %s" % name) - badattr = [attr for attr in ['owner', 'group', 'perms'] if attr in new_entry] + badattr = [attr for attr in ['owner', 'group', 'perms'] + if attr in new_entry] if badattr: metadata_updates = {} metadata_updates.update(self.metadata) @@ -178,12 +193,13 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): infotag = lxml.etree.SubElement(infoxml, 'Info') [infotag.attrib.__setitem__(attr, metadata_updates[attr]) \ for attr in metadata_updates] - ofile = open(self.path + "/info.xml","w") + ofile = open(self.path + "/info.xml", "w") ofile.write(lxml.etree.tostring(infoxml, pretty_print=True)) ofile.close() if log: logger.info("Wrote file %s" % (self.path + "/info.xml")) + class Cfg(Bcfg2.Server.Plugin.GroupSpool, Bcfg2.Server.Plugin.PullTarget): """This generator in the configuration file repository for Bcfg2.""" @@ -197,4 +213,6 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool, return self.entries[entry.get('name')].list_accept_choices(metadata) def AcceptPullData(self, specific, new_entry, log): - return self.entries[new_entry.get('name')].write_update(specific, new_entry, log) + return self.entries[new_entry.get('name')].write_update(specific, + new_entry, + log) -- cgit v1.2.3-1-g7c22 From 0984e097c2f7d973320fa14fcfab36d3122cfaf0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 16 Mar 2011 07:57:25 -0400 Subject: In TGenshi templates, the "name" variable is set to the destination path of the file. If an altsrc attribute was used, then "name" is set to the altsrc value; otherwise, it is set to the "name" attribute of the original tag used to declare the file. In the new Genshi handler functionality of Cfg, this had not been ported over; "name" was always the original name of the file, even if altsrc was specified. Fixed that bug. --- src/lib/Server/Plugins/Cfg.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 585f4f6ac..184bb792c 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -122,10 +122,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): template_cls = NewTextTemplate loader = TemplateLoader() template = loader.load(basefile.name, cls=template_cls, - encoding=self.encoding) - stream = template.generate( \ - name=entry.get('name'), metadata=metadata, - path=basefile.name).filter(removecomment) + encoding=self.encoding) + fname = entry.get('realname', entry.get('name')) + stream = template.generate(name=fname, + metadata=metadata, + path=basefile.name).filter(removecomment) try: data = stream.render('text', strip_whitespace=False) except TypeError: -- cgit v1.2.3-1-g7c22 From 8e876ef13b82bae450b20cda0c684a8671cf425a Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 17 Mar 2011 10:20:23 -0500 Subject: Convert hostbase/reports files to unix file formats Signed-off-by: Sol Jerome --- src/lib/Server/Hostbase/media/base.css | 10 +-- src/lib/Server/Hostbase/media/global.css | 16 ++-- src/lib/Server/Hostbase/media/layout.css | 124 +++++++++++++++---------------- 3 files changed, 75 insertions(+), 75 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Hostbase/media/base.css b/src/lib/Server/Hostbase/media/base.css index 9196c7d51..ddbf02165 100644 --- a/src/lib/Server/Hostbase/media/base.css +++ b/src/lib/Server/Hostbase/media/base.css @@ -1,5 +1,5 @@ - -/* Import other styles */ -@import url('global.css'); -@import url('layout.css'); -@import url('boxypastel.css'); + +/* Import other styles */ +@import url('global.css'); +@import url('layout.css'); +@import url('boxypastel.css'); diff --git a/src/lib/Server/Hostbase/media/global.css b/src/lib/Server/Hostbase/media/global.css index 92d7ce0a3..73451e1bc 100644 --- a/src/lib/Server/Hostbase/media/global.css +++ b/src/lib/Server/Hostbase/media/global.css @@ -1,8 +1,8 @@ -body { - margin:0; - padding:0; - font-size:12px; - font-family:"Lucida Grande","Bitstream Vera Sans",Verdana,Arial,sans-serif; - color:#000; - background:#fff; - } +body { + margin:0; + padding:0; + font-size:12px; + font-family:"Lucida Grande","Bitstream Vera Sans",Verdana,Arial,sans-serif; + color:#000; + background:#fff; + } diff --git a/src/lib/Server/Hostbase/media/layout.css b/src/lib/Server/Hostbase/media/layout.css index 99f61da8f..9085cc220 100644 --- a/src/lib/Server/Hostbase/media/layout.css +++ b/src/lib/Server/Hostbase/media/layout.css @@ -1,62 +1,62 @@ -/* Page Structure */ -#container { position:absolute; top: 3em; margin-left:1em; margin-right:2em; padding:0; margin-top:1.5em; min-width: - 650px; } -#header { width:100%; } -#content-main { float:left; } - -/* HEADER */ -#header { -background:#000; -color:#ffc; -position:absolute; -} -#header a:link, #header a:visited { color:white; } -#header a:hover { text-decoration:underline; } -#branding h1 { padding:0 10px; font-size:18px; margin:8px 0; font-weight:normal; color:#f4f379; } -#branding h2 { padding:0 10px; font-size:14px; margin:-8px 0 8px 0; font-weight:normal; color:#ffc; } -#user-tools { position:absolute; top:0; right:0; padding:1.2em 10px; font-size:11px; text-align:right; } - -/*SIDEBAR*/ -#sidebar { - float:left; - position: relative; - width: auto; - height: 100%; - margin-top: 3em; - padding-right: 1.5em; - padding-left: 1.5em; - padding-top: 1em; - padding-bottom:3em; - background: #000; - color:ffc; -} - -a.sidebar:link {color: #fff;} -a.sidebar:active {color: #fff;} -a.sidebar:visited {color: #fff;} -a.sidebar:hover {color: #fff;} - -ul.sidebar { - color: #ffc; - text-decoration: none; - list-style-type: none; - text-indent: -1em; -} -ul.sidebar-level2 { - text-indent: -2em; - list-style-type: none; - font-size: 11px; -} - -/* ALIGNED FIELDSETS */ -.aligned label { display:block; padding:0 1em 3px 0; float:left; width:8em; } -.aligned label.inline { display:inline; float:none; } -.colMS .aligned .vLargeTextField, .colMS .aligned .vXMLLargeTextField { width:350px; } -form .aligned p, form .aligned ul { margin-left:7em; padding-left:30px; } -form .aligned table p { margin-left:0; padding-left:0; } -form .aligned p.help { padding-left:38px; } -.aligned .vCheckboxLabel { float:none !important; display:inline; padding-left:4px; } -.colM .aligned .vLargeTextField, colM .aligned .vXMLLargeTextField { width:610px; } -.checkbox-row p.help { margin-left:0; padding-left:0 !important; } - - +/* Page Structure */ +#container { position:absolute; top: 3em; margin-left:1em; margin-right:2em; padding:0; margin-top:1.5em; min-width: + 650px; } +#header { width:100%; } +#content-main { float:left; } + +/* HEADER */ +#header { +background:#000; +color:#ffc; +position:absolute; +} +#header a:link, #header a:visited { color:white; } +#header a:hover { text-decoration:underline; } +#branding h1 { padding:0 10px; font-size:18px; margin:8px 0; font-weight:normal; color:#f4f379; } +#branding h2 { padding:0 10px; font-size:14px; margin:-8px 0 8px 0; font-weight:normal; color:#ffc; } +#user-tools { position:absolute; top:0; right:0; padding:1.2em 10px; font-size:11px; text-align:right; } + +/*SIDEBAR*/ +#sidebar { + float:left; + position: relative; + width: auto; + height: 100%; + margin-top: 3em; + padding-right: 1.5em; + padding-left: 1.5em; + padding-top: 1em; + padding-bottom:3em; + background: #000; + color:ffc; +} + +a.sidebar:link {color: #fff;} +a.sidebar:active {color: #fff;} +a.sidebar:visited {color: #fff;} +a.sidebar:hover {color: #fff;} + +ul.sidebar { + color: #ffc; + text-decoration: none; + list-style-type: none; + text-indent: -1em; +} +ul.sidebar-level2 { + text-indent: -2em; + list-style-type: none; + font-size: 11px; +} + +/* ALIGNED FIELDSETS */ +.aligned label { display:block; padding:0 1em 3px 0; float:left; width:8em; } +.aligned label.inline { display:inline; float:none; } +.colMS .aligned .vLargeTextField, .colMS .aligned .vXMLLargeTextField { width:350px; } +form .aligned p, form .aligned ul { margin-left:7em; padding-left:30px; } +form .aligned table p { margin-left:0; padding-left:0; } +form .aligned p.help { padding-left:38px; } +.aligned .vCheckboxLabel { float:none !important; display:inline; padding-left:4px; } +.colM .aligned .vLargeTextField, colM .aligned .vXMLLargeTextField { width:610px; } +.checkbox-row p.help { margin-left:0; padding-left:0 !important; } + + -- cgit v1.2.3-1-g7c22 From a9eb5c16471c3b34e9e89ac1eea3b5bb50478c3e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 30 Mar 2011 13:38:24 -0400 Subject: Use self.log.info() instead of 'print' to produce stats for 'bcfg2-admin reports purge' to make purge honor -q flag. --- src/lib/Server/Admin/Reports.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py index a4dd19064..2a4c13c9c 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Server/Admin/Reports.py @@ -59,12 +59,16 @@ def printStats(fn): fn(*data) - print "Clients removed: %s" % (start_client - Client.objects.count()) - print "Interactions removed: %s" % (start_i - Interaction.objects.count()) - print "Interactions->Entries removed: %s" % \ - (start_ei - Entries_interactions.objects.count()) - print "Metrics removed: %s" % (start_perf - Performance.objects.count()) - print "Ping metrics removed: %s" % (start_ping - Ping.objects.count()) + self.log.info("Clients removed: %s" % + (start_client - Client.objects.count())) + self.log.info("Interactions removed: %s" % + (start_i - Interaction.objects.count())) + self.log.info("Interactions->Entries removed: %s" % + (start_ei - Entries_interactions.objects.count())) + self.log.info("Metrics removed: %s" % + (start_perf - Performance.objects.count())) + self.log.info("Ping metrics removed: %s" % + (start_ping - Ping.objects.count())) return print_stats -- cgit v1.2.3-1-g7c22 From c8ae9cde92dffa3673e524133ab9a42719375eee Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 30 Mar 2011 14:32:45 -0400 Subject: Fixed previous commit. Decorators are hard, let's go shopping. --- src/lib/Server/Admin/Reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py index 2a4c13c9c..1ac94e5e7 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Server/Admin/Reports.py @@ -50,14 +50,14 @@ def printStats(fn): Decorator for purging. Prints database statistics after a run. """ - def print_stats(*data): + def print_stats(self, *data): start_client = Client.objects.count() start_i = Interaction.objects.count() start_ei = Entries_interactions.objects.count() start_perf = Performance.objects.count() start_ping = Ping.objects.count() - fn(*data) + fn(self, *data) self.log.info("Clients removed: %s" % (start_client - Client.objects.count())) -- cgit v1.2.3-1-g7c22 From 3ae6271b798c7bb07d4ea864b081bc145774a704 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 5 Apr 2011 16:07:31 -0400 Subject: sensible defaults for the Django module admin email addresses --- src/lib/Server/Hostbase/settings.py | 2 +- src/lib/Server/Reports/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Server/Hostbase/settings.py index a42fd5b2e..c44c7bf16 100644 --- a/src/lib/Server/Hostbase/settings.py +++ b/src/lib/Server/Hostbase/settings.py @@ -27,7 +27,7 @@ else: DEBUG = True TEMPLATE_DEBUG = DEBUG ADMINS = ( - # ('Your Name', 'your_email@domain.com'), + ('Root', 'root'), ) MANAGERS = ADMINS diff --git a/src/lib/Server/Reports/settings.py b/src/lib/Server/Reports/settings.py index 66da7a8b1..f4deca6aa 100644 --- a/src/lib/Server/Reports/settings.py +++ b/src/lib/Server/Reports/settings.py @@ -18,7 +18,7 @@ else: TEMPLATE_DEBUG = DEBUG ADMINS = ( - ('Bcfg2', 'admin@email.address'), + ('Root', 'root'), ) MANAGERS = ADMINS -- cgit v1.2.3-1-g7c22 From f124f7f6ee5dc8710ba354e0e978a82245c11fbf Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 4 Apr 2011 13:15:14 -0500 Subject: Reports: PY3K + PEP8 fixes Signed-off-by: Sol Jerome --- src/lib/Server/Reports/backends.py | 21 ++-- src/lib/Server/Reports/importscript.py | 116 +++++++++++++--------- src/lib/Server/Reports/manage.py | 2 +- src/lib/Server/Reports/nisauth.py | 19 ++-- src/lib/Server/Reports/reports/models.py | 102 +++++++++++--------- src/lib/Server/Reports/reports/views.py | 161 +++++++++++++++++++------------ src/lib/Server/Reports/updatefix.py | 30 +++--- src/lib/Server/Reports/utils.py | 20 ++-- src/lib/Server/Snapshots/__init__.py | 2 +- src/lib/Server/Snapshots/model.py | 94 +++++++++++++----- 10 files changed, 344 insertions(+), 223 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/backends.py b/src/lib/Server/Reports/backends.py index 9207038ed..9e37a2e6f 100644 --- a/src/lib/Server/Reports/backends.py +++ b/src/lib/Server/Reports/backends.py @@ -1,35 +1,32 @@ from django.contrib.auth.models import User from nisauth import * + class NISBackend(object): def authenticate(self, username=None, password=None): try: - print "start nis authenticate" + print("start nis authenticate") n = nisauth(username, password) temp_pass = User.objects.make_random_password(100) nis_user = dict(username=username, ) - user_session_obj = dict( - email = username, - first_name = None, - last_name = None, - uid = n.uid - ) + user_session_obj = dict(email=username, + first_name=None, + last_name=None, + uid=n.uid) user, created = User.objects.get_or_create(username=username) - + return user except NISAUTHError, e: - print str(e) + print(str(e)) return None - def get_user(self, user_id): try: return User.objects.get(pk=user_id) except User.DoesNotExist, e: - print str(e) + print(str(e)) return None - diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index cdfd8079c..86e176394 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -1,8 +1,13 @@ #! /usr/bin/env python -'''Imports statistics.xml and clients.xml files in to database backend for new statistics engine''' +""" +Imports statistics.xml and clients.xml files in to database backend for +new statistics engine +""" __revision__ = '$Revision$' -import os, sys, binascii +import binascii +import os +import sys try: import Bcfg2.Server.Reports.settings except Exception, e: @@ -29,16 +34,17 @@ import logging import Bcfg2.Logger import platform + def build_reason_kwargs(r_ent): - binary_file=False + binary_file = False if r_ent.get('current_bfile', False): - binary_file=True + binary_file = True rc_diff = r_ent.get('current_bfile') - if len(rc_diff) > 1024*1024: + if len(rc_diff) > 1024 * 1024: rc_diff = '' elif len(rc_diff) == 0: # No point in flagging binary if we have no data - binary_file=False + binary_file = False elif r_ent.get('current_bdiff', False): rc_diff = binascii.a2b_base64(r_ent.get('current_bdiff')) elif r_ent.get('current_diff', False): @@ -57,7 +63,7 @@ def build_reason_kwargs(r_ent): current_to=r_ent.get('current_to', default=""), version=r_ent.get('version', default=""), current_version=r_ent.get('current_version', default=""), - current_exists=r_ent.get('current_exists', default="True").capitalize()=="True", + current_exists=r_ent.get('current_exists', default="True").capitalize() == "True", current_diff=rc_diff, is_binary=binary_file) @@ -75,7 +81,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): name = node.get('name') c_inst, created = Client.objects.get_or_create(name=name) if vlevel > 0: - logger.info("Client %s added to db" % name) + logger.info("Client %s added to db" % name) clients[name] = c_inst try: pingability[name] @@ -93,24 +99,30 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): continue else: newint = Interaction(client=c_inst, - timestamp = timestamp, - state = statistics.get('state', default="unknown"), - repo_rev_code = statistics.get('revision',default="unknown"), - client_version = statistics.get('client_version',default="unknown"), - goodcount = statistics.get('good',default="0"), - totalcount = statistics.get('total',default="0"), - server = location) + timestamp=timestamp, + state=statistics.get('state', + default="unknown"), + repo_rev_code=statistics.get('revision', + default="unknown"), + client_version=statistics.get('client_version', + default="unknown"), + goodcount=statistics.get('good', + default="0"), + totalcount=statistics.get('total', + default="0"), + server=location) newint.save() current_interaction = newint if vlevel > 0: - logger.info("Interaction for %s at %s with id %s INSERTED in to db"%(c_inst.id, + logger.info("Interaction for %s at %s with id %s INSERTED in to db" % (c_inst.id, timestamp, current_interaction.id)) - - counter_fields = { TYPE_CHOICES[0]: 0, TYPE_CHOICES[1]: 0, TYPE_CHOICES[2]: 0 } + counter_fields = {TYPE_CHOICES[0]: 0, + TYPE_CHOICES[1]: 0, + TYPE_CHOICES[2]: 0} pattern = [('Bad/*', TYPE_CHOICES[0]), ('Extra/*', TYPE_CHOICES[2]), - ('Modified/*', TYPE_CHOICES[1]),] + ('Modified/*', TYPE_CHOICES[1])] for (xpath, type) in pattern: for x in statistics.findall(xpath): counter_fields[type] = counter_fields[type] + 1 @@ -131,12 +143,12 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): except Exception, ex: logger.error("Failed to create reason for %s: %s" % (x.get('name'), ex)) rr = Reason(current_exists=x.get('current_exists', - default="True").capitalize()=="True") + default="True").capitalize() == "True") rr.save() entry, created = Entries.objects.get_or_create(\ name=x.get('name'), kind=x.tag) - + Entries_interactions(entry=entry, reason=rr, interaction=current_interaction, type=type[0]).save() @@ -151,7 +163,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): mperfs = [] for times in statistics.findall('OpStamps'): - for metric, value in times.items(): + for metric, value in list(times.items()): mmatch = [] if not quick: mmatch = Performance.objects.filter(metric=metric, value=value) @@ -164,7 +176,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): mperfs.append(mperf) current_interaction.performance_items.add(*mperfs) - for key in pingability.keys(): + for key in list(pingability.keys()): if key not in clients: continue try: @@ -191,27 +203,32 @@ if __name__ == '__main__': clientpath = False statpath = False syslog = False - + try: - opts, args = getopt(argv[1:], "hvudc:s:CS", ["help", "verbose", "updates" , - "debug", "clients=", "stats=", - "config=", "syslog"]) + opts, args = getopt(argv[1:], "hvudc:s:CS", ["help", + "verbose", + "updates", + "debug", + "clients=", + "stats=", + "config=", + "syslog"]) except GetoptError, mesg: # print help information and exit: - print "%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-S] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg) - raise SystemExit, 2 + print("%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-S] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg)) + raise SystemExit(2) for o, a in opts: if o in ("-h", "--help"): - print "Usage:\nimportscript.py [-h] [-v] -c -s \n" - print "h : help; this message" - print "v : verbose; print messages on record insertion/skip" - print "u : updates; print status messages as items inserted semi-verbose" - print "d : debug; print most SQL used to manipulate database" - print "C : path to bcfg2.conf config file." - print "c : clients.xml file" - print "s : statistics.xml file" - print "S : syslog; output to syslog" + print("Usage:\nimportscript.py [-h] [-v] -c -s \n") + print("h : help; this message") + print("v : verbose; print messages on record insertion/skip") + print("u : updates; print status messages as items inserted semi-verbose") + print("d : debug; print most SQL used to manipulate database") + print("C : path to bcfg2.conf config file.") + print("c : clients.xml file") + print("s : statistics.xml file") + print("S : syslog; output to syslog") raise SystemExit if o in ["-C", "--config"]: cpath = a @@ -243,28 +260,33 @@ if __name__ == '__main__': try: statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print "Could not read bcfg2.conf; exiting" - raise SystemExit, 1 + print("Could not read bcfg2.conf; exiting") + raise SystemExit(1) try: statsdata = XML(open(statpath).read()) except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s"%(statpath)) - raise SystemExit, 1 + print("StatReports: Failed to parse %s" % (statpath)) + raise SystemExit(1) if not clientpath: try: clientspath = "%s/Metadata/clients.xml" % \ cf.get('server', 'repository') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print "Could not read bcfg2.conf; exiting" - raise SystemExit, 1 + print("Could not read bcfg2.conf; exiting") + raise SystemExit(1) try: clientsdata = XML(open(clientspath).read()) except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s"%(clientspath)) - raise SystemExit, 1 + print("StatReports: Failed to parse %s" % (clientspath)) + raise SystemExit(1) q = '-O3' in sys.argv # Be sure the database is ready for new schema update_database() - load_stats(clientsdata, statsdata, verb, logger, quick=q, location=platform.node()) + load_stats(clientsdata, + statsdata, + verb, + logger, + quick=q, + location=platform.node()) diff --git a/src/lib/Server/Reports/manage.py b/src/lib/Server/Reports/manage.py index 5e78ea979..4f84f107a 100755 --- a/src/lib/Server/Reports/manage.py +++ b/src/lib/Server/Reports/manage.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from django.core.management import execute_manager try: - import settings # Assumed to be in the same directory. + from . import settings # Assumed to be in the same directory. except ImportError: import sys sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) diff --git a/src/lib/Server/Reports/nisauth.py b/src/lib/Server/Reports/nisauth.py index b4be0e391..6fc346f1e 100644 --- a/src/lib/Server/Reports/nisauth.py +++ b/src/lib/Server/Reports/nisauth.py @@ -1,15 +1,17 @@ -import os -import crypt, nis +import crypt +import nis from Bcfg2.Server.Reports.settings import AUTHORIZED_GROUP """Checks with NIS to see if the current user is in the support group""" __revision__ = "$Revision: $" + class NISAUTHError(Exception): """NISAUTHError is raised when somehting goes boom.""" pass + class nisauth(object): group_test = False samAcctName = None @@ -18,26 +20,27 @@ class nisauth(object): telephoneNumber = None title = None memberOf = None - department = None #this will be a list + department = None # this will be a list mail = None - extensionAttribute1 = None #badgenumber + extensionAttribute1 = None # badgenumber badge_no = None uid = None - def __init__(self,login,passwd=None): + def __init__(self, login, passwd=None): """get user profile from NIS""" try: p = nis.match(login, 'passwd.byname').split(":") - print p + print(p) except: raise NISAUTHError('username') # check user password using crypt and 2 character salt from passwd file if p[1] == crypt.crypt(passwd, p[1][:2]): # check to see if user is in valid support groups # will have to include these groups in a settings file eventually - if not login in nis.match(AUTHORIZED_GROUP, 'group.byname').split(':')[-1].split(','): + if not login in nis.match(AUTHORIZED_GROUP, + 'group.byname').split(':')[-1].split(','): raise NISAUTHError('group') self.uid = p[2] - print self.uid + print(self.uid) else: raise NISAUTHError('password') diff --git a/src/lib/Server/Reports/reports/models.py b/src/lib/Server/Reports/reports/models.py index 1963a9090..c2be40870 100644 --- a/src/lib/Server/Reports/reports/models.py +++ b/src/lib/Server/Reports/reports/models.py @@ -29,6 +29,7 @@ TYPE_CHOICES = ( (TYPE_EXTRA, 'Extra'), ) + def convert_entry_type_to_id(type_name): """Convert a entry type to its entry id""" for e_id, e_name in TYPE_CHOICES: @@ -36,23 +37,25 @@ def convert_entry_type_to_id(type_name): return e_id return -1 + class ClientManager(models.Manager): """Extended client manager functions.""" def active(self, timestamp=None): - """returns a set of clients that have been created and have not yet been - expired as of optional timestmamp argument. Timestamp should be a - datetime object.""" - + """returns a set of clients that have been created and have not + yet been expired as of optional timestmamp argument. Timestamp + should be a datetime object.""" + if timestamp == None: timestamp = datetime.now() elif not isinstance(timestamp, datetime): raise ValueError, 'Expected a datetime object' else: try: - timestamp = datetime(*strptime(timestamp, "%Y-%m-%d %H:%M:%S")[0:6]) + timestamp = datetime(*strptime(timestamp, + "%Y-%m-%d %H:%M:%S")[0:6]) except ValueError: return self.none() - + return self.filter(Q(expiration__gt=timestamp) | Q(expiration__isnull=True), creation__lt=timestamp) @@ -65,25 +68,27 @@ class Client(models.Model): null=True, blank=True, related_name="parent_client") expiration = models.DateTimeField(blank=True, null=True) - + def __str__(self): return self.name objects = ClientManager() - + class Admin: pass + class Ping(models.Model): """Represents a ping of a client (sparsely).""" client = models.ForeignKey(Client, related_name="pings") starttime = models.DateTimeField() endtime = models.DateTimeField() - status = models.CharField(max_length=4, choices=PING_CHOICES)#up/down + status = models.CharField(max_length=4, choices=PING_CHOICES) # up/down class Meta: get_latest_by = 'endtime' - + + class InteractiveManager(models.Manager): """Manages interactions objects.""" @@ -94,31 +99,31 @@ class InteractiveManager(models.Manager): This method uses aggregated queries to return a ValuesQueryDict object. Faster then raw sql since this is executed as a single query. """ - - return self.values('client').annotate(max_timestamp=Max('timestamp')).values() - def interaction_per_client(self, maxdate = None, active_only=True): + return list(self.values('client').annotate(max_timestamp=Max('timestamp')).values()) + + def interaction_per_client(self, maxdate=None, active_only=True): """ Returns the most recent interactions for clients as of a date Arguments: maxdate -- datetime object. Most recent date to pull. (dafault None) active_only -- Include only active clients (default True) - + """ - if maxdate and not isinstance(maxdate,datetime): + if maxdate and not isinstance(maxdate, datetime): raise ValueError, 'Expected a datetime object' - return self.filter(id__in = self.get_interaction_per_client_ids(maxdate, active_only)) + return self.filter(id__in=self.get_interaction_per_client_ids(maxdate, active_only)) - def get_interaction_per_client_ids(self, maxdate = None, active_only=True): + def get_interaction_per_client_ids(self, maxdate=None, active_only=True): """ Returns the ids of most recent interactions for clients as of a date. Arguments: maxdate -- datetime object. Most recent date to pull. (dafault None) active_only -- Include only active clients (default True) - + """ from django.db import connection cursor = connection.cursor() @@ -127,10 +132,10 @@ class InteractiveManager(models.Manager): sql = 'select reports_interaction.id, x.client_id from (select client_id, MAX(timestamp) ' + \ 'as timer from reports_interaction' if maxdate: - if not isinstance(maxdate,datetime): + if not isinstance(maxdate, datetime): raise ValueError, 'Expected a datetime object' sql = sql + " where timestamp <= '%s' " % maxdate - cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate,maxdate) + cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate) sql = sql + ' GROUP BY client_id) x, reports_interaction where ' + \ 'reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer' if active_only: @@ -144,16 +149,17 @@ class InteractiveManager(models.Manager): pass return [] + class Interaction(models.Model): """Models each reconfiguration operation interaction between client and server.""" client = models.ForeignKey(Client, related_name="interactions",) - timestamp = models.DateTimeField()#Timestamp for this record - state = models.CharField(max_length=32)#good/bad/modified/etc - repo_rev_code = models.CharField(max_length=64)#repo revision at time of interaction - client_version = models.CharField(max_length=32)#Client Version - goodcount = models.IntegerField()#of good config-items - totalcount = models.IntegerField()#of total config-items - server = models.CharField(max_length=256) # Name of the server used for the interaction + timestamp = models.DateTimeField() # Timestamp for this record + state = models.CharField(max_length=32) # good/bad/modified/etc + repo_rev_code = models.CharField(max_length=64) # repo revision at time of interaction + client_version = models.CharField(max_length=32) # Client Version + goodcount = models.IntegerField() # of good config-items + totalcount = models.IntegerField() # of total config-items + server = models.CharField(max_length=256) # Name of the server used for the interaction bad_entries = models.IntegerField(default=-1) modified_entries = models.IntegerField(default=-1) extra_entries = models.IntegerField(default=-1) @@ -163,25 +169,25 @@ class Interaction(models.Model): def percentgood(self): if not self.totalcount == 0: - return (self.goodcount/float(self.totalcount))*100 + return (self.goodcount / float(self.totalcount)) * 100 else: return 0 def percentbad(self): if not self.totalcount == 0: - return ((self.totalcount-self.goodcount)/(float(self.totalcount)))*100 + return ((self.totalcount - self.goodcount) / (float(self.totalcount))) * 100 else: return 0 - + def isclean(self): if (self.bad_entry_count() == 0 and self.goodcount == self.totalcount): return True else: return False - + def isstale(self): - if (self == self.client.current_interaction):#Is Mostrecent - if(datetime.now()-self.timestamp > timedelta(hours=25) ): + if (self == self.client.current_interaction): # Is Mostrecent + if(datetime.now() - self.timestamp > timedelta(hours=25)): return True else: return False @@ -194,10 +200,11 @@ class Interaction(models.Model): return True else: return False + def save(self): - super(Interaction, self).save() #call the real save... + super(Interaction, self).save() # call the real save... self.client.current_interaction = self.client.interactions.latest() - self.client.save()#save again post update + self.client.save() # save again post update def delete(self): '''Override the default delete. Allows us to remove Performance items''' @@ -239,35 +246,38 @@ class Interaction(models.Model): self.extra_entries = Entries_interactions.objects.filter(interaction=self, type=TYPE_EXTRA).count() self.save() return self.extra_entries - + objects = InteractiveManager() class Admin: list_display = ('client', 'timestamp', 'state') list_filter = ['client', 'timestamp'] pass + class Meta: get_latest_by = 'timestamp' ordering = ['-timestamp'] unique_together = ("client", "timestamp") + class Reason(models.Model): """reason why modified or bad entry did not verify, or changed.""" owner = models.TextField(max_length=128, blank=True) current_owner = models.TextField(max_length=128, blank=True) group = models.TextField(max_length=128, blank=True) current_group = models.TextField(max_length=128, blank=True) - perms = models.TextField(max_length=4, blank=True)#txt fixes typing issue + perms = models.TextField(max_length=4, blank=True) # txt fixes typing issue current_perms = models.TextField(max_length=4, blank=True) - status = models.TextField(max_length=3, blank=True)#on/off/(None) - current_status = models.TextField(max_length=1, blank=True)#on/off/(None) + status = models.TextField(max_length=3, blank=True) # on/off/(None) + current_status = models.TextField(max_length=1, blank=True) # on/off/(None) to = models.TextField(max_length=256, blank=True) current_to = models.TextField(max_length=256, blank=True) version = models.TextField(max_length=128, blank=True) current_version = models.TextField(max_length=128, blank=True) - current_exists = models.BooleanField()#False means its missing. Default True + current_exists = models.BooleanField() # False means its missing. Default True current_diff = models.TextField(max_length=1280, blank=True) is_binary = models.BooleanField(default=False) + def _str_(self): return "Reason" @@ -278,7 +288,7 @@ class Reason(models.Model): cursor = connection.cursor() cursor.execute('delete from reports_reason where not exists (select rei.id from reports_entries_interactions rei where rei.reason_id = reports_reason.id)') transaction.set_dirty() - + class Entries(models.Model): """Contains all the entries feed by the client.""" @@ -295,19 +305,22 @@ class Entries(models.Model): cursor = connection.cursor() cursor.execute('delete from reports_entries where not exists (select rei.id from reports_entries_interactions rei where rei.entry_id = reports_entries.id)') transaction.set_dirty() - + + class Entries_interactions(models.Model): """Define the relation between the reason, the interaction and the entry.""" entry = models.ForeignKey(Entries) reason = models.ForeignKey(Reason) interaction = models.ForeignKey(Interaction) type = models.IntegerField(choices=TYPE_CHOICES) - + + class Performance(models.Model): """Object representing performance data for any interaction.""" interaction = models.ManyToManyField(Interaction, related_name="performance_items") metric = models.CharField(max_length=128) value = models.DecimalField(max_digits=32, decimal_places=16) + def __str__(self): return self.metric @@ -318,7 +331,8 @@ class Performance(models.Model): cursor = connection.cursor() cursor.execute('delete from reports_performance where not exists (select ri.id from reports_performance_interaction ri where ri.performance_id = reports_performance.id)') transaction.set_dirty() - + + class InternalDatabaseVersion(models.Model): """Object that tell us to witch version is the database.""" version = models.IntegerField() diff --git a/src/lib/Server/Reports/reports/views.py b/src/lib/Server/Reports/reports/views.py index 00d35c092..3cffa68dd 100644 --- a/src/lib/Server/Reports/reports/views.py +++ b/src/lib/Server/Reports/reports/views.py @@ -3,22 +3,26 @@ Report views Functions to handle all of the reporting views. """ -from django.template import Context, RequestContext, loader -from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError, Http404 +from datetime import datetime, timedelta +import sys +from time import strptime + +from django.template import Context, RequestContext +from django.http import \ + HttpResponse, HttpResponseRedirect, HttpResponseServerError, Http404 from django.shortcuts import render_to_response, get_object_or_404 -from django.core.urlresolvers import resolve, reverse, Resolver404, NoReverseMatch +from django.core.urlresolvers import \ + resolve, reverse, Resolver404, NoReverseMatch from django.db import connection -from django.db.backends import util from Bcfg2.Server.Reports.reports.models import * -from datetime import datetime, timedelta -from time import strptime -import sys + class PaginationError(Exception): """This error is raised when pagination cannot be completed.""" pass + def server_error(request): """ 500 error handler. @@ -29,6 +33,7 @@ def server_error(request): from django.views import debug return debug.technical_500_response(request, *sys.exc_info()) + def timeview(fn): """ Setup a timeview view @@ -53,28 +58,32 @@ def timeview(fn): if cal_date.find(' ') > -1: kw['hour'] = timestamp.hour kw['minute'] = timestamp.minute - return HttpResponseRedirect(reverse(view, args=args, kwargs=kw)) + return HttpResponseRedirect(reverse(view, + args=args, + kwargs=kw)) except KeyError: pass except: pass # FIXME - Handle this - + """Extract timestamp from args.""" timestamp = None try: - timestamp = datetime(int(kwargs.pop('year')), int(kwargs.pop('month')), + timestamp = datetime(int(kwargs.pop('year')), + int(kwargs.pop('month')), int(kwargs.pop('day')), int(kwargs.pop('hour', 0)), int(kwargs.pop('minute', 0)), 0) kwargs['timestamp'] = timestamp except KeyError: pass - except: + except: raise return fn(request, **kwargs) return _handle_timeview - + + def config_item(request, pk, type="bad"): """ Display a single entry. @@ -83,30 +92,33 @@ def config_item(request, pk, type="bad"): """ item = get_object_or_404(Entries_interactions, id=pk) - timestamp=item.interaction.timestamp - time_start=item.interaction.timestamp.replace(\ - hour=0, minute=0, second=0, microsecond=0) - time_end=time_start + timedelta(days=1) - - todays_data = Interaction.objects.filter(\ - timestamp__gte=time_start,\ - timestamp__lt=time_end) - shared_entries = Entries_interactions.objects.filter(entry=item.entry,\ - reason=item.reason, type=item.type, - interaction__in=[x['id']\ - for x in todays_data.values('id')]) + timestamp = item.interaction.timestamp + time_start = item.interaction.timestamp.replace(hour=0, + minute=0, + second=0, + microsecond=0) + time_end = time_start + timedelta(days=1) + + todays_data = Interaction.objects.filter(timestamp__gte=time_start, + timestamp__lt=time_end) + shared_entries = Entries_interactions.objects.filter(entry=item.entry, + reason=item.reason, + type=item.type, + interaction__in=[x['id']\ + for x in todays_data.values('id')]) associated_list = Interaction.objects.filter(id__in=[x['interaction']\ for x in shared_entries.values('interaction')])\ - .order_by('client__name','timestamp').select_related().all() + .order_by('client__name', 'timestamp').select_related().all() return render_to_response('config_items/item.html', - {'item':item, - 'isextra': item.type == TYPE_EXTRA, - 'mod_or_bad': type, - 'associated_list':associated_list, - 'timestamp' : timestamp}, - context_instance=RequestContext(request)) + {'item': item, + 'isextra': item.type == TYPE_EXTRA, + 'mod_or_bad': type, + 'associated_list': associated_list, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + @timeview def config_item_list(request, type, timestamp=None): @@ -115,11 +127,12 @@ def config_item_list(request, type, timestamp=None): type = convert_entry_type_to_id(type) if type < 0: raise Http404 - + current_clients = Interaction.objects.get_interaction_per_client_ids(timestamp) item_list_dict = {} seen = dict() - for x in Entries_interactions.objects.filter(interaction__in=current_clients, type=type).select_related(): + for x in Entries_interactions.objects.filter(interaction__in=current_clients, + type=type).select_related(): if (x.entry, x.reason) in seen: continue seen[(x.entry, x.reason)] = 1 @@ -129,13 +142,15 @@ def config_item_list(request, type, timestamp=None): item_list_dict[x.entry.kind] = [x] for kind in item_list_dict: - item_list_dict[kind].sort(lambda a,b: cmp(a.entry.name, b.entry.name)) + item_list_dict[kind].sort(lambda a, b: cmp(a.entry.name, b.entry.name)) - return render_to_response('config_items/listing.html', {'item_list_dict':item_list_dict, - 'mod_or_bad':mod_or_bad, - 'timestamp' : timestamp}, + return render_to_response('config_items/listing.html', + {'item_list_dict': item_list_dict, + 'mod_or_bad': mod_or_bad, + 'timestamp': timestamp}, context_instance=RequestContext(request)) + @timeview def client_index(request, timestamp=None): """ @@ -149,8 +164,10 @@ def client_index(request, timestamp=None): .order_by("client__name").all() return render_to_response('clients/index.html', - { 'inter_list': list, 'timestamp' : timestamp}, - context_instance=RequestContext(request)) + {'inter_list': list, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + @timeview def client_detailed_list(request, timestamp=None, **kwargs): @@ -165,7 +182,8 @@ def client_detailed_list(request, timestamp=None, **kwargs): kwargs['page_limit'] = 0 return render_history_view(request, 'clients/detailed-list.html', **kwargs) -def client_detail(request, hostname = None, pk = None): + +def client_detail(request, hostname=None, pk=None): context = dict() client = get_object_or_404(Client, name=hostname) if(pk == None): @@ -177,6 +195,7 @@ def client_detail(request, hostname = None, pk = None): return render_history_view(request, 'clients/detail.html', page_limit=5, client=client, maxdate=context['interaction'].timestamp, context=context) + def client_manage(request): """Manage client expiration""" message = '' @@ -186,12 +205,12 @@ def client_manage(request): client_action = request.POST.get('client_action', None) client = Client.objects.get(name=client_name) if client_action == 'expire': - client.expiration = datetime.now(); + client.expiration = datetime.now() client.save() message = "Expiration for %s set to %s." % \ (client_name, client.expiration.strftime("%Y-%m-%d %H:%M:%S")) elif client_action == 'unexpire': - client.expiration = None; + client.expiration = None client.save() message = "%s is now active." % client_name else: @@ -205,6 +224,7 @@ def client_manage(request): {'clients': Client.objects.order_by('name').all(), 'message': message}, context_instance=RequestContext(request)) + @timeview def display_summary(request, timestamp=None): """ @@ -216,7 +236,12 @@ def display_summary(request, timestamp=None): if not timestamp: timestamp = datetime.now() - collected_data = dict(clean=[],bad=[],modified=[],extra=[],stale=[],pings=[]) + collected_data = dict(clean=[], + bad=[], + modified=[], + extra=[], + stale=[], + pings=[]) for node in recent_data: if timestamp - node.timestamp > timedelta(hours=24): collected_data['stale'].append(node) @@ -238,42 +263,47 @@ def display_summary(request, timestamp=None): # label, header_text, node_list summary_data = [] - get_dict = lambda name, label: { 'name': name, - 'nodes': collected_data[name], - 'label': label } + get_dict = lambda name, label: {'name': name, + 'nodes': collected_data[name], + 'label': label} if len(collected_data['clean']) > 0: - summary_data.append( get_dict('clean', 'nodes are clean.') ) + summary_data.append(get_dict('clean', + 'nodes are clean.')) if len(collected_data['bad']) > 0: - summary_data.append( get_dict('bad', 'nodes are bad.') ) + summary_data.append(get_dict('bad', + 'nodes are bad.')) if len(collected_data['modified']) > 0: - summary_data.append( get_dict('modified', 'nodes were modified.') ) + summary_data.append(get_dict('modified', + 'nodes were modified.')) if len(collected_data['extra']) > 0: - summary_data.append( get_dict('extra', - 'nodes have extra configurations.') ) + summary_data.append(get_dict('extra', + 'nodes have extra configurations.')) if len(collected_data['stale']) > 0: - summary_data.append( get_dict('stale', - 'nodes did not run within the last 24 hours.') ) + summary_data.append(get_dict('stale', + 'nodes did not run within the last 24 hours.')) if len(collected_data['pings']) > 0: - summary_data.append( get_dict('pings', - 'are down.') ) + summary_data.append(get_dict('pings', + 'are down.')) return render_to_response('displays/summary.html', {'summary_data': summary_data, 'node_count': node_count, 'timestamp': timestamp}, context_instance=RequestContext(request)) + @timeview def display_timing(request, timestamp=None): mdict = dict() inters = Interaction.objects.interaction_per_client(timestamp).select_related().all() [mdict.__setitem__(inter, {'name': inter.client.name}) \ for inter in inters] - for metric in Performance.objects.filter(interaction__in=mdict.keys()).all(): + for metric in Performance.objects.filter(interaction__in=list(mdict.keys())).all(): for i in metric.interaction.all(): mdict[i][metric.metric] = metric.value return render_to_response('displays/timing.html', - {'metrics': mdict.values(), 'timestamp': timestamp}, - context_instance=RequestContext(request)) + {'metrics': list(mdict.values()), + 'timestamp': timestamp}, + context_instance=RequestContext(request)) def render_history_view(request, template='clients/history.html', **kwargs): @@ -303,7 +333,7 @@ def render_history_view(request, template='clients/history.html', **kwargs): max_results = int(kwargs.get('page_limit', 25)) page = int(kwargs.get('page_number', 1)) - client=kwargs.get('client', None) + client = kwargs.get('client', None) if not client and 'hostname' in kwargs: client = get_object_or_404(Client, name=kwargs['hostname']) if client: @@ -333,7 +363,11 @@ def render_history_view(request, template='clients/history.html', **kwargs): entry_list = [] if max_results > 0: try: - rec_start, rec_end = prepare_paginated_list(request, context, iquery, page, max_results) + rec_start, rec_end = prepare_paginated_list(request, + context, + iquery, + page, + max_results) except PaginationError, page_error: if isinstance(page_error[0], HttpResponse): return page_error[0] @@ -345,6 +379,7 @@ def render_history_view(request, template='clients/history.html', **kwargs): return render_to_response(template, context, context_instance=RequestContext(request)) + def prepare_paginated_list(request, context, paged_list, page=1, max_results=25): """ Prepare context and slice an object for pagination. @@ -358,7 +393,7 @@ def prepare_paginated_list(request, context, paged_list, page=1, max_results=25) nitems = paged_list.count() except TypeError: nitems = len(paged_list) - + rec_start = (page - 1) * int(max_results) try: total_pages = (nitems / int(max_results)) + 1 @@ -369,11 +404,11 @@ def prepare_paginated_list(request, context, paged_list, page=1, max_results=25) try: view, args, kwargs = resolve(request.META['PATH_INFO']) kwargs['page_number'] = total_pages - raise PaginationError, HttpResponseRedirect( reverse(view, kwargs=kwargs) ) + raise PaginationError, HttpResponseRedirect(reverse(view, + kwargs=kwargs)) except (Resolver404, NoReverseMatch, ValueError): raise "Accessing beyond last page. Unable to resolve redirect." context['total_pages'] = total_pages context['records_per_page'] = max_results return (rec_start, rec_start + int(max_results)) - diff --git a/src/lib/Server/Reports/updatefix.py b/src/lib/Server/Reports/updatefix.py index f8fca1f90..5c61f599f 100644 --- a/src/lib/Server/Reports/updatefix.py +++ b/src/lib/Server/Reports/updatefix.py @@ -2,12 +2,13 @@ import Bcfg2.Server.Reports.settings from django.db import connection import django.core.management +import logging +import traceback from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion, \ TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA - -import logging, traceback logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix') + # all update function should go here def _merge_database_table_entries(): cursor = connection.cursor() @@ -21,7 +22,7 @@ def _merge_database_table_entries(): select name, kind from reports_extra """) # this fetch could be better done - entries_map={} + entries_map = {} for row in cursor.fetchall(): insert_cursor.execute("insert into reports_entries (name, kind) \ values (%s, %s)", (row[0], row[1])) @@ -48,6 +49,7 @@ def _merge_database_table_entries(): insert_cursor.execute("insert into reports_entries_interactions \ (entry_id, interaction_id, reason_id, type) values (%s, %s, %s, %s)", (entry_id, row[3], row[2], row[4])) + def _interactions_constraint_or_idx(): '''sqlite doesn't support alter tables.. or constraints''' cursor = connection.cursor() @@ -55,17 +57,17 @@ def _interactions_constraint_or_idx(): cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') except: cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') - + def _populate_interaction_entry_counts(): '''Populate up the type totals for the interaction table''' cursor = connection.cursor() - count_field = { TYPE_BAD: 'bad_entries', - TYPE_MODIFIED: 'modified_entries', - TYPE_EXTRA: 'extra_entries' } + count_field = {TYPE_BAD: 'bad_entries', + TYPE_MODIFIED: 'modified_entries', + TYPE_EXTRA: 'extra_entries'} - for type in count_field.keys(): - cursor.execute("select count(type), interaction_id "+ + for type in list(count_field.keys()): + cursor.execute("select count(type), interaction_id " + "from reports_entries_interactions where type = %s group by interaction_id" % type) updates = [] for row in cursor.fetchall(): @@ -73,9 +75,9 @@ def _populate_interaction_entry_counts(): try: cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) except Exception, e: - print e + print(e) cursor.close() - + # be sure to test your upgrade query before reflecting the change in the models # the list of function and sql command to do should go here @@ -104,6 +106,7 @@ _fixes = [_merge_database_table_entries, # this will calculate the last possible version of the database lastversion = len(_fixes) + def rollupdate(current_version): """ function responsible to coordinates all the updates need current_version as integer @@ -119,11 +122,12 @@ def rollupdate(current_version): except: logger.error("Failed to perform db update %s" % (_fixes[i]), exc_info=1) # since array start at 0 but version start at 1 we add 1 to the normal count - ret = InternalDatabaseVersion.objects.create(version=i+1) + ret = InternalDatabaseVersion.objects.create(version=i + 1) return ret else: return None + def dosync(): """Function to do the syncronisation for the models""" # try to detect if it's a fresh new database @@ -164,7 +168,7 @@ def dosync(): def update_database(): ''' methode to search where we are in the revision of the database models and update them ''' - try : + try: logger.debug("Running upgrade of models to the new one") dosync() know_version = InternalDatabaseVersion.objects.order_by('-version') diff --git a/src/lib/Server/Reports/utils.py b/src/lib/Server/Reports/utils.py index b74f09e74..6010f366b 100755 --- a/src/lib/Server/Reports/utils.py +++ b/src/lib/Server/Reports/utils.py @@ -1,11 +1,11 @@ """Helper functions for reports""" -from Bcfg2.Server.Reports.reports.models import TYPE_CHOICES from django.conf.urls.defaults import * import re """List of filters provided by filteredUrls""" filter_list = ('server', 'state') + class BatchFetch(object): """Fetch Django objects in smaller batches to save memory""" @@ -20,7 +20,7 @@ class BatchFetch(object): def __iter__(self): return self - def next(self): + def __next__(self): """Return the next object from our array and fetch from the database when needed""" if self.block_count + self.count - self.step == self.max: @@ -34,11 +34,12 @@ class BatchFetch(object): self.count += 1 return self.data[self.count - 1] + def generateUrls(fn): """ Parse url tuples and send to functions. - Decorator for url generators. Handles url tuple parsing + Decorator for url generators. Handles url tuple parsing before the actual function is called. """ def url_gen(*urls): @@ -51,13 +52,14 @@ def generateUrls(fn): return results return url_gen + @generateUrls def paginatedUrls(pattern, view, kwargs=None, name=None): """ Takes a group of url tuples and adds paginated urls. - Extends a url tuple to include paginated urls. Currently doesn't handle url() compiled - patterns. + Extends a url tuple to include paginated urls. + Currently doesn't handle url() compiled patterns. """ results = [(pattern, view, kwargs, name)] @@ -67,13 +69,15 @@ def paginatedUrls(pattern, view, kwargs=None, name=None): tail = mtail.group(1) pattern = pattern[:len(pattern) - len(tail)] results += [(pattern + "/(?P\d+)" + tail, view, kwargs)] - results += [(pattern + "/(?P\d+)\|(?P\d+)" + tail, view, kwargs)] + results += [(pattern + "/(?P\d+)\|(?P\d+)" + + tail, view, kwargs)] if not kwargs: kwargs = dict() kwargs['page_limit'] = 0 results += [(pattern + "/?\|(?Pall)" + tail, view, kwargs)] return results + @generateUrls def filteredUrls(pattern, view, kwargs=None, name=None): """ @@ -93,7 +97,8 @@ def filteredUrls(pattern, view, kwargs=None, name=None): '/server/(?P[\w\-\.]+)/(?P[A-Za-z]+)'): results += [(pattern + filter + tail, view, kwargs)] return results - + + @generateUrls def timeviewUrls(pattern, view, kwargs=None, name=None): """ @@ -113,4 +118,3 @@ def timeviewUrls(pattern, view, kwargs=None, name=None): '/(?P\d{4})-(?P\d{2})-(?P\d{2})'): results += [(pattern + filter + tail, view, kwargs)] return results - diff --git a/src/lib/Server/Snapshots/__init__.py b/src/lib/Server/Snapshots/__init__.py index 6018377cb..a4d8fadbc 100644 --- a/src/lib/Server/Snapshots/__init__.py +++ b/src/lib/Server/Snapshots/__init__.py @@ -19,7 +19,7 @@ def db_from_config(cfile): db = cp.get('snapshots', 'database') return '%s://%s:%s@%s/%s' % (driver, user, password, host, db) else: - raise Exception, "unsupported db driver %s" % driver + raise Exception("unsupported db driver %s" % driver) def setup_session(cfile, debug=False): diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py index cbb14be79..130d3e8c2 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Server/Snapshots/model.py @@ -33,12 +33,20 @@ class Administrator(Uniquer, Base): email = Column(Unicode(64)) admin_client = Table('admin_client', Base.metadata, - Column('admin_id', Integer, ForeignKey('administrator.id')), - Column('client_id', Integer, ForeignKey('client.id'))) + Column('admin_id', + Integer, + ForeignKey('administrator.id')), + Column('client_id', + Integer, + ForeignKey('client.id'))) admin_group = Table('admin_group', Base.metadata, - Column('admin_id', Integer, ForeignKey('administrator.id')), - Column('group_id', Integer, ForeignKey('group.id'))) + Column('admin_id', + Integer, + ForeignKey('administrator.id')), + Column('group_id', + Integer, + ForeignKey('group.id'))) class Client(Uniquer, Base): @@ -68,12 +76,20 @@ class ConnectorKeyVal(Uniquer, Base): value = Column(UnicodeText) meta_group = Table('meta_group', Base.metadata, - Column('metadata_id', Integer, ForeignKey('metadata.id')), - Column('group_id', Integer, ForeignKey('group.id'))) + Column('metadata_id', + Integer, + ForeignKey('metadata.id')), + Column('group_id', + Integer, + ForeignKey('group.id'))) meta_conn = Table('meta_conn', Base.metadata, - Column('metadata_id', Integer, ForeignKey('metadata.id')), - Column('connkeyval_id', Integer, ForeignKey('connkeyval.id'))) + Column('metadata_id', + Integer, + ForeignKey('metadata.id')), + Column('connkeyval_id', + Integer, + ForeignKey('connkeyval.id'))) class Metadata(Base): @@ -95,7 +111,7 @@ class Metadata(Base): data = getattr(mymetadata, connector) if not isinstance(data, dict): continue - for key, value in data.iteritems(): + for key, value in list(data.items()): if not isinstance(value, str): continue m.keyvals.append(ConnectorKeyVal.by_value(mysession, @@ -143,8 +159,12 @@ class PackageCorrespondence(Base, CorrespondenceType): correct = Column(Boolean) package_snap = Table('package_snap', Base.metadata, - Column('ppair_id', Integer, ForeignKey('package_pair.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('ppair_id', + Integer, + ForeignKey('package_pair.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) class Service(Base, Uniquer): @@ -167,8 +187,12 @@ class ServiceCorrespondence(Base, CorrespondenceType): correct = Column(Boolean) service_snap = Table('service_snap', Base.metadata, - Column('spair_id', Integer, ForeignKey('service_pair.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('spair_id', + Integer, + ForeignKey('service_pair.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) class File(Base, Uniquer): @@ -194,20 +218,36 @@ class FileCorrespondence(Base, CorrespondenceType): correct = Column(Boolean) file_snap = Table('file_snap', Base.metadata, - Column('fpair_id', Integer, ForeignKey('file_pair.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('fpair_id', + Integer, + ForeignKey('file_pair.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) extra_pkg_snap = Table('extra_pkg_snap', Base.metadata, - Column('package_id', Integer, ForeignKey('package.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('package_id', + Integer, + ForeignKey('package.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) extra_file_snap = Table('extra_file_snap', Base.metadata, - Column('file_id', Integer, ForeignKey('file.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('file_id', + Integer, + ForeignKey('file.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) extra_service_snap = Table('extra_service_snap', Base.metadata, - Column('service_id', Integer, ForeignKey('service.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('service_id', + Integer, + ForeignKey('service.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) class Action(Base): @@ -228,7 +268,7 @@ class Snapshot(Base): correct = Column(Boolean) revision = Column(Unicode(36)) metadata_id = Column(Integer, ForeignKey('metadata.id')) - client_metadata = relation(Metadata, primaryjoin=metadata_id==Metadata.id) + client_metadata = relation(Metadata, primaryjoin=metadata_id == Metadata.id) timestamp = Column(DateTime, default=datetime.datetime.now) client_id = Column(Integer, ForeignKey('client.id')) client = relation(Client, backref=backref('snapshots')) @@ -256,23 +296,25 @@ class Snapshot(Base): (cls.e_dispatch, extra)]: for key in dispatch: dest, ecls = dispatch[key] - for edata in data[key].values(): + for edata in list(data[key].values()): getattr(snap, dest).append(ecls.from_record(session, edata)) return snap @classmethod def by_client(cls, session, clientname): - return session.query(cls).join(cls.client_metadata, Metadata.client).filter(Client.name==clientname) + return session.query(cls).join(cls.client_metadata, + Metadata.client).filter(Client.name == clientname) @classmethod def get_current(cls, session, clientname): - return session.query(Snapshot).join(Snapshot.client_metadata, Metadata.client).filter(Client.name==clientname).order_by(desc(Snapshot.timestamp)).first() + return session.query(Snapshot).join(Snapshot.client_metadata, + Metadata.client).filter(Client.name == clientname).order_by(desc(Snapshot.timestamp)).first() @classmethod def get_by_date(cls, session, clientname, timestamp): return session.query(Snapshot)\ .join(Snapshot.client_metadata, Metadata.client)\ .filter(Snapshot.timestamp < timestamp)\ - .filter(Client.name==clientname)\ + .filter(Client.name == clientname)\ .order_by(desc(Snapshot.timestamp))\ .first() -- cgit v1.2.3-1-g7c22 From 9d8f65a64a5319f3e48dd43e2ea3a3ac4c051f9c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 4 Apr 2011 13:28:27 -0500 Subject: Reports: Fix deprecated 'raise' usage Signed-off-by: Sol Jerome --- src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py | 6 +++--- src/lib/Server/Reports/reports/views.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py b/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py index 7fffe289d..2d210cc07 100644 --- a/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py +++ b/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py @@ -219,9 +219,9 @@ def add_url_filter(parser, token): filter_name = filter_name.strip() filter_value = parser.compile_filter(filter_value) except ValueError: - raise template.TemplateSyntaxError, "%r tag requires exactly one argument" % token.contents.split()[0] + raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) if not filter_name or not filter_value: - raise template.TemplateSyntaxError, "argument should be a filter=value pair" + raise template.TemplateSyntaxError("argument should be a filter=value pair") return AddUrlFilter(filter_name, filter_value) @@ -268,7 +268,7 @@ def to_media_url(parser, token): tag_name, filter_value = token.split_contents() filter_value = parser.compile_filter(filter_value) except ValueError: - raise template.TemplateSyntaxError, "%r tag requires exactly one argument" % token.contents.split()[0] + raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) return MediaTag(filter_value) diff --git a/src/lib/Server/Reports/reports/views.py b/src/lib/Server/Reports/reports/views.py index 3cffa68dd..463dec674 100644 --- a/src/lib/Server/Reports/reports/views.py +++ b/src/lib/Server/Reports/reports/views.py @@ -385,9 +385,9 @@ def prepare_paginated_list(request, context, paged_list, page=1, max_results=25) Prepare context and slice an object for pagination. """ if max_results < 1: - raise PaginationError, "Max results less then 1" + raise PaginationError("Max results less then 1") if paged_list == None: - raise PaginationError, "Invalid object" + raise PaginationError("Invalid object") try: nitems = paged_list.count() @@ -404,8 +404,8 @@ def prepare_paginated_list(request, context, paged_list, page=1, max_results=25) try: view, args, kwargs = resolve(request.META['PATH_INFO']) kwargs['page_number'] = total_pages - raise PaginationError, HttpResponseRedirect(reverse(view, - kwargs=kwargs)) + raise PaginationError(HttpResponseRedirect(reverse(view, + kwards=kwargs))) except (Resolver404, NoReverseMatch, ValueError): raise "Accessing beyond last page. Unable to resolve redirect." -- cgit v1.2.3-1-g7c22 From 2d90970d7da08e3c89fe67f4ebf8d0c1cdafeb20 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 5 Apr 2011 14:32:29 -0500 Subject: Plugins: PY3K + PEP8 fixes Signed-off-by: Sol Jerome --- src/lib/Server/Plugin.py | 102 +++++++++++++++++++++----------- src/lib/Server/Plugins/Account.py | 38 +++++++----- src/lib/Server/Plugins/Base.py | 5 +- src/lib/Server/Plugins/Bundler.py | 2 +- src/lib/Server/Plugins/Cfg.py | 8 +-- src/lib/Server/Plugins/Deps.py | 21 ++++--- src/lib/Server/Plugins/Editor.py | 20 +++++-- src/lib/Server/Plugins/GroupPatterns.py | 16 +++-- src/lib/Server/Plugins/Metadata.py | 73 ++++++++++++++--------- src/lib/Server/Plugins/NagiosGen.py | 53 +++++++++-------- src/lib/Server/Plugins/Ohai.py | 8 ++- src/lib/Server/Plugins/Packages.py | 35 +++++++---- src/lib/Server/Plugins/Pkgmgr.py | 49 +++++++++------ src/lib/Server/Plugins/Probes.py | 16 +++-- src/lib/Server/Plugins/SSHbase.py | 32 +++++----- src/lib/Server/Plugins/SSLCA.py | 46 +++++++++----- src/lib/Server/Plugins/Snapshots.py | 5 +- 17 files changed, 328 insertions(+), 201 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 73d054409..5a6f3281b 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -38,14 +38,17 @@ info_regex = re.compile( \ 'paranoid:(\s)*(?P\S+)|' + 'perms:(\s)*(?P\w+)|') + class PluginInitError(Exception): """Error raised in cases of Plugin initialization errors.""" pass + class PluginExecutionError(Exception): """Error raised in case of Plugin execution errors.""" pass + class Plugin(object): """This is the base class for all Bcfg2 Server plugins. Several attributes must be defined in the subclass: @@ -90,6 +93,7 @@ class Plugin(object): def shutdown(self): self.running = False + class Generator(object): """Generator plugins contribute to literal client configurations.""" def HandlesEntry(self, entry, metadata): @@ -100,20 +104,24 @@ class Generator(object): """This is the slow-path handler for configuration entry binding.""" raise PluginExecutionError + class Structure(object): """Structure Plugins contribute to abstract client configurations.""" def BuildStructures(self, metadata): """Return a list of abstract goal structures for client.""" raise PluginExecutionError + class Metadata(object): """Signal metadata capabilities for this plugin""" def add_client(self, client_name, attribs): """Add client.""" pass + def remove_client(self, client_name): """Remove client.""" pass + def viz(self, hosts, bundles, key, colors): """Create viz str for viz admin mode.""" pass @@ -124,6 +132,7 @@ class Metadata(object): def merge_additional_data(self, imd, source, groups, data): raise PluginExecutionError + class Connector(object): """Connector Plugins augment client metadata instances.""" def get_additional_groups(self, metadata): @@ -134,6 +143,7 @@ class Connector(object): """Determine additional data for metadata instances.""" return dict() + class Probing(object): """Signal probe capability for this plugin.""" def GetProbes(self, _): @@ -144,11 +154,13 @@ class Probing(object): """Receive probe results pertaining to client.""" pass + class Statistics(object): """Signal statistics handling capability.""" def process_statistics(self, client, xdata): pass + class ThreadedStatistics(Statistics, threading.Thread): """Threaded statistics handling capability.""" @@ -169,7 +181,7 @@ class ThreadedStatistics(Statistics, while not self.work_queue.empty(): (metadata, data) = self.work_queue.get_nowait() try: - pending_data.append( ( metadata.hostname, lxml.etree.tostring(data) ) ) + pending_data.append((metadata.hostname, lxml.etree.tostring(data))) except: self.logger.warning("Dropping interaction for %s" % metadata.hostname) except Queue.Empty: @@ -202,7 +214,7 @@ class ThreadedStatistics(Statistics, try: while True: try: - metadata = self.core.build_metadata(pmetadata) + metadata = self.core.build_metadata(pmetadata) break except Bcfg2.Server.Plugins.Metadata.MetadataRuntimeError: pass @@ -211,7 +223,7 @@ class ThreadedStatistics(Statistics, if self.terminate.isSet(): return False - self.work_queue.put_nowait( (metadata, lxml.etree.fromstring(pdata)) ) + self.work_queue.put_nowait((metadata, lxml.etree.fromstring(pdata))) except Queue.Full: self.logger.warning("Queue.Full: Failed to load queue data") break @@ -255,6 +267,7 @@ class ThreadedStatistics(Statistics, """Handle stats here.""" pass + class PullSource(object): def GetExtra(self, client): return [] @@ -262,6 +275,7 @@ class PullSource(object): def GetCurrentEntry(self, client, e_type, e_name): raise PluginExecutionError + class PullTarget(object): def AcceptChoices(self, entry, metadata): raise PluginExecutionError @@ -271,31 +285,38 @@ class PullTarget(object): of bcfg2-admin pull.""" raise PluginExecutionError + class Decision(object): """Signal decision handling capability.""" def GetDecisions(self, metadata, mode): return [] + class ValidationError(Exception): pass + class StructureValidator(object): """Validate/modify goal structures.""" def validate_structures(self, metadata, structures): - raise ValidationError, "not implemented" + raise ValidationError("not implemented") + class GoalValidator(object): """Validate/modify configuration goals.""" def validate_goals(self, metadata, goals): - raise ValidationError, "not implemented" + raise ValidationError("not implemented") + class Version(object): """Interact with various version control systems.""" def get_revision(self): return [] + def commit_data(self, file_list, comment=None): pass + # the rest of the file contains classes for coherent file caching class FileBacked(object): @@ -324,6 +345,7 @@ class FileBacked(object): """Update local data structures based on current file state""" pass + class DirectoryBacked(object): """This object is a coherent cache for a filesystem hierarchy of files.""" __child__ = FileBacked @@ -341,7 +363,7 @@ class DirectoryBacked(object): return self.entries[key] def __iter__(self): - return self.entries.iteritems() + return iter(self.entries.items()) def AddEntry(self, name): """Add new entry to data structures upon file creation.""" @@ -380,9 +402,10 @@ class DirectoryBacked(object): elif action in ['endExist']: pass else: - print "Got unknown event %s %s %s" % (event.requestID, + print("Got unknown event %s %s %s" % (event.requestID, event.code2str(), - event.filename) + event.filename)) + class XMLFileBacked(FileBacked): """ @@ -401,7 +424,7 @@ class XMLFileBacked(FileBacked): try: xdata = XML(self.data) except XMLSyntaxError: - logger.error("Failed to parse %s"%(self.name)) + logger.error("Failed to parse %s" % (self.name)) return self.label = xdata.attrib[self.__identifier__] self.entries = xdata.getchildren() @@ -409,12 +432,14 @@ class XMLFileBacked(FileBacked): def __iter__(self): return iter(self.entries) + class SingleXMLFileBacked(XMLFileBacked): """This object is a coherent cache for an independent XML file.""" def __init__(self, filename, fam): XMLFileBacked.__init__(self, filename) fam.AddMonitor(filename, self) + class StructFile(XMLFileBacked): """This file contains a set of structure file formatting logic.""" def __init__(self, name): @@ -429,7 +454,7 @@ class StructFile(XMLFileBacked): logger.error("Failed to parse file %s" % self.name) return self.fragments = {} - work = {lambda x:True: xdata.getchildren()} + work = {lambda x: True: xdata.getchildren()} while work: (predicate, worklist) = work.popitem() self.fragments[predicate] = [item for item in worklist if item.tag != 'Group' @@ -441,26 +466,28 @@ class StructFile(XMLFileBacked): else: cmd = "lambda x:'%s' in x.groups and predicate(x)" - newpred = eval(cmd % (group.get('name')), {'predicate':predicate}) + newpred = eval(cmd % (group.get('name')), {'predicate': predicate}) work[newpred] = group.getchildren() def Match(self, metadata): """Return matching fragments of independent.""" - matching = [frag for (pred, frag) in self.fragments.iteritems() if pred(metadata)] + matching = [frag for (pred, frag) in list(self.fragments.items()) + if pred(metadata)] if matching: - return reduce(lambda x, y:x+y, matching) + return reduce(lambda x, y: x + y, matching) logger.error("File %s got null match" % (self.name)) return [] + class INode: """ LNodes provide lists of things available at a particular group intersection. """ - raw = {'Client':"lambda x:'%s' == x.hostname and predicate(x)", - 'Group':"lambda x:'%s' in x.groups and predicate(x)"} - nraw = {'Client':"lambda x:'%s' != x.hostname and predicate(x)", - 'Group':"lambda x:'%s' not in x.groups and predicate(x)"} + raw = {'Client': "lambda x:'%s' == x.hostname and predicate(x)", + 'Group': "lambda x:'%s' in x.groups and predicate(x)"} + nraw = {'Client': "lambda x:'%s' != x.hostname and predicate(x)", + 'Group': "lambda x:'%s' not in x.groups and predicate(x)"} containers = ['Group', 'Client'] ignore = [] @@ -468,16 +495,16 @@ class INode: self.data = data self.contents = {} if parent == None: - self.predicate = lambda x:True + self.predicate = lambda x: True else: predicate = parent.predicate if data.get('negate', 'false') in ['true', 'True']: psrc = self.nraw else: psrc = self.raw - if data.tag in psrc.keys(): + if data.tag in list(psrc.keys()): self.predicate = eval(psrc[data.tag] % (data.get('name')), - {'predicate':predicate}) + {'predicate': predicate}) else: raise Exception mytype = self.__class__ @@ -491,7 +518,7 @@ class INode: try: self.contents[item.tag][item.get('name')] = item.attrib except KeyError: - self.contents[item.tag] = {item.get('name'):item.attrib} + self.contents[item.tag] = {item.get('name'): item.attrib} if item.text: self.contents[item.tag]['__text__'] = item.text try: @@ -511,6 +538,7 @@ class INode: for child in self.children: child.Match(metadata, data) + class XMLSrc(XMLFileBacked): """XMLSrc files contain a LNode hierarchy that returns matching entries.""" __node__ = INode @@ -557,10 +585,12 @@ class XMLSrc(XMLFileBacked): self.pnode.Match(metadata, cache[1]) self.cache = cache + class XMLDirectoryBacked(DirectoryBacked): """Directorybacked for *.xml.""" patterns = re.compile('.*\.xml') + class PrioDir(Plugin, Generator, XMLDirectoryBacked): """This is a generator that handles package assignments.""" name = 'PrioDir' @@ -579,8 +609,8 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): """Handle events and update dispatch table.""" XMLDirectoryBacked.HandleEvent(self, event) self.Entries = {} - for src in self.entries.values(): - for itype, children in src.items.iteritems(): + for src in list(self.entries.values()): + for itype, children in list(src.items.items()): for child in children: try: self.Entries[itype][child] = self.BindEntry @@ -589,14 +619,14 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): def BindEntry(self, entry, metadata): """Check package lists of package entries.""" - [src.Cache(metadata) for src in self.entries.values()] + [src.Cache(metadata) for src in list(self.entries.values())] name = entry.get('name') if not src.cache: self.logger.error("Called before data loaded") raise PluginExecutionError - matching = [src for src in self.entries.values() + matching = [src for src in list(self.entries.values()) if src.cache and entry.tag in src.cache[1] - and src.cache[1][entry.tag].has_key(name)] + and name in src.cache[1][entry.tag]] if len(matching) == 0: raise PluginExecutionError elif len(matching) == 1: @@ -618,15 +648,17 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): entry.text = data['__text__'] if '__children__' in data: [entry.append(copy.deepcopy(item)) for item in data['__children__']] - [entry.attrib.__setitem__(key, data[key]) for key in data.keys() \ + [entry.attrib.__setitem__(key, data[key]) for key in list(data.keys()) \ if not key.startswith('__')] + # new unified EntrySet backend class SpecificityError(Exception): """Thrown in case of filename parse failure.""" pass + class Specificity: def __init__(self, all=False, group=False, hostname=False, prio=0, delta=False): @@ -665,6 +697,7 @@ class Specificity: return True return False + class SpecificData(object): def __init__(self, name, specific, encoding): self.name = name @@ -678,9 +711,11 @@ class SpecificData(object): except: logger.error("Failed to read file %s" % self.name) + class EntrySet: """Entry sets deal with the host- and group-specific entries.""" ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\\.genshi_include)$") + def __init__(self, basename, path, entry_type, encoding): self.path = path self.entry_type = entry_type @@ -693,7 +728,7 @@ class EntrySet: self.specific = re.compile(pattern) def get_matching(self, metadata): - return [item for item in self.entries.values() \ + return [item for item in list(self.entries.values()) \ if item.specific.matches(metadata)] def handle_event(self, event): @@ -761,11 +796,11 @@ class EntrySet: for line in open(fpath).readlines(): match = info_regex.match(line) if not match: - logger.warning("Failed to match line: %s"%line) + logger.warning("Failed to match line: %s" % line) continue else: mgd = match.groupdict() - for key, value in mgd.iteritems(): + for key, value in list(mgd.items()): if value: self.metadata[key] = value if len(self.metadata['perms']) == 3: @@ -795,7 +830,7 @@ class EntrySet: (entry.get('name'))) raise PluginExecutionError [entry.attrib.__setitem__(key, value) \ - for (key, value) in mdata['Info'][None].iteritems()] + for (key, value) in list(mdata['Info'][None].items())] def bind_entry(self, entry, metadata): """Return the appropriate interpreted template from the set of available templates.""" @@ -817,6 +852,7 @@ class EntrySet: raise PluginExecutionError + class GroupSpool(Plugin, Generator): """Unified interface for handling group-specific data (e.g. .G## files).""" name = 'GroupSpool' @@ -878,9 +914,9 @@ class GroupSpool(Plugin, Generator): if not relative.endswith('/'): relative += '/' name = self.data + relative - if relative not in self.handles.values(): + if relative not in list(self.handles.values()): if not posixpath.isdir(name): - print "Failed to open directory %s" % (name) + print("Failed to open directory %s" % (name)) return reqid = self.core.fam.AddMonitor(name, self) self.handles[reqid] = relative diff --git a/src/lib/Server/Plugins/Account.py b/src/lib/Server/Plugins/Account.py index e3ea58761..f67819b9d 100644 --- a/src/lib/Server/Plugins/Account.py +++ b/src/lib/Server/Plugins/Account.py @@ -3,6 +3,7 @@ __revision__ = '$Revision$' import Bcfg2.Server.Plugin + class Account(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Generator): """This module generates account config files, @@ -21,13 +22,14 @@ class Account(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Generator.__init__(self) - self.Entries = {'ConfigFile':{'/etc/passwd':self.from_yp_cb, - '/etc/group':self.from_yp_cb, - '/etc/security/limits.conf':self.gen_limits_cb, - '/root/.ssh/authorized_keys':self.gen_root_keys_cb, - '/etc/sudoers':self.gen_sudoers}} + self.Entries = {'ConfigFile': {'/etc/passwd': self.from_yp_cb, + '/etc/group': self.from_yp_cb, + '/etc/security/limits.conf': self.gen_limits_cb, + '/root/.ssh/authorized_keys': self.gen_root_keys_cb, + '/etc/sudoers': self.gen_sudoers}} try: - self.repository = Bcfg2.Server.Plugin.DirectoryBacked(self.data, self.core.fam) + self.repository = Bcfg2.Server.Plugin.DirectoryBacked(self.data, + self.core.fam) except: self.logger.error("Failed to load repos: %s, %s" % \ (self.data, "%s/ssh" % (self.data))) @@ -38,9 +40,11 @@ class Account(Bcfg2.Server.Plugin.Plugin, fname = entry.attrib['name'].split('/')[-1] entry.text = self.repository.entries["static.%s" % (fname)].data entry.text += self.repository.entries["dyn.%s" % (fname)].data - perms = {'owner':'root', 'group':'root', 'perms':'0644'} + perms = {'owner': 'root', + 'group': 'root', + 'perms': '0644'} [entry.attrib.__setitem__(key, value) for (key, value) in \ - perms.iteritems()] + list(perms.items())] def gen_limits_cb(self, entry, metadata): """Build limits entries based on current ACLs.""" @@ -50,9 +54,11 @@ class Account(Bcfg2.Server.Plugin.Plugin, self.repository.entries["useraccess"].data.split()] users = [user for (user, host) in \ useraccess if host == metadata.hostname.split('.')[0]] - perms = {'owner':'root', 'group':'root', 'perms':'0600'} + perms = {'owner': 'root', + 'group': 'root', + 'perms': '0600'} [entry.attrib.__setitem__(key, value) for (key, value) in \ - perms.iteritems()] + list(perms.items())] entry.text += "".join(["%s hard maxlogins 1024\n" % uname for uname in superusers + users]) if "*" not in users: entry.text += "* hard maxlogins 0\n" @@ -71,9 +77,11 @@ class Account(Bcfg2.Server.Plugin.Plugin, entry.text = "".join([rdata["%s.key" % user].data for user \ in superusers if \ ("%s.key" % user) in rdata]) - perms = {'owner':'root', 'group':'root', 'perms':'0600'} + perms = {'owner': 'root', + 'group': 'root', + 'perms': '0600'} [entry.attrib.__setitem__(key, value) for (key, value) \ - in perms.iteritems()] + in list(perms.items())] def gen_sudoers(self, entry, metadata): """Build root authorized keys file based on current ACLs.""" @@ -88,6 +96,8 @@ class Account(Bcfg2.Server.Plugin.Plugin, entry.text = self.repository.entries['static.sudoers'].data entry.text += "".join(["%s ALL=(ALL) ALL\n" % uname \ for uname in superusers]) - perms = {'owner':'root', 'group':'root', 'perms':'0440'} + perms = {'owner': 'root', + 'group': 'root', + 'perms': '0440'} [entry.attrib.__setitem__(key, value) for (key, value) \ - in perms.iteritems()] + in list(perms.items())] diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py index 8e5ca1cd9..2b241da9d 100644 --- a/src/lib/Server/Plugins/Base.py +++ b/src/lib/Server/Plugins/Base.py @@ -5,6 +5,7 @@ import Bcfg2.Server.Plugin import copy import lxml.etree + class Base(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Structure, Bcfg2.Server.Plugin.XMLDirectoryBacked): @@ -31,8 +32,8 @@ class Base(Bcfg2.Server.Plugin.Plugin, def BuildStructures(self, metadata): """Build structures for client described by metadata.""" ret = lxml.etree.Element("Independent", version='2.0') - fragments = reduce(lambda x, y: x+y, + fragments = reduce(lambda x, y: x + y, [base.Match(metadata) for base - in self.entries.values()], []) + in list(self.entries.values())], []) [ret.append(copy.deepcopy(frag)) for frag in fragments] return [ret] diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 3f88fe26b..04df8ea86 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -73,7 +73,7 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, """Build all structures for client (metadata).""" bundleset = [] for bundlename in metadata.bundles: - entries = [item for (key, item) in self.entries.iteritems() if \ + entries = [item for (key, item) in list(self.entries.items()) if \ self.patterns.match(key).group('name') == bundlename] if len(entries) == 0: continue diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 184bb792c..6c7a40a52 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -13,9 +13,7 @@ import Bcfg2.Server.Plugin try: import genshi.core import genshi.input - from genshi.template import TemplateLoader, \ - TextTemplate, MarkupTemplate, TemplateError - from genshi.template import NewTextTemplate + from genshi.template import TemplateLoader, NewTextTemplate have_genshi = True except: have_genshi = False @@ -62,7 +60,7 @@ def process_delta(data, delta): output = open(basefile.name, 'r').read() [os.unlink(fname) for fname in [basefile.name, dfile.name]] if ret >> 8 != 0: - raise Bcfg2.Server.Plugin.PluginExecutionError, ('delta', delta) + raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta) return output @@ -96,7 +94,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """return a list of all entries pertinent to a client => [base, delta1, delta2] """ - matching = [ent for ent in self.entries.values() if \ + matching = [ent for ent in list(self.entries.values()) if \ ent.specific.matches(metadata)] matching.sort(self.sort_by_specific) non_delta = [matching.index(m) for m in matching diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py index 088f8cdad..b186258cb 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Server/Plugins/Deps.py @@ -5,20 +5,22 @@ import lxml.etree import Bcfg2.Server.Plugin + class DNode(Bcfg2.Server.Plugin.INode): """DNode provides supports for single predicate types for dependencies.""" - raw = {'Group':"lambda x:'%s' in x.groups and predicate(x)"} + raw = {'Group': "lambda x:'%s' in x.groups and predicate(x)"} containers = ['Group'] def __init__(self, data, idict, parent=None): self.data = data self.contents = {} if parent == None: - self.predicate = lambda x:True + self.predicate = lambda x: True else: predicate = parent.predicate - if data.tag in self.raw.keys(): - self.predicate = eval(self.raw[data.tag] % (data.get('name')), {'predicate':predicate}) + if data.tag in list(self.raw.keys()): + self.predicate = eval(self.raw[data.tag] % (data.get('name')), + {'predicate': predicate}) else: raise Exception mytype = self.__class__ @@ -27,15 +29,18 @@ class DNode(Bcfg2.Server.Plugin.INode): if item.tag in self.containers: self.children.append(mytype(item, idict, self)) else: - data = [(child.tag, child.get('name')) for child in item.getchildren()] + data = [(child.tag, child.get('name')) + for child in item.getchildren()] try: self.contents[item.tag][item.get('name')] = data except KeyError: - self.contents[item.tag] = {item.get('name'):data} + self.contents[item.tag] = {item.get('name'): data} + class DepXMLSrc(Bcfg2.Server.Plugin.XMLSrc): __node__ = DNode + class Deps(Bcfg2.Server.Plugin.PrioDir, Bcfg2.Server.Plugin.StructureValidator): name = 'Deps' @@ -68,12 +73,12 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, if (entries, gdata) in self.cache: prereqs = self.cache[(entries, gdata)] else: - [src.Cache(metadata) for src in self.entries.values()] + [src.Cache(metadata) for src in list(self.entries.values())] toexamine = list(entries[:]) while toexamine: entry = toexamine.pop() - matching = [src for src in self.entries.values() + matching = [src for src in list(self.entries.values()) if src.cache and entry[0] in src.cache[1] and entry[1] in src.cache[1][entry[0]]] if len(matching) > 1: diff --git a/src/lib/Server/Plugins/Editor.py b/src/lib/Server/Plugins/Editor.py index bfd4d6e93..e68d28d8b 100644 --- a/src/lib/Server/Plugins/Editor.py +++ b/src/lib/Server/Plugins/Editor.py @@ -2,6 +2,7 @@ import Bcfg2.Server.Plugin import re import lxml.etree + def linesub(pattern, repl, filestring): """Substitutes instances of pattern with repl in filestring.""" if filestring == None: @@ -12,6 +13,7 @@ def linesub(pattern, repl, filestring): output.append(re.sub(pattern, repl, filestring)) return '\n'.join(output) + class EditDirectives(Bcfg2.Server.Plugin.SpecificData): """This object handles the editing directives.""" def ProcessDirectives(self, input): @@ -22,23 +24,29 @@ class EditDirectives(Bcfg2.Server.Plugin.SpecificData): temp = linesub(directive[0], directive[1], temp) return temp + class EditEntrySet(Bcfg2.Server.Plugin.EntrySet): def __init__(self, basename, path, entry_type, encoding): - self.ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|%s\.H_.*)$" %path.split('/')[-1]) - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) + self.ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|%s\.H_.*)$" % path.split('/')[-1]) + Bcfg2.Server.Plugin.EntrySet.__init__(self, + basename, + path, + entry_type, + encoding) self.inputs = dict() def bind_entry(self, entry, metadata): client = metadata.hostname filename = entry.get('name') - permdata = {'owner':'root', 'group':'root'} - permdata['perms'] = '0644' + permdata = {'owner': 'root', + 'group': 'root' + 'perms': '0644'} [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] entry.text = self.entries['edits'].ProcessDirectives(self.get_client_data(client)) if not entry.text: entry.set('empty', 'true') try: - f = open('%s/%s.H_%s' %(self.path, filename.split('/')[-1], client), 'w') + f = open('%s/%s.H_%s' % (self.path, filename.split('/')[-1], client), 'w') f.write(entry.text) f.close() except: @@ -60,7 +68,7 @@ class Editor(Bcfg2.Server.Plugin.GroupSpool, def GetProbes(self, _): '''Return a set of probes for execution on client''' probelist = list() - for name in self.entries.keys(): + for name in list(self.entries.keys()): probe = lxml.etree.Element('probe') probe.set('name', name) probe.set('source', "Editor") diff --git a/src/lib/Server/Plugins/GroupPatterns.py b/src/lib/Server/Plugins/GroupPatterns.py index 3801a6a08..f99026a9d 100644 --- a/src/lib/Server/Plugins/GroupPatterns.py +++ b/src/lib/Server/Plugins/GroupPatterns.py @@ -3,6 +3,7 @@ import re import Bcfg2.Server.Plugin + class PackedDigitRange(object): def __init__(self, digit_range): self.sparse = list() @@ -18,12 +19,14 @@ class PackedDigitRange(object): if iother in self.sparse: return True for (start, end) in self.ranges: - if iother in xrange(start, end+1): + if iother in range(start, end + 1): return True return False + class PatternMap(object): range_finder = '\\[\\[[\d\-,]+\\]\\]' + def __init__(self, pattern, rangestr, groups): self.pattern = pattern self.rangestr = rangestr @@ -33,8 +36,11 @@ class PatternMap(object): self.process = self.process_re elif rangestr != None: self.process = self.process_range - self.re = re.compile('^' + re.subn(self.range_finder, '(\d+)', rangestr)[0]) - dmatcher = re.compile(re.subn(self.range_finder, '\\[\\[([\d\-,]+)\\]\\]', rangestr)[0]) + self.re = re.compile('^' + re.subn(self.range_finder, '(\d+)', + rangestr)[0]) + dmatcher = re.compile(re.subn(self.range_finder, + '\\[\\[([\d\-,]+)\\]\\]', + rangestr)[0]) self.dranges = [PackedDigitRange(x) for x in dmatcher.match(rangestr).groups()] else: raise Exception @@ -58,10 +64,11 @@ class PatternMap(object): for group in self.groups: newg = group for idx in range(len(sub)): - newg = newg.replace('$%s' % (idx+1), sub[idx]) + newg = newg.replace('$%s' % (idx + 1), sub[idx]) ret.append(newg) return ret + class PatternFile(Bcfg2.Server.Plugin.SingleXMLFileBacked): def __init__(self, filename, fam): Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) @@ -102,6 +109,7 @@ class PatternFile(Bcfg2.Server.Plugin.SingleXMLFileBacked): (pattern.pattern, hostname), exc_info=1) return ret + class GroupPatterns(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): name = "GroupPatterns" diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index aa482e7ed..98caa577d 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -1,4 +1,6 @@ -"""This file stores persistent metadata for the Bcfg2 Configuration Repository.""" +""" +This file stores persistent metadata for the Bcfg2 Configuration Repository. +""" __revision__ = '$Revision$' @@ -12,6 +14,7 @@ import time import Bcfg2.Server.FileMonitor import Bcfg2.Server.Plugin + def locked(fd): """Aquire a lock on a file""" try: @@ -20,14 +23,19 @@ def locked(fd): return True return False + class MetadataConsistencyError(Exception): """This error gets raised when metadata is internally inconsistent.""" pass + class MetadataRuntimeError(Exception): - """This error is raised when the metadata engine is called prior to reading enough data.""" + """This error is raised when the metadata engine + is called prior to reading enough data. + """ pass + class XMLMetadataConfig(object): """Handles xml config files and all XInclude statements""" def __init__(self, metadata, watch_clients, basefile): @@ -39,7 +47,8 @@ class XMLMetadataConfig(object): self.basedata = None self.basedir = metadata.data self.logger = metadata.logger - self.pseudo_monitor = isinstance(metadata.core.fam, Bcfg2.Server.FileMonitor.Pseudo) + self.pseudo_monitor = isinstance(metadata.core.fam, + Bcfg2.Server.FileMonitor.Pseudo) @property def xdata(self): @@ -56,7 +65,8 @@ class XMLMetadataConfig(object): def add_monitor(self, fname): """Add a fam monitor for an included file""" if self.should_monitor: - self.metadata.core.fam.AddMonitor("%s/%s" % (self.basedir, fname), self.metadata) + self.metadata.core.fam.AddMonitor("%s/%s" % (self.basedir, fname), + self.metadata) self.extras.append(fname) def load_xml(self): @@ -81,7 +91,8 @@ class XMLMetadataConfig(object): def write(self): """Write changes to xml back to disk.""" - self.write_xml("%s/%s" % (self.basedir, self.basefile), self.basedata) + self.write_xml("%s/%s" % (self.basedir, self.basefile), + self.basedata) def write_xml(self, fname, xmltree): """Write changes to xml back to disk.""" @@ -182,6 +193,7 @@ class ClientMetadata(object): return grp return '' + class MetadataQuery(object): def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): # resolver is set later @@ -201,6 +213,7 @@ class MetadataQuery(object): def all(self): return [self.by_name(name) for name in self.all_clients()] + class Metadata(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Metadata, Bcfg2.Server.Plugin.Statistics): @@ -220,12 +233,13 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, except: print("Unable to add file monitor for groups.xml or clients.xml") raise Bcfg2.Server.Plugin.PluginInitError - + self.clients_xml = XMLMetadataConfig(self, watch_clients, 'clients.xml') self.groups_xml = XMLMetadataConfig(self, watch_clients, 'groups.xml') self.states = {} if watch_clients: - self.states = {"groups.xml":False, "clients.xml":False} + self.states = {"groups.xml": False, + "clients.xml": False} self.addresses = {} self.auth = dict() self.clients = {} @@ -244,10 +258,11 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, self.session_cache = {} self.default = None self.pdirty = False - self.extra = {'groups.xml':[], 'clients.xml':[]} + self.extra = {'groups.xml': [], + 'clients.xml': []} self.password = core.password self.query = MetadataQuery(core.build_metadata, - lambda:self.clients.keys(), + lambda: list(self.clients.keys()), self.get_client_names_by_groups, self.get_client_names_by_profiles, self.get_all_group_names, @@ -288,7 +303,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, element = lxml.etree.SubElement(self.groups_xml.base_xdata.getroot(), "Group", name=group_name) - for key, val in attribs.iteritems(): + for key, val in list(attribs.items()): element.set(key, val) self.groups_xml.write() @@ -303,7 +318,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, self.logger.error("Unexpected error finding group") raise MetadataConsistencyError - for key, val in attribs.iteritems(): + for key, val in list(attribs.items()): xdict['xquery'][0].set(key, val) self.groups_xml.write_xml(xdict['filename'], xdict['xmltree']) @@ -330,7 +345,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, self.logger.error("Bundle \"%s\" already exists" % (bundle_name)) raise MetadataConsistencyError root.append(element) - group_tree = open(self.data + "/groups.xml","w") + group_tree = open(self.data + "/groups.xml", "w") fd = group_tree.fileno() while True: try: @@ -352,7 +367,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, self.logger.error("Bundle \"%s\" not found" % (bundle_name)) raise MetadataConsistencyError root.remove(node) - group_tree = open(self.data + "/groups.xml","w") + group_tree = open(self.data + "/groups.xml", "w") fd = group_tree.fileno() while True: try: @@ -384,7 +399,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, element = lxml.etree.SubElement(self.clients_xml.base_xdata.getroot(), "Client", name=client_name) - for key, val in attribs.iteritems(): + for key, val in list(attribs.items()): element.set(key, val) self.clients_xml.write() @@ -401,7 +416,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, raise MetadataConsistencyError node = xdict['xquery'][0] - [node.set(key, value) for key, value in attribs.items()] + [node.set(key, value) for key, value in list(attribs.items())] self.clients_xml.write_xml(xdict['filename'], xdict['xmltree']) def HandleEvent(self, event): @@ -506,17 +521,17 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, ggg)) [self.groups[group][0].add(bund) for bund in bundles] self.states['groups.xml'] = True - if False not in self.states.values(): + if False not in list(self.states.values()): # check that all client groups are real and complete - real = self.groups.keys() - for client in self.clients.keys(): + real = list(self.groups.keys()) + for client in list(self.clients.keys()): if self.clients[client] not in self.profiles: self.logger.error("Client %s set as nonexistent or incomplete group %s" \ % (client, self.clients[client])) self.logger.error("Removing client mapping for %s" % (client)) self.bad_clients[client] = self.clients[client] del self.clients[client] - for bclient in self.bad_clients.keys(): + for bclient in list(self.bad_clients.keys()): if self.bad_clients[bclient] in self.profiles: self.logger.info("Restored profile mapping for client %s" % bclient) self.clients[bclient] = self.bad_clients[bclient] @@ -525,7 +540,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def set_profile(self, client, profile, addresspair): """Set group parameter for provided client.""" self.logger.info("Asserting client %s profile to %s" % (client, profile)) - if False in self.states.values(): + if False in list(self.states.values()): raise MetadataRuntimeError if profile not in self.public: self.logger.error("Failed to set client %s to private group %s" % (client, profile)) @@ -579,7 +594,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def get_initial_metadata(self, client): """Return the metadata for a given client.""" - if False in self.states.values(): + if False in list(self.states.values()): raise MetadataRuntimeError client = client.lower() if client in self.aliases: @@ -604,7 +619,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, password = self.passwords[client] else: password = None - uuids = [item for item, value in self.uuid.iteritems() if value == client] + uuids = [item for item, value in list(self.uuid.items()) if value == client] if uuids: uuid = uuids[0] else: @@ -622,7 +637,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def get_all_group_names(self): all_groups = set() - [all_groups.update(g[1]) for g in self.groups.values()] + [all_groups.update(g[1]) for g in list(self.groups.values())] return all_groups def get_all_groups_in_category(self, category): @@ -632,11 +647,12 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, return all_groups def get_client_names_by_profiles(self, profiles): - return [client for client, profile in self.clients.iteritems() \ + return [client for client, profile in list(self.clients.items()) \ if profile in profiles] def get_client_names_by_groups(self, groups): - mdata = [self.core.build_metadata(client) for client in self.clients.keys()] + mdata = [self.core.build_metadata(client) + for client in list(self.clients.keys())] return [md.hostname for md in mdata if md.groups.issuperset(groups)] def merge_additional_groups(self, imd, groups): @@ -766,7 +782,6 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, xdict['xquery'][0].set('auth', 'cert') self.clients_xml.write_xml(xdict['filename'], xdict['xmltree']) - def viz(self, hosts, bundles, key, colors): """Admin mode viz support.""" groups_tree = lxml.etree.parse(self.data + "/groups.xml") @@ -775,7 +790,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, except lxml.etree.XIncludeError: self.logger.error("Failed to process XInclude for file %s" % dest) groups = groups_tree.getroot() - categories = {'default':'grey83'} + categories = {'default': 'grey83'} instances = {} viz_str = "" egroups = groups.findall("Group") + groups.findall('.//Groups/Group') @@ -787,12 +802,12 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, del categories[None] if hosts: clients = self.clients - for client, profile in clients.iteritems(): + for client, profile in list(clients.items()): if profile in instances: instances[profile].append(client) else: instances[profile] = [client] - for profile, clist in instances.iteritems(): + for profile, clist in list(instances.items()): clist.sort() viz_str += '''\t"%s-instances" [ label="%s", shape="record" ];\n''' \ % (profile, '|'.join(clist)) diff --git a/src/lib/Server/Plugins/NagiosGen.py b/src/lib/Server/Plugins/NagiosGen.py index 14277b63d..1724a1c8a 100644 --- a/src/lib/Server/Plugins/NagiosGen.py +++ b/src/lib/Server/Plugins/NagiosGen.py @@ -19,6 +19,7 @@ define host{ address %s ''' + class NagiosGen(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Generator): """NagiosGen is a Bcfg2 plugin that dynamically generates @@ -32,23 +33,23 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Generator.__init__(self) self.Entries = {'Path': - {'/etc/nagiosgen.status' : self.createhostconfig, + {'/etc/nagiosgen.status': self.createhostconfig, '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} - self.client_attrib = {'encoding':'ascii', - 'owner':'root', - 'group':'root', - 'type':'file', - 'perms':'0400'} - self.server_attrib = {'encoding':'ascii', - 'owner':'nagios', - 'group':'nagios', - 'type':'file', - 'perms':'0440'} + self.client_attrib = {'encoding': 'ascii', + 'owner': 'root', + 'group': 'root', + 'type': 'file', + 'perms': '0400'} + self.server_attrib = {'encoding': 'ascii', + 'owner': 'nagios', + 'group': 'nagios', + 'type': 'file', + 'perms': '0440'} def getparents(self, hostname): """Return parents for given hostname.""" - depends=[] + depends = [] if not os.path.isfile('%s/parents.xml' % (self.data)): return depends @@ -88,7 +89,7 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, host_config += '}\n' entry.text = host_config [entry.attrib.__setitem__(key, value) for \ - (key, value) in self.client_attrib.iteritems()] + (key, value) in list(self.client_attrib.items())] try: fileh = open("%s/%s-host.cfg" % \ (self.data, metadata.hostname), 'w') @@ -101,14 +102,14 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, def createserverconfig(self, entry, _): """Build monolithic server configuration file.""" - host_configs = glob.glob('%s/*-host.cfg' % self.data) + host_configs = glob.glob('%s/*-host.cfg' % self.data) group_configs = glob.glob('%s/*-group.cfg' % self.data) host_data = "" group_data = "" for host in host_configs: hostfile = open(host, 'r') - hostname=host.split('/')[-1].replace('-host.cfg','') - parents=self.getparents(hostname) + hostname = host.split('/')[-1].replace('-host.cfg', '') + parents = self.getparents(hostname) if parents: hostlines = hostfile.readlines() else: @@ -116,19 +117,19 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, hostfile.close() if parents: - hostdata='' - addparents=True + hostdata = '' + addparents = True for line in hostlines: - line=line.replace('\n','') + line = line.replace('\n', '') if 'parents' in line: - line+=','+','.join(parents) - addparents=False + line += ',' + ','.join(parents) + addparents = False if '}' in line: - line='' - hostdata+="%s\n" % line + line = '' + hostdata += "%s\n" % line if addparents: - hostdata+=" parents %s\n" % ','.join(parents) - hostdata+="}\n" + hostdata += " parents %s\n" % ','.join(parents) + hostdata += "}\n" host_data += hostdata for group in group_configs: @@ -139,7 +140,7 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, groupfile.close() entry.text = group_data + host_data [entry.attrib.__setitem__(key, value) for \ - (key, value) in self.server_attrib.iteritems()] + (key, value) in list(self.server_attrib.items())] try: fileh = open("%s/nagiosgen.cfg" % (self.data), 'w') fileh.write(group_data + host_data) diff --git a/src/lib/Server/Plugins/Ohai.py b/src/lib/Server/Plugins/Ohai.py index 0f7c7187f..6bd3edc34 100644 --- a/src/lib/Server/Plugins/Ohai.py +++ b/src/lib/Server/Plugins/Ohai.py @@ -37,12 +37,12 @@ class OhaiCache(object): try: data = open("%s/%s.json" % (self.dirname, item)).read() except: - raise KeyError, item + raise KeyError(item) self.cache[item] = json.loads(data) return self.cache[item] def __iter__(self): - data = self.cache.keys() + data = list(self.cache.keys()) data.extend([x[:-5] for x in os.listdir(self.dirname)]) return data.__iter__() @@ -50,7 +50,9 @@ class OhaiCache(object): class Ohai(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Probing, Bcfg2.Server.Plugin.Connector): - """The Ohai plugin is used to detect information about the client operating system.""" + """The Ohai plugin is used to detect information + about the client operating system. + """ name = 'Ohai' experimental = True diff --git a/src/lib/Server/Plugins/Packages.py b/src/lib/Server/Plugins/Packages.py index 438c1d5c0..a84d8dc70 100644 --- a/src/lib/Server/Plugins/Packages.py +++ b/src/lib/Server/Plugins/Packages.py @@ -22,14 +22,17 @@ import Bcfg2.Server.Plugin # build sources.list? # caching for yum + class NoData(Exception): pass + class SomeData(Exception): pass logger = logging.getLogger('Packages') + def source_from_xml(xsource): ret = dict([('rawurl', False), ('url', False)]) for key, tag in [('groups', 'Group'), ('components', 'Component'), @@ -60,6 +63,7 @@ def source_from_xml(xsource): ret['url'] += '/' return ret + def _fetch_url(url): if '@' in url: mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url) @@ -73,6 +77,7 @@ def _fetch_url(url): urllib2.install_opener(urllib2.build_opener(auth)) return urllib2.urlopen(url).read() + class Source(object): basegroups = [] @@ -135,7 +140,7 @@ class Source(object): agroups = ['global'] + [a for a in self.arches if a in meta.groups] vdict = dict() for agrp in agroups: - for key, value in self.provides[agrp].iteritems(): + for key, value in list(self.provides[agrp].items()): if key not in vdict: vdict[key] = set(value) else: @@ -193,6 +198,7 @@ class Source(object): return {'groups': copy.copy(self.groups), \ 'urls': [copy.deepcopy(url) for url in self.url_map]} + class YUMSource(Source): xp = '{http://linux.duke.edu/metadata/common}' rp = '{http://linux.duke.edu/metadata/rpm}' @@ -277,7 +283,7 @@ class YUMSource(Source): fdata = lxml.etree.parse(fname).getroot() self.parse_filelist(fdata, farch) # merge data - sdata = self.packages.values() + sdata = list(self.packages.values()) self.packages['global'] = copy.deepcopy(sdata.pop()) while sdata: self.packages['global'] = self.packages['global'].intersection(sdata.pop()) @@ -337,10 +343,10 @@ class YUMSource(Source): def get_vpkgs(self, metadata): rv = Source.get_vpkgs(self, metadata) - for arch, fmdata in self.filemap.iteritems(): + for arch, fmdata in list(self.filemap.items()): if arch not in metadata.groups and arch != 'global': continue - for filename, pkgs in fmdata.iteritems(): + for filename, pkgs in list(fmdata.items()): rv[filename] = pkgs return rv @@ -348,6 +354,7 @@ class YUMSource(Source): filtered = set([u for u in unknown if u.startswith('rpmlib')]) unknown.difference_update(filtered) + class APTSource(Source): basegroups = ['apt', 'debian', 'ubuntu', 'nexenta'] ptype = 'deb' @@ -449,7 +456,7 @@ class APTSource(Source): for barch in bdeps: self.deps[barch][pkgname] = bdeps[barch][pkgname] provided = set() - for bprovided in bprov.values(): + for bprovided in list(bprov.values()): provided.update(set(bprovided)) for prov in provided: prset = set() @@ -469,6 +476,7 @@ class APTSource(Source): pkg not in self.blacklist and \ (len(self.whitelist) == 0 or pkg in self.whitelist) + class PACSource(Source): basegroups = ['arch', 'parabola'] ptype = 'pacman' @@ -526,7 +534,7 @@ class PACSource(Source): bdeps[barch] = dict() bprov[barch] = dict() try: - print "try to read : " + fname + print("try to read : " + fname) tar = tarfile.open(fname, "r") reader = gzip.GzipFile(fname) except: @@ -536,7 +544,7 @@ class PACSource(Source): for tarinfo in tar: if tarinfo.isdir(): self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0]) - print "added : " + tarinfo.name.rsplit("-", 2)[0] + print("added : " + tarinfo.name.rsplit("-", 2)[0]) tar.close() self.deps['global'] = dict() @@ -556,7 +564,7 @@ class PACSource(Source): for barch in bdeps: self.deps[barch][pkgname] = bdeps[barch][pkgname] provided = set() - for bprovided in bprov.values(): + for bprovided in list(bprov.values()): provided.update(set(bprovided)) for prov in provided: prset = set() @@ -576,6 +584,7 @@ class PACSource(Source): pkg not in self.blacklist and \ (len(self.whitelist) == 0 or pkg in self.whitelist) + class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, @@ -614,7 +623,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, vpkgs = dict() for source in self.get_matching_sources(meta): s_vpkgs = source.get_vpkgs(meta) - for name, prov_set in s_vpkgs.iteritems(): + for name, prov_set in list(s_vpkgs.items()): if name not in vpkgs: vpkgs[name] = set(prov_set) else: @@ -726,7 +735,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, satisfied_vpkgs.add(current) elif [item for item in vpkg_cache[current] if item in packages]: if debug: - self.logger.debug("Packages: requirement %s satisfied by %s" % (current, [item for item in vpkg_cache[current] if item in packages])) + self.logger.debug("Packages: requirement %s satisfied by %s" % (current, + [item for item in vpkg_cache[current] + if item in packages])) satisfied_vpkgs.add(current) vpkgs.difference_update(satisfied_vpkgs) @@ -736,7 +747,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # allow use of virt through explicit specification, then fall back to forcing current on last pass if [item for item in vpkg_cache[current] if item in packages]: if debug: - self.logger.debug("Packages: requirement %s satisfied by %s" % (current, [item for item in vpkg_cache[current] if item in packages])) + self.logger.debug("Packages: requirement %s satisfied by %s" % (current, + [item for item in vpkg_cache[current] + if item in packages])) satisfied_both.add(current) elif current in input_requirements or final_pass: pkgs.add(current) diff --git a/src/lib/Server/Plugins/Pkgmgr.py b/src/lib/Server/Plugins/Pkgmgr.py index b58a7c91d..dc4a5f37f 100644 --- a/src/lib/Server/Plugins/Pkgmgr.py +++ b/src/lib/Server/Plugins/Pkgmgr.py @@ -7,15 +7,17 @@ import Bcfg2.Server.Plugin logger = logging.getLogger('Bcfg2.Plugins.Pkgmgr') + class FuzzyDict(dict): fuzzy = re.compile('(?P.*):(?P\S+(,\S+)*)') + def __getitem__(self, key): if isinstance(key, str): mdata = self.fuzzy.match(key) if mdata: return dict.__getitem__(self, mdata.groupdict()['name']) else: - print "got non-string key %s" % str(key) + print("got non-string key %s" % str(key)) return dict.__getitem__(self, key) def has_key(self, key): @@ -33,11 +35,14 @@ class FuzzyDict(dict): return default raise + class PNode(Bcfg2.Server.Plugin.INode): - """PNode has a list of packages available at a particular group intersection.""" - splitters = {'rpm':re.compile('^(.*/)?(?P[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \ + """PNode has a list of packages available at a + particular group intersection. + """ + splitters = {'rpm': re.compile('^(.*/)?(?P[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \ '(?P[\w\d\.]+-([\w\d\.]+))\.(?P\S+)\.rpm$'), - 'encap':re.compile('^(?P[\w-]+)-(?P[\w\d\.+-]+).encap.*$')} + 'encap': re.compile('^(?P[\w-]+)-(?P[\w\d\.+-]+).encap.*$')} ignore = ['Package'] def Match(self, metadata, data): @@ -54,41 +59,44 @@ class PNode(Bcfg2.Server.Plugin.INode): def __init__(self, data, pdict, parent=None): # copy local attributes to all child nodes if no local attribute exists - if not pdict.has_key('Package'): + if 'Package' not in pdict: pdict['Package'] = set() for child in data.getchildren(): - for attr in [key for key in data.attrib.keys() \ - if key != 'name' and not child.attrib.has_key(key)]: + for attr in [key for key in data.attrib.keys() + if key != 'name' and key not in child.attrib]: try: child.set(attr, data.get(attr)) except: # don't fail on things like comments and other immutable elements pass Bcfg2.Server.Plugin.INode.__init__(self, data, pdict, parent) - if not self.contents.has_key('Package'): + if 'Package' not in self.contents: self.contents['Package'] = FuzzyDict() for pkg in data.findall('./Package'): - if pkg.attrib.has_key('name') and pkg.get('name') not in pdict['Package']: + if 'name' in pkg.attrib and pkg.get('name') not in pdict['Package']: pdict['Package'].add(pkg.get('name')) if pkg.get('name') != None: self.contents['Package'][pkg.get('name')] = {} if pkg.getchildren(): self.contents['Package'][pkg.get('name')]['__children__'] \ = pkg.getchildren() - if pkg.attrib.has_key('simplefile'): + if 'simplefile' in pkg.attrib: pkg.set('url', "%s/%s" % (pkg.get('uri'), pkg.get('simplefile'))) self.contents['Package'][pkg.get('name')].update(pkg.attrib) else: - if pkg.attrib.has_key('file'): - if pkg.attrib.has_key('multiarch'): + if 'file' in pkg.attrib: + if 'multiarch' in pkg.attrib: archs = pkg.get('multiarch').split() srcs = pkg.get('srcs', pkg.get('multiarch')).split() - url = ' '.join(["%s/%s" % (pkg.get('uri'), pkg.get('file') % {'src':srcs[idx], 'arch':archs[idx]}) + url = ' '.join(["%s/%s" % (pkg.get('uri'), + pkg.get('file') % {'src':srcs[idx], + 'arch':archs[idx]}) for idx in range(len(archs))]) pkg.set('url', url) else: - pkg.set('url', '%s/%s' % (pkg.get('uri'), pkg.get('file'))) - if self.splitters.has_key(pkg.get('type')) and pkg.get('file') != None: + pkg.set('url', '%s/%s' % (pkg.get('uri'), + pkg.get('file'))) + if pkg.get('type') in self.splitters and pkg.get('file') != None: mdata = self.splitters[pkg.get('type')].match(pkg.get('file')) if not mdata: logger.error("Failed to match pkg %s" % pkg.get('file')) @@ -112,10 +120,13 @@ class PNode(Bcfg2.Server.Plugin.INode): class PkgSrc(Bcfg2.Server.Plugin.XMLSrc): - """PkgSrc files contain a PNode hierarchy that returns matching package entries.""" + """PkgSrc files contain a PNode hierarchy that + returns matching package entries. + """ __node__ = PNode __cacheobj__ = FuzzyDict + class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles package assignments.""" name = 'Pkgmgr' @@ -127,8 +138,8 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): def HandleEvent(self, event): '''Handle events and update dispatch table''' Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - for src in self.entries.values(): - for itype, children in src.items.iteritems(): + for src in list(self.entries.values()): + for itype, children in list(src.items.items()): for child in children: try: self.Entries[itype][child] = self.BindEntry @@ -149,7 +160,7 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): if inst.get('arch') not in arches] def HandlesEntry(self, entry, metadata): - return entry.tag == 'Package' and entry.get('name').split(':')[0] in self.Entries['Package'].keys() + return entry.tag == 'Package' and entry.get('name').split(':')[0] in list(self.Entries['Package'].keys()) def HandleEntry(self, entry, metadata): self.BindEntry(entry, metadata) diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index 57dd4f698..ea2e79ccc 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -6,8 +6,10 @@ import Bcfg2.Server.Plugin specific_probe_matcher = re.compile("(.*/)?(?P\S+)(.(?P[GH](\d\d)?)_\S+)") probe_matcher = re.compile("(.*/)?(?P\S+)") + class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") + def __init__(self, path, fam, encoding, plugin_name): fpattern = '[0-9A-Za-z_\-]+' self.plugin_name = plugin_name @@ -34,7 +36,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): if pname not in build: build[pname] = entry - for (name, entry) in build.iteritems(): + for (name, entry) in list(build.items()): probe = lxml.etree.Element('probe') probe.set('name', name.split('/')[-1]) probe.set('source', self.plugin_name) @@ -47,6 +49,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ret.append(probe) return ret + class Probes(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Probing, Bcfg2.Server.Plugin.Connector): @@ -80,7 +83,8 @@ class Probes(Bcfg2.Server.Plugin.Plugin, value=self.probedata[client][probe]) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(cx, "Group", name=group) - data = lxml.etree.tostring(top, encoding='UTF-8', xml_declaration=True, + data = lxml.etree.tostring(top, encoding='UTF-8', + xml_declaration=True, pretty_print='true') try: datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') @@ -98,7 +102,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.cgroups = {} for client in data.getchildren(): self.probedata[client.get('name')] = {} - self.cgroups[client.get('name')]=[] + self.cgroups[client.get('name')] = [] for pdata in client: if (pdata.tag == 'Probe'): self.probedata[client.get('name')][pdata.get('name')] = pdata.get('value') @@ -118,7 +122,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, def ReceiveDataItem(self, client, data): """Receive probe results pertaining to client.""" - if not self.cgroups.has_key(client.hostname): + if client.hostname not in self.cgroups: self.cgroups[client.hostname] = [] if data.text == None: self.logger.error("Got null response to probe %s from %s" % \ @@ -139,9 +143,9 @@ class Probes(Bcfg2.Server.Plugin.Plugin, dlines.remove(line) dtext = "\n".join(dlines) try: - self.probedata[client.hostname].update({data.get('name'):dtext}) + self.probedata[client.hostname].update({data.get('name'): dtext}) except KeyError: - self.probedata[client.hostname] = {data.get('name'):dtext} + self.probedata[client.hostname] = {data.get('name'): dtext} def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index 96a444875..77c5e008f 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -73,7 +73,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, """Build memory cache of the ssh known hosts file.""" if not self.__skn: self.__skn = "\n".join([value.data for key, value in \ - self.entries.iteritems() if \ + list(self.entries.items()) if \ key.endswith('.static')]) names = dict() # if no metadata is registered yet, defer @@ -103,7 +103,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, continue names[cmeta.hostname] = sorted(names[cmeta.hostname]) # now we have our name cache - pubkeys = [pubk for pubk in self.entries.keys() \ + pubkeys = [pubk for pubk in list(self.entries.keys()) \ if pubk.find('.pub.H_') != -1] pubkeys.sort() badnames = set() @@ -131,7 +131,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, if event and event.filename.endswith('.static'): self.skn = False if not self.__skn: - if (len(self.entries.keys())) >= (len(os.listdir(self.data))-1): + if (len(list(self.entries.keys()))) >= (len(os.listdir(self.data)) - 1): _ = self.skn def HandlesEntry(self, entry, _): @@ -205,26 +205,26 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for hostkey in hostkeys: entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % ( self.entries[hostkey].data) - permdata = {'owner':'root', - 'group':'root', - 'type':'file', - 'perms':'0644'} + permdata = {'owner': 'root', + 'group': 'root', + 'type': 'file', + 'perms': '0644'} [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] def build_hk(self, entry, metadata): """This binds host key data into entries.""" client = metadata.hostname filename = "%s.H_%s" % (entry.get('name').split('/')[-1], client) - if filename not in self.entries.keys(): + if filename not in list(self.entries.keys()): self.GenerateHostKeys(client) if not filename in self.entries: self.logger.error("%s still not registered" % filename) raise Bcfg2.Server.Plugin.PluginExecutionError keydata = self.entries[filename].data - permdata = {'owner':'root', - 'group':'root', - 'type':'file', - 'perms':'0600'} + permdata = {'owner': 'root', + 'group': 'root', + 'type': 'file', + 'perms': '0600'} if entry.get('name')[-4:] == '.pub': permdata['perms'] = '0644' [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] @@ -245,7 +245,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: keytype = 'rsa1' - if hostkey not in self.entries.keys(): + if hostkey not in list(self.entries.keys()): fileloc = "%s/%s" % (self.data, hostkey) publoc = self.data + '/' + ".".join([hostkey.split('.')[0], 'pub', @@ -257,8 +257,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, shutil.copy(temploc, fileloc) shutil.copy("%s.pub" % temploc, publoc) self.AddEntry(hostkey) - self.AddEntry(".".join([hostkey.split('.')[0]]+['pub', "H_%s" \ - % client])) + self.AddEntry(".".join([hostkey.split('.')[0]] + ['pub', "H_%s" \ + % client])) try: os.unlink(temploc) os.unlink("%s.pub" % temploc) @@ -277,7 +277,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, try: open(filename, 'w').write(entry['text']) if log: - print "Wrote file %s" % filename + print("Wrote file %s" % filename) except KeyError: self.logger.error("Failed to pull %s. This file does not currently " "exist on the client" % entry.get('name')) diff --git a/src/lib/Server/Plugins/SSLCA.py b/src/lib/Server/Plugins/SSLCA.py index 1c9e1b59d..00f67834d 100644 --- a/src/lib/Server/Plugins/SSLCA.py +++ b/src/lib/Server/Plugins/SSLCA.py @@ -41,14 +41,14 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): if event.filename.endswith('.xml'): if action in ['exists', 'created', 'changed']: if event.filename.endswith('key.xml'): - key_spec = dict(lxml.etree.parse(epath).find('Key').items()) + key_spec = dict(list(lxml.etree.parse(epath).find('Key').items())) self.key_specs[ident] = { 'bits': key_spec.get('bits', 2048), 'type': key_spec.get('type', 'rsa') } self.Entries['Path'][ident] = self.get_key elif event.filename.endswith('cert.xml'): - cert_spec = dict(lxml.etree.parse(epath).find('Cert').items()) + cert_spec = dict(list(lxml.etree.parse(epath).find('Cert').items())) ca = cert_spec.get('ca', 'default') self.cert_specs[ident] = { 'ca': ca, @@ -64,7 +64,7 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): } cp = ConfigParser() cp.read(self.core.cfile) - self.CAs[ca] = dict(cp.items('sslca_'+ca)) + self.CAs[ca] = dict(cp.items('sslca_' + ca)) self.Entries['Path'][ident] = self.get_cert if action == 'deleted': if ident in self.Entries['Path']: @@ -99,12 +99,14 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): # check if we already have a hostfile, or need to generate a new key # TODO: verify key fits the specs path = entry.get('name') - filename = "".join([path, '/', path.rsplit('/', 1)[1], '.H_', metadata.hostname]) - if filename not in self.entries.keys(): + filename = "".join([path, '/', path.rsplit('/', 1)[1], + '.H_', metadata.hostname]) + if filename not in list(self.entries.keys()): key = self.build_key(filename, entry, metadata) open(self.data + filename, 'w').write(key) entry.text = key - self.entries[filename] = self.__child__("%s%s" % (self.data, filename)) + self.entries[filename] = self.__child__("%s%s" % (self.data, + filename)) self.entries[filename].HandleEvent() else: entry.text = self.entries[filename].data @@ -135,23 +137,28 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] path = entry.get('name') - filename = "".join([path, '/', path.rsplit('/', 1)[1], '.H_', metadata.hostname]) + filename = "".join([path, '/', path.rsplit('/', 1)[1], + '.H_', metadata.hostname]) # first - ensure we have a key to work with key = self.cert_specs[entry.get('name')].get('key') - key_filename = "".join([key, '/', key.rsplit('/', 1)[1], '.H_', metadata.hostname]) + key_filename = "".join([key, '/', key.rsplit('/', 1)[1], + '.H_', metadata.hostname]) if key_filename not in self.entries: e = lxml.etree.Element('Path') e.attrib['name'] = key self.core.Bind(e, metadata) # check if we have a valid hostfile - if filename in self.entries.keys() and self.verify_cert(filename, key_filename, entry): + if filename in list(self.entries.keys()) and self.verify_cert(filename, + key_filename, + entry): entry.text = self.entries[filename].data else: cert = self.build_cert(key_filename, entry, metadata) open(self.data + filename, 'w').write(cert) - self.entries[filename] = self.__child__("%s%s" % (self.data, filename)) + self.entries[filename] = self.__child__("%s%s" % (self.data, + filename)) self.entries[filename].HandleEvent() entry.text = cert @@ -188,7 +195,6 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): return True return False - def build_cert(self, key_filename, entry, metadata): """ creates a new certificate according to the specification @@ -200,9 +206,14 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): days = self.cert_specs[entry.get('name')]['days'] passphrase = self.CAs[ca].get('passphrase') if passphrase: - cmd = "openssl ca -config %s -in %s -days %s -batch -passin pass:%s" % (ca_config, req, days, passphrase) + cmd = "openssl ca -config %s -in %s -days %s -batch -passin pass:%s" % (ca_config, + req, + days, + passphrase) else: - cmd = "openssl ca -config %s -in %s -days %s -batch" % (ca_config, req, days) + cmd = "openssl ca -config %s -in %s -days %s -batch" % (ca_config, + req, + days) cert = Popen(cmd, shell=True, stdout=PIPE).stdout.read() try: os.unlink(req_config) @@ -234,7 +245,7 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): }, 'alt_names': {} } - for section in defaults.keys(): + for section in list(defaults.keys()): cp.add_section(section) for key in defaults[section]: cp.set(section, key, defaults[section][key]) @@ -242,7 +253,7 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): altnames = list(metadata.aliases) altnames.append(metadata.hostname) for altname in altnames: - cp.set('alt_names', 'DNS.'+str(x), altname) + cp.set('alt_names', 'DNS.' + str(x), altname) x += 1 for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']: if self.cert_specs[entry.get('name')][item]: @@ -259,6 +270,9 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): req = tempfile.mkstemp()[1] days = self.cert_specs[entry.get('name')]['days'] key = self.data + key_filename - cmd = "openssl req -new -config %s -days %s -key %s -text -out %s" % (req_config, days, key, req) + cmd = "openssl req -new -config %s -days %s -key %s -text -out %s" % (req_config, + days, + key, + req) res = Popen(cmd, shell=True, stdout=PIPE).stdout.read() return req diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py index a4489ae95..a1f72ba3e 100644 --- a/src/lib/Server/Plugins/Snapshots.py +++ b/src/lib/Server/Plugins/Snapshots.py @@ -24,6 +24,7 @@ datafields = { 'SymLink': ['to'], } + def build_snap_ent(entry): basefields = [] if entry.tag in ['Package', 'Service']: @@ -119,12 +120,12 @@ class Snapshots(Bcfg2.Server.Plugin.Statistics, data['name'] = ename extra[entry.tag][ename] = data else: - print "extra", entry.tag, entry.get('name') + print("extra", entry.tag, entry.get('name')) t2 = time.time() snap = Snapshot.from_data(self.session, correct, revision, metadata, entries, extra) self.session.add(snap) self.session.commit() t3 = time.time() - logger.info("Snapshot storage took %fs" % (t3-t2)) + logger.info("Snapshot storage took %fs" % (t3 - t2)) return True -- cgit v1.2.3-1-g7c22 From e17608a61575ec9d50cfed34a80540257254840b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 5 Apr 2011 14:48:41 -0500 Subject: Hostbase: PY3K + PEP8 fixes Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Hostbase.py | 79 +++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 35 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Hostbase.py b/src/lib/Server/Plugins/Hostbase.py index 65992596d..b1cbb9dfc 100644 --- a/src/lib/Server/Plugins/Hostbase.py +++ b/src/lib/Server/Plugins/Hostbase.py @@ -1,4 +1,7 @@ -'''This file provides the Hostbase plugin. It manages dns/dhcp/nis host information''' +""" +This file provides the Hostbase plugin. +It manages dns/dhcp/nis host information +""" __revision__ = '$Revision$' import os @@ -13,6 +16,7 @@ from django.db import connection import re import cStringIO + class Hostbase(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Structure, Bcfg2.Server.Plugin.Generator): @@ -23,24 +27,29 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, filepath = '/my/adm/hostbase/files/bind' def __init__(self, core, datastore): - + self.ready = False Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Structure.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) - files = ['zone.tmpl', 'reversesoa.tmpl', 'named.tmpl', 'reverseappend.tmpl', - 'dhcpd.tmpl', 'hosts.tmpl', 'hostsappend.tmpl'] + files = ['zone.tmpl', + 'reversesoa.tmpl', + 'named.tmpl', + 'reverseappend.tmpl', + 'dhcpd.tmpl', + 'hosts.tmpl', + 'hostsappend.tmpl'] self.filedata = {} self.dnsservers = [] self.dhcpservers = [] - self.templates = {'zone':loader.get_template('zone.tmpl'), - 'reversesoa':loader.get_template('reversesoa.tmpl'), - 'named':loader.get_template('named.tmpl'), - 'namedviews':loader.get_template('namedviews.tmpl'), - 'reverseapp':loader.get_template('reverseappend.tmpl'), - 'dhcp':loader.get_template('dhcpd.tmpl'), - 'hosts':loader.get_template('hosts.tmpl'), - 'hostsapp':loader.get_template('hostsappend.tmpl'), + self.templates = {'zone': loader.get_template('zone.tmpl'), + 'reversesoa': loader.get_template('reversesoa.tmpl'), + 'named': loader.get_template('named.tmpl'), + 'namedviews': loader.get_template('namedviews.tmpl'), + 'reverseapp': loader.get_template('reverseappend.tmpl'), + 'dhcp': loader.get_template('dhcpd.tmpl'), + 'hosts': loader.get_template('hosts.tmpl'), + 'hostsapp': loader.get_template('hostsappend.tmpl'), } self.Entries['ConfigFile'] = {} self.__rmi__ = ['rebuildState'] @@ -48,14 +57,17 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, self.rebuildState(None) except: raise PluginInitError - + def FetchFile(self, entry, metadata): """Return prebuilt file data.""" fname = entry.get('name').split('/')[-1] if not fname in self.filedata: raise PluginExecutionError - perms = {'owner':'root', 'group':'root', 'perms':'644'} - [entry.attrib.__setitem__(key, value) for (key, value) in perms.iteritems()] + perms = {'owner': 'root', + 'group': 'root', + 'perms': '644'} + [entry.attrib.__setitem__(key, value) + for (key, value) in list(perms.items())] entry.text = self.filedata[fname] def BuildStructures(self, metadata): @@ -173,29 +185,29 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, ('', 'MX', host[4], host[5])) elif not prevhost[5] == host[5]: zonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) + ('', 'MX', host[4], host[5])) if host[6] == 'global': externalzonefile.write("%-32s%-10s%-3s%s.\n" % ('', 'MX', host[4], host[5])) - + if host[3]: try: if host[3].split(".", 1)[1] == zone[1]: cnames.write("%-32s%-10s%-32s\n" % (host[3].split(".", 1)[0], - 'CNAME',host[2].split(".", 1)[0])) + 'CNAME', host[2].split(".", 1)[0])) if host[6] == 'global': cnamesexternal.write("%-32s%-10s%-32s\n" % (host[3].split(".", 1)[0], - 'CNAME',host[2].split(".", 1)[0])) + 'CNAME', host[2].split(".", 1)[0])) else: cnames.write("%-32s%-10s%-32s\n" % - (host[3]+".", + (host[3] + ".", 'CNAME', host[2].split(".", 1)[0])) if host[6] == 'global': cnamesexternal.write("%-32s%-10s%-32s\n" % - (host[3]+".", + (host[3] + ".", 'CNAME', host[2].split(".", 1)[0])) @@ -215,9 +227,9 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, cursor.execute("SELECT * FROM hostbase_zone WHERE zone LIKE \'%%.rev\' AND zone <> \'.rev\'") reversezones = cursor.fetchall() - + reversenames = [] - for reversezone in reversezones: + for reversezone in reversezones: cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id WHERE z.zone_id = \'%s\'""" % reversezone[0]) @@ -236,7 +248,7 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, subnet = reversezone[1].split(".") subnet.reverse() - reversenames.append((reversezone[1].rstrip('.rev'),".".join(subnet[1:]))) + reversenames.append((reversezone[1].rstrip('.rev'), ".".join(subnet[1:]))) for filename in reversenames: cursor.execute(""" @@ -268,13 +280,13 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, 'hosts': hosts, 'inaddr': origin[0], 'fileorigin': filename[0], - }) + }) zonefile.write(self.templates['reverseapp'].render(context)) context = Context({ 'hosts': hosts_external, 'inaddr': origin[0], 'fileorigin': filename[0], - }) + }) externalzonefile.write(self.templates['reverseapp'].render(context)) else: originlist = [filename[0]] @@ -289,7 +301,7 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, 'hosts': hosts, 'inaddr': filename[0], 'fileorigin': None, - }) + }) zonefile.write(self.templates['reverseapp'].render(context)) context = Context({ 'hosts': hosts_external, @@ -308,13 +320,12 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, context = Context({ 'zones': zones, 'reverses': reversenames, - }) + }) self.filedata['named.conf'] = self.templates['named'].render(context) self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf'] = self.FetchFile self.filedata['named.conf.views'] = self.templates['namedviews'].render(context) self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf.views'] = self.FetchFile - def buildDHCP(self): """Pre-build dhcpd.conf and stash in the filedata table.""" @@ -362,7 +373,6 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, self.filedata['dhcpd.conf'] = self.templates['dhcp'].render(context) self.Entries['ConfigFile']['/my/adm/hostbase/files/dhcpd.conf'] = self.FetchFile - def buildHosts(self): """Pre-build and stash /etc/hosts file.""" @@ -490,7 +500,7 @@ Name Room User Type def buildHostsLPD(self): """Creates the /mcs/etc/hosts.lpd file""" - + # this header needs to be changed to be more generic header = """+@machines +@all-machines @@ -503,7 +513,7 @@ delphi.esh.anl.gov anlcv1.ctd.anl.gov anlvms.ctd.anl.gov olivia.ctd.anl.gov\n\n""" - + cursor = connection.cursor() cursor.execute(""" SELECT hostname FROM hostbase_host WHERE netgroup=\"red\" AND status = 'active' @@ -534,7 +544,6 @@ olivia.ctd.anl.gov\n\n""" self.filedata['hosts.lpd'] = hostslpdfile self.Entries['ConfigFile']['/mcs/etc/hosts.lpd'] = self.FetchFile - def buildNetgroups(self): """Makes the *-machine files""" header = """################################################################### @@ -557,11 +566,11 @@ olivia.ctd.anl.gov\n\n""" nameslist = cursor.fetchall() # gets the first host and initializes the hash hostdata = nameslist[0] - netgroups = {hostdata[2]:[hostdata[0]]} + netgroups = {hostdata[2]: [hostdata[0]]} for row in nameslist: # if new netgroup, create it if row[2] not in netgroups: - netgroups.update({row[2]:[]}) + netgroups.update({row[2]: []}) # if it belongs in the netgroup and has multiple interfaces, put them in if hostdata[0] == row[0] and row[3]: netgroups[row[2]].append(row[1]) -- cgit v1.2.3-1-g7c22 From 742fc83dcdb82639b97723ce4cbfade75fb1aa71 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 6 Apr 2011 19:34:30 -0500 Subject: bcfg2-admin: PY3K + PEP8 fixes Signed-off-by: Sol Jerome --- src/lib/Server/Admin/Backup.py | 3 +- src/lib/Server/Admin/Bundle.py | 50 +++++++++-------- src/lib/Server/Admin/Client.py | 15 ++--- src/lib/Server/Admin/Compare.py | 59 +++++++++++--------- src/lib/Server/Admin/Group.py | 15 ++--- src/lib/Server/Admin/Init.py | 116 +++++++++++++++++++++++++++++++-------- src/lib/Server/Admin/Perf.py | 11 ++-- src/lib/Server/Admin/Pull.py | 46 ++++++++++------ src/lib/Server/Admin/Query.py | 15 ++--- src/lib/Server/Admin/Reports.py | 74 +++++++++++++------------ src/lib/Server/Admin/Tidy.py | 14 +++-- src/lib/Server/Admin/Viz.py | 13 +++-- src/lib/Server/Admin/Xcmd.py | 20 ++++--- src/lib/Server/Admin/__init__.py | 25 ++++++--- 14 files changed, 296 insertions(+), 180 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Backup.py b/src/lib/Server/Admin/Backup.py index fefc9fc9e..9bd644ff9 100644 --- a/src/lib/Server/Admin/Backup.py +++ b/src/lib/Server/Admin/Backup.py @@ -5,6 +5,7 @@ import tarfile import Bcfg2.Server.Admin import Bcfg2.Options + class Backup(Bcfg2.Server.Admin.MetadataCore): __shorthelp__ = "Make a backup of the Bcfg2 repository" __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin backup\n") @@ -29,4 +30,4 @@ class Backup(Bcfg2.Server.Admin.MetadataCore): out = tarfile.open(self.datastore + '/' + filename, mode=mode) out.add(self.datastore, os.path.basename(self.datastore)) out.close() - print "Archive %s was stored under %s" % (filename, self.datastore) + print("Archive %s was stored under %s" % (filename, self.datastore)) diff --git a/src/lib/Server/Admin/Bundle.py b/src/lib/Server/Admin/Bundle.py index 96a7ba59d..9b2a71783 100644 --- a/src/lib/Server/Admin/Bundle.py +++ b/src/lib/Server/Admin/Bundle.py @@ -6,11 +6,11 @@ import Bcfg2.Server.Admin import Bcfg2.Options from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError + class Bundle(Bcfg2.Server.Admin.MetadataCore): __shorthelp__ = "Create or delete bundle entries" - __longhelp__ = (__shorthelp__ + #"\n\nbcfg2-admin bundle add " - #"\n\nbcfg2-admin bundle del " - "\n\nbcfg2-admin bundle list-xml" + # TODO: add/del functions + __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin bundle list-xml" "\nbcfg2-admin bundle list-genshi" "\nbcfg2-admin bundle show\n") __usage__ = ("bcfg2-admin bundle [options] [add|del] [group]") @@ -21,7 +21,7 @@ class Bundle(Bcfg2.Server.Admin.MetadataCore): def __call__(self, args): Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - reg='((?:[a-z][a-z\\.\\d\\-]+)\\.(?:[a-z][a-z\\-]+))(?![\\w\\.])' + reg = '((?:[a-z][a-z\\.\\d\\-]+)\\.(?:[a-z][a-z\\-]+))(?![\\w\\.])' # Get all bundles out of the Bundle/ directory opts = {'repo': Bcfg2.Options.SERVER_REPOSITORY} @@ -38,31 +38,31 @@ class Bundle(Bcfg2.Server.Admin.MetadataCore): # try: # self.metadata.add_bundle(args[1]) # except MetadataConsistencyError: -# print "Error in adding bundle." +# print("Error in adding bundle.") # raise SystemExit(1) # elif args[0] in ['delete', 'remove', 'del', 'rm']: # try: # self.metadata.remove_bundle(args[1]) # except MetadataConsistencyError: -# print "Error in deleting bundle." +# print("Error in deleting bundle.") # raise SystemExit(1) # Lists all available xml bundles elif args[0] in ['list-xml', 'ls-xml']: bundle_name = [] for bundle_path in xml_list: - rg = re.compile(reg,re.IGNORECASE|re.DOTALL) + rg = re.compile(reg, re.IGNORECASE | re.DOTALL) bundle_name.append(rg.search(bundle_path).group(1)) for bundle in bundle_name: - print bundle.split('.')[0] + print(bundle.split('.')[0]) # Lists all available genshi bundles elif args[0] in ['list-genshi', 'ls-gen']: bundle_name = [] for bundle_path in genshi_list: - rg = re.compile(reg,re.IGNORECASE|re.DOTALL) + rg = re.compile(reg, re.IGNORECASE | re.DOTALL) bundle_name.append(rg.search(bundle_path).group(1)) for bundle in bundle_name: - print bundle.split('.')[0] - # Shows a list of all available bundles and prints bundle + print(bundle.split('.')[0]) + # Shows a list of all available bundles and prints bundle # details after the user choose one bundle. # FIXME: Add support for detailed output of genshi bundles # FIXME: This functionality is almost identical with @@ -71,32 +71,34 @@ class Bundle(Bcfg2.Server.Admin.MetadataCore): bundle_name = [] bundle_list = xml_list + genshi_list for bundle_path in bundle_list: - rg = re.compile(reg,re.IGNORECASE|re.DOTALL) + rg = re.compile(reg, re.IGNORECASE | re.DOTALL) bundle_name.append(rg.search(bundle_path).group(1)) text = "Available bundles (Number of bundles: %s)" % \ (len(bundle_list)) - print text - print "%s" % (len(text) * "-") + print(text) + print("%s" % (len(text) * "-")) for i in range(len(bundle_list)): - print "[%i]\t%s" % (i, bundle_name[i]) - print "Enter the line number of a bundle for details:", - lineno = raw_input() + print("[%i]\t%s" % (i, bundle_name[i])) + try: + lineno = raw_input("Enter the line number of a bundle for details: ") + except NameError: + lineno = input("Enter the line number of a bundle for details: ") if int(lineno) >= int(len(bundle_list)): - print "No line with this number." + print("No line with this number.") else: if '%s/Bundler/%s' % \ (repo, bundle_name[int(lineno)]) in genshi_list: - print "Detailed output for *.genshi bundles is not supported." + print("Detailed output for *.genshi bundles is not supported.") else: - print 'Details for the "%s" bundle:' % \ - (bundle_name[int(lineno)].split('.')[0]) + print('Details for the "%s" bundle:' % \ + (bundle_name[int(lineno)].split('.')[0])) tree = lxml.etree.parse(bundle_list[int(lineno)]) #Prints bundle content - #print lxml.etree.tostring(tree) + #print(lxml.etree.tostring(tree)) names = ['Action', 'Package', 'Path', 'Service'] for name in names: for node in tree.findall("//" + name): - print "%s:\t%s" % (name, node.attrib["name"]) + print("%s:\t%s" % (name, node.attrib["name"])) else: - print "No command specified" + print("No command specified") raise SystemExit(1) diff --git a/src/lib/Server/Admin/Client.py b/src/lib/Server/Admin/Client.py index 08bd34151..3af25b15a 100644 --- a/src/lib/Server/Admin/Client.py +++ b/src/lib/Server/Admin/Client.py @@ -2,6 +2,7 @@ import lxml.etree import Bcfg2.Server.Admin from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError + class Client(Bcfg2.Server.Admin.MetadataCore): __shorthelp__ = "Create, delete, or modify client entries" __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin client add " @@ -27,13 +28,13 @@ class Client(Bcfg2.Server.Admin.MetadataCore): attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', 'location', 'secure', 'address']: - print "Attribute %s unknown" % attr + print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val try: self.metadata.add_client(args[1], attr_d) except MetadataConsistencyError: - print "Error in adding client" + print("Error in adding client") raise SystemExit(1) elif args[0] in ['update', 'up']: attr_d = {} @@ -41,24 +42,24 @@ class Client(Bcfg2.Server.Admin.MetadataCore): attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', 'location', 'secure', 'address']: - print "Attribute %s unknown" % attr + print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val try: self.metadata.update_client(args[1], attr_d) except MetadataConsistencyError: - print "Error in updating client" + print("Error in updating client") raise SystemExit(1) elif args[0] in ['delete', 'remove', 'del', 'rm']: try: self.metadata.remove_client(args[1]) except MetadataConsistencyError: - print "Error in deleting client" + print("Error in deleting client") raise SystemExit(1) elif args[0] in ['list', 'ls']: tree = lxml.etree.parse(self.metadata.data + "/clients.xml") for node in tree.findall("//Client"): - print node.attrib["name"] + print(node.attrib["name"]) else: - print "No command specified" + print("No command specified") raise SystemExit(1) diff --git a/src/lib/Server/Admin/Compare.py b/src/lib/Server/Admin/Compare.py index f97233b0e..4c751b55a 100644 --- a/src/lib/Server/Admin/Compare.py +++ b/src/lib/Server/Admin/Compare.py @@ -1,6 +1,9 @@ -import lxml.etree, os +import lxml.etree +import os + import Bcfg2.Server.Admin + class Compare(Bcfg2.Server.Admin.Mode): __shorthelp__ = ("Determine differences between files or " "directories of client specification instances") @@ -11,30 +14,30 @@ class Compare(Bcfg2.Server.Admin.Mode): def __init__(self, configfile): Bcfg2.Server.Admin.Mode.__init__(self, configfile) - self.important = {'Package':['name', 'version'], - 'Service':['name', 'status'], - 'Directory':['name', 'owner', 'group', 'perms'], - 'SymLink':['name', 'to'], - 'ConfigFile':['name', 'owner', 'group', 'perms'], - 'Permissions':['name', 'perms'], - 'PostInstall':['name']} + self.important = {'Package': ['name', 'version'], + 'Service': ['name', 'status'], + 'Directory': ['name', 'owner', 'group', 'perms'], + 'SymLink': ['name', 'to'], + 'ConfigFile': ['name', 'owner', 'group', 'perms'], + 'Permissions': ['name', 'perms'], + 'PostInstall': ['name']} def compareStructures(self, new, old): for child in new.getchildren(): equiv = old.xpath('%s[@name="%s"]' % (child.tag, child.get('name'))) if child.tag in self.important: - print "tag type %s not handled" % (child.tag) + print("tag type %s not handled" % (child.tag)) continue if len(equiv) == 0: - print ("didn't find matching %s %s" % - (child.tag, child.get('name'))) + print("didn't find matching %s %s" % + (child.tag, child.get('name'))) continue elif len(equiv) >= 1: if child.tag == 'ConfigFile': if child.text != equiv[0].text: - print " %s %s contents differ" \ - % (child.tag, child.get('name')) + print(" %s %s contents differ" \ + % (child.tag, child.get('name'))) continue noattrmatch = [field for field in self.important[child.tag] if \ child.get(field) != equiv[0].get(field)] @@ -42,8 +45,8 @@ class Compare(Bcfg2.Server.Admin.Mode): new.remove(child) old.remove(equiv[0]) else: - print " %s %s attributes %s do not match" % \ - (child.tag, child.get('name'), noattrmatch) + print(" %s %s attributes %s do not match" % \ + (child.tag, child.get('name'), noattrmatch)) if len(old.getchildren()) == 0 and len(new.getchildren()) == 0: return True if new.tag == 'Independent': @@ -59,24 +62,26 @@ class Compare(Bcfg2.Server.Admin.Mode): newl.remove(entry) oldl.remove(entry) for entry in both: - print " %s differs (in bundle %s)" % (entry, name) + print(" %s differs (in bundle %s)" % (entry, name)) for entry in oldl: - print " %s only in old configuration (in bundle %s)" % (entry, name) + print(" %s only in old configuration (in bundle %s)" % (entry, + name)) for entry in newl: - print " %s only in new configuration (in bundle %s)" % (entry, name) + print(" %s only in new configuration (in bundle %s)" % (entry, + name)) return False def compareSpecifications(self, path1, path2): try: new = lxml.etree.parse(path1).getroot() except IOError: - print "Failed to read %s" % (path1) + print("Failed to read %s" % (path1)) raise SystemExit(1) try: old = lxml.etree.parse(path2).getroot() except IOError: - print "Failed to read %s" % (path2) + print("Failed to read %s" % (path2)) raise SystemExit(1) for src in [new, old]: @@ -88,7 +93,7 @@ class Compare(Bcfg2.Server.Admin.Mode): for bundle in new.findall('./Bundle'): equiv = old.xpath('Bundle[@name="%s"]' % (bundle.get('name'))) if len(equiv) == 0: - print "couldnt find matching bundle for %s" % bundle.get('name') + print("couldnt find matching bundle for %s" % bundle.get('name')) continue if len(equiv) == 1: if self.compareStructures(bundle, equiv[0]): @@ -98,7 +103,7 @@ class Compare(Bcfg2.Server.Admin.Mode): else: rcs.append(False) else: - print "Unmatched bundle %s" % (bundle.get('name')) + print("Unmatched bundle %s" % (bundle.get('name'))) rcs.append(False) i1 = new.find('./Independent') i2 = old.find('./Independent') @@ -120,18 +125,18 @@ class Compare(Bcfg2.Server.Admin.Mode): (oldd, newd) = args (old, new) = [os.listdir(spot) for spot in args] for item in old: - print "Entry:", item + print("Entry:", item) state = self.__call__([oldd + '/' + item, newd + '/' + item]) new.remove(item) if state: - print "Entry:", item, "good" + print("Entry:", item, "good") else: - print "Entry:", item, "bad" + print("Entry:", item, "bad") if new: - print "new has extra entries", new + print("new has extra entries", new) return try: (old, new) = args except IndexError: - print self.__call__.__doc__ + print(self.__call__.__doc__) raise SystemExit(1) diff --git a/src/lib/Server/Admin/Group.py b/src/lib/Server/Admin/Group.py index 4b2db28ec..1c5d0c12f 100644 --- a/src/lib/Server/Admin/Group.py +++ b/src/lib/Server/Admin/Group.py @@ -2,6 +2,7 @@ import lxml.etree import Bcfg2.Server.Admin from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError + class Group(Bcfg2.Server.Admin.MetadataCore): __shorthelp__ = "Create, delete, or modify group entries" __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin group add " @@ -28,13 +29,13 @@ class Group(Bcfg2.Server.Admin.MetadataCore): if attr not in ['profile', 'public', 'default', 'name', 'auth', 'toolset', 'category', 'comment']: - print "Attribute %s unknown" % attr + print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val try: self.metadata.add_group(args[1], attr_d) except MetadataConsistencyError: - print "Error in adding group" + print("Error in adding group") raise SystemExit(1) elif args[0] in ['update', 'up']: attr_d = {} @@ -43,24 +44,24 @@ class Group(Bcfg2.Server.Admin.MetadataCore): if attr not in ['profile', 'public', 'default', 'name', 'auth', 'toolset', 'category', 'comment']: - print "Attribute %s unknown" % attr + print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val try: self.metadata.update_group(args[1], attr_d) except MetadataConsistencyError: - print "Error in updating group" + print("Error in updating group") raise SystemExit(1) elif args[0] in ['delete', 'remove', 'del', 'rm']: try: self.metadata.remove_group(args[1]) except MetadataConsistencyError: - print "Error in deleting group" + print("Error in deleting group") raise SystemExit(1) elif args[0] in ['list', 'ls']: tree = lxml.etree.parse(self.metadata.data + "/groups.xml") for node in tree.findall("//Group"): - print node.attrib["name"] + print(node.attrib["name"]) else: - print "No command specified" + print("No command specified") raise SystemExit(1) diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index 8f54b836e..eddbd732a 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -137,20 +137,33 @@ def create_key(hostname, keypath, certpath, country, state, location): keypath, certpath)) subprocess.call((ccstr), shell=True) - os.chmod(keypath, 0600) + # py3k compatibility + try: + os.chmod(keypath, 0600) + except SyntaxError: + os.chmod(keypath, 0o600) def create_conf(confpath, confdata): # Don't overwrite existing bcfg2.conf file if os.path.exists(confpath): - result = raw_input("\nWarning: %s already exists. " - "Overwrite? [y/N]: " % confpath) + # py3k compatibility + try: + result = raw_input("\nWarning: %s already exists. " + "Overwrite? [y/N]: " % confpath) + except NameError: + result = input("\nWarning: %s already exists. " + "Overwrite? [y/N]: " % confpath) if result not in ['Y', 'y']: print("Leaving %s unchanged" % confpath) return try: open(confpath, "w").write(confdata) - os.chmod(confpath, 0600) + # py3k compatibility + try: + os.chmod(keypath, 0600) + except SyntaxError: + os.chmod(keypath, 0o600) except Exception, e: print("Error %s occured while trying to write configuration " "file to '%s'.\n" % @@ -204,7 +217,12 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_hostname(self): """Ask for the server hostname.""" - data = raw_input("What is the server's hostname [%s]: " % + # py3k compatibility + try: + data = raw_input("What is the server's hostname [%s]: " % + socket.getfqdn()) + except NameError: + data = input("What is the server's hostname [%s]: " % socket.getfqdn()) if data != '': self.shostname = data @@ -213,21 +231,36 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_config(self): """Ask for the configuration file path.""" - newconfig = raw_input("Store Bcfg2 configuration in [%s]: " % - self.configfile) + # py3k compatibility + try: + newconfig = raw_input("Store Bcfg2 configuration in [%s]: " % + self.configfile) + except NameError: + newconfig = input("Store Bcfg2 configuration in [%s]: " % + self.configfile) if newconfig != '': self.configfile = newconfig def _prompt_repopath(self): """Ask for the repository path.""" while True: - newrepo = raw_input("Location of Bcfg2 repository [%s]: " % - self.repopath) + # py3k compatibility + try: + newrepo = raw_input("Location of Bcfg2 repository [%s]: " % + self.repopath) + except NameError: + newrepo = input("Location of Bcfg2 repository [%s]: " % + self.repopath) if newrepo != '': self.repopath = newrepo if os.path.isdir(self.repopath): - response = raw_input("Directory %s exists. Overwrite? [y/N]:" \ - % self.repopath) + # py3k compatibility + try: + response = raw_input("Directory %s exists. Overwrite? [y/N]:" \ + % self.repopath) + except NameError: + response = input("Directory %s exists. Overwrite? [y/N]:" \ + % self.repopath) if response.lower().strip() == 'y': break else: @@ -243,8 +276,13 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_server(self): """Ask for the server name.""" - newserver = raw_input("Input the server location [%s]: " % - self.server_uri) + # py3k compatibility + try: + newserver = raw_input("Input the server location [%s]: " % + self.server_uri) + except NameError: + newserver = input("Input the server location [%s]: " % + self.server_uri) if newserver != '': self.server_uri = newserver @@ -256,51 +294,81 @@ class Init(Bcfg2.Server.Admin.Mode): prompt += ': ' while True: try: - self.os_sel = os_list[int(raw_input(prompt))-1][1] + # py3k compatibility + try: + osidx = int(raw_input(prompt)) + except NameError: + osidx = int(input(prompt)) + self.os_sel = os_list[osidx - 1][1] break except ValueError: continue def _prompt_plugins(self): - default = raw_input("Use default plugins? (%s) [Y/n]: " % + # py3k compatibility + try: + default = raw_input("Use default plugins? (%s) [Y/n]: " % + ''.join(default_plugins)).lower() + except NameError: + default = input("Use default plugins? (%s) [Y/n]: " % ''.join(default_plugins)).lower() if default != 'y' or default != '': while True: plugins_are_valid = True - plug_str = raw_input("Specify plugins: ") + # py3k compatibility + try: + plug_str = raw_input("Specify plugins: ") + except NameError: + plug_str = input("Specify plugins: ") plugins = plug_str.split(',') for plugin in plugins: plugin = plugin.strip() if not plugin in plugin_list: plugins_are_valid = False - print "ERROR: Plugin %s not recognized" % plugin + print("ERROR: Plugin %s not recognized" % plugin) if plugins_are_valid: break def _prompt_certificate(self): """Ask for the key details (country, state, and location).""" - print "The following questions affect SSL certificate generation." - print "If no data is provided, the default values are used." - newcountry = raw_input("Country name (2 letter code) for certificate: ") + print("The following questions affect SSL certificate generation.") + print("If no data is provided, the default values are used.") + # py3k compatibility + try: + newcountry = raw_input("Country name (2 letter code) for certificate: ") + except NameError: + newcountry = input("Country name (2 letter code) for certificate: ") if newcountry != '': if len(newcountry) == 2: self.country = newcountry else: while len(newcountry) != 2: - newcountry = raw_input("2 letter country code (eg. US): ") + # py3k compatibility + try: + newcountry = raw_input("2 letter country code (eg. US): ") + except NameError: + newcountry = input("2 letter country code (eg. US): ") if len(newcountry) == 2: self.country = newcountry break else: self.country = 'US' - newstate = raw_input("State or Province Name (full name) for certificate: ") + # py3k compatibility + try: + newstate = raw_input("State or Province Name (full name) for certificate: ") + except NameError: + newstate = input("State or Province Name (full name) for certificate: ") if newstate != '': self.state = newstate else: self.state = 'Illinois' - newlocation = raw_input("Locality Name (eg, city) for certificate: ") + # py3k compatibility + try: + newlocation = raw_input("Locality Name (eg, city) for certificate: ") + except NameError: + newlocation = input("Locality Name (eg, city) for certificate: ") if newlocation != '': self.location = newlocation else: @@ -349,6 +417,6 @@ class Init(Bcfg2.Server.Admin.Mode): try: os.makedirs(path) self._init_plugins() - print "Repository created successfuly in %s" % (self.repopath) + print("Repository created successfuly in %s" % (self.repopath)) except OSError: print("Failed to create %s." % path) diff --git a/src/lib/Server/Admin/Perf.py b/src/lib/Server/Admin/Perf.py index 095180592..af1c83072 100644 --- a/src/lib/Server/Admin/Perf.py +++ b/src/lib/Server/Admin/Perf.py @@ -1,8 +1,9 @@ +import sys + import Bcfg2.Options import Bcfg2.Proxy import Bcfg2.Server.Admin -import sys class Perf(Bcfg2.Server.Admin.Mode): __shorthelp__ = ("Query server for performance data") @@ -27,11 +28,11 @@ class Perf(Bcfg2.Server.Admin.Mode): proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], setup['user'], setup['password'], - key = setup['key'], - cert = setup['certificate'], - ca = setup['ca']) + key=setup['key'], + cert=setup['certificate'], + ca=setup['ca']) data = proxy.get_statistics() - for key, value in data.iteritems(): + for key, value in list(data.items()): data = tuple(["%.06f" % (item) for item in value[:-1]] + [value[-1]]) output.append((key, ) + data) self.print_table(output) diff --git a/src/lib/Server/Admin/Pull.py b/src/lib/Server/Admin/Pull.py index 926eda1b3..47a8be253 100644 --- a/src/lib/Server/Admin/Pull.py +++ b/src/lib/Server/Admin/Pull.py @@ -1,7 +1,9 @@ import getopt import sys + import Bcfg2.Server.Admin + class Pull(Bcfg2.Server.Admin.MetadataCore): """Pull mode retrieves entries from clients and integrates the information into the repository. @@ -38,7 +40,7 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): try: opts, gargs = getopt.getopt(args, 'vfIs') except: - print self.__shorthelp__ + print(self.__shorthelp__) raise SystemExit(1) for opt in opts: if opt[0] == '-v': @@ -55,18 +57,20 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): try: self.PullEntry(*line.split(None, 3)) except SystemExit: - print " for %s" % line + print(" for %s" % line) except: - print "Bad entry: %s" % line.strip() + print("Bad entry: %s" % line.strip()) elif len(gargs) < 3: - print self.__longhelp__ + print(self.__longhelp__) raise SystemExit(1) else: self.PullEntry(gargs[0], gargs[1], gargs[2]) def BuildNewEntry(self, client, etype, ename): - """Construct a new full entry for given client/entry from statistics.""" - new_entry = {'type':etype, 'name':ename} + """Construct a new full entry for + given client/entry from statistics. + """ + new_entry = {'type': etype, 'name': ename} for plugin in self.bcore.pull_sources: try: (owner, group, perms, contents) = \ @@ -74,16 +78,19 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): break except Bcfg2.Server.Plugin.PluginExecutionError: if plugin == self.bcore.pull_sources[-1]: - print "Pull Source failure; could not fetch current state" + print("Pull Source failure; could not fetch current state") raise SystemExit(1) try: - data = {'owner':owner, 'group':group, 'perms':perms, 'text':contents} + data = {'owner': owner, + 'group': group, + 'perms': perms, + 'text': contents} except UnboundLocalError: print("Unable to build entry. " "Do you have a statistics plugin enabled?") raise SystemExit(1) - for k, v in data.iteritems(): + for k, v in list(data.items()): if v: new_entry[k] = v #print new_entry @@ -93,17 +100,22 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): """Determine where to put pull data.""" if self.mode == 'interactive': for choice in choices: - print "Plugin returned choice:" + print("Plugin returned choice:") if id(choice) == id(choices[0]): - print "(current entry)", + print("(current entry) ") if choice.all: - print " => global entry" + print(" => global entry") elif choice.group: - print (" => group entry: %s (prio %d)" % - (choice.group, choice.prio)) + print(" => group entry: %s (prio %d)" % + (choice.group, choice.prio)) else: - print " => host entry: %s" % (choice.hostname) - if raw_input("Use this entry? [yN]: ") in ['y', 'Y']: + print(" => host entry: %s" % (choice.hostname)) + # py3k compatibility + try: + ans = raw_input("Use this entry? [yN]: ") in ['y', 'Y'] + except NameError: + ans = input("Use this entry? [yN]: ") in ['y', 'Y'] + if ans: return choice return False else: @@ -136,7 +148,7 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): self.errExit("Configuration upload not supported by plugin %s" \ % (plugin.name)) # Commit if running under a VCS - for vcsplugin in self.bcore.plugins.values(): + for vcsplugin in list(self.bcore.plugins.values()): if isinstance(vcsplugin, Bcfg2.Server.Plugin.Version): files = "%s/%s" % (plugin.data, ename) comment = 'file "%s" pulled from host %s' % (files, client) diff --git a/src/lib/Server/Admin/Query.py b/src/lib/Server/Admin/Query.py index b5af9bad2..207b65035 100644 --- a/src/lib/Server/Admin/Query.py +++ b/src/lib/Server/Admin/Query.py @@ -2,6 +2,7 @@ import logging import Bcfg2.Logger import Bcfg2.Server.Admin + class Query(Bcfg2.Server.Admin.Mode): __shorthelp__ = "Query clients" __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin query [-n] [-c] " @@ -32,7 +33,7 @@ class Query(Bcfg2.Server.Admin.Mode): def __call__(self, args): Bcfg2.Server.Admin.Mode.__call__(self, args) - clients = self.meta.clients.keys() + clients = list(self.meta.clients.keys()) filename_arg = False filename = None for arg in args: @@ -48,7 +49,7 @@ class Query(Bcfg2.Server.Admin.Mode): try: k, v = arg.split('=') except: - print "Unknown argument %s" % arg + print("Unknown argument %s" % arg) continue if k == 'p': nc = self.meta.get_client_names_by_profiles(v.split(',')) @@ -57,22 +58,22 @@ class Query(Bcfg2.Server.Admin.Mode): # add probed groups (if present) for conn in self.bcore.connectors: if isinstance(conn, Bcfg2.Server.Plugins.Probes.Probes): - for c, glist in conn.cgroups.items(): + for c, glist in list(conn.cgroups.items()): for g in glist: if g in v.split(','): nc.append(c) else: - print "One of g= or p= must be specified" + print("One of g= or p= must be specified") raise SystemExit(1) clients = [c for c in clients if c in nc] if '-n' in args: for client in clients: - print client + print(client) else: - print ','.join(clients) + print(','.join(clients)) if '-f' in args: f = open(filename, "w") for client in clients: f.write(client + "\n") f.close() - print "Wrote results to %s" % (filename) + print("Wrote results to %s" % (filename)) diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py index 1ac94e5e7..39c9eb71e 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Server/Admin/Reports.py @@ -42,7 +42,8 @@ from django.db import connection, transaction from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, \ Entries_interactions, Performance, \ - Reason, Ping, TYPE_CHOICES, InternalDatabaseVersion + Reason, Ping + def printStats(fn): """ @@ -72,6 +73,7 @@ def printStats(fn): return print_stats + class Reports(Bcfg2.Server.Admin.Mode): '''Admin interface for dynamic reports''' __shorthelp__ = "Manage dynamic reports" @@ -97,7 +99,7 @@ class Reports(Bcfg2.Server.Admin.Mode): def __init__(self, cfile): Bcfg2.Server.Admin.Mode.__init__(self, cfile) self.log.setLevel(logging.INFO) - self.django_commands = [ 'syncdb', 'sqlall', 'validate' ] + self.django_commands = ['syncdb', 'sqlall', 'validate'] self.__usage__ = self.__usage__ + " Django commands:\n " + \ "\n ".join(self.django_commands) @@ -127,54 +129,54 @@ class Reports(Bcfg2.Server.Admin.Mode): update_database() elif args[0] == 'load_stats': quick = '-O3' in args - stats_file=None - clients_file=None - i=1 + stats_file = None + clients_file = None + i = 1 while i < len(args): if args[i] == '-s' or args[i] == '--stats': - stats_file = args[i+1] + stats_file = args[i + 1] if stats_file[0] == '-': self.errExit("Invalid statistics file: %s" % stats_file) elif args[i] == '-c' or args[i] == '--clients-file': - clients_file = args[i+1] + clients_file = args[i + 1] if clients_file[0] == '-': self.errExit("Invalid clients file: %s" % clients_file) i = i + 1 self.load_stats(stats_file, clients_file, verb, quick) elif args[0] == 'purge': - expired=False - client=None - maxdate=None - state=None - i=1 + expired = False + client = None + maxdate = None + state = None + i = 1 while i < len(args): if args[i] == '-c' or args[i] == '--client': if client: self.errExit("Only one client per run") - client = args[i+1] - print client + client = args[i + 1] + print(client) i = i + 1 elif args[i] == '--days': if maxdate: self.errExit("Max date specified multiple times") try: - maxdate = datetime.datetime.now() - datetime.timedelta(days=int(args[i+1])) + maxdate = datetime.datetime.now() - datetime.timedelta(days=int(args[i + 1])) except: - self.log.error("Invalid number of days: %s" % args[i+1]) - raise SystemExit, -1 + self.log.error("Invalid number of days: %s" % args[i + 1]) + raise SystemExit(-1) i = i + 1 elif args[i] == '--expired': - expired=True + expired = True i = i + 1 if expired: if state: self.log.error("--state is not valid with --expired") - raise SystemExit, -1 + raise SystemExit(-1) self.purge_expired(maxdate) else: self.purge(client, maxdate, state) else: - print "Unknown command: %s" % args[0] + print("Unknown command: %s" % args[0]) @transaction.commit_on_success def scrub(self): @@ -187,7 +189,7 @@ class Reports(Bcfg2.Server.Admin.Mode): self.log.error("Failed to load reason objects: %s" % e) return dup_reasons = [] - + cmp_reasons = dict() batch_update = [] for reason in BatchFetch(Reason.objects): @@ -196,7 +198,7 @@ class Reports(Bcfg2.Server.Admin.Mode): comparisons ''' id = reason.id reason.id = None - key=md5(pickle.dumps(reason)).hexdigest() + key = md5(pickle.dumps(reason)).hexdigest() reason.id = id if key in cmp_reasons: @@ -207,7 +209,7 @@ class Reports(Bcfg2.Server.Admin.Mode): else: cmp_reasons[key] = reason.id self.log.debug("key %d" % reason.id) - + self.log.debug("Done with updates, deleting dupes") try: cursor = connection.cursor() @@ -248,7 +250,7 @@ class Reports(Bcfg2.Server.Admin.Mode): try: statsdata = XML(open(stats_file).read()) except (IOError, XMLSyntaxError): - self.errExit("StatReports: Failed to parse %s"%(stats_file)) + self.errExit("StatReports: Failed to parse %s" % (stats_file)) if not clientspath: try: @@ -259,10 +261,15 @@ class Reports(Bcfg2.Server.Admin.Mode): try: clientsdata = XML(open(clientspath).read()) except (IOError, XMLSyntaxError): - self.errExit("StatReports: Failed to parse %s"%(clientspath)) + self.errExit("StatReports: Failed to parse %s" % (clientspath)) try: - load_stats(clientsdata, statsdata, verb, self.log, quick=quick, location=platform.node()) + load_stats(clientsdata, + statsdata, + verb, + self.log, + quick=quick, + location=platform.node()) except: pass @@ -270,7 +277,7 @@ class Reports(Bcfg2.Server.Admin.Mode): def purge(self, client=None, maxdate=None, state=None): '''Purge historical data from the database''' - filtered = False # indicates whether or not a client should be deleted + filtered = False # indicates whether or not a client should be deleted if not client and not maxdate and not state: self.errExit("Reports.prune: Refusing to prune all data") @@ -282,13 +289,13 @@ class Reports(Bcfg2.Server.Admin.Mode): ipurge = ipurge.filter(client=cobj) except Client.DoesNotExist: self.log.error("Client %s not in database" % client) - raise SystemExit, -1 + raise SystemExit(-1) self.log.debug("Filtering by client: %s" % client) if maxdate: filtered = True if not isinstance(maxdate, datetime.datetime): - raise TypeError, "maxdate is not a DateTime object" + raise TypeError("maxdate is not a DateTime object") self.log.debug("Filtering by maxdate: %s" % maxdate) ipurge = ipurge.filter(timestamp__lt=maxdate) @@ -300,9 +307,9 @@ class Reports(Bcfg2.Server.Admin.Mode): if state: filtered = True - if state not in ('dirty','clean','modified'): - raise TypeError, "state is not one of the following values " + \ - "('dirty','clean','modified')" + if state not in ('dirty', 'clean', 'modified'): + raise TypeError("state is not one of the following values " + \ + "('dirty','clean','modified')") self.log.debug("Filtering by state: %s" % state) ipurge = ipurge.filter(state=state) @@ -346,7 +353,7 @@ class Reports(Bcfg2.Server.Admin.Mode): if maxdate: if not isinstance(maxdate, datetime.datetime): - raise TypeError, "maxdate is not a DateTime object" + raise TypeError("maxdate is not a DateTime object") self.log.debug("Filtering by maxdate: %s" % maxdate) clients = Client.objects.filter(expiration__lt=maxdate) else: @@ -358,4 +365,3 @@ class Reports(Bcfg2.Server.Admin.Mode): client.delete() self.log.debug("Pruning orphan Performance objects") Performance.prune_orphans() - diff --git a/src/lib/Server/Admin/Tidy.py b/src/lib/Server/Admin/Tidy.py index cc8ab4f5e..f79991fd9 100644 --- a/src/lib/Server/Admin/Tidy.py +++ b/src/lib/Server/Admin/Tidy.py @@ -4,6 +4,7 @@ import socket import Bcfg2.Server.Admin + class Tidy(Bcfg2.Server.Admin.Mode): __shorthelp__ = "Clean up useless files in the repo" __longhelp__ = __shorthelp__ + "\n\nbcfg2-admin tidy [-f] [-I]\n" @@ -24,17 +25,21 @@ class Tidy(Bcfg2.Server.Admin.Mode): if '-f' in args or '-I' in args: if '-I' in args: for name in badfiles[:]: - answer = raw_input("Unlink file %s? [yN] " % name) + # py3k compatibility + try: + answer = raw_input("Unlink file %s? [yN] " % name) + except NameError: + answer = input("Unlink file %s? [yN] " % name) if answer not in ['y', 'Y']: badfiles.remove(name) for name in badfiles: try: os.unlink(name) except IOError: - print "Failed to unlink %s" % name + print("Failed to unlink %s" % name) else: for name in badfiles: - print name + print(name) def buildTidyList(self): """Clean up unused or unusable files from the repository.""" @@ -56,7 +61,8 @@ class Tidy(Bcfg2.Server.Admin.Mode): bad.append(hostname) for name in os.listdir("%s/SSHbase" % (self.get_repo_path())): if not hostmatcher.match(name): - to_remove.append("%s/SSHbase/%s" % (self.get_repo_path(), name)) + to_remove.append("%s/SSHbase/%s" % (self.get_repo_path(), + name)) else: if hostmatcher.match(name).group(1) in bad: to_remove.append("%s/SSHbase/%s" % diff --git a/src/lib/Server/Admin/Viz.py b/src/lib/Server/Admin/Viz.py index e3daea84b..a77502b5d 100644 --- a/src/lib/Server/Admin/Viz.py +++ b/src/lib/Server/Admin/Viz.py @@ -1,7 +1,9 @@ import getopt from subprocess import Popen, PIPE + import Bcfg2.Server.Admin + class Viz(Bcfg2.Server.Admin.MetadataCore): __shorthelp__ = "Produce graphviz diagrams of metadata structures" __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin viz [--includehosts] " @@ -27,7 +29,8 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): plugin_blacklist = ['DBStats', 'Snapshots', 'Cfg', 'Pkgmgr', 'Packages', 'Rules', 'Account', 'Decisions', 'Deps', 'Git', 'Svn', - 'Fossil', 'Bzr', 'Bundler', 'TGenshi', 'SGenshi', 'Base'] + 'Fossil', 'Bzr', 'Bundler', 'TGenshi', 'SGenshi', + 'Base'] def __init__(self, cfile): @@ -43,7 +46,7 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): ['includehosts', 'includebundles', 'includekey', 'outfile=']) except getopt.GetoptError, msg: - print msg + print(msg) #FIXME: is this for --raw? #rset = False @@ -63,8 +66,8 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): data = self.Visualize(self.get_repo_path(), hset, bset, kset, outputfile) - print data - raise SystemExit, 0 + print(data) + raise SystemExit(0) def Visualize(self, repopath, hosts=False, bundles=False, key=False, output=False): @@ -82,7 +85,7 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): try: dotpipe.stdin.write("digraph groups {\n") except: - print "write to dot process failed. Is graphviz installed?" + print("write to dot process failed. Is graphviz installed?") raise SystemExit(1) dotpipe.stdin.write('\trankdir="LR";\n') dotpipe.stdin.write(self.metadata.viz(hosts, bundles, diff --git a/src/lib/Server/Admin/Xcmd.py b/src/lib/Server/Admin/Xcmd.py index 8ea98b79c..e761a5e3d 100644 --- a/src/lib/Server/Admin/Xcmd.py +++ b/src/lib/Server/Admin/Xcmd.py @@ -1,9 +1,10 @@ +import sys +import xmlrpclib + import Bcfg2.Options import Bcfg2.Proxy import Bcfg2.Server.Admin -import sys -import xmlrpclib class Xcmd(Bcfg2.Server.Admin.Mode): __shorthelp__ = ("XML-RPC Command Interface") @@ -16,8 +17,8 @@ class Xcmd(Bcfg2.Server.Admin.Mode): 'user': Bcfg2.Options.CLIENT_USER, 'password': Bcfg2.Options.SERVER_PASSWORD, 'key': Bcfg2.Options.SERVER_KEY, - 'certificate' : Bcfg2.Options.CLIENT_CERT, - 'ca' : Bcfg2.Options.CLIENT_CA + 'certificate': Bcfg2.Options.CLIENT_CERT, + 'ca': Bcfg2.Options.CLIENT_CA } setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[2:]) @@ -25,9 +26,10 @@ class Xcmd(Bcfg2.Server.Admin.Mode): proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], setup['user'], setup['password'], - key = setup['key'], - cert = setup['certificate'], - ca = setup['ca'], timeout=180) + key=setup['key'], + cert=setup['certificate'], + ca=setup['ca'], + timeout=180) if len(setup['args']) == 0: print("Usage: xcmd ") return @@ -36,7 +38,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode): if len(setup['args']) > 1: args = tuple(setup['args'][1:]) try: - data = apply(getattr(proxy, cmd), args) + data = getattr(proxy, cmd)(*args) except xmlrpclib.Fault, flt: if flt.faultCode == 7: print("Unknown method %s" % cmd) @@ -46,4 +48,4 @@ class Xcmd(Bcfg2.Server.Admin.Mode): else: raise if data != None: - print data + print(data) diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py index dc3dc8c01..411f909ee 100644 --- a/src/lib/Server/Admin/__init__.py +++ b/src/lib/Server/Admin/__init__.py @@ -27,14 +27,17 @@ import sys import Bcfg2.Server.Core import Bcfg2.Options + class ModeOperationError(Exception): pass + class Mode(object): """Help message has not yet been added for mode.""" __shorthelp__ = 'Shorthelp not defined yet' __longhelp__ = 'Longhelp not defined yet' __args__ = [] + def __init__(self, configfile): self.configfile = configfile self.__cfp = False @@ -50,11 +53,11 @@ class Mode(object): def __call__(self, args): if len(args) > 0 and args[0] == 'help': - print self.__longhelp__ + print(self.__longhelp__) raise SystemExit(0) def errExit(self, emsg): - print emsg + print(emsg) raise SystemExit(1) def get_repo_path(self): @@ -80,9 +83,9 @@ class Mode(object): """ hdelim = "=" - justify = {'left':str.ljust, - 'center':str.center, - 'right':str.rjust}[justify.lower()] + justify = {'left': str.ljust, + 'center': str.center, + 'right': str.rjust}[justify.lower()] """ Calculate column widths (longest item in each column @@ -90,9 +93,9 @@ class Mode(object): """ cols = list(zip(*rows)) - colWidths = [max([len(str(item))+2*padding for \ + colWidths = [max([len(str(item)) + 2 * padding for \ item in col]) for col in cols] - borderline = vdelim.join([w*hdelim for w in colWidths]) + borderline = vdelim.join([w * hdelim for w in colWidths]) # Print out the table print(borderline) @@ -103,6 +106,7 @@ class Mode(object): print(borderline) hdr = False + class MetadataCore(Mode): """Base class for admin-modes that handle metadata.""" def __init__(self, configfile, usage, pwhitelist=None, pblacklist=None): @@ -113,9 +117,11 @@ class MetadataCore(Mode): setup.hm = usage setup.parse(sys.argv[1:]) if pwhitelist is not None: - setup['plugins'] = [x for x in setup['plugins'] if x in pwhitelist] + setup['plugins'] = [x for x in setup['plugins'] + if x in pwhitelist] elif pblacklist is not None: - setup['plugins'] = [x for x in setup['plugins'] if x not in pblacklist] + setup['plugins'] = [x for x in setup['plugins'] + if x not in pblacklist] try: self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(), setup['plugins'], @@ -125,5 +131,6 @@ class MetadataCore(Mode): self.bcore.fam.handle_events_in_interval(5) self.metadata = self.bcore.metadata + class StructureMode(MetadataCore): pass -- cgit v1.2.3-1-g7c22 From 2e1a79fe401bea5b33551b9f94689524bf43cdca Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 7 Apr 2011 14:04:36 -0500 Subject: PY3K + PEP8 fixes for remaining files Signed-off-by: Sol Jerome --- src/lib/Server/Core.py | 64 ++++++++++++++----------- src/lib/Server/FileMonitor.py | 18 +++++-- src/lib/Server/Hostbase/ldapauth.py | 82 +++++++++++++++++--------------- src/lib/Server/Plugin.py | 2 +- src/lib/Server/Plugins/Pkgmgr.py | 2 +- src/lib/Server/Reports/reports/models.py | 6 +-- 6 files changed, 96 insertions(+), 78 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index ac67b8a69..e82e05a89 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -16,6 +16,7 @@ import Bcfg2.Server.Plugins.Metadata logger = logging.getLogger('Bcfg2.Server.Core') + def critical_error(operation): """Log and err, traceback and return an xmlrpc fault to client.""" logger.error(operation, exc_info=1) @@ -27,12 +28,16 @@ try: except: pass + class CoreInitError(Exception): """This error is raised when the core cannot be initialized.""" pass + class Core(Component): - """The Core object is the container for all Bcfg2 Server logic and modules.""" + """The Core object is the container for all + Bcfg2 Server logic and modules. + """ name = 'bcfg2-server' implementation = 'bcfg2-server' @@ -42,15 +47,16 @@ class Core(Component): Component.__init__(self) self.datastore = repo if filemonitor not in Bcfg2.Server.FileMonitor.available: - logger.error("File monitor driver %s not available; forcing to default" % filemonitor) + logger.error("File monitor driver %s not available; " + "forcing to default" % filemonitor) filemonitor = 'default' try: self.fam = Bcfg2.Server.FileMonitor.available[filemonitor]() except IOError: logger.error("Failed to instantiate fam driver %s" % filemonitor, exc_info=1) - raise CoreInitError, "failed to instantiate fam driver (used %s)" % \ - filemonitor + raise CoreInitError("failed to instantiate fam driver (used %s)" % \ + filemonitor) self.pubspace = {} self.cfile = cfile self.cron = {} @@ -70,44 +76,43 @@ class Core(Component): if not plugin in self.plugins: self.init_plugins(plugin) # Remove blacklisted plugins - for p, bl in self.plugin_blacklist.items(): + for p, bl in list(self.plugin_blacklist.items()): if len(bl) > 0: logger.error("The following plugins conflict with %s;" "Unloading %s" % (p, bl)) for plug in bl: del self.plugins[plug] # This section loads the experimental plugins - expl = [plug for (name, plug) in self.plugins.iteritems() + expl = [plug for (name, plug) in list(self.plugins.items()) if plug.experimental] if expl: logger.info("Loading experimental plugin(s): %s" % \ (" ".join([x.name for x in expl]))) logger.info("NOTE: Interfaces subject to change") - depr = [plug for (name, plug) in self.plugins.iteritems() + depr = [plug for (name, plug) in list(self.plugins.items()) if plug.deprecated] # This section loads the deprecated plugins if depr: logger.info("Loading deprecated plugin(s): %s" % \ (" ".join([x.name for x in depr]))) - - mlist = [p for p in self.plugins.values() if \ + mlist = [p for p in list(self.plugins.values()) if \ isinstance(p, Bcfg2.Server.Plugin.Metadata)] if len(mlist) == 1: self.metadata = mlist[0] else: logger.error("No Metadata Plugin loaded; failed to instantiate Core") - raise CoreInitError, "No Metadata Plugin" - self.statistics = [plugin for plugin in self.plugins.values() if \ - isinstance(plugin, Bcfg2.Server.Plugin.Statistics)] - self.pull_sources = [plugin for plugin in self.statistics if \ - isinstance(plugin, Bcfg2.Server.Plugin.PullSource)] - self.generators = [plugin for plugin in self.plugins.values() if \ - isinstance(plugin, Bcfg2.Server.Plugin.Generator)] - self.structures = [plugin for plugin in self.plugins.values() if \ - isinstance(plugin, Bcfg2.Server.Plugin.Structure)] - self.connectors = [plugin for plugin in self.plugins.values() if \ - isinstance(plugin, Bcfg2.Server.Plugin.Connector)] + raise CoreInitError("No Metadata Plugin") + self.statistics = [plugin for plugin in list(self.plugins.values()) + if isinstance(plugin, Bcfg2.Server.Plugin.Statistics)] + self.pull_sources = [plugin for plugin in self.statistics + if isinstance(plugin, Bcfg2.Server.Plugin.PullSource)] + self.generators = [plugin for plugin in list(self.plugins.values()) + if isinstance(plugin, Bcfg2.Server.Plugin.Generator)] + self.structures = [plugin for plugin in list(self.plugins.values()) + if isinstance(plugin, Bcfg2.Server.Plugin.Structure)] + self.connectors = [plugin for plugin in list(self.plugins.values()) + if isinstance(plugin, Bcfg2.Server.Plugin.Connector)] self.ca = ca self.fam_thread = threading.Thread(target=self._file_monitor_thread) if start_fam_thread: @@ -128,7 +133,7 @@ class Core(Component): except: continue # VCS plugin periodic updates - for plugin in self.plugins.values(): + for plugin in list(self.plugins.values()): if isinstance(plugin, Bcfg2.Server.Plugin.Version): self.revision = plugin.get_revision() @@ -157,15 +162,15 @@ class Core(Component): (plugin), exc_info=1) def shutdown(self): - """Shuting down the plugins.""" + """Shutting down the plugins.""" if not self.terminate.isSet(): self.terminate.set() - for plugin in self.plugins.values(): + for plugin in list(self.plugins.values()): plugin.shutdown() def validate_data(self, metadata, data, base_cls): """Checks the data structure.""" - for plugin in self.plugins.values(): + for plugin in list(self.plugins.values()): if isinstance(plugin, base_cls): try: if base_cls == Bcfg2.Server.Plugin.StructureValidator: @@ -182,7 +187,7 @@ class Core(Component): def GetStructures(self, metadata): """Get all structures for client specified by metadata.""" - structures = reduce(lambda x, y:x+y, + structures = reduce(lambda x, y: x + y, [struct.BuildStructures(metadata) for struct \ in self.structures], []) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] @@ -232,7 +237,8 @@ class Core(Component): glist = [gen for gen in self.generators if entry.get('name') in gen.Entries.get(entry.tag, {})] if len(glist) == 1: - return glist[0].Entries[entry.tag][entry.get('name')](entry, metadata) + return glist[0].Entries[entry.tag][entry.get('name')](entry, + metadata) elif len(glist) > 1: generators = ", ".join([gen.name for gen in glist]) logger.error("%s %s served by multiple generators: %s" % \ @@ -242,7 +248,7 @@ class Core(Component): if len(g2list) == 1: return g2list[0].HandleEntry(entry, metadata) entry.set('failure', 'no matching generator') - raise PluginExecutionError, (entry.tag, entry.get('name')) + raise PluginExecutionError(entry.tag, entry.get('name')) def BuildConfiguration(self, client): """Build configuration for clients.""" @@ -290,7 +296,7 @@ class Core(Component): def GetDecisions(self, metadata, mode): """Get data for the decision list.""" result = [] - for plugin in self.plugins.values(): + for plugin in list(self.plugins.values()): try: if isinstance(plugin, Bcfg2.Server.Plugin.Decision): result += plugin.GetDecisions(metadata, mode) @@ -300,7 +306,7 @@ class Core(Component): return result def build_metadata(self, client_name): - """Build the metadata structure.""" + """Build the metadata structure.""" if not hasattr(self, 'metadata'): # some threads start before metadata is even loaded raise Bcfg2.Server.Plugins.Metadata.MetadataRuntimeError diff --git a/src/lib/Server/FileMonitor.py b/src/lib/Server/FileMonitor.py index 0f09f7751..d6b313e6b 100644 --- a/src/lib/Server/FileMonitor.py +++ b/src/lib/Server/FileMonitor.py @@ -7,6 +7,7 @@ from time import sleep, time logger = logging.getLogger('Bcfg2.Server.FileMonitor') + def ShouldIgnore(event): """Test if the event should be suppresed.""" # FIXME should move event suppression out of the core @@ -18,6 +19,7 @@ def ShouldIgnore(event): return True return False + class Event(object): def __init__(self, request_id, filename, code): self.requestID = request_id @@ -29,6 +31,8 @@ class Event(object): return self.action available = {} + + class FileMonitor(object): """File Monitor baseclass.""" def __init__(self, debug=False): @@ -78,7 +82,7 @@ class FileMonitor(object): if lock: lock.release() end = time() - logger.info("Handled %d events in %.03fs" % (count, (end-start))) + logger.info("Handled %d events in %.03fs" % (count, (end - start))) def handle_events_in_interval(self, interval): end = time() + interval @@ -91,7 +95,9 @@ class FileMonitor(object): class FamFam(object): - """The fam object is a set of callbacks for file alteration events (FAM support).""" + """The fam object is a set of callbacks for + file alteration events (FAM support). + """ def __init__(self): object.__init__(self) @@ -164,7 +170,6 @@ class FamFam(object): return count - class Fam(FileMonitor): """ The fam object is a set of callbacks for @@ -195,6 +200,7 @@ class Fam(FileMonitor): def get_event(self): return self.fm.nextEvent() + class Pseudo(FileMonitor): """ The fam object is a set of callbacks for @@ -213,14 +219,16 @@ class Pseudo(FileMonitor): def AddMonitor(self, path, obj): """add a monitor to path, installing a callback to obj.HandleEvent""" - handleID = len(self.handles.keys()) + handleID = len(list(self.handles.keys())) mode = os.stat(path)[stat.ST_MODE] handle = Event(handleID, path, 'exists') if stat.S_ISDIR(mode): dirList = os.listdir(path) self.pending_events.append(handle) for includedFile in dirList: - self.pending_events.append(Event(handleID, includedFile, 'exists')) + self.pending_events.append(Event(handleID, + includedFile, + 'exists')) self.pending_events.append(Event(handleID, path, 'endExist')) else: self.pending_events.append(Event(handleID, path, 'exists')) diff --git a/src/lib/Server/Hostbase/ldapauth.py b/src/lib/Server/Hostbase/ldapauth.py index f2148181f..21b462c86 100644 --- a/src/lib/Server/Hostbase/ldapauth.py +++ b/src/lib/Server/Hostbase/ldapauth.py @@ -1,16 +1,18 @@ -"""Checks with LDAP (ActiveDirectory) to see if the current user is an LDAP(AD) user, -and returns a subset of the user's profile that is needed by Argonne/CIS to -to set user level privleges in Django""" - -__revision__ = '$Revision: 2456 $' +""" +Checks with LDAP (ActiveDirectory) to see if the current user is an LDAP(AD) +user, and returns a subset of the user's profile that is needed by Argonne/CIS +to set user level privleges in Django +""" import os import ldap + class LDAPAUTHError(Exception): """LDAPAUTHError is raised when somehting goes boom.""" pass + class ldapauth(object): group_test = False check_member_of = os.environ['LDAP_CHECK_MBR_OF_GRP'] @@ -20,35 +22,35 @@ class ldapauth(object): telephoneNumber = None title = None memberOf = None - department = None #this will be a list + department = None # this will be a list mail = None - extensionAttribute1 = None #badgenumber + extensionAttribute1 = None # badgenumber badge_no = None - def __init__(self,login,passwd): + def __init__(self, login, passwd): """get username (if using ldap as auth the apache env var REMOTE_USER should be used) from username get user profile from AD/LDAP """ #p = self.user_profile(login,passwd) - d = self.user_dn(login) #success, distname - print d[1] + d = self.user_dn(login) # success, distname + print(d[1]) if d[0] == 'success': pass - p = self.user_bind(d[1],passwd) + p = self.user_bind(d[1], passwd) if p[0] == 'success': #parse results parsed = self.parse_results(p[2]) - print self.department + print(self.department) self.group_test = self.member_of() securitylevel = self.security_level() - print "ACCESS LEVEL: " + str(securitylevel) + print("ACCESS LEVEL: " + str(securitylevel)) else: raise LDAPAUTHError(p[2]) else: raise LDAPAUTHError(p[2]) - def user_profile(self,login,passwd=None): + def user_profile(self, login, passwd=None): """NOT USED RIGHT NOW""" ldap_login = "CN=%s" % login svc_acct = os.environ['LDAP_SVC_ACCT_NAME'] @@ -60,33 +62,35 @@ class ldapauth(object): try: conn = ldap.initialize(os.environ['LDAP_URI']) - conn.bind(svc_acct,svc_pass,ldap.AUTH_SIMPLE) + conn.bind(svc_acct, svc_pass, ldap.AUTH_SIMPLE) result_id = conn.search(search_pth, - ldap.SCOPE_SUBTREE, - ldap_login,None) - result_type,result_data = conn.result(result_id,0) - return ('success','User profile found',result_data,) - except ldap.LDAPError,e: + ldap.SCOPE_SUBTREE, + ldap_login, + None) + result_type, result_data = conn.result(result_id, 0) + return ('success', 'User profile found', result_data,) + except ldap.LDAPError, e: #connection failed - return ('error','LDAP connect failed',e,) + return ('error', 'LDAP connect failed', e,) - def user_bind(self,distinguishedName,passwd): + def user_bind(self, distinguishedName, passwd): """Binds to LDAP Server""" search_pth = os.environ['LDAP_SEARCH_PTH'] try: conn = ldap.initialize(os.environ['LDAP_URI']) - conn.bind(distinguishedName,passwd,ldap.AUTH_SIMPLE) + conn.bind(distinguishedName, passwd, ldap.AUTH_SIMPLE) cn = distinguishedName.split(",") result_id = conn.search(search_pth, - ldap.SCOPE_SUBTREE, - cn[0],None) - result_type,result_data = conn.result(result_id,0) - return ('success','User profile found',result_data,) - except ldap.LDAPError,e: + ldap.SCOPE_SUBTREE, + cn[0], + None) + result_type, result_data = conn.result(result_id, 0) + return ('success', 'User profile found', result_data,) + except ldap.LDAPError, e: #connection failed - return ('error','LDAP connect failed',e,) + return ('error', 'LDAP connect failed', e,) - def user_dn(self,cn): + def user_dn(self, cn): """Uses Service Account to get distinguishedName""" ldap_login = "CN=%s" % cn svc_acct = os.environ['LDAP_SVC_ACCT_NAME'] @@ -95,19 +99,20 @@ class ldapauth(object): try: conn = ldap.initialize(os.environ['LDAP_URI']) - conn.bind(svc_acct,svc_pass,ldap.AUTH_SIMPLE) + conn.bind(svc_acct, svc_pass, ldap.AUTH_SIMPLE) result_id = conn.search(search_pth, - ldap.SCOPE_SUBTREE, - ldap_login,None) - result_type,result_data = conn.result(result_id,0) + ldap.SCOPE_SUBTREE, + ldap_login, + None) + result_type, result_data = conn.result(result_id, 0) raw_obj = result_data[0][1] distinguishedName = raw_obj['distinguishedName'] - return ('success',distinguishedName[0],) - except ldap.LDAPError,e: + return ('success', distinguishedName[0],) + except ldap.LDAPError, e: #connection failed - return ('error','LDAP connect failed',e,) + return ('error', 'LDAP connect failed', e,) - def parse_results(self,user_obj): + def parse_results(self, user_obj): """Clean up the huge ugly object handed to us in the LDAP query""" #user_obj is a list formatted like this: #[('LDAP_DN',{user_dict},),] @@ -169,4 +174,3 @@ class ldapauth(object): level = 4 return level - diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 5a6f3281b..ac9479d44 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -363,7 +363,7 @@ class DirectoryBacked(object): return self.entries[key] def __iter__(self): - return iter(self.entries.items()) + return iter(list(self.entries.items())) def AddEntry(self, name): """Add new entry to data structures upon file creation.""" diff --git a/src/lib/Server/Plugins/Pkgmgr.py b/src/lib/Server/Plugins/Pkgmgr.py index dc4a5f37f..b96e7ea7d 100644 --- a/src/lib/Server/Plugins/Pkgmgr.py +++ b/src/lib/Server/Plugins/Pkgmgr.py @@ -62,7 +62,7 @@ class PNode(Bcfg2.Server.Plugin.INode): if 'Package' not in pdict: pdict['Package'] = set() for child in data.getchildren(): - for attr in [key for key in data.attrib.keys() + for attr in [key for key in list(data.attrib.keys()) if key != 'name' and key not in child.attrib]: try: child.set(attr, data.get(attr)) diff --git a/src/lib/Server/Reports/reports/models.py b/src/lib/Server/Reports/reports/models.py index c2be40870..d94b2e1ba 100644 --- a/src/lib/Server/Reports/reports/models.py +++ b/src/lib/Server/Reports/reports/models.py @@ -48,7 +48,7 @@ class ClientManager(models.Manager): if timestamp == None: timestamp = datetime.now() elif not isinstance(timestamp, datetime): - raise ValueError, 'Expected a datetime object' + raise ValueError('Expected a datetime object') else: try: timestamp = datetime(*strptime(timestamp, @@ -113,7 +113,7 @@ class InteractiveManager(models.Manager): """ if maxdate and not isinstance(maxdate, datetime): - raise ValueError, 'Expected a datetime object' + raise ValueError('Expected a datetime object') return self.filter(id__in=self.get_interaction_per_client_ids(maxdate, active_only)) def get_interaction_per_client_ids(self, maxdate=None, active_only=True): @@ -133,7 +133,7 @@ class InteractiveManager(models.Manager): 'as timer from reports_interaction' if maxdate: if not isinstance(maxdate, datetime): - raise ValueError, 'Expected a datetime object' + raise ValueError('Expected a datetime object') sql = sql + " where timestamp <= '%s' " % maxdate cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate) sql = sql + ' GROUP BY client_id) x, reports_interaction where ' + \ -- cgit v1.2.3-1-g7c22 From dc1262d01800e2cb8560fc81d0cdb6827dd0432f Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Mon, 11 Apr 2011 13:46:01 -0500 Subject: Reports: delete orphaned Reasons after the purge command is run --- src/lib/Server/Admin/Reports.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py index 39c9eb71e..ee3a4473f 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Server/Admin/Reports.py @@ -334,6 +334,8 @@ class Reports(Bcfg2.Server.Admin.Mode): # bulk operations bypass the Interaction.delete method self.log.debug("Pruning orphan Performance objects") Performance.prune_orphans() + self.log.debug("Pruning orphan Reason objects") + Reason.prune_orphans() if client and not filtered: '''Delete the client, ping data is automatic''' -- cgit v1.2.3-1-g7c22 From c16ed3a800cc6dd926c80bc75aaced5f3cf939d0 Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Mon, 11 Apr 2011 14:01:54 -0500 Subject: Reports: Use the newer DATABASES option in settings.py --- src/lib/Server/Reports/settings.py | 47 +++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 19 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/settings.py b/src/lib/Server/Reports/settings.py index f4deca6aa..121d55a27 100644 --- a/src/lib/Server/Reports/settings.py +++ b/src/lib/Server/Reports/settings.py @@ -23,26 +23,35 @@ ADMINS = ( MANAGERS = ADMINS -DATABASE_ENGINE = c.get('statistics', 'database_engine') -# 'postgresql', 'mysql', 'sqlite3' or 'ado_mssql'. +db_engine = c.get('statistics', 'database_engine') +db_name = '' if c.has_option('statistics', 'database_name'): - DATABASE_NAME = c.get('statistics', 'database_name') -else: - DATABASE_NAME = '' -# Or path to database file if using sqlite3. -#/etc/brpt.sqlite is default path - -if DATABASE_ENGINE != 'sqlite3': - DATABASE_USER = c.get('statistics', 'database_user') - # Not used with sqlite3. - DATABASE_PASSWORD = c.get('statistics', 'database_password') - # Not used with sqlite3. - DATABASE_HOST = c.get('statistics', 'database_host') - # Set to empty string for localhost. Not used with sqlite3. - DATABASE_PORT = c.get('statistics', 'database_port') - # Set to empty string for default. Not used with sqlite3. -if DATABASE_ENGINE == 'sqlite3' and DATABASE_NAME == '': - DATABASE_NAME = "%s/etc/brpt.sqlite" % c.get('server', 'repository') + db_name = c.get('statistics', 'database_name') +if db_engine == 'sqlite3' and db_name == '': + db_name = "%s/etc/brpt.sqlite" % c.get('server', 'repository') + +DATABASES = { + 'default': { + 'ENGINE': "django.db.backends.%s" % db_engine, + 'NAME': db_name + } +} + +if db_engine != 'sqlite3': + DATABASES['default']['USER'] = c.get('statistics', 'database_user') + DATABASES['default']['PASSWORD'] = c.get('statistics', 'database_password') + DATABASES['default']['HOST'] = c.get('statistics', 'database_host') + DATABASES['default']['PORT'] = c.get('statistics', 'database_port') + +if django.VERSION[0] == 1 and django.VERSION[1] < 2: + DATABASE_ENGINE = db_engine + DATABASE_NAME = DATABASES['default']['NAME'] + if DATABASE_ENGINE != 'sqlite3': + DATABASE_USER = DATABASES['default']['USER'] + DATABASE_PASSWORD = DATABASES['default']['PASSWORD'] + DATABASE_HOST = DATABASES['default']['HOST'] + DATABASE_PORT = DATABASES['default']['PORT'] + # Local time zone for this installation. All choices can be found here: # http://docs.djangoproject.com/en/dev/ref/settings/#time-zone -- cgit v1.2.3-1-g7c22 From 0cd16d39169d0cab01b7ffe19426d26086500b3e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 12 Apr 2011 08:12:17 -0400 Subject: added next() method to Reports BatchFetch iterator for compatibility with pre-3.0 versions of python --- src/lib/Server/Reports/utils.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/utils.py b/src/lib/Server/Reports/utils.py index 6010f366b..e0b6ead59 100755 --- a/src/lib/Server/Reports/utils.py +++ b/src/lib/Server/Reports/utils.py @@ -20,6 +20,10 @@ class BatchFetch(object): def __iter__(self): return self + def next(self): + """Provide compatibility with python < 3.0""" + return self.__next__() + def __next__(self): """Return the next object from our array and fetch from the database when needed""" -- cgit v1.2.3-1-g7c22 From 73effe023b731a7252c6ab75e84f154ed939479a Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 12 Apr 2011 08:18:27 -0500 Subject: Editor: Add missing comma (Reported by emias) Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Editor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Editor.py b/src/lib/Server/Plugins/Editor.py index e68d28d8b..76a03a325 100644 --- a/src/lib/Server/Plugins/Editor.py +++ b/src/lib/Server/Plugins/Editor.py @@ -39,7 +39,7 @@ class EditEntrySet(Bcfg2.Server.Plugin.EntrySet): client = metadata.hostname filename = entry.get('name') permdata = {'owner': 'root', - 'group': 'root' + 'group': 'root', 'perms': '0644'} [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] entry.text = self.entries['edits'].ProcessDirectives(self.get_client_data(client)) -- cgit v1.2.3-1-g7c22 From 33d4e0178b26dc4756a4b933a9f966379ce99453 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 12 Apr 2011 08:37:23 -0500 Subject: Init: Fix SyntaxError (Reported by emias on IRC) Python < 2.6 uses the '0600' format for specifying the mode while 2.6 and later allow the use of '0o600'. Since python 3 forces the latter, we can use the stat module to maintain compatibility with both. Signed-off-by: Sol Jerome --- src/lib/Server/Admin/Init.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index eddbd732a..9771fd10b 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -2,6 +2,7 @@ import getpass import os import random import socket +import stat import string import subprocess import Bcfg2.Server.Admin @@ -137,11 +138,7 @@ def create_key(hostname, keypath, certpath, country, state, location): keypath, certpath)) subprocess.call((ccstr), shell=True) - # py3k compatibility - try: - os.chmod(keypath, 0600) - except SyntaxError: - os.chmod(keypath, 0o600) + os.chmod(keypath, stat.S_IRUSR|stat.S_IWUSR) # 0600 def create_conf(confpath, confdata): @@ -159,11 +156,7 @@ def create_conf(confpath, confdata): return try: open(confpath, "w").write(confdata) - # py3k compatibility - try: - os.chmod(keypath, 0600) - except SyntaxError: - os.chmod(keypath, 0o600) + os.chmod(keypath, stat.S_IRUSR|stat.S_IWUSR) # 0600 except Exception, e: print("Error %s occured while trying to write configuration " "file to '%s'.\n" % -- cgit v1.2.3-1-g7c22 From 7cebd3f47926453bdac2b3f2c41804ac83358388 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 14 Apr 2011 08:47:24 -0400 Subject: added Client tag to StructFile; made PropertyFile inherit from StructFile --- src/lib/Server/Plugin.py | 34 +++++++++++++++++++++++----------- src/lib/Server/Plugins/Properties.py | 10 ++-------- 2 files changed, 25 insertions(+), 19 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index ac9479d44..69b38c4ff 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -457,17 +457,29 @@ class StructFile(XMLFileBacked): work = {lambda x: True: xdata.getchildren()} while work: (predicate, worklist) = work.popitem() - self.fragments[predicate] = [item for item in worklist if item.tag != 'Group' - and not isinstance(item, lxml.etree._Comment)] - for group in [item for item in worklist if item.tag == 'Group']: - # if only python had forceable early-binding - if group.get('negate', 'false') in ['true', 'True']: - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - - newpred = eval(cmd % (group.get('name')), {'predicate': predicate}) - work[newpred] = group.getchildren() + self.fragments[predicate] = \ + [item for item in worklist + if (item.tag != 'Group' and + item.tag != 'Client' and + not isinstance(item, + lxml.etree._Comment))] + for item in worklist: + cmd = None + if item.tag == 'Group': + if item.get('negate', 'false').lower() == 'true': + cmd = "lambda x:'%s' not in x.groups and predicate(x)" + else: + cmd = "lambda x:'%s' in x.groups and predicate(x)" + elif item.tag == 'Client': + if item.get('negate', 'false').lower() == 'true': + cmd = "lambda x:x.hostname != '%s' and predicate(x)" + else: + cmd = "lambda x:x.hostname == '%s' and predicate(x)" + # else, ignore item + if cmd is not None: + newpred = eval(cmd % item.get('name'), + {'predicate':predicate}) + work[newpred] = item.getchildren() def Match(self, metadata): """Return matching fragments of independent.""" diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index 2888ef1d1..c5d2acc44 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -4,15 +4,9 @@ import lxml.etree import Bcfg2.Server.Plugin -class PropertyFile(Bcfg2.Server.Plugin.XMLFileBacked): +class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" - - def Index(self): - """Build data into an xml object.""" - try: - self.data = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % self.name) + pass class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): -- cgit v1.2.3-1-g7c22 From 6e7ba79340635aba26ee7f1980ce91714b853188 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 14 Apr 2011 11:20:58 -0400 Subject: fixed PropertyFile.Index() to populate .data as a side effect --- src/lib/Server/Plugins/Properties.py | 38 +++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index c5d2acc44..b34bde998 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -6,7 +6,43 @@ import Bcfg2.Server.Plugin class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" - pass + def Index(self): + """Build internal data structures.""" + if type(self.data) is not lxml.etree._Element: + try: + self.data = lxml.etree.XML(self.data) + except lxml.etree.XMLSyntaxError: + Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % + self.name) + + self.fragments = {} + work = {lambda x: True: self.data.getchildren()} + while work: + (predicate, worklist) = work.popitem() + self.fragments[predicate] = \ + [item for item in worklist + if (item.tag != 'Group' and + item.tag != 'Client' and + not isinstance(item, + lxml.etree._Comment))] + for item in worklist: + cmd = None + if item.tag == 'Group': + if item.get('negate', 'false').lower() == 'true': + cmd = "lambda x:'%s' not in x.groups and predicate(x)" + else: + cmd = "lambda x:'%s' in x.groups and predicate(x)" + elif item.tag == 'Client': + if item.get('negate', 'false').lower() == 'true': + cmd = "lambda x:x.hostname != '%s' and predicate(x)" + else: + cmd = "lambda x:x.hostname == '%s' and predicate(x)" + # else, ignore item + if cmd is not None: + newpred = eval(cmd % item.get('name'), + {'predicate':predicate}) + work[newpred] = item.getchildren() + class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): -- cgit v1.2.3-1-g7c22 From fe0076bb34aaf77b773c0951e2b5141af2d08331 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 19 Apr 2011 07:37:31 -0500 Subject: Reports: Fix import error (Reported by emias on IRC) Signed-off-by: Sol Jerome --- src/lib/Server/Reports/manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/manage.py b/src/lib/Server/Reports/manage.py index 4f84f107a..858bddeca 100755 --- a/src/lib/Server/Reports/manage.py +++ b/src/lib/Server/Reports/manage.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from django.core.management import execute_manager try: - from . import settings # Assumed to be in the same directory. + import settings # Assumed to be in the same directory. except ImportError: import sys sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) -- cgit v1.2.3-1-g7c22 From b5810882e8c6b1e6b76a8239f70a129d415ecee6 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 20 Apr 2011 09:41:07 -0400 Subject: Rewrote bcfg2-repo-validate as bcfg2-lint, which uses a plugin interface to be lots more flexible and extensible. Added several more tests. If bcfg2-lint is run as bcfg2-repo-validate, it roughly emulates the functionality of that program. TODO: Need to figure out correct way to symlink bcfg2-repo-validate to bcfg2-lint on install. --- src/lib/Server/Lint/Bundles.py | 56 +++++++++++ src/lib/Server/Lint/Comments.py | 183 ++++++++++++++++++++++++++++++++++ src/lib/Server/Lint/Duplicates.py | 79 +++++++++++++++ src/lib/Server/Lint/InfoXML.py | 38 +++++++ src/lib/Server/Lint/Pkgmgr.py | 33 +++++++ src/lib/Server/Lint/RequiredAttrs.py | 69 +++++++++++++ src/lib/Server/Lint/Validate.py | 185 +++++++++++++++++++++++++++++++++++ src/lib/Server/Lint/__init__.py | 90 +++++++++++++++++ 8 files changed, 733 insertions(+) create mode 100644 src/lib/Server/Lint/Bundles.py create mode 100644 src/lib/Server/Lint/Comments.py create mode 100644 src/lib/Server/Lint/Duplicates.py create mode 100644 src/lib/Server/Lint/InfoXML.py create mode 100644 src/lib/Server/Lint/Pkgmgr.py create mode 100644 src/lib/Server/Lint/RequiredAttrs.py create mode 100644 src/lib/Server/Lint/Validate.py create mode 100644 src/lib/Server/Lint/__init__.py (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Bundles.py b/src/lib/Server/Lint/Bundles.py new file mode 100644 index 000000000..a1ce631c9 --- /dev/null +++ b/src/lib/Server/Lint/Bundles.py @@ -0,0 +1,56 @@ +import lxml.etree +import Bcfg2.Server.Lint + +class Bundles(Bcfg2.Server.Lint.ServerPlugin): + """ Perform various bundle checks """ + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + """ run plugin """ + self.missing_bundles() + self.bundle_names() + self.sgenshi_groups() + + def missing_bundles(self): + """ find bundles listed in Metadata but not implemented in Bundler """ + groupdata = self.metadata.groups_xml.xdata + ref_bundles = set([b.get("name") + for b in groupdata.findall("//Bundle")]) + + allbundles = self.core.plugins['Bundler'].entries.keys() + for bundle in ref_bundles: + xmlbundle = "%s.xml" % bundle + genshibundle = "%s.genshi" % bundle + if xmlbundle not in allbundles and genshibundle not in allbundles: + self.LintError("Bundle %s referenced, but does not exist" % + bundle) + + def bundle_names(self): + """ verify bundle name attribute matches filename """ + for bundle in self.core.plugins['Bundler'].entries.values(): + if self.HandlesFile(bundle.name): + try: + xdata = lxml.etree.XML(bundle.data) + except AttributeError: + # genshi template + xdata = lxml.etree.parse(bundle.template.filepath).getroot() + + fname = bundle.name.split('Bundler/')[1].split('.')[0] + bname = xdata.get('name') + if fname != bname: + self.LintWarning("Inconsistent bundle name: filename is %s, bundle name is %s" % + (fname, bname)) + + def sgenshi_groups(self): + """ ensure that Genshi Bundles do not include tags, + which are not supported """ + for bundle in self.core.plugins['Bundler'].entries.values(): + if self.HandlesFile(bundle.name): + if (type(bundle) is + Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile): + xdata = lxml.etree.parse(bundle.name) + groups = [self.RenderXML(g) + for g in xdata.getroottree().findall("//Group")] + if groups: + self.LintError(" tag is not allowed in SGenshi Bundle:\n%s" % + "\n".join(groups)) diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py new file mode 100644 index 000000000..0b50df373 --- /dev/null +++ b/src/lib/Server/Lint/Comments.py @@ -0,0 +1,183 @@ +import os.path +import lxml.etree +import Bcfg2.Server.Lint + +class Comments(Bcfg2.Server.Lint.ServerPlugin): + """ check files for various required headers """ + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) + self.config_cache = {} + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + self.check_bundles() + self.check_properties() + self.check_metadata() + self.check_cfg() + self.check_infoxml() + self.check_probes() + + def required_keywords(self, rtype): + """ given a file type, fetch the list of required VCS keywords + from the bcfg2-lint config """ + return self.required_items(rtype, "keyword", default=["Id"]) + + def required_comments(self, rtype): + """ given a file type, fetch the list of required comments + from the bcfg2-lint config """ + return self.required_items(rtype, "comment") + + def required_items(self, rtype, itype, default=None): + """ given a file type and item type (comment or keyword), + fetch the list of required items from the bcfg2-lint config """ + if itype not in self.config_cache: + self.config_cache[itype] = {} + + if rtype not in self.config_cache[itype]: + rv = [] + global_item = "global_%ss" % itype + if global_item in self.config: + rv.extend(self.config[global_item].split(",")) + elif default is not None: + rv.extend(default) + + item = "%s_%ss" % (rtype.lower(), itype) + if item in self.config: + if self.config[item]: + rv.extend(self.config[item].split(",")) + else: + # config explicitly specifies nothing + rv = [] + self.config_cache[itype][rtype] = rv + return self.config_cache[itype][rtype] + + def check_bundles(self): + """ check bundle files for required headers """ + for bundle in self.core.plugins['Bundler'].entries.values(): + xdata = None + rtype = "" + try: + xdata = lxml.etree.XML(bundle.data) + rtype = "bundler" + except AttributeError: + xdata = lxml.etree.parse(bundle.template.filepath).getroot() + rtype = "sgenshi" + + self.check_xml(bundle.name, xdata, rtype) + + def check_properties(self): + """ check properties files for required headers """ + if 'Properties' in self.core.plugins: + props = self.core.plugins['Properties'] + for propfile, pdata in props.store.entries.items(): + if os.path.splitext(propfile)[1] == ".xml": + self.check_xml(pdata.name, pdata.data, 'properties') + + def check_metadata(self): + """ check metadata files for required headers """ + metadata = self.core.plugins['Metadata'] + if self.has_all_xincludes("groups.xml"): + self.check_xml(os.path.join(metadata.data, "groups.xml"), + metadata.groups_xml.data, + "metadata") + if self.has_all_xincludes("clients.xml"): + self.check_xml(os.path.join(metadata.data, "clients.xml"), + metadata.clients_xml.data, + "metadata") + + def check_cfg(self): + """ check Cfg files for required headers """ + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if entry.name.endswith(".genshi"): + rtype = "tgenshi" + else: + rtype = "cfg" + self.check_plaintext(entry.name, entry.data, rtype) + + def check_infoxml(self): + """ check info.xml files for required headers """ + for entryset in self.core.plugins['Cfg'].entries.items(): + if hasattr(entryset, "infoxml") and entryset.infoxml is not None: + self.check_xml(entryset.infoxml.name, + entryset.infoxml.pnode.data, + "infoxml") + + def check_probes(self): + """ check probes for required headers """ + if 'Probes' in self.core.plugins: + for probe in self.core.plugins['Probes'].probes.entries.values(): + self.check_plaintext(probe.name, probe.data, "probes") + + def check_xml(self, filename, xdata, rtype): + """ check generic XML files for required headers """ + self.check_lines(filename, + [str(el) + for el in xdata.getiterator(lxml.etree.Comment)], + rtype) + + def check_plaintext(self, filename, data, rtype): + """ check generic plaintex files for required headers """ + self.check_lines(filename, data.splitlines(), rtype) + + def check_lines(self, filename, lines, rtype): + """ generic header check for a set of lines """ + if self.HandlesFile(filename): + # found is trivalent: + # False == not found + # None == found but not expanded + # True == found and expanded + found = dict((k, False) for k in self.required_keywords(rtype)) + + for line in lines: + # we check for both '$:' and '$$' to see + # if the keyword just hasn't been expanded + for (keyword, status) in found.items(): + if not status: + if '$%s:' % keyword in line: + found[keyword] = True + elif '$%s$' % keyword in line: + found[keyword] = None + + unexpanded = [keyword for (keyword, status) in found.items() + if status is None] + if unexpanded: + self.LintError("%s: Required keywords(s) found but not expanded: %s" % + (filename, ", ".join(unexpanded))) + missing = [keyword for (keyword, status) in found.items() + if status is False] + if missing: + self.LintError("%s: Required keywords(s) not found: %s" % + (filename, ", ".join(missing))) + + # next, check for required comments. found is just + # boolean + found = dict((k, False) for k in self.required_comments(rtype)) + + for line in lines: + for (comment, status) in found.items(): + if not status: + found[comment] = comment in line + + missing = [comment for (comment, status) in found.items() + if status is False] + if missing: + self.LintError("%s: Required comments(s) not found: %s" % + (filename, ", ".join(missing))) + + def has_all_xincludes(self, mfile): + """ return true if self.files includes all XIncludes listed in + the specified metadata type, false otherwise""" + if self.files is None: + return True + else: + path = os.path.join(self.metadata.data, mfile) + if path in self.files: + xdata = lxml.etree.parse(path) + for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): + if not self.has_all_xincludes(el.get('href')): + self.LintWarning("Broken XInclude chain: could not include %s" % path) + return False + + return True + diff --git a/src/lib/Server/Lint/Duplicates.py b/src/lib/Server/Lint/Duplicates.py new file mode 100644 index 000000000..c8b542025 --- /dev/null +++ b/src/lib/Server/Lint/Duplicates.py @@ -0,0 +1,79 @@ +import os.path +import lxml.etree +import Bcfg2.Server.Lint + +class Duplicates(Bcfg2.Server.Lint.ServerPlugin): + """ Find duplicate clients, groups, etc. """ + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) + self.groups_xdata = None + self.clients_xdata = None + self.load_xdata() + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + """ run plugin """ + # only run this plugin if we were not given a list of files. + # not only is it marginally silly to run this plugin with a + # partial list of files, it turns out to be really freaking + # hard to get only a fragment of group or client metadata + if self.groups_xdata is not None: + self.duplicate_groups() + self.duplicate_defaults() + if self.clients_xdata is not None: + self.duplicate_clients() + + def load_xdata(self): + """ attempt to load XML data for groups and clients. only + actually load data if all documents reference in XIncludes can + be found in self.files""" + if self.has_all_xincludes("groups.xml"): + self.groups_xdata = self.metadata.clients_xml.xdata + if self.has_all_xincludes("clients.xml"): + self.clients_xdata = self.metadata.clients_xml.xdata + + def duplicate_groups(self): + """ find duplicate groups """ + self.duplicate_entries(self.clients_xdata.xpath('//Groups/Group'), + 'group') + + def duplicate_clients(self): + """ find duplicate clients """ + self.duplicate_entries(self.clients_xdata.xpath('//Clients/Client'), + 'client') + + def duplicate_entries(self, data, etype): + """ generic duplicate entry finder """ + seen = {} + for el in data: + if el.get('name') not in seen: + seen[el.get('name')] = el + else: + self.LintError("Duplicate %s '%s':\n%s\n%s" % + (etype, el.get('name'), + self.RenderXML(seen[el.get('name')]), + self.RenderXML(el))) + + def duplicate_defaults(self): + """ check for multiple default group definitions """ + default_groups = [g for g in self.groups_xdata.findall('.//Group') + if g.get('default') == 'true'] + if len(default_groups) > 1: + self.LintError("Multiple default groups defined: %s" % + ",".join(default_groups)) + + def has_all_xincludes(self, mfile): + """ return true if self.files includes all XIncludes listed in + the specified metadata type, false otherwise""" + if self.files is None: + return True + else: + path = os.path.join(self.metadata.data, mfile) + if path in self.files: + xdata = lxml.etree.parse(path) + for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): + if not self.has_all_xincludes(el.get('href')): + self.LintWarning("Broken XInclude chain: could not include %s" % path) + return False + + return True diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py new file mode 100644 index 000000000..097c2d6f9 --- /dev/null +++ b/src/lib/Server/Lint/InfoXML.py @@ -0,0 +1,38 @@ +import os.path +import Bcfg2.Options +import Bcfg2.Server.Lint + +class InfoXML(Bcfg2.Server.Lint.ServerPlugin): + """ ensure that all config files have an info.xml file""" + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + for filename, entryset in self.core.plugins['Cfg'].entries.items(): + infoxml_fname = os.path.join(entryset.path, "info.xml") + if self.HandlesFile(infoxml_fname): + if (hasattr(entryset, "infoxml") and + entryset.infoxml is not None): + xdata = entryset.infoxml.pnode.data + for info in xdata.getroottree().findall("//Info"): + required = ["owner", "group", "perms"] + if "required" in self.config: + required = self.config["required"].split(",") + + missing = [attr for attr in required + if info.get(attr) is None] + if missing: + self.LintError("Required attribute(s) %s not found in %s:%s" % + (",".join(missing), infoxml_fname, + self.RenderXML(info))) + + if ("require_paranoid" in self.config and + self.config["require_paranoid"].lower() == "true" and + not Bcfg2.Options.MDATA_PARANOID.value and + info.get("paranoid").lower() != "true"): + self.LintError("Paranoid must be true in %s:%s" % + (infoxml_fname, + self.RenderXML(info))) + elif ("require" in self.config and + self.config["require"].lower != "false"): + self.LintError("No info.xml found for %s" % filename) + diff --git a/src/lib/Server/Lint/Pkgmgr.py b/src/lib/Server/Lint/Pkgmgr.py new file mode 100644 index 000000000..28ca698bd --- /dev/null +++ b/src/lib/Server/Lint/Pkgmgr.py @@ -0,0 +1,33 @@ +import Bcfg2.Server.Lint + +class Pkgmgr(Bcfg2.Server.Lint.ServerPlugin): + """ find duplicate Pkgmgr entries with the same priority """ + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + pset = set() + for plist in self.core.plugins['Pkgmgr'].entries.values(): + if self.HandlesFile(plist.name): + xdata = plist.data + # get priority, type, group + priority = xdata.getroot().get('priority') + ptype = xdata.getroot().get('type') + for pkg in xdata.findall("//Package"): + if pkg.getparent().tag == 'Group': + grp = pkg.getparent().get('name') + if (type(grp) is not str and + grp.getparent().tag == 'Group'): + pgrp = grp.getparent().get('name') + else: + pgrp = 'none' + else: + grp = 'none' + pgrp = 'none' + ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) + # check if package is already listed with same + # priority, type, grp + if ptuple in pset: + self.LintWarning("Duplicate Package %s, priority:%s, type:%s" % + (pkg.get('name'), priority, ptype)) + else: + pset.add(ptuple) diff --git a/src/lib/Server/Lint/RequiredAttrs.py b/src/lib/Server/Lint/RequiredAttrs.py new file mode 100644 index 000000000..7215fe163 --- /dev/null +++ b/src/lib/Server/Lint/RequiredAttrs.py @@ -0,0 +1,69 @@ +import os.path +import lxml.etree +import Bcfg2.Server.Lint + +class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): + """ verify attributes for configuration entries (as defined in + doc/server/configurationentries) """ + + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) + self.required_attrs = { + 'device': ['name', 'owner', 'group', 'dev_type'], + 'directory': ['name', 'owner', 'group', 'perms'], + 'file': ['name', 'owner', 'group', 'perms'], + 'hardlink': ['name', 'to'], + 'symlink': ['name', 'to'], + 'ignore': ['name'], + 'nonexistent': ['name'], + 'permissions': ['name', 'owner', 'group', 'perms']} + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + self.check_rules() + self.check_bundles() + + def check_rules(self): + """ check Rules for Path entries with missing attrs """ + if 'Rules' in self.core.plugins: + for rules in self.core.plugins['Rules'].entries.values(): + xdata = rules.pnode.data + for path in xdata.xpath("//Path"): + self.check_entry(path, os.path.join(self.config['repo'], + rules.name)) + + def check_bundles(self): + """ check bundles for BoundPath entries with missing attrs """ + for bundle in self.core.plugins['Bundler'].entries.values(): + try: + xdata = lxml.etree.XML(bundle.data) + except AttributeError: + xdata = lxml.etree.parse(bundle.template.filepath).getroot() + + for path in xdata.xpath("//BoundPath"): + self.check_entry(path, bundle.name) + + def check_entry(self, entry, filename): + """ generic entry check """ + if self.HandlesFile(filename): + pathname = entry.get('name') + pathtype = entry.get('type') + pathset = set(entry.attrib.keys()) + try: + required_attrs = set(self.required_attrs[pathtype] + ['type']) + except KeyError: + self.LintError("Unknown path type %s: %s" % + (pathtype, self.RenderXML(entry))) + + if 'dev_type' in required_attrs: + dev_type = entry.get('dev_type') + if dev_type in ['block', 'char']: + # check if major/minor are specified + required_attrs |= set(['major', 'minor']) + if not pathset.issuperset(required_attrs): + self.LintError("The required attributes %s are missing for %s %sin %s:\n%s" % + (",".join([attr + for attr in + required_attrs.difference(pathset)]), + entry.tag, pathname, filename, + self.RenderXML(entry))) diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py new file mode 100644 index 000000000..bb5af93f4 --- /dev/null +++ b/src/lib/Server/Lint/Validate.py @@ -0,0 +1,185 @@ +import glob +import lxml.etree +import os +import fnmatch +import Bcfg2.Options +import Bcfg2.Server.Lint +from subprocess import Popen, PIPE, STDOUT + +class Validate(Bcfg2.Server.Lint.ServerlessPlugin): + """ Ensure that the repo validates """ + + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + self.filesets = {"metadata:groups":"%s/metadata.xsd", + "metadata:clients":"%s/clients.xsd", + "info":"%s/info.xsd", + "%s/Bundler/*.{xml,genshi}":"%s/bundle.xsd", + "%s/Pkgmgr/*.xml":"%s/pkglist.xsd", + "%s/Base/*.xml":"%s/base.xsd", + "%s/Rules/*.xml":"%s/rules.xsd", + "%s/etc/report-configuration.xml":"%s/report-configuration.xsd", + "%s/Svcmgr/*.xml":"%s/services.xsd", + "%s/Deps/*.xml":"%s/deps.xsd", + "%s/Decisions/*.xml":"%s/decisions.xsd", + "%s/Packages/config.xml":"%s/packages.xsd", + "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd"} + + self.filelists = {} + self.get_filelists() + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + self.schemadir = self.config['schema'] + + for schemaname, path in self.filesets.items(): + try: + filelist = self.filelists[path] + except KeyError: + filelist = [] + + if filelist: + # avoid loading schemas for empty file lists + try: + schema = lxml.etree.XMLSchema(lxml.etree.parse(schemaname % + schemadir)) + except: + self.LintWarning("Failed to process schema %s", + schemaname % schemadir) + continue + for filename in filelist: + self.validate(filename, schemaname % schemadir, + schema=schema) + + self.check_properties() + + def check_properties(self): + """ check Properties files against their schemas """ + alert = self.logger.debug + if "properties_schema" in self.config: + if self.config['properties_schema'].lower().startswith('warn'): + alert = self.LintWarning + elif self.config['properties_schema'].lower().startswith('require'): + alert = self.LintError + + for filename in self.filelists['props']: + schemafile = "%s.xsd" % os.path.splitext(filename)[0] + if os.path.exists(schemafile): + self.validate(filename, schemafile) + else: + alert("No schema found for %s" % filename) + + def validate(self, filename, schemafile, schema=None): + """validate a file against the given lxml.etree.Schema. + return True on success, False on failure """ + if schema is None: + # if no schema object was provided, instantiate one + try: + schema = lxml.etree.XMLSchema(lxml.etree.parse(schemafile)) + except: + self.LintWarning("Failed to process schema %s" % schemafile) + return False + + try: + datafile = lxml.etree.parse(filename) + except SyntaxError: + lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT) + self.LintError("%s fails to parse:\n%s" % (filename, + lint.communicate()[0])) + lint.wait() + return False + except IOError: + self.LintError("Failed to open file %s" % filename) + return False + + if not schema.validate(datafile): + cmd = ["xmllint"] + if self.files is None: + cmd.append("--xinclude") + cmd.extend(["--noout", "--schema", schemafile, filename]) + lint = Popen(cmd, stdout=PIPE, stderr=STDOUT) + output = lint.communicate()[0] + if lint.wait(): + self.LintError("%s fails to verify:\n%s" % (filename, output)) + return False + return True + + def get_filelists(self): + """ get lists of different kinds of files to validate """ + if self.files is not None: + listfiles = lambda p: fnmatch.filter(self.files, p % "*") + else: + listfiles = lambda p: glob.glob(p % self.config['repo']) + + for path in self.filesets.keys(): + if path.startswith("metadata:"): + mtype = path.split(":")[1] + self.filelists[path] = self.get_metadata_list(mtype) + elif path == "info": + if self.files is not None: + self.filelists[path] = \ + [f for f in self.files + if os.path.basename(f) == 'info.xml'] + else: # self.files is None + self.filelists[path] = [] + for infodir in ['Cfg', 'TGenshi', 'TCheetah']: + for root, dirs, files in os.walk('%s/%s' % + (self.config['repo'], + infodir)): + self.filelists[path].extend([os.path.join(root, f) + for f in files + if f == 'info.xml']) + else: + self.filelists[path] = listfiles(path) + + self.filelists['props'] = listfiles("%s/Properties/*.xml") + all_metadata = listfiles("%s/Metadata/*.xml") + + # if there are other files in Metadata that aren't xincluded + # from clients.xml or groups.xml, we can't verify them. warn + # about those. + for fname in all_metadata: + if (fname not in self.filelists['metadata:groups'] and + fname not in self.filelists['metadata:clients']): + self.LintWarning("Broken XInclude chain: Could not determine file type of %s" % fname) + + def get_metadata_list(self, mtype): + """ get all metadata files for the specified type (clients or + group) """ + if self.files is not None: + rv = fnmatch.filter(self.files, "*/Metadata/%s.xml" % mtype) + else: + rv = glob.glob("%s/Metadata/%s.xml" % (self.config['repo'], mtype)) + + # attempt to follow XIncludes. if the top-level files aren't + # listed in self.files, though, there's really nothing we can + # do to guess what a file in Metadata is + if rv: + rv.extend(self.follow_xinclude(rv[0])) + + return rv + + def follow_xinclude(self, xfile): + """ follow xincludes in the given file """ + xdata = lxml.etree.parse(xfile) + included = set([ent.get('href') for ent in + xdata.findall('./{http://www.w3.org/2001/XInclude}include')]) + rv = [] + + while included: + try: + filename = included.pop() + except KeyError: + continue + + path = os.path.join(os.path.dirname(xfile), filename) + if self.HandlesFile(path): + rv.append(path) + groupdata = lxml.etree.parse(path) + [included.add(el.get('href')) + for el in + groupdata.findall('./{http://www.w3.org/2001/XInclude}include')] + included.discard(filename) + + return rv + diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py new file mode 100644 index 000000000..4e6d03fb5 --- /dev/null +++ b/src/lib/Server/Lint/__init__.py @@ -0,0 +1,90 @@ +__revision__ = '$Revision$' + +__all__ = ['Bundles', + 'Comments', + 'Duplicates', + 'InfoXML', + 'Pkgmgr', + 'RequiredAttrs', + 'Validate'] + +import logging +import os.path +from copy import copy +import lxml.etree +import Bcfg2.Logger + +def returnErrors(fn): + """ Decorator for Run method that returns error counts """ + def run(self, *args, **kwargs): + fn(self, *args, **kwargs) + return (self.error_count, self.warning_count) + + return run + +class Plugin (object): + """ base class for ServerlessPlugin and ServerPlugin """ + def __init__(self, config, files=None): + self.files = files + self.error_count = 0 + self.warning_count = 0 + self.config = config + Bcfg2.Logger.setup_logging('bcfg2-info', to_syslog=False) + self.logger = logging.getLogger('bcfg2-lint') + + def Run(self): + """ run the plugin. must be overloaded by child classes """ + pass + + def HandlesFile(self, fname): + """ returns true if the given file should be handled by the + plugin according to the files list, false otherwise """ + return (self.files is None or + fname in self.files or + os.path.join(self.config['repo'], fname) in self.files or + os.path.abspath(fname) in self.files or + os.path.abspath(os.path.join(self.config['repo'], + fname)) in self.files) + + def LintError(self, msg): + """ log an error condition """ + self.error_count += 1 + lines = msg.splitlines() + self.logger.error("ERROR: %s" % lines.pop()) + [self.logger.error(" %s" % l) for l in lines] + + def LintWarning(self, msg): + """ log a warning condition """ + self.warning_count += 1 + lines = msg.splitlines() + self.logger.warning("WARNING: %s" % lines.pop()) + [self.logger.warning(" %s" % l) for l in lines] + + def RenderXML(self, element): + """render an XML element for error output -- line number + prefixed, no children""" + xml = None + if len(element) or element.text: + el = copy(element) + if el.text: + el.text = '...' + [el.remove(c) for c in el.iterchildren()] + xml = lxml.etree.tostring(el).strip() + else: + xml = lxml.etree.tostring(element).strip() + return " line %s: %s" % (element.sourceline, xml) + +class ServerlessPlugin (Plugin): + """ base class for plugins that are run before the server starts + up (i.e., plugins that check things that may prevent the server + from starting up) """ + pass + +class ServerPlugin (Plugin): + """ base class for plugins that check things that require the + running Bcfg2 server """ + def __init__(self, lintCore, config, files=None): + Plugin.__init__(self, config, files=files) + self.core = lintCore + self.logger = self.core.logger + self.metadata = self.core.metadata -- cgit v1.2.3-1-g7c22 From e84d86449811cf4278aeaf9c7ba46e78d12decfe Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 20 Apr 2011 11:40:28 -0400 Subject: short-circuit Pkgmgr lint checks if Pkgmgr plugin not enabled --- src/lib/Server/Lint/Pkgmgr.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Pkgmgr.py b/src/lib/Server/Lint/Pkgmgr.py index 28ca698bd..f2eb2a5f6 100644 --- a/src/lib/Server/Lint/Pkgmgr.py +++ b/src/lib/Server/Lint/Pkgmgr.py @@ -5,6 +5,10 @@ class Pkgmgr(Bcfg2.Server.Lint.ServerPlugin): @Bcfg2.Server.Lint.returnErrors def Run(self): + if 'Pkgmgr' not in self.core.plugins: + self.LintWarning("Pkgmgr server plugin is not enabled, skipping Pkgmgr lint checks") + return + pset = set() for plist in self.core.plugins['Pkgmgr'].entries.values(): if self.HandlesFile(plist.name): -- cgit v1.2.3-1-g7c22 From 34af400ef16018e0753413454ba55af4c94856b0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 20 Apr 2011 13:31:46 -0400 Subject: fixed duplicate group detection in Metadata --- src/lib/Server/Plugins/Metadata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index 98caa577d..ff723941f 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -478,8 +478,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, grouptmp = {} self.categories = {} groupseen = list() - for group in xdata.xpath('//Groups/Group') \ - + xdata.xpath('Group'): + for group in xdata.xpath('//Groups/Group'): if group.get('name') not in groupseen: groupseen.append(group.get('name')) else: -- cgit v1.2.3-1-g7c22 From 6da7d24710fe67c80c4a71f227cd01675eebca88 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 21 Apr 2011 08:50:06 -0400 Subject: Lots of cleanup for bcfg2-repo-validate rewrite: * Changed all references to bcfg2-repo-validate in the documentation to bcfg2-lint * Wrote man pages for bcfg2-lint and bcfg2-lint.conf * Cleaned up straggling references to bcfg2-repo-validate in Makefiles, spec files, and the POSIX tool * A few minor bug fixes --- src/lib/Server/Lint/InfoXML.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py index 097c2d6f9..42679109a 100644 --- a/src/lib/Server/Lint/InfoXML.py +++ b/src/lib/Server/Lint/InfoXML.py @@ -15,8 +15,8 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): xdata = entryset.infoxml.pnode.data for info in xdata.getroottree().findall("//Info"): required = ["owner", "group", "perms"] - if "required" in self.config: - required = self.config["required"].split(",") + if "required_attrs" in self.config: + required = self.config["required_attrs"].split(",") missing = [attr for attr in required if info.get(attr) is None] @@ -27,8 +27,12 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if ("require_paranoid" in self.config and self.config["require_paranoid"].lower() == "true" and - not Bcfg2.Options.MDATA_PARANOID.value and - info.get("paranoid").lower() != "true"): + (Bcfg2.Options.MDATA_PARANOID.value and + info.get("paranoid") is not None and + info.get("paranoid").lower() == "false") or + (not Bcfg2.Options.MDATA_PARANOID.value and + (info.get("paranoid") is None or + info.get("paranoid").lower() != "true"))): self.LintError("Paranoid must be true in %s:%s" % (infoxml_fname, self.RenderXML(info))) -- cgit v1.2.3-1-g7c22 From 24f6403f1ba483b813e30ea15446a59090d90d90 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 21 Apr 2011 14:43:39 -0400 Subject: Misc. bcfg2-lint fixes and tweaks: * fixed bcfg2-lint bug with older pythons * made bcfg2-lint silent by default on success * adjusted bcfg2-lint defaults and alerting levels to work better out-of-the-box --- src/lib/Server/Lint/Bundles.py | 2 +- src/lib/Server/Lint/Comments.py | 6 ++---- src/lib/Server/Lint/InfoXML.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Bundles.py b/src/lib/Server/Lint/Bundles.py index a1ce631c9..b242239ae 100644 --- a/src/lib/Server/Lint/Bundles.py +++ b/src/lib/Server/Lint/Bundles.py @@ -52,5 +52,5 @@ class Bundles(Bcfg2.Server.Lint.ServerPlugin): groups = [self.RenderXML(g) for g in xdata.getroottree().findall("//Group")] if groups: - self.LintError(" tag is not allowed in SGenshi Bundle:\n%s" % + self.LintWarning(" tag is not allowed in SGenshi Bundle:\n%s" % "\n".join(groups)) diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 0b50df373..1b75bb25e 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -20,14 +20,14 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): def required_keywords(self, rtype): """ given a file type, fetch the list of required VCS keywords from the bcfg2-lint config """ - return self.required_items(rtype, "keyword", default=["Id"]) + return self.required_items(rtype, "keyword") def required_comments(self, rtype): """ given a file type, fetch the list of required comments from the bcfg2-lint config """ return self.required_items(rtype, "comment") - def required_items(self, rtype, itype, default=None): + def required_items(self, rtype, itype): """ given a file type and item type (comment or keyword), fetch the list of required items from the bcfg2-lint config """ if itype not in self.config_cache: @@ -38,8 +38,6 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): global_item = "global_%ss" % itype if global_item in self.config: rv.extend(self.config[global_item].split(",")) - elif default is not None: - rv.extend(default) item = "%s_%ss" % (rtype.lower(), itype) if item in self.config: diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py index 42679109a..798d8c208 100644 --- a/src/lib/Server/Lint/InfoXML.py +++ b/src/lib/Server/Lint/InfoXML.py @@ -14,7 +14,7 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): entryset.infoxml is not None): xdata = entryset.infoxml.pnode.data for info in xdata.getroottree().findall("//Info"): - required = ["owner", "group", "perms"] + required = [] if "required_attrs" in self.config: required = self.config["required_attrs"].split(",") -- cgit v1.2.3-1-g7c22 From 17b8ceb17e0ee775a667d2f92b2b192e567b2df6 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 25 Apr 2011 10:45:41 -0400 Subject: Various bcfg2-lint fixes: * check for all plugins before referencing them, since in --stdin mode even plugins like Bundler may not be instantiated * formatting fixes * made Bundles plugin work with or without genshi installed * fixed name of plugin in example bcfg2-lint.conf --- src/lib/Server/Lint/Bundles.py | 81 +++++++++++++++++++----------------- src/lib/Server/Lint/Comments.py | 63 +++++++++++++++------------- src/lib/Server/Lint/InfoXML.py | 61 ++++++++++++++------------- src/lib/Server/Lint/RequiredAttrs.py | 15 +++---- 4 files changed, 115 insertions(+), 105 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Bundles.py b/src/lib/Server/Lint/Bundles.py index b242239ae..417f76c2d 100644 --- a/src/lib/Server/Lint/Bundles.py +++ b/src/lib/Server/Lint/Bundles.py @@ -1,56 +1,61 @@ import lxml.etree import Bcfg2.Server.Lint - + class Bundles(Bcfg2.Server.Lint.ServerPlugin): """ Perform various bundle checks """ @Bcfg2.Server.Lint.returnErrors def Run(self): """ run plugin """ - self.missing_bundles() - self.bundle_names() - self.sgenshi_groups() + if 'Bundler' in self.core.plugins: + self.missing_bundles() + for bundle in self.core.plugins['Bundler'].entries.values(): + if self.HandlesFile(bundle.name): + if (Bcfg2.Server.Plugins.Bundler.have_genshi and + type(bundle) is + Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile): + self.sgenshi_groups(bundle) + else: + self.bundle_names(bundle) def missing_bundles(self): """ find bundles listed in Metadata but not implemented in Bundler """ - groupdata = self.metadata.groups_xml.xdata - ref_bundles = set([b.get("name") - for b in groupdata.findall("//Bundle")]) + if self.files is None: + # when given a list of files on stdin, this check is + # useless, so skip it + groupdata = self.metadata.groups_xml.xdata + ref_bundles = set([b.get("name") + for b in groupdata.findall("//Bundle")]) - allbundles = self.core.plugins['Bundler'].entries.keys() - for bundle in ref_bundles: - xmlbundle = "%s.xml" % bundle - genshibundle = "%s.genshi" % bundle - if xmlbundle not in allbundles and genshibundle not in allbundles: - self.LintError("Bundle %s referenced, but does not exist" % - bundle) + allbundles = self.core.plugins['Bundler'].entries.keys() + for bundle in ref_bundles: + xmlbundle = "%s.xml" % bundle + genshibundle = "%s.genshi" % bundle + if (xmlbundle not in allbundles and + genshibundle not in allbundles): + self.LintError("Bundle %s referenced, but does not exist" % + bundle) - def bundle_names(self): + def bundle_names(self, bundle): """ verify bundle name attribute matches filename """ - for bundle in self.core.plugins['Bundler'].entries.values(): - if self.HandlesFile(bundle.name): - try: - xdata = lxml.etree.XML(bundle.data) - except AttributeError: - # genshi template - xdata = lxml.etree.parse(bundle.template.filepath).getroot() + try: + xdata = lxml.etree.XML(bundle.data) + except AttributeError: + # genshi template + xdata = lxml.etree.parse(bundle.template.filepath).getroot() - fname = bundle.name.split('Bundler/')[1].split('.')[0] - bname = xdata.get('name') - if fname != bname: - self.LintWarning("Inconsistent bundle name: filename is %s, bundle name is %s" % - (fname, bname)) + fname = bundle.name.split('Bundler/')[1].split('.')[0] + bname = xdata.get('name') + if fname != bname: + self.LintWarning("Inconsistent bundle name: filename is %s, bundle name is %s" % + (fname, bname)) - def sgenshi_groups(self): + def sgenshi_groups(self, bundle): """ ensure that Genshi Bundles do not include tags, which are not supported """ - for bundle in self.core.plugins['Bundler'].entries.values(): - if self.HandlesFile(bundle.name): - if (type(bundle) is - Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile): - xdata = lxml.etree.parse(bundle.name) - groups = [self.RenderXML(g) - for g in xdata.getroottree().findall("//Group")] - if groups: - self.LintWarning(" tag is not allowed in SGenshi Bundle:\n%s" % - "\n".join(groups)) + xdata = lxml.etree.parse(bundle.name) + groups = [self.RenderXML(g) + for g in xdata.getroottree().findall("//Group")] + if groups: + self.LintWarning(" tag is not allowed in SGenshi Bundle:\n%s" % + "\n".join(groups)) diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 1b75bb25e..587f20603 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -51,17 +51,18 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): def check_bundles(self): """ check bundle files for required headers """ - for bundle in self.core.plugins['Bundler'].entries.values(): - xdata = None - rtype = "" - try: - xdata = lxml.etree.XML(bundle.data) - rtype = "bundler" - except AttributeError: - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - rtype = "sgenshi" - - self.check_xml(bundle.name, xdata, rtype) + if 'Bundler' in self.core.plugins: + for bundle in self.core.plugins['Bundler'].entries.values(): + xdata = None + rtype = "" + try: + xdata = lxml.etree.XML(bundle.data) + rtype = "bundler" + except AttributeError: + xdata = lxml.etree.parse(bundle.template.filepath).getroot() + rtype = "sgenshi" + + self.check_xml(bundle.name, xdata, rtype) def check_properties(self): """ check properties files for required headers """ @@ -73,33 +74,35 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): def check_metadata(self): """ check metadata files for required headers """ - metadata = self.core.plugins['Metadata'] if self.has_all_xincludes("groups.xml"): - self.check_xml(os.path.join(metadata.data, "groups.xml"), - metadata.groups_xml.data, + self.check_xml(os.path.join(self.metadata.data, "groups.xml"), + self.metadata.groups_xml.data, "metadata") if self.has_all_xincludes("clients.xml"): - self.check_xml(os.path.join(metadata.data, "clients.xml"), - metadata.clients_xml.data, + self.check_xml(os.path.join(self.metadata.data, "clients.xml"), + self.metadata.clients_xml.data, "metadata") def check_cfg(self): """ check Cfg files for required headers """ - for entryset in self.core.plugins['Cfg'].entries.values(): - for entry in entryset.entries.values(): - if entry.name.endswith(".genshi"): - rtype = "tgenshi" - else: - rtype = "cfg" - self.check_plaintext(entry.name, entry.data, rtype) + if 'Cfg' in self.core.plugins: + for entryset in self.core.plugins['Cfg'].entries.values(): + for entry in entryset.entries.values(): + if entry.name.endswith(".genshi"): + rtype = "tgenshi" + else: + rtype = "cfg" + self.check_plaintext(entry.name, entry.data, rtype) def check_infoxml(self): """ check info.xml files for required headers """ - for entryset in self.core.plugins['Cfg'].entries.items(): - if hasattr(entryset, "infoxml") and entryset.infoxml is not None: - self.check_xml(entryset.infoxml.name, - entryset.infoxml.pnode.data, - "infoxml") + if 'Cfg' in self.core.plugins: + for entryset in self.core.plugins['Cfg'].entries.items(): + if (hasattr(entryset, "infoxml") and + entryset.infoxml is not None): + self.check_xml(entryset.infoxml.name, + entryset.infoxml.pnode.data, + "infoxml") def check_probes(self): """ check probes for required headers """ @@ -145,8 +148,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): missing = [keyword for (keyword, status) in found.items() if status is False] if missing: - self.LintError("%s: Required keywords(s) not found: %s" % - (filename, ", ".join(missing))) + self.LintError("%s: Required keywords(s) not found: $%s$" % + (filename, "$, $".join(missing))) # next, check for required comments. found is just # boolean diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py index 798d8c208..25f609902 100644 --- a/src/lib/Server/Lint/InfoXML.py +++ b/src/lib/Server/Lint/InfoXML.py @@ -7,36 +7,37 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): @Bcfg2.Server.Lint.returnErrors def Run(self): - for filename, entryset in self.core.plugins['Cfg'].entries.items(): - infoxml_fname = os.path.join(entryset.path, "info.xml") - if self.HandlesFile(infoxml_fname): - if (hasattr(entryset, "infoxml") and - entryset.infoxml is not None): - xdata = entryset.infoxml.pnode.data - for info in xdata.getroottree().findall("//Info"): - required = [] - if "required_attrs" in self.config: - required = self.config["required_attrs"].split(",") + if 'Cfg' in self.core.plugins: + for filename, entryset in self.core.plugins['Cfg'].entries.items(): + infoxml_fname = os.path.join(entryset.path, "info.xml") + if self.HandlesFile(infoxml_fname): + if (hasattr(entryset, "infoxml") and + entryset.infoxml is not None): + self.check_infoxml(entryset.infoxml.pnode.data) + elif ("require" in self.config and + self.config["require"].lower != "false"): + self.LintError("No info.xml found for %s" % filename) - missing = [attr for attr in required - if info.get(attr) is None] - if missing: - self.LintError("Required attribute(s) %s not found in %s:%s" % - (",".join(missing), infoxml_fname, - self.RenderXML(info))) + def check_infoxml(self, xdata): + for info in xdata.getroottree().findall("//Info"): + required = [] + if "required_attrs" in self.config: + required = self.config["required_attrs"].split(",") - if ("require_paranoid" in self.config and - self.config["require_paranoid"].lower() == "true" and - (Bcfg2.Options.MDATA_PARANOID.value and - info.get("paranoid") is not None and - info.get("paranoid").lower() == "false") or - (not Bcfg2.Options.MDATA_PARANOID.value and - (info.get("paranoid") is None or - info.get("paranoid").lower() != "true"))): - self.LintError("Paranoid must be true in %s:%s" % - (infoxml_fname, - self.RenderXML(info))) - elif ("require" in self.config and - self.config["require"].lower != "false"): - self.LintError("No info.xml found for %s" % filename) + missing = [attr for attr in required if info.get(attr) is None] + if missing: + self.LintError("Required attribute(s) %s not found in %s:%s" % + (",".join(missing), infoxml_fname, + self.RenderXML(info))) + + if ("require_paranoid" in self.config and + self.config["require_paranoid"].lower() == "true" and + (Bcfg2.Options.MDATA_PARANOID.value and + info.get("paranoid") is not None and + info.get("paranoid").lower() == "false") or + (not Bcfg2.Options.MDATA_PARANOID.value and + (info.get("paranoid") is None or + info.get("paranoid").lower() != "true"))): + self.LintError("Paranoid must be true in %s:%s" % + (infoxml_fname, self.RenderXML(info))) diff --git a/src/lib/Server/Lint/RequiredAttrs.py b/src/lib/Server/Lint/RequiredAttrs.py index 7215fe163..70ce4fe0a 100644 --- a/src/lib/Server/Lint/RequiredAttrs.py +++ b/src/lib/Server/Lint/RequiredAttrs.py @@ -34,14 +34,15 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): def check_bundles(self): """ check bundles for BoundPath entries with missing attrs """ - for bundle in self.core.plugins['Bundler'].entries.values(): - try: - xdata = lxml.etree.XML(bundle.data) - except AttributeError: - xdata = lxml.etree.parse(bundle.template.filepath).getroot() + if 'Bundler' in self.core.plugins: + for bundle in self.core.plugins['Bundler'].entries.values(): + try: + xdata = lxml.etree.XML(bundle.data) + except AttributeError: + xdata = lxml.etree.parse(bundle.template.filepath).getroot() - for path in xdata.xpath("//BoundPath"): - self.check_entry(path, bundle.name) + for path in xdata.xpath("//BoundPath"): + self.check_entry(path, bundle.name) def check_entry(self, entry, filename): """ generic entry check """ -- cgit v1.2.3-1-g7c22 From 5f7092b061cb200afef2eff2aa39fc150a6ea838 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 25 Apr 2011 10:58:48 -0400 Subject: unexpanded vcs keywords raise warning, not error --- src/lib/Server/Lint/Comments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 587f20603..8c83545b3 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -143,8 +143,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): unexpanded = [keyword for (keyword, status) in found.items() if status is None] if unexpanded: - self.LintError("%s: Required keywords(s) found but not expanded: %s" % - (filename, ", ".join(unexpanded))) + self.LintWarning("%s: Required keywords(s) found but not expanded: %s" % + (filename, ", ".join(unexpanded))) missing = [keyword for (keyword, status) in found.items() if status is False] if missing: -- cgit v1.2.3-1-g7c22 From 3598405308c007d5e2edbb4d51c66def6187a468 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 25 Apr 2011 11:21:13 -0500 Subject: Version bump to 1.2.0pre2 Signed-off-by: Sol Jerome --- src/lib/Server/Reports/reports/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/reports/templates/base.html b/src/lib/Server/Reports/reports/templates/base.html index a64f1c76a..6ef4c9aff 100644 --- a/src/lib/Server/Reports/reports/templates/base.html +++ b/src/lib/Server/Reports/reports/templates/base.html @@ -87,7 +87,7 @@
-- cgit v1.2.3-1-g7c22 From e1c5dcea61d36c589b639e080a314c871b1b129f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Apr 2011 10:50:34 -0400 Subject: Better error messages when Svn2 fails for predictable reasons Other misc. Svn2 tweaks and fixes --- src/lib/Server/Plugins/Svn2.py | 44 ++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Svn2.py b/src/lib/Server/Plugins/Svn2.py index 875e9e6a6..35f555294 100644 --- a/src/lib/Server/Plugins/Svn2.py +++ b/src/lib/Server/Plugins/Svn2.py @@ -1,4 +1,3 @@ -import os try: import pysvn missing = False @@ -7,7 +6,7 @@ except: import Bcfg2.Server.Plugin class Svn2(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): + Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" name = 'Svn2' __version__ = '$Id$' @@ -36,7 +35,7 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, if not self.revision: raise Bcfg2.Server.Plugin.PluginInitError - self.logger.debug("Initialized svn plugin with svn root %s at revision %s" \ + self.logger.debug("Initialized svn plugin with svn root %s at revision %s" % (self.svn_root, revision)) def get_revision(self): @@ -63,25 +62,50 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, #FIXME - look for conflicts? - for file in file_list: - stat = self.client.status(file) + for fname in file_list: + stat = self.client.status(fname) self.client.add([f.path for f in stat \ if f.text_status == pysvn.wc_status_kind.unversioned]) try: self.revision = self.client.checkin([self.datastore], comment, recurse=True) self.revision = self.client.update(self.datastore, recurse=True)[0] - self.logger.info("Svn2: Commited changes. At %s" % self.revision.number) - except: - self.logger.error("Svn2: Failed to commit changes", exc_info=1) + self.logger.info("Svn2: Commited changes. At %s" % + self.revision.number) + except Exception, err: + # try to be smart about the error we got back + details = None + if "callback_ssl_server_trust_prompt" in err.message: + details = "SVN server certificate is not trusted" + elif "callback_get_login" in err.message: + details = "SVN credentials not cached" + + if details is None: + self.logger.error("Svn2: Failed to commit changes", + exc_info=1) + else: + self.logger.error("Svn2: Failed to commit changes: %s" % + details) def Update(self): '''Svn2.Update() => True|False\nUpdate svn working copy\n''' try: old_revision = self.revision.number self.revision = self.client.update(self.datastore, recurse=True)[0] - except: - self.logger.error("Svn2: Failed to update server repository", exc_info=1) + except Exception, err: + # try to be smart about the error we got back + details = None + if "callback_ssl_server_trust_prompt" in err.message: + details = "SVN server certificate is not trusted" + elif "callback_get_login" in err.message: + details = "SVN credentials not cached" + + if details is None: + self.logger.error("Svn2: Failed to update server repository", + exc_info=1) + else: + self.logger.error("Svn2: Failed to update server repository: %s" % + details) return False if old_revision == self.revision.number: -- cgit v1.2.3-1-g7c22 From 25576cd076d66dfed4bbd98ce1bbb3bc86a6230e Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 26 Apr 2011 12:13:04 -0500 Subject: Reports: Add full PY3K compatibility Note that Django still doesn't yet support version 3 so this won't necessarily work until there is a compatible version of django available for use. Signed-off-by: Sol Jerome --- src/lib/Server/Reports/backends.py | 10 +++++---- src/lib/Server/Reports/importscript.py | 13 ++++++++---- src/lib/Server/Reports/py3kcompat.py | 24 ++++++++++++++++++++++ .../Reports/reports/templatetags/bcfg2_tags.py | 14 +++++++------ .../reports/templatetags/syntax_coloring.py | 13 +++++++++--- src/lib/Server/Reports/reports/views.py | 3 ++- src/lib/Server/Reports/settings.py | 2 +- src/lib/Server/Reports/updatefix.py | 3 ++- 8 files changed, 62 insertions(+), 20 deletions(-) create mode 100644 src/lib/Server/Reports/py3kcompat.py (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/backends.py b/src/lib/Server/Reports/backends.py index 9e37a2e6f..85241932f 100644 --- a/src/lib/Server/Reports/backends.py +++ b/src/lib/Server/Reports/backends.py @@ -20,13 +20,15 @@ class NISBackend(object): return user - except NISAUTHError, e: - print(str(e)) + except NISAUTHError: + e = sys.exc_info()[1] + print(e) return None def get_user(self, user_id): try: return User.objects.get(pk=user_id) - except User.DoesNotExist, e: - print(str(e)) + except User.DoesNotExist: + e = sys.exc_info()[1] + print(e) return None diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index 86e176394..0766ffa94 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -10,7 +10,8 @@ import os import sys try: import Bcfg2.Server.Reports.settings -except Exception, e: +except Exception: + e = sys.exc_info()[1] sys.stderr.write("Failed to load configuration settings. %s\n" % e) sys.exit(1) @@ -29,11 +30,13 @@ from datetime import datetime from time import strptime from django.db import connection from Bcfg2.Server.Reports.updatefix import update_database -import ConfigParser import logging import Bcfg2.Logger import platform +# Compatibility imports +from py3kcompat import ConfigParser + def build_reason_kwargs(r_ent): binary_file = False @@ -140,7 +143,8 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): rr.save() if vlevel > 0: logger.info("Created reason: %s" % rr.id) - except Exception, ex: + except Exception: + ex = sys.exc_info()[1] logger.error("Failed to create reason for %s: %s" % (x.get('name'), ex)) rr = Reason(current_exists=x.get('current_exists', default="True").capitalize() == "True") @@ -213,7 +217,8 @@ if __name__ == '__main__': "stats=", "config=", "syslog"]) - except GetoptError, mesg: + except GetoptError: + mesg = sys.exc_info()[1] # print help information and exit: print("%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-S] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg)) raise SystemExit(2) diff --git a/src/lib/Server/Reports/py3kcompat.py b/src/lib/Server/Reports/py3kcompat.py new file mode 100644 index 000000000..b334c4466 --- /dev/null +++ b/src/lib/Server/Reports/py3kcompat.py @@ -0,0 +1,24 @@ +try: + from email.Utils import formatdate +except ImportError: + from email.utils import formatdate + +# urllib imports +try: + from urllib import urlopen +except ImportError: + from urllib.request import urlopen +try: + from urlparse import urljoin +except ImportError: + from urllib.parse import urljoin + +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + +try: + import ConfigParser +except ImportError: + import configparser as ConfigParser diff --git a/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py b/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py index 2d210cc07..629984f26 100644 --- a/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py +++ b/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py @@ -21,9 +21,9 @@ def page_navigator(context): path = context['request'].META['PATH_INFO'] total_pages = int(context['total_pages']) records_per_page = int(context['records_per_page']) - except KeyError, e: + except KeyError: return fragment - except ValueError, e: + except ValueError: return fragment if total_pages < 2: @@ -84,7 +84,8 @@ def page_navigator(context): except Resolver404: path = "404" - except NoReverseMatch, nr: + except NoReverseMatch: + nr = sys.exc_info()[1] path = "NoReverseMatch: %s" % nr except ValueError: path = "ValueError" @@ -193,12 +194,13 @@ class AddUrlFilter(template.Node): del kwargs['server'] try: link = reverse(view, args=args, kwargs=kwargs) - except NoReverseMatch, rm: + except NoReverseMatch: link = reverse(self.fallback_view, args=None, kwargs={ filter_name: filter_value }) - except NoReverseMatch, rm: + except NoReverseMatch: + rm = sys.exc_info()[1] raise rm - except (Resolver404, ValueError), e: + except (Resolver404, ValueError): pass return link diff --git a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py index 43dafb262..291528e2e 100644 --- a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py +++ b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py @@ -1,3 +1,4 @@ +import sys from django import template from django.utils.encoding import smart_unicode, smart_str from django.utils.html import conditional_escape @@ -14,6 +15,12 @@ try: except: colorize = False +def u_str(string): + if sys.hexversion >= 0x03000000: + return string + else: + return unicode(string) + @register.filter def syntaxhilight(value, arg="diff", autoescape=None): """ @@ -26,9 +33,9 @@ def syntaxhilight(value, arg="diff", autoescape=None): if colorize: try: - output = u'' + + u_str('') lexer = get_lexer_by_name(arg) output += highlight(value, lexer, HtmlFormatter()) @@ -36,6 +43,6 @@ def syntaxhilight(value, arg="diff", autoescape=None): except: return value else: - return mark_safe(u'
Tip: Install pygments for highlighting
%s
' % value) + return mark_safe(u_str('
Tip: Install pygments for highlighting
%s
') % value) syntaxhilight.needs_autoescape = True diff --git a/src/lib/Server/Reports/reports/views.py b/src/lib/Server/Reports/reports/views.py index 463dec674..ccd71a60e 100644 --- a/src/lib/Server/Reports/reports/views.py +++ b/src/lib/Server/Reports/reports/views.py @@ -368,7 +368,8 @@ def render_history_view(request, template='clients/history.html', **kwargs): iquery, page, max_results) - except PaginationError, page_error: + except PaginationError: + page_error = sys.exc_info()[1] if isinstance(page_error[0], HttpResponse): return page_error[0] return HttpResponseServerError(page_error) diff --git a/src/lib/Server/Reports/settings.py b/src/lib/Server/Reports/settings.py index 121d55a27..0b9d25776 100644 --- a/src/lib/Server/Reports/settings.py +++ b/src/lib/Server/Reports/settings.py @@ -1,7 +1,7 @@ import django # Django settings for bcfg2 reports project. -from ConfigParser import ConfigParser, NoSectionError, NoOptionError +from py3kcompat import ConfigParser c = ConfigParser() c.read(['/etc/bcfg2.conf', '/etc/bcfg2-web.conf']) diff --git a/src/lib/Server/Reports/updatefix.py b/src/lib/Server/Reports/updatefix.py index 5c61f599f..4d3c964f5 100644 --- a/src/lib/Server/Reports/updatefix.py +++ b/src/lib/Server/Reports/updatefix.py @@ -74,7 +74,8 @@ def _populate_interaction_entry_counts(): updates.append(row) try: cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) - except Exception, e: + except Exception: + e = sys.exc_info()[1] print(e) cursor.close() -- cgit v1.2.3-1-g7c22 From 944df5470f9d30717baccf7b716fd4847b31da27 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 26 Apr 2011 12:40:32 -0500 Subject: Consolidate PY3K compatibility libraries Signed-off-by: Sol Jerome --- src/lib/Server/Reports/importscript.py | 4 ++-- src/lib/Server/Reports/py3kcompat.py | 24 ------------------------ src/lib/Server/Reports/settings.py | 3 ++- src/lib/Server/Snapshots/__init__.py | 3 ++- src/lib/Server/Snapshots/model.py | 18 +++++++++++++----- 5 files changed, 19 insertions(+), 33 deletions(-) delete mode 100644 src/lib/Server/Reports/py3kcompat.py (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index 0766ffa94..1781e2fac 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -34,8 +34,8 @@ import logging import Bcfg2.Logger import platform -# Compatibility imports -from py3kcompat import ConfigParser +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser def build_reason_kwargs(r_ent): diff --git a/src/lib/Server/Reports/py3kcompat.py b/src/lib/Server/Reports/py3kcompat.py deleted file mode 100644 index b334c4466..000000000 --- a/src/lib/Server/Reports/py3kcompat.py +++ /dev/null @@ -1,24 +0,0 @@ -try: - from email.Utils import formatdate -except ImportError: - from email.utils import formatdate - -# urllib imports -try: - from urllib import urlopen -except ImportError: - from urllib.request import urlopen -try: - from urlparse import urljoin -except ImportError: - from urllib.parse import urljoin - -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -try: - import ConfigParser -except ImportError: - import configparser as ConfigParser diff --git a/src/lib/Server/Reports/settings.py b/src/lib/Server/Reports/settings.py index 0b9d25776..5a9e4ae4b 100644 --- a/src/lib/Server/Reports/settings.py +++ b/src/lib/Server/Reports/settings.py @@ -1,7 +1,8 @@ import django +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Django settings for bcfg2 reports project. -from py3kcompat import ConfigParser c = ConfigParser() c.read(['/etc/bcfg2.conf', '/etc/bcfg2-web.conf']) diff --git a/src/lib/Server/Snapshots/__init__.py b/src/lib/Server/Snapshots/__init__.py index a4d8fadbc..7c901adb2 100644 --- a/src/lib/Server/Snapshots/__init__.py +++ b/src/lib/Server/Snapshots/__init__.py @@ -2,7 +2,8 @@ __all__ = ['models', 'db_from_config', 'setup_session'] import sqlalchemy import sqlalchemy.orm -import ConfigParser +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser def db_from_config(cfile): diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py index 130d3e8c2..a2395d168 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Server/Snapshots/model.py @@ -1,3 +1,4 @@ +import sys from sqlalchemy import Table, Column, Integer, Unicode, ForeignKey, Boolean, \ DateTime, UnicodeText, desc import datetime @@ -6,6 +7,13 @@ from sqlalchemy.orm import relation, backref from sqlalchemy.ext.declarative import declarative_base +def u_str(string) + if sys.hexversion >= 0x03000000: + return string + else: + return unicode(string) + + class Uniquer(object): force_rt = True @@ -103,10 +111,10 @@ class Metadata(Base): @classmethod def from_metadata(cls, mysession, mymetadata): - client = Client.by_value(mysession, name=unicode(mymetadata.hostname)) + client = Client.by_value(mysession, name=u_str(mymetadata.hostname)) m = cls(client=client) for group in mymetadata.groups: - m.groups.append(Group.by_value(mysession, name=unicode(group))) + m.groups.append(Group.by_value(mysession, name=u_str(group))) for connector in mymetadata.connectors: data = getattr(mymetadata, connector) if not isinstance(data, dict): @@ -115,9 +123,9 @@ class Metadata(Base): if not isinstance(value, str): continue m.keyvals.append(ConnectorKeyVal.by_value(mysession, - connector=unicode(connector), - key=unicode(key), - value=unicode(value))) + connector=u_str(connector), + key=u_str(key), + value=u_str(value))) return m -- cgit v1.2.3-1-g7c22 From bef2d0c73eb0b3fd071f616aa43a945ae2d103b7 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 26 Apr 2011 15:18:04 -0500 Subject: Plugins: Add full PY3K compatibility Signed-off-by: Sol Jerome --- src/lib/Server/Plugin.py | 27 ++++++++++++++++-------- src/lib/Server/Plugins/Base.py | 7 ++++++- src/lib/Server/Plugins/Bundler.py | 3 ++- src/lib/Server/Plugins/Cfg.py | 16 ++++++++++++--- src/lib/Server/Plugins/DBStats.py | 6 ++++-- src/lib/Server/Plugins/Decisions.py | 3 ++- src/lib/Server/Plugins/Hostbase.py | 25 +++++++++++----------- src/lib/Server/Plugins/Ldap.py | 3 ++- src/lib/Server/Plugins/Metadata.py | 3 ++- src/lib/Server/Plugins/NagiosGen.py | 6 ++++-- src/lib/Server/Plugins/Packages.py | 26 +++++++++++++++-------- src/lib/Server/Plugins/Properties.py | 5 +++-- src/lib/Server/Plugins/SGenshi.py | 9 +++++--- src/lib/Server/Plugins/SSHbase.py | 3 ++- src/lib/Server/Plugins/SSLCA.py | 3 ++- src/lib/Server/Plugins/Snapshots.py | 40 ++++++++++++++++++++++-------------- src/lib/Server/Plugins/Statistics.py | 4 ++-- src/lib/Server/Plugins/TCheetah.py | 8 ++++++-- src/lib/Server/Plugins/TGenshi.py | 18 +++++++++++----- 19 files changed, 143 insertions(+), 72 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 69b38c4ff..e18bf09a1 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -8,13 +8,19 @@ import os import pickle import posixpath import re -import Queue import threading from lxml.etree import XML, XMLSyntaxError import Bcfg2.Options +# py3k compatibility +if sys.hexversion >= 0x03000000: + from functools import reduce +from Bcfg2.Bcfg2Py3k import Queue +from Bcfg2.Bcfg2Py3k import Empty +from Bcfg2.Bcfg2Py3k import Full + # grab default metadata info from bcfg2.conf opts = {'owner': Bcfg2.Options.MDATA_OWNER, 'group': Bcfg2.Options.MDATA_GROUP, @@ -169,7 +175,7 @@ class ThreadedStatistics(Statistics, threading.Thread.__init__(self) # Event from the core signaling an exit self.terminate = core.terminate - self.work_queue = Queue.Queue(100000) + self.work_queue = Queue(100000) self.pending_file = "%s/etc/%s.pending" % (datastore, self.__class__.__name__) self.daemon = True self.start() @@ -184,7 +190,7 @@ class ThreadedStatistics(Statistics, pending_data.append((metadata.hostname, lxml.etree.tostring(data))) except: self.logger.warning("Dropping interaction for %s" % metadata.hostname) - except Queue.Empty: + except Empty: pass try: @@ -204,7 +210,8 @@ class ThreadedStatistics(Statistics, savefile = open(self.pending_file, 'r') pending_data = pickle.load(savefile) savefile.close() - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.logger.warning("Failed to load pending data: %s" % e) for (pmetadata, pdata) in pending_data: # check that shutdown wasnt called early @@ -224,10 +231,11 @@ class ThreadedStatistics(Statistics, return False self.work_queue.put_nowait((metadata, lxml.etree.fromstring(pdata))) - except Queue.Full: + except Full: self.logger.warning("Queue.Full: Failed to load queue data") break - except lxml.etree.LxmlError, lxml_error: + except lxml.etree.LxmlError: + lxml_error = sys.exc_info()[1] self.logger.error("Unable to load save interaction: %s" % lxml_error) except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: self.logger.error("Unable to load metadata for save interaction: %s" % pmetadata) @@ -244,9 +252,10 @@ class ThreadedStatistics(Statistics, while not self.terminate.isSet(): try: (xdata, client) = self.work_queue.get(block=True, timeout=2) - except Queue.Empty: + except Empty: continue - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.logger.error("ThreadedStatistics: %s" % e) continue self.handle_statistic(xdata, client) @@ -258,7 +267,7 @@ class ThreadedStatistics(Statistics, try: self.work_queue.put_nowait((metadata, copy.deepcopy(data))) warned = False - except Queue.Full: + except Full: if not warned: self.logger.warning("%s: Queue is full. Dropping interactions." % self.__class__.__name__) warned = True diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py index 2b241da9d..5e7d89727 100644 --- a/src/lib/Server/Plugins/Base.py +++ b/src/lib/Server/Plugins/Base.py @@ -1,9 +1,14 @@ """This module sets up a base list of configuration entries.""" __revision__ = '$Revision$' -import Bcfg2.Server.Plugin import copy import lxml.etree +import sys +# py3k compatibility +if sys.hexversion >= 0x03000000: + from functools import reduce + +import Bcfg2.Server.Plugin class Base(Bcfg2.Server.Plugin.Plugin, diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 04df8ea86..1a8e7348b 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -80,7 +80,8 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, elif len(entries) == 1: try: bundleset.append(entries[0].get_xml_value(metadata)) - except genshi.template.base.TemplateError, t: + except genshi.template.base.TemplateError: + t = sys.exc_info()[1] self.logger.error("Bundler: Failed to template genshi bundle %s" \ % (bundlename)) self.logger.error(t) diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 6c7a40a52..41cf6c9c1 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -6,6 +6,7 @@ import logging import lxml import os import re +import sys import tempfile import Bcfg2.Server.Plugin @@ -21,6 +22,13 @@ except: logger = logging.getLogger('Bcfg2.Plugins.Cfg') +def u_str(string, encoding): + if sys.hexversion >= 0x03000000: + return str(string, encoding) + else: + return unicode(string, encoding) + + # snipped from TGenshi def removecomment(stream): """A genshi filter that removes comments from the stream.""" @@ -131,7 +139,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): data = stream.render('text') if data == '': entry.set('empty', 'true') - except Exception, e: + except Exception: + e = sys.exc_info()[1] logger.error("Cfg: genshi exception: %s" % e) raise Bcfg2.Server.Plugin.PluginExecutionError else: @@ -145,8 +154,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): entry.text = binascii.b2a_base64(data) else: try: - entry.text = unicode(data, self.encoding) - except UnicodeDecodeError, e: + entry.text = u_str(data, self.encoding) + except UnicodeDecodeError: + e = sys.exc_info()[1] logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) logger.error("Please verify you are using the proper encoding.") raise Bcfg2.Server.Plugin.PluginExecutionError diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py index 27696a978..5ef1920e1 100644 --- a/src/lib/Server/Plugins/DBStats.py +++ b/src/lib/Server/Plugins/DBStats.py @@ -33,7 +33,8 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, logger.debug("Searching for new models to add to the statistics database") try: update_database() - except Exception, inst: + except Exception: + inst = sys.exc_info()[1] logger.debug(str(inst)) logger.debug(str(type(inst))) @@ -61,7 +62,8 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, logger.info("Imported data for %s in %s seconds" \ % (metadata.hostname, time.time() - start)) return - except MultipleObjectsReturned, e: + except MultipleObjectsReturned: + e = sys.exc_info()[1] logger.error("DBStats: MultipleObjectsReturned while handling %s: %s" % \ (metadata.hostname, e)) logger.error("DBStats: Data is inconsistent") diff --git a/src/lib/Server/Plugins/Decisions.py b/src/lib/Server/Plugins/Decisions.py index 1f9525a0e..e239be5ee 100644 --- a/src/lib/Server/Plugins/Decisions.py +++ b/src/lib/Server/Plugins/Decisions.py @@ -26,7 +26,8 @@ class DecisionSet(Bcfg2.Server.Plugin.EntrySet): DecisionFile, encoding) try: fam.AddMonitor(path, self) - except OSError, e: + except OSError: + e = sys.exc_info()[1] logger.error('Adding filemonitor for %s failed. ' 'Make sure directory exists' % path) raise Bcfg2.Server.Plugin.PluginInitError(e) diff --git a/src/lib/Server/Plugins/Hostbase.py b/src/lib/Server/Plugins/Hostbase.py index b1cbb9dfc..4180fd716 100644 --- a/src/lib/Server/Plugins/Hostbase.py +++ b/src/lib/Server/Plugins/Hostbase.py @@ -14,7 +14,8 @@ from sets import Set from django.template import Context, loader from django.db import connection import re -import cStringIO +# Compatibility imports +from Bcfg2.Bcfg2Py3k import StringIO class Hostbase(Bcfg2.Server.Plugin.Plugin, @@ -122,8 +123,8 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, hosts = {} for zone in zones: - zonefile = cStringIO.StringIO() - externalzonefile = cStringIO.StringIO() + zonefile = StringIO() + externalzonefile = StringIO() cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id WHERE z.zone_id = \'%s\'""" % zone[0]) @@ -160,20 +161,20 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, cursor.execute(querystring) zonehosts = cursor.fetchall() prevhost = (None, None, None, None) - cnames = cStringIO.StringIO() - cnamesexternal = cStringIO.StringIO() + cnames = StringIO() + cnamesexternal = StringIO() for host in zonehosts: if not host[2].split(".", 1)[1] == zone[1]: zonefile.write(cnames.getvalue()) externalzonefile.write(cnamesexternal.getvalue()) - cnames = cStringIO.StringIO() - cnamesexternal = cStringIO.StringIO() + cnames = StringIO() + cnamesexternal = StringIO() continue if not prevhost[1] == host[1] or not prevhost[2] == host[2]: zonefile.write(cnames.getvalue()) externalzonefile.write(cnamesexternal.getvalue()) - cnames = cStringIO.StringIO() - cnamesexternal = cStringIO.StringIO() + cnames = StringIO() + cnamesexternal = StringIO() zonefile.write("%-32s%-10s%-32s\n" % (host[2].split(".", 1)[0], 'A', host[1])) zonefile.write("%-32s%-10s%-3s%s.\n" % @@ -259,8 +260,8 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, WHERE p.ip_addr LIKE '%s%%%%' AND h.status = 'active' ORDER BY p.ip_addr """ % filename[1]) reversehosts = cursor.fetchall() - zonefile = cStringIO.StringIO() - externalzonefile = cStringIO.StringIO() + zonefile = StringIO() + externalzonefile = StringIO() if len(filename[0].split(".")) == 2: originlist = [] [originlist.append((".".join([ip[1].split(".")[2], filename[0]]), @@ -581,7 +582,7 @@ olivia.ctd.anl.gov\n\n""" hostdata = row for netgroup in netgroups: - fileoutput = cStringIO.StringIO() + fileoutput = StringIO() fileoutput.write(header % (netgroup, netgroup, len(netgroups[netgroup]))) for each in netgroups[netgroup]: fileoutput.write(each + "\n") diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py index 4f10d8ca6..b904dbe02 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Server/Plugins/Ldap.py @@ -81,7 +81,8 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): self.debug_log("LdapPlugin debug: query '" + query.name + "' not applicable to host '" + metadata.hostname + "'") return data - except Exception, error_msg: + except Exception: + error_msg = sys.exc_info()[1] if self.debug_flag: raise else: diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index ff723941f..ca6e43851 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -99,7 +99,8 @@ class XMLMetadataConfig(object): tmpfile = "%s.new" % fname try: datafile = open("%s" % tmpfile, 'w') - except IOError, e: + except IOError: + e = sys.exc_info()[1] self.logger.error("Failed to write %s: %s" % (tmpfile, e)) raise MetadataRuntimeError # prep data diff --git a/src/lib/Server/Plugins/NagiosGen.py b/src/lib/Server/Plugins/NagiosGen.py index 1724a1c8a..ca70ba80c 100644 --- a/src/lib/Server/Plugins/NagiosGen.py +++ b/src/lib/Server/Plugins/NagiosGen.py @@ -95,7 +95,8 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, (self.data, metadata.hostname), 'w') fileh.write(host_config) fileh.close() - except OSError, ioerr: + except OSError: + ioerr = sys.exc_info()[1] LOGGER.error("Failed to write %s/%s-host.cfg" % \ (self.data, metadata.hostname)) LOGGER.error(ioerr) @@ -145,6 +146,7 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, fileh = open("%s/nagiosgen.cfg" % (self.data), 'w') fileh.write(group_data + host_data) fileh.close() - except OSError, ioerr: + except OSError: + ioerr = sys.exc_info()[1] LOGGER.error("Failed to write %s/nagiosgen.cfg" % (self.data)) LOGGER.error(ioerr) diff --git a/src/lib/Server/Plugins/Packages.py b/src/lib/Server/Plugins/Packages.py index a84d8dc70..5346111c4 100644 --- a/src/lib/Server/Plugins/Packages.py +++ b/src/lib/Server/Plugins/Packages.py @@ -1,4 +1,3 @@ -import cPickle import copy import gzip import tarfile @@ -8,7 +7,15 @@ import lxml.etree import os import re import sys -import urllib2 + +# Compatibility imports +from Bcfg2.Bcfg2Py3k import cPickle +from Bcfg2.Bcfg2Py3k import HTTPBasicAuthHandler +from Bcfg2.Bcfg2Py3k import HTTPPasswordMgrWithDefaultRealm +from Bcfg2.Bcfg2Py3k import HTTPError +from Bcfg2.Bcfg2Py3k import install_opener +from Bcfg2.Bcfg2Py3k import build_opener +from Bcfg2.Bcfg2Py3k import urlopen # FIXME: Remove when server python dep is 2.5 or greater if sys.version_info >= (2, 5): @@ -72,10 +79,10 @@ def _fetch_url(url): user = mobj.group(2) passwd = mobj.group(3) url = mobj.group(1) + mobj.group(4) - auth = urllib2.HTTPBasicAuthHandler(urllib2.HTTPPasswordMgrWithDefaultRealm()) + auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm()) auth.add_password(None, url, user, passwd) - urllib2.install_opener(urllib2.build_opener(auth)) - return urllib2.urlopen(url).read() + install_opener(build_opener(auth)) + return urlopen(url).read() class Source(object): @@ -165,7 +172,8 @@ class Source(object): except ValueError: logger.error("Packages: Bad url string %s" % url) continue - except urllib2.HTTPError, h: + except HTTPError: + h = sys.exc_info()[1] logger.error("Packages: Failed to fetch url %s. code=%s" \ % (url, h.code)) continue @@ -256,7 +264,8 @@ class YUMSource(Source): except ValueError: logger.error("Packages: Bad url string %s" % rmdurl) continue - except urllib2.HTTPError, h: + except HTTPError: + h = sys.exc_info()[1] logger.error("Packages: Failed to fetch url %s. code=%s" \ % (rmdurl, h.code)) continue @@ -841,7 +850,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, xdata.xinclude() xdata = xdata.getroot() except (lxml.etree.XIncludeError, \ - lxml.etree.XMLSyntaxError), xmlerr: + lxml.etree.XMLSyntaxError): + xmlerr = sys.exc_info()[1] self.logger.error("Package: Error processing xml: %s" % xmlerr) raise Bcfg2.Server.Plugin.PluginInitError except IOError: diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index b34bde998..dea797a10 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -63,9 +63,10 @@ class Properties(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector.__init__(self) try: self.store = PropDirectoryBacked(self.data, core.fam) - except OSError, e: + except OSError: + e = sys.exc_info()[1] Bcfg2.Server.Plugin.logger.error("Error while creating Properties " - "store: %s %s" % (e.strerror,e.filename)) + "store: %s %s" % (e.strerror, e.filename)) raise Bcfg2.Server.Plugin.PluginInitError def get_additional_data(self, _): diff --git a/src/lib/Server/Plugins/SGenshi.py b/src/lib/Server/Plugins/SGenshi.py index cead06e34..04942c2bd 100644 --- a/src/lib/Server/Plugins/SGenshi.py +++ b/src/lib/Server/Plugins/SGenshi.py @@ -23,11 +23,14 @@ class SGenshiTemplateFile(Bcfg2.Server.Plugins.TGenshi.TemplateFile): Bcfg2.Server.Plugins.TGenshi.removecomment) data = stream.render('xml', strip_whitespace=False) return lxml.etree.XML(data) - except LookupError, lerror: + except LookupError: + lerror = sys.exc_info()[1] logger.error('Genshi lookup error: %s' % lerror) - except genshi.template.TemplateError, terror: + except genshi.template.TemplateError: + terror = sys.exc_info()[1] logger.error('Genshi template error: %s' % terror) - except genshi.input.ParseError, perror: + except genshi.input.ParseError: + perror = sys.exc_info()[1] logger.error('Genshi parse error: %s' % perror) raise diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index 77c5e008f..ce08235eb 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -52,7 +52,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, try: Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, self.core.fam) - except OSError, ioerr: + except OSError: + ioerr = sys.exc_info()[1] self.logger.error("Failed to load SSHbase repository from %s" \ % (self.data)) self.logger.error(ioerr) diff --git a/src/lib/Server/Plugins/SSLCA.py b/src/lib/Server/Plugins/SSLCA.py index 00f67834d..baaa14ba9 100644 --- a/src/lib/Server/Plugins/SSLCA.py +++ b/src/lib/Server/Plugins/SSLCA.py @@ -5,7 +5,8 @@ import posixpath import tempfile import os from subprocess import Popen, PIPE, STDOUT -from ConfigParser import ConfigParser +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser class SSLCA(Bcfg2.Server.Plugin.GroupSpool): diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py index a1f72ba3e..8b6bad574 100644 --- a/src/lib/Server/Plugins/Snapshots.py +++ b/src/lib/Server/Plugins/Snapshots.py @@ -8,10 +8,13 @@ import Bcfg2.Server.Plugin import Bcfg2.Server.Snapshots import Bcfg2.Logger from Bcfg2.Server.Snapshots.model import Snapshot -import Queue +import sys import time import threading +# Compatibility import +from Bcfg2.Bcfg2Py3k import Queue + logger = logging.getLogger('Snapshots') ftypes = ['ConfigFile', 'SymLink', 'Directory'] @@ -25,13 +28,20 @@ datafields = { } +def u_str(string): + if sys.hexversion >= 0x03000000: + return string + else: + return unicode(string) + + def build_snap_ent(entry): basefields = [] if entry.tag in ['Package', 'Service']: basefields += ['type'] - desired = dict([(key, unicode(entry.get(key))) for key in basefields]) - state = dict([(key, unicode(entry.get(key))) for key in basefields]) - desired.update([(key, unicode(entry.get(key))) for key in \ + desired = dict([(key, u_str(entry.get(key))) for key in basefields]) + state = dict([(key, u_str(entry.get(key))) for key in basefields]) + desired.update([(key, u_str(entry.get(key))) for key in \ datafields[entry.tag]]) if entry.tag == 'ConfigFile' or \ ((entry.tag == 'Path') and (entry.get('type') == 'file')): @@ -39,19 +49,19 @@ def build_snap_ent(entry): desired['contents'] = None else: if entry.get('encoding', 'ascii') == 'ascii': - desired['contents'] = unicode(entry.text) + desired['contents'] = u_str(entry.text) else: - desired['contents'] = unicode(binascii.a2b_base64(entry.text)) + desired['contents'] = u_str(binascii.a2b_base64(entry.text)) if 'current_bfile' in entry.attrib: - state['contents'] = unicode(binascii.a2b_base64( \ + state['contents'] = u_str(binascii.a2b_base64( \ entry.get('current_bfile'))) elif 'current_bdiff' in entry.attrib: diff = binascii.a2b_base64(entry.get('current_bdiff')) - state['contents'] = unicode( \ + state['contents'] = u_str( \ '\n'.join(difflib.restore(diff.split('\n'), 1))) - state.update([(key, unicode(entry.get('current_' + key, entry.get(key)))) \ + state.update([(key, u_str(entry.get('current_' + key, entry.get(key)))) \ for key in datafields[entry.tag]]) if entry.tag in ['ConfigFile', 'Path'] and entry.get('exists', 'true') == 'false': state = None @@ -67,7 +77,7 @@ class Snapshots(Bcfg2.Server.Plugin.Statistics, Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Statistics.__init__(self) self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile) - self.work_queue = Queue.Queue() + self.work_queue = Queue() self.loader = threading.Thread(target=self.load_snapshot) self.loader.start() @@ -93,9 +103,9 @@ class Snapshots(Bcfg2.Server.Plugin.Statistics, bad = [] state = xdata.find('.//Statistics') correct = state.get('state') == 'clean' - revision = unicode(state.get('revision', '-1')) + revision = u_str(state.get('revision', '-1')) for entry in state.find('.//Bad'): - data = [False, False, unicode(entry.get('name'))] \ + data = [False, False, u_str(entry.get('name'))] \ + build_snap_ent(entry) if entry.tag in ftypes: etag = 'Path' @@ -108,15 +118,15 @@ class Snapshots(Bcfg2.Server.Plugin.Statistics, else: etag = entry.tag if entry.get('name') in entries[etag]: - data = [True, False, unicode(entry.get('name'))] + \ + data = [True, False, u_str(entry.get('name'))] + \ build_snap_ent(entry) else: - data = [True, False, unicode(entry.get('name'))] + \ + data = [True, False, u_str(entry.get('name'))] + \ build_snap_ent(entry) for entry in state.find('.//Extra'): if entry.tag in datafields: data = build_snap_ent(entry)[1] - ename = unicode(entry.get('name')) + ename = u_str(entry.get('name')) data['name'] = ename extra[entry.tag][ename] = data else: diff --git a/src/lib/Server/Plugins/Statistics.py b/src/lib/Server/Plugins/Statistics.py index c7fa0e534..f4f4c7175 100644 --- a/src/lib/Server/Plugins/Statistics.py +++ b/src/lib/Server/Plugins/Statistics.py @@ -8,7 +8,6 @@ import logging from lxml.etree import XML, SubElement, Element, XMLSyntaxError import lxml.etree import os -import Queue from time import asctime, localtime, time, strptime, mktime import threading @@ -33,7 +32,8 @@ class StatisticsStore(object): or force: try: fout = open(self.filename + '.new', 'w') - except IOError, ioerr: + except IOError: + ioerr = sys.exc_info()[1] self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr)) else: fout.write(lxml.etree.tostring(self.element, encoding='UTF-8', xml_declaration=True)) diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py index d40f4baf3..151cc6543 100644 --- a/src/lib/Server/Plugins/TCheetah.py +++ b/src/lib/Server/Plugins/TCheetah.py @@ -36,7 +36,8 @@ class TemplateFile: self.template = Cheetah.Template.Template(open(self.name).read(), compilerSettings=s, searchList=self.searchlist) - except Cheetah.Parser.ParseError, perror: + except Cheetah.Parser.ParseError: + perror = sys.exc_info()[1] logger.error("Cheetah parse error for file %s" % (self.name)) logger.error(perror.report()) @@ -52,11 +53,14 @@ class TemplateFile: if entry.tag == 'Path': entry.set('type', 'file') try: + # py3k compatibility + if sys.hexversion >= 0x03000000: + unicode = str if type(self.template) == unicode: entry.text = self.template else: if entry.get('encoding') == 'base64': - # take care of case where file needs base64 encoding + # take care of case where file needs base64 encoding entry.text = binascii.b2a_base64(self.template) else: entry.text = unicode(str(self.template), self.encoding) diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py index 2a12672cc..cd268e967 100644 --- a/src/lib/Server/Plugins/TGenshi.py +++ b/src/lib/Server/Plugins/TGenshi.py @@ -63,16 +63,22 @@ class TemplateFile: try: self.template = loader.load(self.name, cls=self.template_cls, encoding=self.encoding) - except LookupError, lerror: + except LookupError: + lerror = sys.exc_info()[1] logger.error('Genshi lookup error: %s' % lerror) - except TemplateError, terror: + except TemplateError: + terror = sys.exc_info()[1] logger.error('Genshi template error: %s' % terror) - except genshi.input.ParseError, perror: + except genshi.input.ParseError: + perror = sys.exc_info()[1] logger.error('Genshi parse error: %s' % perror) def bind_entry(self, entry, metadata): """Build literal file information.""" fname = entry.get('realname', entry.get('name')) + # py3k compatibility + if sys.hexversion >= 0x03000000: + unicode = str if entry.tag == 'Path': entry.set('type', 'file') try: @@ -107,10 +113,12 @@ class TemplateFile: entry.text = unicode(xmldata, self.encoding) if entry.text == '': entry.set('empty', 'true') - except TemplateError, terror: + except TemplateError: + terror = sys.exc_info()[1] logger.error('Genshi template error: %s' % terror) raise Bcfg2.Server.Plugin.PluginExecutionError - except AttributeError, err: + except AttributeError: + err = sys.exc_info()[1] logger.error('Genshi template loading error: %s' % err) raise Bcfg2.Server.Plugin.PluginExecutionError -- cgit v1.2.3-1-g7c22 From 6e974603a397fd881d7b87fefd01ba66d5b6b83c Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 26 Apr 2011 15:19:40 -0500 Subject: Plugins: Fix import and indent errors Signed-off-by: Sol Jerome --- src/lib/Server/Plugin.py | 1 + src/lib/Server/Plugins/Packages.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index e18bf09a1..6351a9eb1 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -8,6 +8,7 @@ import os import pickle import posixpath import re +import sys import threading from lxml.etree import XML, XMLSyntaxError diff --git a/src/lib/Server/Plugins/Packages.py b/src/lib/Server/Plugins/Packages.py index 5346111c4..1178aadaa 100644 --- a/src/lib/Server/Plugins/Packages.py +++ b/src/lib/Server/Plugins/Packages.py @@ -851,7 +851,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, xdata = xdata.getroot() except (lxml.etree.XIncludeError, \ lxml.etree.XMLSyntaxError): - xmlerr = sys.exc_info()[1] + xmlerr = sys.exc_info()[1] self.logger.error("Package: Error processing xml: %s" % xmlerr) raise Bcfg2.Server.Plugin.PluginInitError except IOError: -- cgit v1.2.3-1-g7c22 From 33ab18de87eb12e893690f7fec193f2ffffe0dde Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 26 Apr 2011 16:19:33 -0500 Subject: Snapshots: Fix missing colon (Reported by emias on IRC) Signed-off-by: Sol Jerome --- src/lib/Server/Snapshots/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py index a2395d168..2aa35f1ec 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Server/Snapshots/model.py @@ -7,7 +7,7 @@ from sqlalchemy.orm import relation, backref from sqlalchemy.ext.declarative import declarative_base -def u_str(string) +def u_str(string): if sys.hexversion >= 0x03000000: return string else: -- cgit v1.2.3-1-g7c22 From cca5e6a474adf1453408909979143233d54ed2be Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 26 Apr 2011 19:11:21 -0500 Subject: Reports: Fix settings parser Signed-off-by: Sol Jerome --- src/lib/Server/Reports/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/settings.py b/src/lib/Server/Reports/settings.py index 5a9e4ae4b..fff30d30a 100644 --- a/src/lib/Server/Reports/settings.py +++ b/src/lib/Server/Reports/settings.py @@ -3,7 +3,7 @@ import django # Compatibility import from Bcfg2.Bcfg2Py3k import ConfigParser # Django settings for bcfg2 reports project. -c = ConfigParser() +c = ConfigParser.ConfigParser() c.read(['/etc/bcfg2.conf', '/etc/bcfg2-web.conf']) try: -- cgit v1.2.3-1-g7c22 From 7f68ebb4ce4cb65850a63660f2f1cec06100064a Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 26 Apr 2011 20:07:47 -0500 Subject: Admin: Add full PY3K compatibility Signed-off-by: Sol Jerome --- src/lib/Server/Admin/Init.py | 6 ++++-- src/lib/Server/Admin/Query.py | 3 ++- src/lib/Server/Admin/Reports.py | 13 +++++++++---- src/lib/Server/Admin/Snapshots.py | 8 +++++--- src/lib/Server/Admin/Viz.py | 3 ++- src/lib/Server/Admin/Xcmd.py | 7 +++++-- src/lib/Server/Admin/__init__.py | 6 ++++-- 7 files changed, 31 insertions(+), 15 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index 9771fd10b..9fd408585 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -157,7 +157,8 @@ def create_conf(confpath, confdata): try: open(confpath, "w").write(confdata) os.chmod(keypath, stat.S_IRUSR|stat.S_IWUSR) # 0600 - except Exception, e: + except Exception: + e = sys.exc_info()[1] print("Error %s occured while trying to write configuration " "file to '%s'.\n" % (e, confpath)) @@ -381,7 +382,8 @@ class Init(Bcfg2.Server.Admin.Mode): '', ["Bcfg2.Server.Plugins"]) cls = getattr(module, plugin) cls.init_repo(self.repopath) - except Exception, e: + except Exception: + e = sys.exc_info()[1] print("Plugin setup for %s failed: %s\n" "Check that dependencies are installed?" % (plugin, e)) diff --git a/src/lib/Server/Admin/Query.py b/src/lib/Server/Admin/Query.py index 207b65035..9e1d7cc88 100644 --- a/src/lib/Server/Admin/Query.py +++ b/src/lib/Server/Admin/Query.py @@ -26,7 +26,8 @@ class Query(Bcfg2.Server.Admin.Mode): self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(), ['Metadata', 'Probes'], 'foo', False, 'UTF-8') - except Bcfg2.Server.Core.CoreInitError, msg: + except Bcfg2.Server.Core.CoreInitError: + msg = sys.exc_info()[1] self.errExit("Core load failed because %s" % msg) self.bcore.fam.handle_events_in_interval(1) self.meta = self.bcore.metadata diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py index ee3a4473f..942477a49 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Server/Admin/Reports.py @@ -1,7 +1,6 @@ '''Admin interface for dynamic reports''' import Bcfg2.Logger import Bcfg2.Server.Admin -import ConfigParser import datetime import os import logging @@ -14,6 +13,9 @@ from Bcfg2.Server.Reports.updatefix import update_database from Bcfg2.Server.Reports.utils import * from lxml.etree import XML, XMLSyntaxError +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser + # FIXME: Remove when server python dep is 2.5 or greater if sys.version_info >= (2, 5): from hashlib import md5 @@ -26,7 +28,8 @@ import django.core.management # FIXME - settings file uses a hardcoded path for /etc/bcfg2.conf try: import Bcfg2.Server.Reports.settings -except Exception, e: +except Exception: + e = sys.exc_info()[1] sys.stderr.write("Failed to load configuration settings. %s\n" % e) sys.exit(1) @@ -185,7 +188,8 @@ class Reports(Bcfg2.Server.Admin.Mode): # Currently only reasons are a problem try: start_count = Reason.objects.count() - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.log.error("Failed to load reason objects: %s" % e) return dup_reasons = [] @@ -216,7 +220,8 @@ class Reports(Bcfg2.Server.Admin.Mode): cursor.executemany('update reports_entries_interactions set reason_id=%s where reason_id=%s', batch_update) cursor.executemany('delete from reports_reason where id = %s', dup_reasons) transaction.set_dirty() - except Exception, ex: + except Exception: + ex = sys.exc_info()[1] self.log.error("Failed to delete reasons: %s" % ex) raise diff --git a/src/lib/Server/Admin/Snapshots.py b/src/lib/Server/Admin/Snapshots.py index d58873174..052545b61 100644 --- a/src/lib/Server/Admin/Snapshots.py +++ b/src/lib/Server/Admin/Snapshots.py @@ -8,6 +8,8 @@ import Bcfg2.Server.Snapshots import Bcfg2.Server.Snapshots.model from Bcfg2.Server.Snapshots.model import Snapshot, Client, Metadata, Base, \ File, Group, Package, Service +# Compatibility import +from Bcfg2.Bcfg2Py3k import u_str class Snapshots(Bcfg2.Server.Admin.Mode): __shorthelp__ = "Interact with the Snapshots system" @@ -71,7 +73,7 @@ class Snapshots(Bcfg2.Server.Admin.Mode): session.commit() elif args[0] == 'dump': client = args[1] - snap = Snapshot.get_current(self.session, unicode(client)) + snap = Snapshot.get_current(self.session, u_str(client)) if not snap: print("Current snapshot for %s not found" % client) sys.exit(1) @@ -105,7 +107,7 @@ class Snapshots(Bcfg2.Server.Admin.Mode): print("Usage: bcfg2-admin snapshots -b ") return client = args[2] - snap = Snapshot.get_current(self.session, unicode(client)) + snap = Snapshot.get_current(self.session, u_str(client)) if not snap: print("Current snapshot for %s not found" % client) sys.exit(1) @@ -128,7 +130,7 @@ class Snapshots(Bcfg2.Server.Admin.Mode): elif '-e' in args[1:]: # Query a single host for extra entries client = args[2] - snap = Snapshot.get_current(self.session, unicode(client)) + snap = Snapshot.get_current(self.session, u_str(client)) if not snap: print("Current snapshot for %s not found" % client) sys.exit(1) diff --git a/src/lib/Server/Admin/Viz.py b/src/lib/Server/Admin/Viz.py index a77502b5d..f39e6d7a8 100644 --- a/src/lib/Server/Admin/Viz.py +++ b/src/lib/Server/Admin/Viz.py @@ -45,7 +45,8 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): opts, args = getopt.getopt(args, 'Hbko:', ['includehosts', 'includebundles', 'includekey', 'outfile=']) - except getopt.GetoptError, msg: + except getopt.GetoptError: + msg = sys.exc_info()[1] print(msg) #FIXME: is this for --raw? diff --git a/src/lib/Server/Admin/Xcmd.py b/src/lib/Server/Admin/Xcmd.py index e761a5e3d..fd5794f88 100644 --- a/src/lib/Server/Admin/Xcmd.py +++ b/src/lib/Server/Admin/Xcmd.py @@ -1,10 +1,12 @@ import sys -import xmlrpclib import Bcfg2.Options import Bcfg2.Proxy import Bcfg2.Server.Admin +# Compatibility import +from Bcfg2.Bcfg2Py3k import xmlrpclib + class Xcmd(Bcfg2.Server.Admin.Mode): __shorthelp__ = ("XML-RPC Command Interface") @@ -39,7 +41,8 @@ class Xcmd(Bcfg2.Server.Admin.Mode): args = tuple(setup['args'][1:]) try: data = getattr(proxy, cmd)(*args) - except xmlrpclib.Fault, flt: + except xmlrpclib.Fault: + flt = sys.exc_info()[1] if flt.faultCode == 7: print("Unknown method %s" % cmd) return diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py index 411f909ee..8915492a3 100644 --- a/src/lib/Server/Admin/__init__.py +++ b/src/lib/Server/Admin/__init__.py @@ -19,13 +19,14 @@ __all__ = [ 'Xcmd' ] -import ConfigParser import logging import lxml.etree import sys import Bcfg2.Server.Core import Bcfg2.Options +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser class ModeOperationError(Exception): @@ -126,7 +127,8 @@ class MetadataCore(Mode): self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(), setup['plugins'], 'foo', 'UTF-8') - except Bcfg2.Server.Core.CoreInitError, msg: + except Bcfg2.Server.Core.CoreInitError: + msg = sys.exc_info()[1] self.errExit("Core load failed because %s" % msg) self.bcore.fam.handle_events_in_interval(5) self.metadata = self.bcore.metadata -- cgit v1.2.3-1-g7c22 From d3348a34c78ba13d4d4c3e96db19faeeeefac11b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 27 Apr 2011 11:40:08 -0500 Subject: Common: Add full PY3K compatibility Signed-off-by: Sol Jerome --- src/lib/Server/Core.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index e82e05a89..4018c92ef 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -7,12 +7,15 @@ import lxml.etree import select import threading import time -import xmlrpclib from Bcfg2.Component import Component, exposed from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError import Bcfg2.Server.FileMonitor import Bcfg2.Server.Plugins.Metadata +# Compatibility imports +from Bcfg2.Bcfg2Py3k import xmlrpclib +if sys.hexversion >= 0x03000000: + from functools import reduce logger = logging.getLogger('Bcfg2.Server.Core') @@ -142,7 +145,7 @@ class Core(Component): try: mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % (plugin)).Server.Plugins, plugin) - except ImportError, e: + except ImportError: try: mod = __import__(plugin) except: @@ -177,7 +180,8 @@ class Core(Component): plugin.validate_structures(metadata, data) elif base_cls == Bcfg2.Server.Plugin.GoalValidator: plugin.validate_goals(metadata, data) - except Bcfg2.Server.Plugin.ValidationError, err: + except Bcfg2.Server.Plugin.ValidationError: + err = sys.exc_info()[1] logger.error("Plugin %s structure validation failed: %s" \ % (plugin.name, err.message)) raise -- cgit v1.2.3-1-g7c22 From 6045754bb05e0b01b3325b4f616029a96c2cbb56 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 27 Apr 2011 15:11:26 -0500 Subject: Fix some syntax issues Signed-off-by: Sol Jerome --- src/lib/Server/Admin/Init.py | 4 ++-- src/lib/Server/Plugins/TGenshi.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index 9fd408585..e69412a5e 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -138,7 +138,7 @@ def create_key(hostname, keypath, certpath, country, state, location): keypath, certpath)) subprocess.call((ccstr), shell=True) - os.chmod(keypath, stat.S_IRUSR|stat.S_IWUSR) # 0600 + os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR) # 0600 def create_conf(confpath, confdata): @@ -156,7 +156,7 @@ def create_conf(confpath, confdata): return try: open(confpath, "w").write(confdata) - os.chmod(keypath, stat.S_IRUSR|stat.S_IWUSR) # 0600 + os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR) # 0600 except Exception: e = sys.exc_info()[1] print("Error %s occured while trying to write configuration " diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py index cd268e967..83b60c958 100644 --- a/src/lib/Server/Plugins/TGenshi.py +++ b/src/lib/Server/Plugins/TGenshi.py @@ -98,7 +98,7 @@ class TemplateFile: entry.text = textdata else: if entry.get('encoding') == 'base64': - # take care of case where file needs base64 encoding + # take care of case where file needs base64 encoding entry.text = binascii.b2a_base64(textdata) else: entry.text = unicode(textdata, self.encoding) -- cgit v1.2.3-1-g7c22 From 8bc620dd945a001754d01c87974a71ff078dc85a Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 3 May 2011 12:50:22 -0500 Subject: Fix regressions with new py3k code Signed-off-by: Sol Jerome --- src/lib/Server/Core.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index 4018c92ef..ac0a51e23 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -5,6 +5,7 @@ import atexit import logging import lxml.etree import select +import sys import threading import time -- cgit v1.2.3-1-g7c22 From 1f5b1dd445ca673e26c1a5723fa820726aa48094 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 3 May 2011 14:04:35 -0500 Subject: Plugins: Handle builtin file type for PY3K Signed-off-by: Sol Jerome --- src/lib/Server/Plugin.py | 7 +++++-- src/lib/Server/Plugins/Packages.py | 22 ++++++++++++++-------- 2 files changed, 19 insertions(+), 10 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 6351a9eb1..cd2b63656 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -18,6 +18,9 @@ import Bcfg2.Options # py3k compatibility if sys.hexversion >= 0x03000000: from functools import reduce + from io import FileIO as BUILTIN_FILE_TYPE +else: + BUILTIN_FILE_TYPE = file from Bcfg2.Bcfg2Py3k import Queue from Bcfg2.Bcfg2Py3k import Empty from Bcfg2.Bcfg2Py3k import Full @@ -346,7 +349,7 @@ class FileBacked(object): if event and event.code2str() not in ['exists', 'changed', 'created']: return try: - self.data = file(self.name).read() + self.data = BUILTIN_FILE_TYPE(self.name).read() self.Index() except IOError: logger.error("Failed to read file %s" % (self.name)) @@ -577,7 +580,7 @@ class XMLSrc(XMLFileBacked): def HandleEvent(self, _=None): """Read file upon update.""" try: - data = file(self.name).read() + data = BUILTIN_FILE_TYPE(self.name).read() except IOError: logger.error("Failed to read file %s" % (self.name)) return diff --git a/src/lib/Server/Plugins/Packages.py b/src/lib/Server/Plugins/Packages.py index 1178aadaa..19152074e 100644 --- a/src/lib/Server/Plugins/Packages.py +++ b/src/lib/Server/Plugins/Packages.py @@ -17,6 +17,12 @@ from Bcfg2.Bcfg2Py3k import install_opener from Bcfg2.Bcfg2Py3k import build_opener from Bcfg2.Bcfg2Py3k import urlopen +# py3k compatibility +if sys.hexversion >= 0x03000000: + from io import FileIO as BUILTIN_FILE_TYPE +else: + BUILTIN_FILE_TYPE = file + # FIXME: Remove when server python dep is 2.5 or greater if sys.version_info >= (2, 5): from hashlib import md5 @@ -177,7 +183,7 @@ class Source(object): logger.error("Packages: Failed to fetch url %s. code=%s" \ % (url, h.code)) continue - file(fname, 'w').write(data) + BUILTIN_FILE_TYPE(fname, 'w').write(data) def applies(self, metadata): return len([g for g in self.basegroups if g in metadata.groups]) != 0 and \ @@ -231,13 +237,13 @@ class YUMSource(Source): self.file_to_arch = dict() def save_state(self): - cache = file(self.cachefile, 'wb') + cache = BUILTIN_FILE_TYPE(self.cachefile, 'wb') cPickle.dump((self.packages, self.deps, self.provides, self.filemap, self.url_map), cache, 2) cache.close() def load_state(self): - data = file(self.cachefile) + data = BUILTIN_FILE_TYPE(self.cachefile) (self.packages, self.deps, self.provides, \ self.filemap, self.url_map) = cPickle.load(data) @@ -378,13 +384,13 @@ class APTSource(Source): 'components': self.components, 'arches': self.arches, 'groups': self.groups}] def save_state(self): - cache = file(self.cachefile, 'wb') + cache = BUILTIN_FILE_TYPE(self.cachefile, 'wb') cPickle.dump((self.pkgnames, self.deps, self.provides), cache, 2) cache.close() def load_state(self): - data = file(self.cachefile) + data = BUILTIN_FILE_TYPE(self.cachefile) self.pkgnames, self.deps, self.provides = cPickle.load(data) def filter_unknown(self, unknown): @@ -423,7 +429,7 @@ class APTSource(Source): print("Failed to read file %s" % fname) raise for line in reader.readlines(): - words = line.strip().split(':', 1) + words = line.strip().split(b':', 1) if words[0] == 'Package': pkgname = words[1].strip().rstrip() self.pkgnames.add(pkgname) @@ -500,13 +506,13 @@ class PACSource(Source): 'components': self.components, 'arches': self.arches, 'groups': self.groups}] def save_state(self): - cache = file(self.cachefile, 'wb') + cache = BUILTIN_FILE_TYPE(self.cachefile, 'wb') cPickle.dump((self.pkgnames, self.deps, self.provides), cache, 2) cache.close() def load_state(self): - data = file(self.cachefile) + data = BUILTIN_FILE_TYPE(self.cachefile) self.pkgnames, self.deps, self.provides = cPickle.load(data) def filter_unknown(self, unknown): -- cgit v1.2.3-1-g7c22 From 9d68095482ffe50fd0f86ab6119fc55064dcb56f Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 3 May 2011 14:36:27 -0500 Subject: Packages: Solve byte string incompatibility in a different way Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/Packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Packages.py b/src/lib/Server/Plugins/Packages.py index 19152074e..4e47f8549 100644 --- a/src/lib/Server/Plugins/Packages.py +++ b/src/lib/Server/Plugins/Packages.py @@ -429,7 +429,7 @@ class APTSource(Source): print("Failed to read file %s" % fname) raise for line in reader.readlines(): - words = line.strip().split(b':', 1) + words = str(line.strip()).split(':', 1) if words[0] == 'Package': pkgname = words[1].strip().rstrip() self.pkgnames.add(pkgname) -- cgit v1.2.3-1-g7c22 From 23ae3d201af82292ad4e939569a50f2e32c689a3 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 5 May 2011 08:16:51 -0400 Subject: made bcfg2-lint error handling configurable on a much more granular level --- src/lib/Server/Lint/Bundles.py | 13 ++-- src/lib/Server/Lint/Comments.py | 14 +++-- src/lib/Server/Lint/Duplicates.py | 9 ++- src/lib/Server/Lint/InfoXML.py | 16 ++--- src/lib/Server/Lint/Pkgmgr.py | 5 +- src/lib/Server/Lint/RequiredAttrs.py | 6 +- src/lib/Server/Lint/Validate.py | 30 ++++----- src/lib/Server/Lint/__init__.py | 115 +++++++++++++++++++++++++++-------- 8 files changed, 143 insertions(+), 65 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Bundles.py b/src/lib/Server/Lint/Bundles.py index 417f76c2d..e90159f7c 100644 --- a/src/lib/Server/Lint/Bundles.py +++ b/src/lib/Server/Lint/Bundles.py @@ -33,7 +33,8 @@ class Bundles(Bcfg2.Server.Lint.ServerPlugin): genshibundle = "%s.genshi" % bundle if (xmlbundle not in allbundles and genshibundle not in allbundles): - self.LintError("Bundle %s referenced, but does not exist" % + self.LintError("bundle-not-found", + "Bundle %s referenced, but does not exist" % bundle) def bundle_names(self, bundle): @@ -47,8 +48,9 @@ class Bundles(Bcfg2.Server.Lint.ServerPlugin): fname = bundle.name.split('Bundler/')[1].split('.')[0] bname = xdata.get('name') if fname != bname: - self.LintWarning("Inconsistent bundle name: filename is %s, bundle name is %s" % - (fname, bname)) + self.LintError("inconsistent-bundle-name", + "Inconsistent bundle name: filename is %s, bundle name is %s" % + (fname, bname)) def sgenshi_groups(self, bundle): """ ensure that Genshi Bundles do not include tags, @@ -57,5 +59,6 @@ class Bundles(Bcfg2.Server.Lint.ServerPlugin): groups = [self.RenderXML(g) for g in xdata.getroottree().findall("//Group")] if groups: - self.LintWarning(" tag is not allowed in SGenshi Bundle:\n%s" % - "\n".join(groups)) + self.LintError("group-tag-not-allowed", + " tag is not allowed in SGenshi Bundle:\n%s" % + "\n".join(groups)) diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 8c83545b3..8e86cc564 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -143,12 +143,14 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): unexpanded = [keyword for (keyword, status) in found.items() if status is None] if unexpanded: - self.LintWarning("%s: Required keywords(s) found but not expanded: %s" % - (filename, ", ".join(unexpanded))) + self.LintError("unexpanded-keywords", + "%s: Required keywords(s) found but not expanded: %s" % + (filename, ", ".join(unexpanded))) missing = [keyword for (keyword, status) in found.items() if status is False] if missing: - self.LintError("%s: Required keywords(s) not found: $%s$" % + self.LintError("keywords-not-found", + "%s: Required keywords(s) not found: $%s$" % (filename, "$, $".join(missing))) # next, check for required comments. found is just @@ -163,7 +165,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): missing = [comment for (comment, status) in found.items() if status is False] if missing: - self.LintError("%s: Required comments(s) not found: %s" % + self.LintError("comments-not-found", + "%s: Required comments(s) not found: %s" % (filename, ", ".join(missing))) def has_all_xincludes(self, mfile): @@ -177,7 +180,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): xdata = lxml.etree.parse(path) for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): if not self.has_all_xincludes(el.get('href')): - self.LintWarning("Broken XInclude chain: could not include %s" % path) + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: could not include %s" % path) return False return True diff --git a/src/lib/Server/Lint/Duplicates.py b/src/lib/Server/Lint/Duplicates.py index c8b542025..517f0dd7b 100644 --- a/src/lib/Server/Lint/Duplicates.py +++ b/src/lib/Server/Lint/Duplicates.py @@ -49,7 +49,8 @@ class Duplicates(Bcfg2.Server.Lint.ServerPlugin): if el.get('name') not in seen: seen[el.get('name')] = el else: - self.LintError("Duplicate %s '%s':\n%s\n%s" % + self.LintError("duplicate-%s" % etype, + "Duplicate %s '%s':\n%s\n%s" % (etype, el.get('name'), self.RenderXML(seen[el.get('name')]), self.RenderXML(el))) @@ -59,7 +60,8 @@ class Duplicates(Bcfg2.Server.Lint.ServerPlugin): default_groups = [g for g in self.groups_xdata.findall('.//Group') if g.get('default') == 'true'] if len(default_groups) > 1: - self.LintError("Multiple default groups defined: %s" % + self.LintError("multiple-default-groups", + "Multiple default groups defined: %s" % ",".join(default_groups)) def has_all_xincludes(self, mfile): @@ -73,7 +75,8 @@ class Duplicates(Bcfg2.Server.Lint.ServerPlugin): xdata = lxml.etree.parse(path) for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): if not self.has_all_xincludes(el.get('href')): - self.LintWarning("Broken XInclude chain: could not include %s" % path) + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: could not include %s" % path) return False return True diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py index 25f609902..7725ad748 100644 --- a/src/lib/Server/Lint/InfoXML.py +++ b/src/lib/Server/Lint/InfoXML.py @@ -14,9 +14,9 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): if (hasattr(entryset, "infoxml") and entryset.infoxml is not None): self.check_infoxml(entryset.infoxml.pnode.data) - elif ("require" in self.config and - self.config["require"].lower != "false"): - self.LintError("No info.xml found for %s" % filename) + else: + self.LintError("no-infoxml", + "No info.xml found for %s" % filename) def check_infoxml(self, xdata): for info in xdata.getroottree().findall("//Info"): @@ -26,18 +26,18 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): missing = [attr for attr in required if info.get(attr) is None] if missing: - self.LintError("Required attribute(s) %s not found in %s:%s" % + self.LintError("required-infoxml-attrs-missing", + "Required attribute(s) %s not found in %s:%s" % (",".join(missing), infoxml_fname, self.RenderXML(info))) - if ("require_paranoid" in self.config and - self.config["require_paranoid"].lower() == "true" and - (Bcfg2.Options.MDATA_PARANOID.value and + if ((Bcfg2.Options.MDATA_PARANOID.value and info.get("paranoid") is not None and info.get("paranoid").lower() == "false") or (not Bcfg2.Options.MDATA_PARANOID.value and (info.get("paranoid") is None or info.get("paranoid").lower() != "true"))): - self.LintError("Paranoid must be true in %s:%s" % + self.LintError("paranoid-false", + "Paranoid must be true in %s:%s" % (infoxml_fname, self.RenderXML(info))) diff --git a/src/lib/Server/Lint/Pkgmgr.py b/src/lib/Server/Lint/Pkgmgr.py index f2eb2a5f6..39c601617 100644 --- a/src/lib/Server/Lint/Pkgmgr.py +++ b/src/lib/Server/Lint/Pkgmgr.py @@ -6,7 +6,7 @@ class Pkgmgr(Bcfg2.Server.Lint.ServerPlugin): @Bcfg2.Server.Lint.returnErrors def Run(self): if 'Pkgmgr' not in self.core.plugins: - self.LintWarning("Pkgmgr server plugin is not enabled, skipping Pkgmgr lint checks") + self.logger.info("Pkgmgr server plugin is not enabled, skipping Pkgmgr lint checks") return pset = set() @@ -31,7 +31,8 @@ class Pkgmgr(Bcfg2.Server.Lint.ServerPlugin): # check if package is already listed with same # priority, type, grp if ptuple in pset: - self.LintWarning("Duplicate Package %s, priority:%s, type:%s" % + self.LintError("duplicate-package", + "Duplicate Package %s, priority:%s, type:%s" % (pkg.get('name'), priority, ptype)) else: pset.add(ptuple) diff --git a/src/lib/Server/Lint/RequiredAttrs.py b/src/lib/Server/Lint/RequiredAttrs.py index 70ce4fe0a..cbb4395c4 100644 --- a/src/lib/Server/Lint/RequiredAttrs.py +++ b/src/lib/Server/Lint/RequiredAttrs.py @@ -53,7 +53,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): try: required_attrs = set(self.required_attrs[pathtype] + ['type']) except KeyError: - self.LintError("Unknown path type %s: %s" % + self.LintError("unknown-path-type", + "Unknown path type %s: %s" % (pathtype, self.RenderXML(entry))) if 'dev_type' in required_attrs: @@ -62,7 +63,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): # check if major/minor are specified required_attrs |= set(['major', 'minor']) if not pathset.issuperset(required_attrs): - self.LintError("The required attributes %s are missing for %s %sin %s:\n%s" % + self.LintError("required-attrs-missing", + "The required attributes %s are missing for %s %sin %s:\n%s" % (",".join([attr for attr in required_attrs.difference(pathset)]), diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index bb5af93f4..c7a77a4fb 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -44,8 +44,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): schema = lxml.etree.XMLSchema(lxml.etree.parse(schemaname % schemadir)) except: - self.LintWarning("Failed to process schema %s", - schemaname % schemadir) + self.LintError("schema-failed-to-parse", + "Failed to process schema %s", + schemaname % schemadir) continue for filename in filelist: self.validate(filename, schemaname % schemadir, @@ -55,19 +56,13 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): def check_properties(self): """ check Properties files against their schemas """ - alert = self.logger.debug - if "properties_schema" in self.config: - if self.config['properties_schema'].lower().startswith('warn'): - alert = self.LintWarning - elif self.config['properties_schema'].lower().startswith('require'): - alert = self.LintError - for filename in self.filelists['props']: schemafile = "%s.xsd" % os.path.splitext(filename)[0] if os.path.exists(schemafile): self.validate(filename, schemafile) else: - alert("No schema found for %s" % filename) + self.LintError("properties-schema-not-found", + "No schema found for %s" % filename) def validate(self, filename, schemafile, schema=None): """validate a file against the given lxml.etree.Schema. @@ -77,19 +72,22 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): try: schema = lxml.etree.XMLSchema(lxml.etree.parse(schemafile)) except: - self.LintWarning("Failed to process schema %s" % schemafile) + self.LintError("schema-failed-to-parse", + "Failed to process schema %s" % schemafile) return False try: datafile = lxml.etree.parse(filename) except SyntaxError: lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT) - self.LintError("%s fails to parse:\n%s" % (filename, + self.LintError("xml-failed-to-parse", + "%s fails to parse:\n%s" % (filename, lint.communicate()[0])) lint.wait() return False except IOError: - self.LintError("Failed to open file %s" % filename) + self.LintError("xml-failed-to-read", + "Failed to open file %s" % filename) return False if not schema.validate(datafile): @@ -100,7 +98,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): lint = Popen(cmd, stdout=PIPE, stderr=STDOUT) output = lint.communicate()[0] if lint.wait(): - self.LintError("%s fails to verify:\n%s" % (filename, output)) + self.LintError("xml-failed-to-verify", + "%s fails to verify:\n%s" % (filename, output)) return False return True @@ -141,7 +140,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): for fname in all_metadata: if (fname not in self.filelists['metadata:groups'] and fname not in self.filelists['metadata:clients']): - self.LintWarning("Broken XInclude chain: Could not determine file type of %s" % fname) + self.LintError("broken-xinclude-chain", + "Broken XInclude chain: Could not determine file type of %s" % fname) def get_metadata_list(self, mtype): """ get all metadata files for the specified type (clients or diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index 4e6d03fb5..3b89d1f9e 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -16,21 +16,19 @@ import Bcfg2.Logger def returnErrors(fn): """ Decorator for Run method that returns error counts """ - def run(self, *args, **kwargs): - fn(self, *args, **kwargs) - return (self.error_count, self.warning_count) - - return run + return fn class Plugin (object): """ base class for ServerlessPlugin and ServerPlugin """ - def __init__(self, config, files=None): + + def __init__(self, config, errorhandler=None, files=None): self.files = files - self.error_count = 0 - self.warning_count = 0 self.config = config - Bcfg2.Logger.setup_logging('bcfg2-info', to_syslog=False) self.logger = logging.getLogger('bcfg2-lint') + if errorhandler is None: + self.errorHandler = ErrorHandler() + else: + self.errorHandler = errorhandler def Run(self): """ run the plugin. must be overloaded by child classes """ @@ -45,21 +43,10 @@ class Plugin (object): os.path.abspath(fname) in self.files or os.path.abspath(os.path.join(self.config['repo'], fname)) in self.files) - - def LintError(self, msg): - """ log an error condition """ - self.error_count += 1 - lines = msg.splitlines() - self.logger.error("ERROR: %s" % lines.pop()) - [self.logger.error(" %s" % l) for l in lines] - - def LintWarning(self, msg): - """ log a warning condition """ - self.warning_count += 1 - lines = msg.splitlines() - self.logger.warning("WARNING: %s" % lines.pop()) - [self.logger.warning(" %s" % l) for l in lines] + def LintError(self, err, msg): + self.errorHandler.dispatch(err, msg) + def RenderXML(self, element): """render an XML element for error output -- line number prefixed, no children""" @@ -74,17 +61,95 @@ class Plugin (object): xml = lxml.etree.tostring(element).strip() return " line %s: %s" % (element.sourceline, xml) + +class ErrorHandler (object): + # how to handle different errors by default + _errors = {"no-infoxml":"warning", + "paranoid-false":"warning", + "bundle-not-found":"error", + "inconsistent-bundle-name":"warning", + "group-tag-not-allowed":"error", + "unexpanded-keywords":"warning", + "keywords-not-found":"warning", + "comments-not-found":"warning", + "broken-xinclude-chain":"warning", + "duplicate-client":"error", + "duplicate-group":"error", + "duplicate-package":"error", + "multiple-default-groups":"error", + "required-infoxml-attrs-missing":"error", + "unknown-path-type":"error", + "required-attrs-missing":"error", + "schema-failed-to-parse":"warning", + "properties-schema-not-found":"warning", + "xml-failed-to-parse":"error", + "xml-failed-to-read":"error", + "xml-failed-to-verify":"error",} + + def __init__(self, config=None): + self.errors = 0 + self.warnings = 0 + + self.logger = logging.getLogger('bcfg2-lint') + + self._handlers = {} + if config is not None: + for err, action in config.items(): + if "warn" in action: + self._handlers[err] = self.warn + elif "err" in action: + self._handlers[err] = self.error + else: + self._handlers[err] = self.debug + + for err, action in self._errors.items(): + if err not in self._handlers: + if "warn" in action: + self._handlers[err] = self.warn + elif "err" in action: + self._handlers[err] = self.error + else: + self._handlers[err] = self.debug + + def dispatch(self, err, msg): + if err in self._handlers: + self._handlers[err](msg) + self.logger.debug(" (%s)" % err) + else: + self.logger.info("Unknown error %s" % err) + + def error(self, msg): + """ log an error condition """ + self.errors += 1 + lines = msg.splitlines() + self.logger.error("ERROR: %s" % lines.pop()) + [self.logger.error(" %s" % l) for l in lines] + + def warn(self, msg): + """ log a warning condition """ + self.warnings += 1 + lines = msg.splitlines() + self.logger.warning("WARNING: %s" % lines.pop()) + [self.logger.warning(" %s" % l) for l in lines] + + def debug(self, msg): + """ log a silent/debug condition """ + lines = msg.splitlines() + [self.logger.debug("%s" % l) for l in lines] + + class ServerlessPlugin (Plugin): """ base class for plugins that are run before the server starts up (i.e., plugins that check things that may prevent the server from starting up) """ pass + class ServerPlugin (Plugin): """ base class for plugins that check things that require the running Bcfg2 server """ - def __init__(self, lintCore, config, files=None): - Plugin.__init__(self, config, files=files) + def __init__(self, lintCore, config, **kwargs): + Plugin.__init__(self, config, **kwargs) self.core = lintCore self.logger = self.core.logger self.metadata = self.core.metadata -- cgit v1.2.3-1-g7c22 From b6c9928eced9b725d936260d003026f1ebfa455d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 5 May 2011 11:10:29 -0500 Subject: Core: Raise error on failed lxml import (Reported by Raul Cuza on IRC) Signed-off-by: Sol Jerome --- src/lib/Server/Core.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index ac0a51e23..8f9d3e746 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -3,11 +3,15 @@ __revision__ = '$Revision$' import atexit import logging -import lxml.etree import select import sys import threading import time +try: + import lxml.etree +except ImportError: + print("Failed to import lxml dependency. Shutting down server.") + raise SystemExit(1) from Bcfg2.Component import Component, exposed from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError -- cgit v1.2.3-1-g7c22 From a4de0e44bd10761f5ec7d948fd0a3824720a90f2 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 6 May 2011 08:13:20 -0400 Subject: Rewrote NagiosGen config to use NagiosGen/config.xml, which understands and tags, rather than the client-specific Properties/NagiosGen.xml and the group-specific but limited NagiosGen/parents.xml. Includes schema and bcfg2-lint updates necessary. Wrote conversion tool, nagiosgen-convert.py, which converts everything but the tag in the old NagiosGen.xml, which cannot be reasonably converted to StructFile format. Also removed a _lot_ of string modification in NagiosGen.py, which should make it a fair bit faster. --- src/lib/Server/Lint/Validate.py | 3 +- src/lib/Server/Plugins/NagiosGen.py | 177 ++++++++++++++++++------------------ 2 files changed, 91 insertions(+), 89 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index c7a77a4fb..c87c55ee9 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -23,7 +23,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "%s/Deps/*.xml":"%s/deps.xsd", "%s/Decisions/*.xml":"%s/decisions.xsd", "%s/Packages/config.xml":"%s/packages.xsd", - "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd"} + "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd", + "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd"} self.filelists = {} self.get_filelists() diff --git a/src/lib/Server/Plugins/NagiosGen.py b/src/lib/Server/Plugins/NagiosGen.py index ca70ba80c..0f17a8015 100644 --- a/src/lib/Server/Plugins/NagiosGen.py +++ b/src/lib/Server/Plugins/NagiosGen.py @@ -1,40 +1,43 @@ '''This module implements a Nagios configuration generator''' -import glob -import logging -import lxml.etree import os import re +import sys +import glob import socket +import logging +import lxml.etree import Bcfg2.Server.Plugin LOGGER = logging.getLogger('Bcfg2.Plugins.NagiosGen') -host_config_fmt = \ -''' -define host{ - host_name %s - alias %s - address %s -''' +line_fmt = '\t%-32s %s' +class NagiosGenConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, + Bcfg2.Server.Plugin.StructFile): + def __init__(self, filename, fam): + Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) + Bcfg2.Server.Plugin.StructFile.__init__(self, filename) + class NagiosGen(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Generator): """NagiosGen is a Bcfg2 plugin that dynamically generates Nagios configuration file based on Bcfg2 data. """ name = 'NagiosGen' - __version__ = '0.6' + __version__ = '0.7' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Generator.__init__(self) + self.config = NagiosGenConfig(os.path.join(self.data, 'config.xml'), + core.fam) self.Entries = {'Path': - {'/etc/nagiosgen.status': self.createhostconfig, - '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} + {'/etc/nagiosgen.status': self.createhostconfig, + '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} self.client_attrib = {'encoding': 'ascii', 'owner': 'root', @@ -47,106 +50,104 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, 'type': 'file', 'perms': '0440'} - def getparents(self, hostname): - """Return parents for given hostname.""" - depends = [] - if not os.path.isfile('%s/parents.xml' % (self.data)): - return depends - - tree = lxml.etree.parse('%s/parents.xml' % (self.data)) - for entry in tree.findall('.//Depend'): - if entry.attrib['name'] == hostname: - depends.append(entry.attrib['on']) - return depends - def createhostconfig(self, entry, metadata): """Build host specific configuration file.""" host_address = socket.gethostbyname(metadata.hostname) - host_groups = [grp for grp in metadata.groups if \ - os.path.isfile('%s/%s-group.cfg' % (self.data, grp))] - host_config = host_config_fmt % \ - (metadata.hostname, metadata.hostname, host_address) + host_groups = [grp for grp in metadata.groups + if os.path.isfile('%s/%s-group.cfg' % (self.data, grp))] + host_config = [] + host_config.append(line_fmt % ('host_name', metadata.hostname)) + host_config.append(line_fmt % ('alias', metadata.hostname)) + host_config.append(line_fmt % ('address', host_address)) if host_groups: - host_config += ' hostgroups %s\n' % (",".join(host_groups)) - - xtra = None - if hasattr(metadata, 'Properties') and \ - 'NagiosGen.xml' in metadata.Properties: - for q in (metadata.hostname, 'default'): - xtra = metadata.Properties['NagiosGen.xml'].data.find(q) - if xtra is not None: - break - - if xtra is not None: - directives = list(xtra) - for item in directives: - host_config += ' %-32s %s\n' % (item.tag, item.text) - + host_config.append(line_fmt % ("hostgroups", + ",".join(host_groups))) + + # read the old-style Properties config, but emit a warning. + xtra = dict() + props = None + if (hasattr(metadata, 'Properties') and + 'NagiosGen.xml' in metadata.Properties): + props = metadata.Properties['NagiosGen.xml'].data + if props is not None: + LOGGER.warn("Parsing deprecated Properties/NagiosGen.xml. " + "Update to the new-style config with " + "nagiosgen-convert.py.") + xtra = dict((el.tag, el.text) + for el in props.find(metadata.hostname)) + # hold off on parsing the defaults until we've checked for + # a new-style config + + # read the old-style parents.xml, but emit a warning + pfile = os.path.join(self.data, "parents.xml") + if os.path.exists(pfile): + LOGGER.warn("Parsing deprecated NagiosGen/parents.xml. " + "Update to the new-style config with " + "nagiosgen-convert.py.") + parents = lxml.etree.parse(pfile) + for el in parents.xpath("//Depend[@name='%s']" % metadata.hostname): + if 'parent' in xtra: + xtra['parent'] += "," + el.get("on") + else: + xtra['parent'] = el.get("on") + + # read the new-style config and overwrite the old-style config + for el in self.config.Match(): + if el.tag == 'Option': + xtra[el.get("name")] = el.text + + # if we haven't found anything in the new- or old-style + # configs, finally read defaults from old-style config + if (not xtra and + hasattr(metadata, 'Properties') and + 'NagiosGen.xml' in metadata.Properties): + xtra = dict((el.tag, el.text) for el in props.find('default')) + + if xtra: + host_config.extend([line_fmt % (opt, val) + for opt, val in list(xtra.items)]) else: - host_config += ' use default\n' + host_config.append(line_fmt % ('use', 'default')) - host_config += '}\n' - entry.text = host_config - [entry.attrib.__setitem__(key, value) for \ - (key, value) in list(self.client_attrib.items())] + entry.text = "define host {\n%s\n}" % "\n".join(host_config) + [entry.attrib.__setitem__(key, value) + for (key, value) in list(self.client_attrib.items())] try: - fileh = open("%s/%s-host.cfg" % \ - (self.data, metadata.hostname), 'w') + fileh = open("%s/%s-host.cfg" % + (self.data, metadata.hostname), 'w') fileh.write(host_config) fileh.close() except OSError: ioerr = sys.exc_info()[1] - LOGGER.error("Failed to write %s/%s-host.cfg" % \ - (self.data, metadata.hostname)) + LOGGER.error("Failed to write %s/%s-host.cfg" % + (self.data, metadata.hostname)) LOGGER.error(ioerr) def createserverconfig(self, entry, _): """Build monolithic server configuration file.""" host_configs = glob.glob('%s/*-host.cfg' % self.data) group_configs = glob.glob('%s/*-group.cfg' % self.data) - host_data = "" - group_data = "" + host_data = [] + group_data = [] for host in host_configs: - hostfile = open(host, 'r') - hostname = host.split('/')[-1].replace('-host.cfg', '') - parents = self.getparents(hostname) - if parents: - hostlines = hostfile.readlines() - else: - hostdata = hostfile.read() - hostfile.close() - - if parents: - hostdata = '' - addparents = True - for line in hostlines: - line = line.replace('\n', '') - if 'parents' in line: - line += ',' + ','.join(parents) - addparents = False - if '}' in line: - line = '' - hostdata += "%s\n" % line - if addparents: - hostdata += " parents %s\n" % ','.join(parents) - hostdata += "}\n" - - host_data += hostdata + host_data.append(open(host, 'r').read()) + for group in group_configs: group_name = re.sub("(-group.cfg|.*/(?=[^/]+))", "", group) - if host_data.find(group_name) != -1: + if "\n".join(host_data).find(group_name) != -1: groupfile = open(group, 'r') - group_data += groupfile.read() + group_data.append(groupfile.read()) groupfile.close() - entry.text = group_data + host_data - [entry.attrib.__setitem__(key, value) for \ - (key, value) in list(self.server_attrib.items())] + + entry.text = "%s\n\n%s" % ("\n".join(group_data), "\n".join(host_data)) + [entry.attrib.__setitem__(key, value) + for (key, value) in list(self.server_attrib.items())] try: - fileh = open("%s/nagiosgen.cfg" % (self.data), 'w') - fileh.write(group_data + host_data) + fileh = open("%s/nagiosgen.cfg" % self.data, 'w') + fileh.write(entry.text) fileh.close() except OSError: ioerr = sys.exc_info()[1] - LOGGER.error("Failed to write %s/nagiosgen.cfg" % (self.data)) + LOGGER.error("Failed to write %s/nagiosgen.cfg" % self.data) LOGGER.error(ioerr) -- cgit v1.2.3-1-g7c22 From 4c101414df7e224efb8bd52b75781df52c0fb26a Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 6 May 2011 09:30:03 -0400 Subject: Fixed several bugs with last commit. Sorry, I'm a doofus. --- src/lib/Server/Plugins/NagiosGen.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/NagiosGen.py b/src/lib/Server/Plugins/NagiosGen.py index 0f17a8015..8a76c130d 100644 --- a/src/lib/Server/Plugins/NagiosGen.py +++ b/src/lib/Server/Plugins/NagiosGen.py @@ -55,10 +55,10 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, host_address = socket.gethostbyname(metadata.hostname) host_groups = [grp for grp in metadata.groups if os.path.isfile('%s/%s-group.cfg' % (self.data, grp))] - host_config = [] - host_config.append(line_fmt % ('host_name', metadata.hostname)) - host_config.append(line_fmt % ('alias', metadata.hostname)) - host_config.append(line_fmt % ('address', host_address)) + host_config = ['define host {', + line_fmt % ('host_name', metadata.hostname), + line_fmt % ('alias', metadata.hostname), + line_fmt % ('address', host_address)] if host_groups: host_config.append(line_fmt % ("hostgroups", @@ -93,30 +93,29 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, xtra['parent'] = el.get("on") # read the new-style config and overwrite the old-style config - for el in self.config.Match(): + for el in self.config.Match(metadata): if el.tag == 'Option': xtra[el.get("name")] = el.text # if we haven't found anything in the new- or old-style # configs, finally read defaults from old-style config - if (not xtra and - hasattr(metadata, 'Properties') and - 'NagiosGen.xml' in metadata.Properties): + if not xtra and props is not None: xtra = dict((el.tag, el.text) for el in props.find('default')) if xtra: host_config.extend([line_fmt % (opt, val) - for opt, val in list(xtra.items)]) + for opt, val in list(xtra.items())]) else: host_config.append(line_fmt % ('use', 'default')) - entry.text = "define host {\n%s\n}" % "\n".join(host_config) + host_config.append('}') + entry.text = "%s\n" % "\n".join(host_config) [entry.attrib.__setitem__(key, value) for (key, value) in list(self.client_attrib.items())] try: fileh = open("%s/%s-host.cfg" % (self.data, metadata.hostname), 'w') - fileh.write(host_config) + fileh.write(entry.text) fileh.close() except OSError: ioerr = sys.exc_info()[1] -- cgit v1.2.3-1-g7c22 From f9cf76fd1f52777ac10ac6ad69eadfbff3f70ad5 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 6 May 2011 08:35:54 -0500 Subject: TGenshi: Fix local variable bug reported by trehn on IRC Signed-off-by: Sol Jerome (cherry picked from commit 071ef1a1fbe6368b1abb81855e1ab95e316e6911) --- src/lib/Server/Plugins/TGenshi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py index 83b60c958..bc5e00400 100644 --- a/src/lib/Server/Plugins/TGenshi.py +++ b/src/lib/Server/Plugins/TGenshi.py @@ -3,7 +3,11 @@ __revision__ = '$Revision$' import binascii import logging +import sys import Bcfg2.Server.Plugin +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str logger = logging.getLogger('Bcfg2.Plugins.TGenshi') @@ -76,9 +80,6 @@ class TemplateFile: def bind_entry(self, entry, metadata): """Build literal file information.""" fname = entry.get('realname', entry.get('name')) - # py3k compatibility - if sys.hexversion >= 0x03000000: - unicode = str if entry.tag == 'Path': entry.set('type', 'file') try: -- cgit v1.2.3-1-g7c22 From ab4ab921d96ef9ed1f3246d57ad888b46b17b668 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 6 May 2011 12:46:57 -0500 Subject: TCheetah: Fix local variable bug reported by trehn on IRC Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/TCheetah.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Server/Plugins/TCheetah.py index 151cc6543..49be88881 100644 --- a/src/lib/Server/Plugins/TCheetah.py +++ b/src/lib/Server/Plugins/TCheetah.py @@ -6,6 +6,9 @@ import logging import sys import traceback import Bcfg2.Server.Plugin +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str logger = logging.getLogger('Bcfg2.Plugins.TCheetah') @@ -53,9 +56,6 @@ class TemplateFile: if entry.tag == 'Path': entry.set('type', 'file') try: - # py3k compatibility - if sys.hexversion >= 0x03000000: - unicode = str if type(self.template) == unicode: entry.text = self.template else: -- cgit v1.2.3-1-g7c22 From 5db084d815c9ddb7e12219e501d38ed11b9db632 Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:43:43 -0400 Subject: Add missing import. --- src/lib/Server/Admin/Init.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index e69412a5e..f450e5303 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -4,6 +4,7 @@ import random import socket import stat import string +import sys import subprocess import Bcfg2.Server.Admin import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From 372347c63082fbb2e0c09f0b4e42f307a45c1fe6 Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:48:49 -0400 Subject: Pass keypath to create_conf function. --- src/lib/Server/Admin/Init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index f450e5303..fff8bcd1c 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -142,7 +142,7 @@ def create_key(hostname, keypath, certpath, country, state, location): os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR) # 0600 -def create_conf(confpath, confdata): +def create_conf(confpath, confdata, keypath): # Don't overwrite existing bcfg2.conf file if os.path.exists(confpath): # py3k compatibility @@ -402,7 +402,7 @@ class Init(Bcfg2.Server.Admin.Mode): self.server_uri) # Create the configuration file and SSL key - create_conf(self.configfile, confdata) + create_conf(self.configfile, confdata, keypath) kpath = keypath + '/bcfg2.key' cpath = keypath + '/bcfg2.crt' create_key(self.shostname, kpath, cpath, self.country, -- cgit v1.2.3-1-g7c22 From 2c4111694b6c2eb94ddeb945174b58e1595e2c84 Mon Sep 17 00:00:00 2001 From: Raul Cuza Date: Fri, 6 May 2011 23:53:00 -0400 Subject: Add missing import sys. --- src/lib/Server/Plugins/SSHbase.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index ce08235eb..97238f4f3 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -5,6 +5,7 @@ import binascii import os import socket import shutil +import sys import tempfile from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From 045daf4275cad402685b08ac52e8801626305755 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 15:47:46 +0200 Subject: add missing sys import --- src/lib/Server/Plugins/Ldap.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py index b904dbe02..7d6d0b609 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Server/Plugins/Ldap.py @@ -1,4 +1,5 @@ import imp +import sys import time import ldap import Bcfg2.Options -- cgit v1.2.3-1-g7c22 From 51aa48695413ccfa069b9c5bf9441dbd51d09f21 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 15:54:42 +0200 Subject: log more useful error message if python-ldap is not installed --- src/lib/Server/Plugins/Ldap.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py index 7d6d0b609..06ecaed7b 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Server/Plugins/Ldap.py @@ -1,10 +1,18 @@ import imp +import logging import sys import time -import ldap import Bcfg2.Options import Bcfg2.Server.Plugin +logger = logging.getLogger('Bcfg2.Plugins.Ldap') + +try: + import ldap +except: + logger.error("Unable to load ldap module. Is python-ldap installed?") + raise ImportError + # time in seconds between retries after failed LDAP connection RETRY_DELAY = 5 # how many times to try reaching the LDAP server if a connection is broken -- cgit v1.2.3-1-g7c22 From 87ee710edf4c771740696b31e7bd4bbac2955fab Mon Sep 17 00:00:00 2001 From: Tim Laszlo Date: Mon, 25 Apr 2011 21:33:46 -0500 Subject: DBStats: Stop duplicating data in reports_reason --- src/lib/Server/Reports/importscript.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index 1781e2fac..b6a3c2599 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -133,12 +133,9 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): try: rr = None - if not quick: - try: - rr = Reason.objects.filter(**kargs)[0] - except IndexError: - pass - if not rr: + try: + rr = Reason.objects.filter(**kargs)[0] + except IndexError: rr = Reason(**kargs) rr.save() if vlevel > 0: -- cgit v1.2.3-1-g7c22 From 0035ec1448dcac6664ed50e427b05b5bd7ce2aa1 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 16:32:44 +0200 Subject: add another missing sys import --- src/lib/Server/Plugins/SGenshi.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/SGenshi.py b/src/lib/Server/Plugins/SGenshi.py index 04942c2bd..efd981956 100644 --- a/src/lib/Server/Plugins/SGenshi.py +++ b/src/lib/Server/Plugins/SGenshi.py @@ -5,6 +5,7 @@ import genshi.input import genshi.template import lxml.etree import logging +import sys import Bcfg2.Server.Plugin import Bcfg2.Server.Plugins.TGenshi -- cgit v1.2.3-1-g7c22 From 00a6e2c458851dfa895ddfc680ab3a8e5ad11570 Mon Sep 17 00:00:00 2001 From: Torsten Rehn Date: Mon, 9 May 2011 16:53:44 +0200 Subject: add yet another missing sys import --- src/lib/Server/Plugins/Bundler.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 1a8e7348b..01ad3c78b 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -4,6 +4,7 @@ __revision__ = '$Revision$' import copy import lxml.etree import re +import sys import Bcfg2.Server.Plugin -- cgit v1.2.3-1-g7c22 From 9e4a32fbe8588eace80f98078fd93cc3f0344009 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 10 May 2011 08:56:09 -0500 Subject: SSHbase: Fix PY3K tracebacks for known_hosts file Signed-off-by: Sol Jerome --- src/lib/Server/Plugins/SSHbase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Server') diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index 97238f4f3..cf0998aaa 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -74,7 +74,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([value.data for key, value in \ + self.__skn = "\n".join([str(value.data) for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() -- cgit v1.2.3-1-g7c22