diff options
author | Chris St. Pierre <chris.a.st.pierre@gmail.com> | 2013-06-27 08:19:09 -0400 |
---|---|---|
committer | Chris St. Pierre <chris.a.st.pierre@gmail.com> | 2013-06-27 08:19:09 -0400 |
commit | 317d3087459538877100032733477362f456f550 (patch) | |
tree | 27fabdff3649c75bcdd9de8ccde5c5826c679e5e /src/lib/Bcfg2 | |
parent | 9d5e8170292e17d3b087878918562dcf502f70d4 (diff) | |
parent | a519dc9298317b678bca43597892df5aa13d874d (diff) | |
download | bcfg2-317d3087459538877100032733477362f456f550.tar.gz bcfg2-317d3087459538877100032733477362f456f550.tar.bz2 bcfg2-317d3087459538877100032733477362f456f550.zip |
Merge branch 'maint'
Conflicts:
doc/server/plugins/generators/cfg.txt
doc/server/plugins/generators/tcheetah.txt
src/lib/Bcfg2/Server/Admin/Xcmd.py
src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
Diffstat (limited to 'src/lib/Bcfg2')
47 files changed, 526 insertions, 287 deletions
diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index b24b46dbc..a668a0870 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -111,8 +111,8 @@ class Frame(object): self.logger.warning(deprecated) experimental = [tool.name for tool in self.tools if tool.experimental] if experimental: - self.logger.warning("Loaded experimental tool drivers:") - self.logger.warning(experimental) + self.logger.info("Loaded experimental tool drivers:") + self.logger.info(experimental) # find entries not handled by any tools self.unhandled = [entry for struct in config diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index fd2c467d7..05e35befc 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -31,10 +31,17 @@ class Action(Bcfg2.Client.Tools.Tool): def RunAction(self, entry): """This method handles command execution and status return.""" + shell = False + shell_string = '' + if entry.get('shell', 'false') == 'true': + shell = True + shell_string = '(in shell) ' + if not self.setup['dryrun']: if self.setup['interactive']: - prompt = ('Run Action %s, %s: (y/N): ' % - (entry.get('name'), entry.get('command'))) + prompt = ('Run Action %s%s, %s: (y/N): ' % + (shell_string, entry.get('name'), + entry.get('command'))) # flush input buffer while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: @@ -47,8 +54,9 @@ class Action(Bcfg2.Client.Tools.Tool): self.logger.debug("Action: Deferring execution of %s due " "to build mode" % entry.get('command')) return False - self.logger.debug("Running Action %s" % (entry.get('name'))) - rv = self.cmd.run(entry.get('command')) + self.logger.debug("Running Action %s %s" % + (shell_string, entry.get('name'))) + rv = self.cmd.run(entry.get('command'), shell=shell) self.logger.debug("Action: %s got return code %s" % (entry.get('command'), rv.retval)) entry.set('rc', str(rv.retval)) diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index 0f5f32302..156f76159 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -57,27 +57,29 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): return True current_bootstatus = self.verify_bootstatus(entry, bootstatus) - svcstatus = self.check_service(entry) - if entry.get('status') == 'on': - if svcstatus: - current_srvstatus = True - else: - current_srvstatus = False - elif entry.get('status') == 'off': - if svcstatus: - current_srvstatus = False - else: - current_srvstatus = True - else: + if entry.get('status') == 'ignore': # 'ignore' should verify - current_srvstatus = True + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True if svcstatus: entry.set('current_status', 'on') else: entry.set('current_status', 'off') - return current_bootstatus and current_srvstatus + return current_bootstatus and current_svcstatus def InstallService(self, entry): """Install Service entry.""" diff --git a/src/lib/Bcfg2/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index 116d4f8b0..761c51db7 100644 --- a/src/lib/Bcfg2/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -18,6 +18,9 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): svcre = \ re.compile(r'/etc/.*/(?P<action>[SK])(?P<sequence>\d+)(?P<name>\S+)') + def get_svc_command(self, service, action): + return '/usr/sbin/invoke-rc.d %s %s' % (service.get('name'), action) + def verify_bootstatus(self, entry, bootstatus): """Verify bootstatus for entry.""" rawfiles = glob.glob("/etc/rc*.d/[SK]*%s" % (entry.get('name'))) @@ -78,27 +81,29 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): return True current_bootstatus = self.verify_bootstatus(entry, bootstatus) - svcstatus = self.check_service(entry) - if entry.get('status') == 'on': - if svcstatus: - current_srvstatus = True - else: - current_srvstatus = False - elif entry.get('status') == 'off': - if svcstatus: - current_srvstatus = False - else: - current_srvstatus = True - else: + if entry.get('status') == 'ignore': # 'ignore' should verify - current_srvstatus = True + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True if svcstatus: entry.set('current_status', 'on') else: entry.set('current_status', 'off') - return current_bootstatus and current_srvstatus + return current_bootstatus and current_svcstatus def InstallService(self, entry): """Install Service entry.""" @@ -165,6 +170,3 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): # Extra service removal is nonsensical # Extra services need to be reflected in the config return - - def get_svc_command(self, service, action): - return '/usr/sbin/invoke-rc.d %s %s' % (service.get('name'), action) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 168c35c98..9f47fb53a 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -146,8 +146,8 @@ class POSIXFile(POSIXTool): return POSIXTool.install(self, entry) and rv - def _get_diffs(self, entry, interactive=False, sensitive=False, - is_binary=False, content=None): + def _get_diffs(self, entry, interactive=False, # pylint: disable=R0912 + sensitive=False, is_binary=False, content=None): """ generate the necessary diffs for entry """ if not interactive and sensitive: return @@ -163,6 +163,8 @@ class POSIXFile(POSIXTool): # prompts for -I and the reports try: content = open(entry.get('name')).read() + except UnicodeDecodeError: + content = open(entry.get('name'), encoding='utf-8').read() except IOError: self.logger.error("POSIX: Failed to read %s: %s" % (entry.get("name"), sys.exc_info()[1])) diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index d6329256e..8e9626521 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -62,27 +62,29 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): entry.get('name')) return False - svcstatus = self.check_service(entry) - if entry.get('status') == 'on': - if svcstatus: - current_srvstatus = True - else: - current_srvstatus = False - elif entry.get('status') == 'off': - if svcstatus: - current_srvstatus = False - else: - current_srvstatus = True - else: + if entry.get('status') == 'ignore': # 'ignore' should verify - current_srvstatus = True + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True if svcstatus: entry.set('current_status', 'on') else: entry.set('current_status', 'off') - return current_bootstatus and current_srvstatus + return current_bootstatus and current_svcstatus def InstallService(self, entry): """Install Service entry.""" diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 1ab867215..aca5dbbc7 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -1,14 +1,15 @@ """VCS support.""" # TODO: -# * git_write_index # * add svn support # * integrate properly with reports missing = [] +import errno import os import shutil import sys +import stat # python-dulwich git imports try: @@ -26,6 +27,38 @@ except ImportError: import Bcfg2.Client.Tools +def cleanup_mode(mode): + """Cleanup a mode value. + + This will return a mode that can be stored in a tree object. + + :param mode: Mode to clean up. + """ + if stat.S_ISLNK(mode): + return stat.S_IFLNK + elif stat.S_ISDIR(mode): + return stat.S_IFDIR + elif dulwich.index.S_ISGITLINK(mode): + return dulwich.index.S_IFGITLINK + ret = stat.S_IFREG | int('644', 8) + ret |= (mode & int('111', 8)) + return ret + + +def index_entry_from_stat(stat_val, hex_sha, flags, mode=None): + """Create a new index entry from a stat value. + + :param stat_val: POSIX stat_result instance + :param hex_sha: Hex sha of the object + :param flags: Index flags + """ + if mode is None: + mode = cleanup_mode(stat_val.st_mode) + return (stat_val.st_ctime, stat_val.st_mtime, stat_val.st_dev, + stat_val.st_ino, mode, stat_val.st_uid, + stat_val.st_gid, stat_val.st_size, hex_sha, flags) + + class VCS(Bcfg2.Client.Tools.Tool): """VCS support.""" __handles__ = [('Path', 'vcs')] @@ -47,11 +80,24 @@ class VCS(Bcfg2.Client.Tools.Tool): self.logger.info("Repository %s does not exist" % entry.get('name')) return False - cur_rev = repo.head() - if cur_rev != entry.get('revision'): + try: + expected_rev = entry.get('revision') + cur_rev = repo.head() + except: + return False + + try: + client, path = dulwich.client.get_transport_and_path(entry.get('sourceurl')) + remote_refs = client.fetch_pack(path, (lambda x: None), None, None, None) + if expected_rev in remote_refs: + expected_rev = remote_refs[expected_rev] + except: + pass + + if cur_rev != expected_rev: self.logger.info("At revision %s need to go to revision %s" % - (cur_rev, entry.get('revision'))) + (cur_rev.strip(), expected_rev.strip())) return False return True @@ -71,45 +117,64 @@ class VCS(Bcfg2.Client.Tools.Tool): destname) return False - destr = dulwich.repo.Repo.init(destname, mkdir=True) + dulwich.file.ensure_dir_exists(destname) + destr = dulwich.repo.Repo.init(destname) cl, host_path = dulwich.client.get_transport_and_path(entry.get('sourceurl')) remote_refs = cl.fetch(host_path, destr, determine_wants=destr.object_store.determine_wants_all, progress=sys.stdout.write) - destr.refs['refs/heads/master'] = entry.get('revision') - dtree = destr[entry.get('revision')].tree - obj_store = destr.object_store - for fname, mode, sha in obj_store.iter_tree_contents(dtree): - fullpath = os.path.join(destname, fname) - try: - f = open(os.path.join(destname, fname), 'wb') - except IOError: - dir = os.path.split(fullpath)[0] - os.makedirs(dir) - f = open(os.path.join(destname, fname), 'wb') - f.write(destr[sha].data) - f.close() - os.chmod(os.path.join(destname, fname), mode) + + if entry.get('revision') in remote_refs: + destr.refs['HEAD'] = remote_refs[entry.get('revision')] + else: + destr.refs['HEAD'] = entry.get('revision') + + dtree = destr['HEAD'].tree + index = dulwich.index.Index(destr.index_path()) + for fname, mode, sha in destr.object_store.iter_tree_contents(dtree): + full_path = os.path.join(destname, fname) + dulwich.file.ensure_dir_exists(os.path.dirname(full_path)) + + if stat.S_ISLNK(mode): + src_path = destr[sha].as_raw_string() + try: + os.symlink(src_path, full_path) + except OSError: + e = sys.exc_info()[1] + if e.errno == errno.EEXIST: + os.unlink(full_path) + os.symlink(src_path, full_path) + else: + raise + else: + file = open(full_path, 'wb') + file.write(destr[sha].as_raw_string()) + file.close() + os.chmod(full_path, mode) + + st = os.lstat(full_path) + index[fname] = index_entry_from_stat(st, sha, 0) + + index.write() return True - # FIXME: figure out how to write the git index properly - #iname = "%s/.git/index" % entry.get('name') - #f = open(iname, 'w+') - #entries = obj_store[sha].iteritems() - #try: - # dulwich.index.write_index(f, entries) - #finally: - # f.close() def Verifysvn(self, entry, _): """Verify svn repositories""" + headrev = pysvn.Revision( pysvn.opt_revision_kind.head ) client = pysvn.Client() try: cur_rev = str(client.info(entry.get('name')).revision.number) + server = client.info2(entry.get('sourceurl'), headrev, recurse=False) + if server: + server_rev = str(server[0][1].rev.number) except: self.logger.info("Repository %s does not exist" % entry.get('name')) return False + if entry.get('revision') == 'latest' and cur_rev == server_rev: + return True + if cur_rev != entry.get('revision'): self.logger.info("At revision %s need to go to revision %s" % (cur_rev, entry.get('revision'))) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 3bc261f2f..25603186e 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -22,8 +22,5 @@ def prompt(msg): ans = input(msg) return ans in ['y', 'Y'] except EOFError: - # python 2.4.3 on CentOS doesn't like ^C for some reason - return False - except: - print("Error while reading input: %s" % sys.exc_info()[1]) - return False + # handle ^C on rhel-based platforms + raise SystemExit(1) diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index d034c0777..049236e03 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -79,10 +79,7 @@ except NameError: def u_str(string, encoding=None): """ print to file compatibility """ if sys.hexversion >= 0x03000000: - if encoding is not None: - return string.encode(encoding) - else: - return string + return string else: if encoding is not None: return unicode(string, encoding) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index df82248d0..3d224432e 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -125,7 +125,9 @@ class ReportingCollector(object): # this wil be missing if called from bcfg2-admin self.terminate.set() if self.transport: - self.transport.shutdown() + try: + self.transport.shutdown() + except OSError: + pass if self.storage: self.storage.shutdown() - diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index c73339911..7f1fcba3b 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -1,4 +1,8 @@ {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} <?xml version="1.0"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> @@ -25,8 +29,9 @@ <div id="header"> <a href="http://bcfg2.org"><img src='{% to_media_url bcfg2_logo.png %}' - height='115' width='300' alt='Bcfg2' style='float:left; height: 115px' /></a> - </div> + height='115' width='300' alt='Bcfg2' + style='float:left; height: 115px' /></a> + </div> <div id="document"> <div id="content"><div id="contentwrapper"> @@ -46,26 +51,26 @@ <li>Overview</li> </ul> <ul class='menu-level2'> - <li><a href="{% url reports_summary %}">Summary</a></li> - <li><a href="{% url reports_history %}">Recent Interactions</a></li> - <li><a href="{% url reports_timing %}">Timing</a></li> + <li><a href="{% url "reports_summary" %}">Summary</a></li> + <li><a href="{% url "reports_history" %}">Recent Interactions</a></li> + <li><a href="{% url "reports_timing" %}">Timing</a></li> </ul> <ul class='menu-level1'> <li>Clients</li> </ul> <ul class='menu-level2'> - <li><a href="{% url reports_grid_view %}">Grid View</a></li> - <li><a href="{% url reports_detailed_list %}">Detailed List</a></li> - <li><a href="{% url reports_client_manage %}">Manage</a></li> + <li><a href="{% url "reports_grid_view" %}">Grid View</a></li> + <li><a href="{% url "reports_detailed_list" %}">Detailed List</a></li> + <li><a href="{% url "reports_client_manage" %}">Manage</a></li> </ul> <ul class='menu-level1'> <li>Entries Configured</li> </ul> <ul class='menu-level2'> - <li><a href="{% url reports_common_problems %}">Common problems</a></li> - <li><a href="{% url reports_item_list "bad" %}">Bad</a></li> - <li><a href="{% url reports_item_list "modified" %}">Modified</a></li> - <li><a href="{% url reports_item_list "extra" %}">Extra</a></li> + <li><a href="{% url "reports_common_problems" %}">Common problems</a></li> + <li><a href="{% url "reports_item_list" "bad" %}">Bad</a></li> + <li><a href="{% url "reports_item_list" "modified" %}">Modified</a></li> + <li><a href="{% url "reports_item_list" "extra" %}">Extra</a></li> </ul> {% comment %} TODO diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detail.html b/src/lib/Bcfg2/Reporting/templates/clients/detail.html index 4608ce6f1..e890589a7 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detail.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detail.html @@ -1,24 +1,28 @@ {% extends "base.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Client {{client.name}}{% endblock %} {% block extra_header_info %} <style type="text/css"> .node_data { - border: 1px solid #98DBCC; - margin: 10px; - padding-left: 18px; + border: 1px solid #98DBCC; + margin: 10px; + padding-left: 18px; } .node_data td { - padding: 1px 20px 1px 2px; + padding: 1px 20px 1px 2px; } span.history_links { - font-size: 90%; - margin-left: 50px; + font-size: 90%; + margin-left: 50px; } span.history_links a { - font-size: 90%; + font-size: 90%; } </style> {% endblock %} @@ -30,12 +34,12 @@ span.history_links a { {% block content %} <div class='detail_header'> <h2>{{client.name}}</h2> - <a href='{% url reports_client_manage %}#{{ client.name }}'>[manage]</a> - <span class='history_links'><a href="{% url reports_client_history client.name %}">View History</a> | Jump to + <a href='{% url "reports_client_manage" %}#{{ client.name }}'>[manage]</a> + <span class='history_links'><a href="{% url "reports_client_history" client.name %}">View History</a> | Jump to <select id="quick" name="quick" onchange="javascript:pageJump('quick');"> <option value="" selected="selected">--- Time ---</option> {% for i in client.interactions.all|slice:":25" %} - <option value="{% url reports_client_detail_pk hostname=client.name, pk=i.id %}">{{i.timestamp|date:"c"}}</option> + <option value="{% url "reports_client_detail_pk" hostname=client.name pk=i.id %}">{{i.timestamp|date:"c"}}</option> {% endfor %} </select></span> </div> @@ -110,7 +114,7 @@ span.history_links a { {% for entry in entry_list %} <tr class='{% cycle listview,listview_alt %}'> <td class='entry_list_type'>{{entry.entry_type}}</td> - <td><a href="{% url reports_item entry.class_name entry.pk interaction.pk %}"> + <td><a href="{% url "reports_item" entry.class_name entry.pk interaction.pk %}"> {{entry.name}}</a></td> </tr> {% endfor %} @@ -129,7 +133,7 @@ span.history_links a { {% for failure in interaction.failures.all %} <tr class='{% cycle listview,listview_alt %}'> <td class='entry_list_type'>{{failure.entry_type}}</td> - <td><a href="{% url reports_item failure.class_name failure.pk interaction.pk %}"> + <td><a href="{% url "reports_item" failure.class_name failure.pk interaction.pk %}"> {{failure.name}}</a></td> </tr> {% endfor %} @@ -140,11 +144,11 @@ span.history_links a { {% if entry_list %} <div class="entry_list recent_history_wrapper"> <div class="entry_list_head" style="border-bottom: 2px solid #98DBCC;"> - <h4 style="display: inline"><a href="{% url reports_client_history client.name %}">Recent Interactions</a></h4> + <h4 style="display: inline"><a href="{% url "reports_client_history" client.name %}">Recent Interactions</a></h4> </div> <div class='recent_history_box'> {% include "widgets/interaction_list.inc" %} - <div style='padding-left: 5px'><a href="{% url reports_client_history client.name %}">more...</a></div> + <div style='padding-left: 5px'><a href="{% url "reports_client_history" client.name %}">more...</a></div> </div> </div> {% endif %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index fd9a545ce..33c78a5f0 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Detailed Client Listing{% endblock %} {% block pagebanner %}Clients - Detailed View{% endblock %} @@ -21,7 +25,7 @@ </tr> {% for entry in entry_list %} <tr class='{% cycle listview,listview_alt %}'> - <td class='left_column'><a href='{% url Bcfg2.Reporting.views.client_detail hostname=entry.client.name, pk=entry.id %}'>{{ entry.client.name }}</a></td> + <td class='left_column'><a href='{% url "Bcfg2.Reporting.views.client_detail" hostname=entry.client.name pk=entry.id %}'>{{ entry.client.name }}</a></td> <td class='right_column' style='width:75px'><a href='{% add_url_filter state=entry.state %}' class='{{entry|determine_client_state}}'>{{ entry.state }}</a></td> <td class='right_column_narrow'>{{ entry.good_count }}</td> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/index.html b/src/lib/Bcfg2/Reporting/templates/clients/index.html index d9c415c20..eba83670b 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/index.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/index.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block extra_header_info %} {% endblock%} @@ -17,9 +21,9 @@ <td class='{{ inter|determine_client_state }}'> <a href="{% spaceless %} {% if not timestamp %} - {% url reports_client_detail inter.client.name %} + {% url "reports_client_detail" inter.client.name %} {% else %} - {% url reports_client_detail_pk inter.client.name,inter.id %} + {% url "reports_client_detail_pk" inter.client.name inter.id %} {% endif %} {% endspaceless %}">{{ inter.client.name }}</a> </td> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/manage.html b/src/lib/Bcfg2/Reporting/templates/clients/manage.html index 443ec8ccb..03918aad7 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/manage.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/manage.html @@ -1,4 +1,8 @@ {% extends "base.html" %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block extra_header_info %} {% endblock%} @@ -24,10 +28,10 @@ <td><span id="{{ client.name }}"> </span> <span id="ttag-{{ client.name }}"> </span> <span id="s-ttag-{{ client.name }}"> </span> - <a href="{% url reports_client_detail client.name %}">{{ client.name }}</a></td> + <a href='{% url "reports_client_detail" client.name %}'>{{ client.name }}</a></td> <td>{% firstof client.expiration 'Active' %}</td> <td> - <form method="post" action="{% url reports_client_manage %}"> + <form method="post" action='{% url "reports_client_manage" %}'> <div> {# here for no reason other then to validate #} <input type="hidden" name="client_name" value="{{ client.name }}" /> <input type="hidden" name="client_action" value="{% if client.expiration %}unexpire{% else %}expire{% endif %}" /> diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/common.html b/src/lib/Bcfg2/Reporting/templates/config_items/common.html index 57191ec39..91f37d7dc 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/common.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/common.html @@ -1,5 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% load url from future %} {% block title %}Bcfg2 - Common Problems{% endblock %} @@ -29,9 +30,9 @@ {% for item in type_list %} <tr class='{% cycle listview,listview_alt %}'> <td>{{ item.ENTRY_TYPE }}</td> - <td><a href="{% url reports_entry item.class_name, item.pk %}">{{ item.name }}</a></td> + <td><a href='{% url "reports_entry" item.class_name item.pk %}'>{{ item.name }}</a></td> <td>{{ item.num_entries }}</td> - <td><a href="{% url reports_item item.ENTRY_TYPE, item.pk %}">{{ item.short_list|join:"," }}</a></td> + <td><a href='{% url "reports_item" item.ENTRY_TYPE item.pk %}'>{{ item.short_list|join:"," }}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html index e940889ab..e3befb0eb 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Entry Status{% endblock %} @@ -17,10 +21,10 @@ {% for item, inters in items %} {% for inter in inters %} <tr class='{% cycle listview,listview_alt %}'> - <td><a href='{% url reports_client_detail hostname=inter.client.name %}'>{{inter.client.name}}</a></td> - <td><a href='{% url reports_client_detail_pk hostname=inter.client.name, pk=inter.pk %}'>{{inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe}}</a></td> + <td><a href='{% url "reports_client_detail" hostname=inter.client.name %}'>{{inter.client.name}}</a></td> + <td><a href='{% url "reports_client_detail_pk" hostname=inter.client.name pk=inter.pk %}'>{{inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe}}</a></td> <td>{{ item.get_state_display }}</td> - <td style='white-space: nowrap'><a href="{% url reports_item entry_type=item.class_name pk=item.pk %}">({{item.pk}}) {{item.short_list|join:","}}</a></td> + <td style='white-space: nowrap'><a href='{% url "reports_item" entry_type=item.class_name pk=item.pk %}'>({{item.pk}}) {{item.short_list|join:","}}</a></td> </tr> {% endfor %} {% endfor %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index 259414399..b03d48045 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -1,6 +1,10 @@ {% extends "base.html" %} {% load split %} {% load syntax_coloring %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Element Details{% endblock %} @@ -9,20 +13,20 @@ {% block extra_header_info %} <style type="text/css"> #table_list_header { - font-size: 100%; + font-size: 100%; } table.entry_list { - width: auto; + width: auto; } div.information_wrapper { - margin: 15px; + margin: 15px; } div.diff_wrapper { - overflow: auto; + overflow: auto; } div.entry_list h3 { - font-size: 90%; - padding: 5px; + font-size: 90%; + padding: 5px; } </style> {% endblock%} @@ -131,9 +135,9 @@ div.entry_list h3 { {% if associated_list %} <table class="entry_list" cellpadding="3"> {% for inter in associated_list %} - <tr><td><a href="{% url reports_client_detail inter.client.name %}" + <tr><td><a href='{% url "reports_client_detail" inter.client.name %}' >{{inter.client.name}}</a></td> - <td><a href="{% url reports_client_detail_pk hostname=inter.client.name,pk=inter.id %}" + <td><a href='{% url "reports_client_detail_pk" hostname=inter.client.name pk=inter.id %}' >{{inter.timestamp}}</a></td> </tr> {% endfor %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html index 864392754..0e4812e85 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Element Listing{% endblock %} @@ -21,9 +25,9 @@ <tr style='text-align: left' ><th>Name</th><th>Count</th><th>Reason</th></tr> {% for entry in type_data %} <tr class='{% cycle listview,listview_alt %}'> - <td><a href="{% url reports_entry entry.class_name entry.pk %}">{{entry.name}}</a></td> + <td><a href='{% url "reports_entry" entry.class_name entry.pk %}'>{{entry.name}}</a></td> <td>{{entry.num_entries}}</td> - <td><a href="{% url reports_item entry.class_name entry.pk %}">{{entry.short_list|join:","}}</a></td> + <td><a href='{% url "reports_item" entry.class_name entry.pk %}'>{{entry.short_list|join:","}}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Bcfg2/Reporting/templates/displays/summary.html b/src/lib/Bcfg2/Reporting/templates/displays/summary.html index b9847cf96..ffafd52e0 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/summary.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/summary.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Client Summary{% endblock %} {% block pagebanner %}Clients - Summary{% endblock %} @@ -30,7 +34,7 @@ hide_tables[{{ forloop.counter0 }}] = "table_{{ summary.name }}"; <table id='table_{{ summary.name }}' class='entry_list'> {% for node in summary.nodes|sort_interactions_by_name %} <tr class='{% cycle listview,listview_alt %}'> - <td><a href="{% url reports_client_detail_pk hostname=node.client.name,pk=node.id %}">{{ node.client.name }}</a></td> + <td><a href='{% url "reports_client_detail_pk" hostname=node.client.name pk=node.id %}'>{{ node.client.name }}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Bcfg2/Reporting/templates/displays/timing.html b/src/lib/Bcfg2/Reporting/templates/displays/timing.html index ff775ded5..8ac5e49bb 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/timing.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/timing.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Performance Metrics{% endblock %} {% block pagebanner %}Performance Metrics{% endblock %} @@ -12,7 +16,7 @@ <div class='client_list_box'> {% if metrics %} <table cellpadding="3"> - <tr id='table_list_header' class='listview'> + <tr id='table_list_header' class='listview'> <td>Name</td> <td>Parse</td> <td>Probe</td> @@ -21,15 +25,15 @@ <td>Config</td> <td>Total</td> </tr> - {% for metric in metrics|dictsort:"name" %} + {% for metric in metrics|dictsort:"name" %} <tr class='{% cycle listview,listview_alt %}'> <td><a style='font-size: 100%' - href="{% url reports_client_detail hostname=metric.name %}">{{ metric.name }}</a></td> + href='{% url "reports_client_detail" hostname=metric.name %}'>{{ metric.name }}</a></td> {% for mitem in metric|build_metric_list %} <td>{{ mitem }}</td> {% endfor %} - </tr> - {% endfor %} + </tr> + {% endfor %} </table> {% else %} <p>No metric data available</p> diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index f5f2e7528..489682f30 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -5,9 +5,8 @@ from django import template from django.conf import settings from django.core.urlresolvers import resolve, reverse, \ Resolver404, NoReverseMatch -from django.template.loader import get_template, \ - get_template_from_string,TemplateDoesNotExist -from django.utils.encoding import smart_unicode, smart_str +from django.template.loader import get_template_from_string +from django.utils.encoding import smart_str from django.utils.safestring import mark_safe from datetime import datetime, timedelta from Bcfg2.Reporting.utils import filter_list @@ -133,19 +132,22 @@ def filter_navigator(context): del myargs[filter] filters.append((filter, reverse(view, args=args, kwargs=myargs) + qs)) - filters.sort(lambda x, y: cmp(x[0], y[0])) + filters.sort(key=lambda x: x[0]) myargs = kwargs.copy() - selected=True + selected = True if 'group' in myargs: del myargs['group'] - selected=False - groups = [('---', reverse(view, args=args, kwargs=myargs) + qs, selected)] + selected = False + groups = [('---', + reverse(view, args=args, kwargs=myargs) + qs, + selected)] for group in Group.objects.values('name'): myargs['group'] = group['name'] - groups.append((group['name'], reverse(view, args=args, kwargs=myargs) + qs, - group['name'] == kwargs.get('group', ''))) - + groups.append((group['name'], + reverse(view, args=args, kwargs=myargs) + qs, + group['name'] == kwargs.get('group', ''))) + return {'filters': filters, 'groups': groups} except (Resolver404, NoReverseMatch, ValueError, KeyError): pass @@ -205,7 +207,7 @@ def sort_interactions_by_name(value): Sort an interaction list by client name """ inters = list(value) - inters.sort(lambda a, b: cmp(a.client.name, b.client.name)) + inters.sort(key=lambda a: a.client.name) return inters @@ -223,7 +225,7 @@ class AddUrlFilter(template.Node): filter_value = self.filter_value.resolve(context, True) if filter_value: filter_name = smart_str(self.filter_name) - filter_value = smart_unicode(filter_value) + filter_value = smart_str(filter_value) kwargs[filter_name] = filter_value # These two don't make sense if filter_name == 'server' and 'hostname' in kwargs: @@ -306,6 +308,7 @@ def to_media_url(parser, token): return MediaTag(filter_value) + @register.filter def determine_client_state(entry): """ @@ -338,10 +341,11 @@ def do_qs(parser, token): try: tag, name, value = token.split_contents() except ValueError: - raise template.TemplateSyntaxError, "%r tag requires exactly two arguments" \ - % token.contents.split()[0] + raise template.TemplateSyntaxError("%r tag requires exactly two arguments" + % token.contents.split()[0]) return QsNode(name, value) + class QsNode(template.Node): def __init__(self, name, value): self.name = template.Variable(name) @@ -359,7 +363,7 @@ class QsNode(template.Node): return '' except KeyError: if settings.TEMPLATE_DEBUG: - raise Exception, "'qs' tag requires context['request']" + raise Exception("'qs' tag requires context['request']") return '' except: return '' @@ -380,6 +384,7 @@ def sort_link(parser, token): return SortLinkNode(sort_key, text) + class SortLinkNode(template.Node): __TMPL__ = "{% load bcfg2_tags %}<a href='{% qs 'sort' key %}'>{{ text }}</a>" @@ -420,4 +425,3 @@ class SortLinkNode(template.Node): raise raise return '' - diff --git a/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py b/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py index 2712d6395..22700689f 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py +++ b/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py @@ -1,11 +1,8 @@ -import sys from django import template -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_str from django.utils.html import conditional_escape from django.utils.safestring import mark_safe -from Bcfg2.Compat import u_str - register = template.Library() # pylint: disable=E0611 @@ -33,9 +30,9 @@ def syntaxhilight(value, arg="diff", autoescape=None): if colorize: try: - output = u_str('<style type="text/css">') \ - + smart_unicode(HtmlFormatter().get_style_defs('.highlight')) \ - + u_str('</style>') + output = smart_str('<style type="text/css">') \ + + smart_str(HtmlFormatter().get_style_defs('.highlight')) \ + + smart_str('</style>') lexer = get_lexer_by_name(arg) output += highlight(value, lexer, HtmlFormatter()) @@ -43,6 +40,7 @@ def syntaxhilight(value, arg="diff", autoescape=None): except: return value else: - return mark_safe(u_str('<div class="note-box">Tip: Install pygments ' - 'for highlighting</div><pre>%s</pre>') % value) + return mark_safe(smart_str( + '<div class="note-box">Tip: Install pygments ' + 'for highlighting</div><pre>%s</pre>') % value) syntaxhilight.needs_autoescape = True diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index 6cba7bf8c..c7c2a503f 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -338,6 +338,8 @@ def client_detail(request, hostname=None, pk=None): for label in etypes.values(): edict[label] = [] for ekind in inter.entry_types: + if ekind == 'failures': + continue for ent in getattr(inter, ekind).all(): edict[etypes[ent.state]].append(ent) context['entry_types'] = edict diff --git a/src/lib/Bcfg2/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py index 570e993ed..187ccfd71 100644 --- a/src/lib/Bcfg2/Server/Admin/Client.py +++ b/src/lib/Bcfg2/Server/Admin/Client.py @@ -18,19 +18,15 @@ class Client(Bcfg2.Server.Admin.MetadataCore): try: self.metadata.add_client(args[1]) except MetadataConsistencyError: - err = sys.exc_info()[1] - print("Error in adding client: %s" % err) - raise SystemExit(1) + self.errExit("Error in adding client: %s" % sys.exc_info()[1]) elif args[0] in ['delete', 'remove', 'del', 'rm']: try: self.metadata.remove_client(args[1]) except MetadataConsistencyError: - err = sys.exc_info()[1] - print("Error in deleting client: %s" % err) - raise SystemExit(1) + self.errExit("Error in deleting client: %s" % + sys.exc_info()[1]) elif args[0] in ['list', 'ls']: for client in self.metadata.list_clients(): print(client) else: - print("No command specified") - raise SystemExit(1) + self.errExit("No command specified") diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py index d0831362c..6bb15cafd 100644 --- a/src/lib/Bcfg2/Server/Admin/Compare.py +++ b/src/lib/Bcfg2/Server/Admin/Compare.py @@ -144,5 +144,4 @@ class Compare(Bcfg2.Server.Admin.Mode): (old, new) = args return self.compareSpecifications(new, old) except IndexError: - print(self.__call__.__doc__) - raise SystemExit(1) + self.errExit(self.__call__.__doc__) diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index 93e42305c..37ca74894 100644 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -20,9 +20,8 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): "Please see bcfg2-admin minestruct help for usage.") try: (opts, args) = getopt.getopt(args, 'f:g:h') - except: - self.log.error(self.__doc__) - raise SystemExit(1) + except getopt.GetoptError: + self.errExit(self.__doc__) client = args[0] output = sys.stdout @@ -33,8 +32,7 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): try: output = open(optarg, 'w') except IOError: - self.log.error("Failed to open file: %s" % (optarg)) - raise SystemExit(1) + self.errExit("Failed to open file: %s" % (optarg)) elif opt == '-g': groups = optarg.split(':') @@ -43,10 +41,9 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): for source in self.bcore.plugins_by_type(PullSource): for item in source.GetExtra(client): extra.add(item) - except: - self.log.error("Failed to find extra entry info for client %s" % - client) - raise SystemExit(1) + except: # pylint: disable=W0702 + self.errExit("Failed to find extra entry info for client %s" % + client) root = lxml.etree.Element("Base") self.log.info("Found %d extra entries" % (len(extra))) add_point = root diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index e883c432f..8f84cd87d 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -32,9 +32,8 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): use_stdin = False try: opts, gargs = getopt.getopt(args, 'vfIs') - except: - print(self.__doc__) - raise SystemExit(1) + except getopt.GetoptError: + self.errExit(self.__doc__) for opt in opts: if opt[0] == '-v': self.log = True diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index bb5ee352b..d21d66a22 100644 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -79,8 +79,7 @@ class Reports(Bcfg2.Server.Admin.Mode): def __call__(self, args): if len(args) == 0 or args[0] == '-h': - print(self.__usage__) - raise SystemExit(0) + self.errExit(self.__usage__) # FIXME - dry run @@ -101,9 +100,7 @@ class Reports(Bcfg2.Server.Admin.Mode): management.call_command("syncdb", verbosity=vrb) management.call_command("migrate", verbosity=vrb) except: - print("Update failed: %s" % - traceback.format_exc().splitlines()[-1]) - raise SystemExit(1) + self.errExit("Update failed: %s" % sys.exc_info()[1]) elif args[0] == 'purge': expired = False client = None @@ -124,22 +121,20 @@ class Reports(Bcfg2.Server.Admin.Mode): maxdate = datetime.datetime.now() - \ datetime.timedelta(days=int(args[i + 1])) except: - self.log.error("Invalid number of days: %s" % - args[i + 1]) - raise SystemExit(-1) + self.errExit("Invalid number of days: %s" % + args[i + 1]) i = i + 1 elif args[i] == '--expired': expired = True i = i + 1 if expired: if state: - self.log.error("--state is not valid with --expired") - raise SystemExit(-1) + self.errExit("--state is not valid with --expired") self.purge_expired(maxdate) else: self.purge(client, maxdate, state) else: - print("Unknown command: %s" % args[0]) + self.errExit("Unknown command: %s" % args[0]) @transaction.commit_on_success def scrub(self): @@ -155,8 +150,7 @@ class Reports(Bcfg2.Server.Admin.Mode): (start_count - cls.objects.count(), cls.__class__.__name__)) except: print("Failed to prune %s: %s" % - (cls.__class__.__name__, - traceback.format_exc().splitlines()[-1])) + (cls.__class__.__name__, sys.exc_info()[1])) def django_command_proxy(self, command): '''Call a django command''' @@ -180,8 +174,7 @@ class Reports(Bcfg2.Server.Admin.Mode): cobj = Client.objects.get(name=client) ipurge = ipurge.filter(client=cobj) except Client.DoesNotExist: - self.log.error("Client %s not in database" % client) - raise SystemExit(-1) + self.errExit("Client %s not in database" % client) self.log.debug("Filtering by client: %s" % client) if maxdate: diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py index 84ad93ae0..2722364f7 100644 --- a/src/lib/Bcfg2/Server/Admin/Syncdb.py +++ b/src/lib/Bcfg2/Server/Admin/Syncdb.py @@ -3,6 +3,7 @@ import Bcfg2.settings import Bcfg2.Options import Bcfg2.Server.Admin import Bcfg2.Server.models +from django.core.exceptions import ImproperlyConfigured from django.core.management import setup_environ, call_command @@ -24,10 +25,7 @@ class Syncdb(Bcfg2.Server.Admin.Mode): call_command("syncdb", interactive=False, verbosity=0) self._database_available = True except ImproperlyConfigured: - err = sys.exc_info()[1] - self.log.error("Django configuration problem: %s" % err) - raise SystemExit(1) + self.errExit("Django configuration problem: %s" % + sys.exc_info()[1]) except: - err = sys.exc_info()[1] - self.log.error("Database update failed: %s" % err) - raise SystemExit(1) + self.errExit("Database update failed: %s" % sys.exc_info()[1]) diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py index ba4777c93..2613f74ac 100644 --- a/src/lib/Bcfg2/Server/Admin/Xcmd.py +++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py @@ -1,10 +1,10 @@ """ XML-RPC Command Interface for bcfg2-admin""" import sys +import xmlrpclib import Bcfg2.Options import Bcfg2.Client.Proxy import Bcfg2.Server.Admin -from Bcfg2.Compat import xmlrpclib class Xcmd(Bcfg2.Server.Admin.Mode): @@ -32,8 +32,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode): ca=setup['ca'], timeout=setup['timeout']) if len(setup['args']) == 0: - print("Usage: xcmd <xmlrpc method> <optional arguments>") - return + self.errExit("Usage: xcmd <xmlrpc method> <optional arguments>") cmd = args[0] try: data = getattr(proxy, cmd)(*args[1:]) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 8ef9e3e96..7aa07f2a2 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -2,14 +2,14 @@ implementations inherit from. """ import os -import sys -import time +import pwd import atexit -import select -import signal import logging -import inspect +import select +import sys import threading +import time +import inspect import lxml.etree import Bcfg2.Server import Bcfg2.Logger @@ -243,14 +243,6 @@ class BaseCore(object): #: The CA that signed the server cert self.ca = self.setup['ca'] - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - self.shutdown() - os._exit(1) # pylint: disable=W0212 - - signal.signal(signal.SIGINT, hdlr) - #: The FAM :class:`threading.Thread`, #: :func:`_file_monitor_thread` self.fam_thread = \ @@ -762,6 +754,11 @@ class BaseCore(object): os.chmod(piddir, 493) # 0775 if not self._daemonize(): return False + + # rewrite $HOME. pulp stores its auth creds in ~/.pulp, so + # this is necessary to make that work when privileges are + # dropped + os.environ['HOME'] = pwd.getpwuid(self.setup['daemon_uid'])[5] else: os.umask(int(self.setup['umask'], 8)) @@ -789,7 +786,8 @@ class BaseCore(object): while self.fam.pending() != 0: time.sleep(1) - self.set_debug(None, self.debug_flag) + if self.debug_flag: + self.set_debug(None, self.debug_flag) self._block() def _daemonize(self): diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index 7c3b2d9cc..f028e225e 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -143,10 +143,11 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.check_xml(os.path.join(self.metadata.data, "groups.xml"), self.metadata.groups_xml.data, "metadata") - if self.has_all_xincludes("clients.xml"): - self.check_xml(os.path.join(self.metadata.data, "clients.xml"), - self.metadata.clients_xml.data, - "metadata") + if hasattr(self.metadata, "clients_xml"): + if self.has_all_xincludes("clients.xml"): + self.check_xml(os.path.join(self.metadata.data, "clients.xml"), + self.metadata.clients_xml.data, + "metadata") def check_cfg(self): """ Check Cfg files and ``info.xml`` files for required diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index a1d0b7fa1..da8da1aa4 100755 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -34,6 +34,12 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): err = sys.exc_info()[1] self.LintError("genshi-syntax-error", "Genshi syntax error: %s" % err) + except: + etype, err = sys.exc_info()[:2] + self.LintError( + "genshi-syntax-error", + "Unexpected Genshi error on %s: %s: %s" % + (entry.name, etype.__name__, err)) def check_bundler(self): """ Check templates in Bundler for syntax errors. """ diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 83b00bcb3..3bf76765b 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -70,7 +70,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): permissions=dict(name=is_filename, owner=is_username, group=is_username, mode=is_octal_mode), vcs=dict(vcstype=lambda v: (v != 'Path' and - hasattr(Bcfg2.Client.Tools.VCS, + hasattr(Bcfg2.Client.Tools.VCS.VCS, "Install%s" % v)), revision=None, sourceurl=None)), Service={"__any__": dict(name=None), diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index ecd970b54..c825a57b5 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -87,6 +87,10 @@ class Plugin(Debuggable): #: alphabetically by their name. sort_order = 500 + #: Whether or not to automatically create a data directory for + #: this plugin + create = True + #: List of names of methods to be exposed as XML-RPC functions __rmi__ = Debuggable.__rmi__ @@ -107,7 +111,7 @@ class Plugin(Debuggable): self.Entries = {} self.core = core self.data = os.path.join(datastore, self.name) - if not os.path.exists(self.data): + if self.create and not os.path.exists(self.data): self.logger.warning("%s: %s does not exist, creating" % (self.name, self.data)) os.makedirs(self.data) diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index d460cc45d..7909eaa03 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -286,6 +286,8 @@ class Statistics(Plugin): you should avoid using Statistics and use :class:`ThreadedStatistics` instead.""" + create = False + def process_statistics(self, client, xdata): """ Process the given XML statistics data for the specified client. @@ -526,6 +528,8 @@ class GoalValidator(object): class Version(Plugin): """ Version plugins interact with various version control systems. """ + create = False + #: The path to the VCS metadata file or directory, relative to the #: base of the Bcfg2 repository. E.g., for Subversion this would #: be ".svn" diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 7af69ec81..8a787751c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -24,6 +24,24 @@ from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ #: facility for passing it otherwise. CFG = None +_HANDLERS = [] + + +def handlers(): + """ A list of Cfg handler classes. Loading the handlers must + be done at run-time, not at compile-time, or it causes a + circular import and Bad Things Happen.""" + if not _HANDLERS: + for submodule in walk_packages(path=__path__, prefix=__name__ + "."): + mname = submodule[1].rsplit('.', 1)[-1] + module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, + mname) + hdlr = getattr(module, mname) + if issubclass(hdlr, CfgBaseFileMatcher): + _HANDLERS.append(hdlr) + _HANDLERS.sort(key=operator.attrgetter("__priority__")) + return _HANDLERS + class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, Bcfg2.Server.Plugin.Debuggable): @@ -432,24 +450,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, entry.set_debug(debug) return rv - @property - def handlers(self): - """ A list of Cfg handler classes. Loading the handlers must - be done at run-time, not at compile-time, or it causes a - circular import and Bad Things Happen.""" - if self._handlers is None: - self._handlers = [] - for submodule in walk_packages(path=__path__, - prefix=__name__ + "."): - mname = submodule[1].rsplit('.', 1)[-1] - module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, - mname) - hdlr = getattr(module, mname) - if CfgBaseFileMatcher in hdlr.__mro__: - self._handlers.append(hdlr) - self._handlers.sort(key=operator.attrgetter("__priority__")) - return self._handlers - def handle_event(self, event): """ Dispatch a FAM event to :func:`entry_init` or the appropriate child handler object. @@ -466,7 +466,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # process a bogus changed event like a created return - for hdlr in self.handlers: + for hdlr in handlers(): if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but @@ -546,10 +546,18 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, def bind_entry(self, entry, metadata): self.bind_info_to_entry(entry, metadata) - data = self._generate_data(entry, metadata) - - for fltr in self.get_handlers(metadata, CfgFilter): - data = fltr.modify_data(entry, metadata, data) + data, generator = self._generate_data(entry, metadata) + + if generator is not None: + # apply no filters if the data was created by a CfgCreator + for fltr in self.get_handlers(metadata, CfgFilter): + if fltr.specific <= generator.specific: + # only apply filters that are as specific or more + # specific than the generator used for this entry. + # Note that specificity comparison is backwards in + # this sense, since it's designed to sort from + # most specific to least specific. + data = fltr.modify_data(entry, metadata, data) if self.setup['validate']: try: @@ -658,7 +666,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, :type entry: lxml.etree._Element :param metadata: The client metadata to generate data for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :returns: string - the data for the entry + :returns: tuple of (string, generator) - the data for the + entry and the generator used to generate it (or + None, if data was created) """ try: generator = self.best_matching(metadata, @@ -667,10 +677,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, except PluginExecutionError: # if no creators or generators exist, _create_data() # raises an appropriate exception - return self._create_data(entry, metadata) + return (self._create_data(entry, metadata), None) try: - return generator.get_data(entry, metadata) + return (generator.get_data(entry, metadata), generator) except: msg = "Cfg: Error rendering %s: %s" % (entry.get("name"), sys.exc_info()[1]) @@ -837,10 +847,13 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): def Run(self): for basename, entry in list(self.core.plugins['Cfg'].entries.items()): self.check_pubkey(basename, entry) + self.check_missing_files() @classmethod def Errors(cls): - return {"no-pubkey-xml": "warning"} + return {"no-pubkey-xml": "warning", + "unknown-cfg-files": "error", + "extra-cfg-files": "error"} def check_pubkey(self, basename, entry): """ check that privkey.xml files have corresponding pubkey.xml @@ -862,3 +875,41 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): self.LintError("no-pubkey-xml", "%s has no corresponding pubkey.xml at %s" % (basename, pubkey)) + + def check_missing_files(self): + """ check that all files on the filesystem are known to Cfg """ + cfg = self.core.plugins['Cfg'] + + # first, collect ignore patterns from handlers + ignore = [] + for hdlr in handlers(): + ignore.extend(hdlr.__ignore__) + + # next, get a list of all non-ignored files on the filesystem + all_files = set() + for root, _, files in os.walk(cfg.data): + all_files.update(os.path.join(root, fname) + for fname in files + if not any(fname.endswith("." + i) + for i in ignore)) + + # next, get a list of all files known to Cfg + cfg_files = set() + for root, eset in cfg.entries.items(): + cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) + for fname in eset.entries.keys()) + + # finally, compare the two + unknown_files = all_files - cfg_files + extra_files = cfg_files - all_files + if unknown_files: + self.LintError( + "unknown-cfg-files", + "Files on the filesystem could not be understood by Cfg: %s" % + "; ".join(unknown_files)) + if extra_files: + self.LintError( + "extra-cfg-files", + "Cfg has entries for files that do not exist on the " + "filesystem: %s\nThis is probably a bug." % + "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 507973fa6..a9b622637 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -40,6 +40,8 @@ if HAS_DJANGO: """ dict-like object to make it easier to access client bcfg2 versions from the database """ + create = False + def __getitem__(self, key): try: return MetadataClientModel.objects.get(hostname=key).version diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py index 1736becc7..71128d64c 100644 --- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py @@ -9,6 +9,8 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.GoalValidator): """POSIXCompat is a goal validator plugin for POSIX entries.""" + create = False + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.GoalValidator.__init__(self) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index a4b17f05a..98add2ccf 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -674,7 +674,10 @@ class YumCollection(Collection): gdicts.append(dict(group=group, type=ptype)) if self.use_yum: - return self.call_helper("get_groups", inputdata=gdicts) + try: + return self.call_helper("get_groups", inputdata=gdicts) + except ValueError: + return dict() else: pkgs = dict() for gdict in gdicts: @@ -837,12 +840,13 @@ class YumCollection(Collection): return Collection.complete(self, packagelist) if packagelist: - result = \ - self.call_helper("complete", - dict(packages=list(packagelist), - groups=list(self.get_relevant_groups()))) - if not result: - # some sort of error, reported by call_helper() + try: + result = self.call_helper( + "complete", + dict(packages=list(packagelist), + groups=list(self.get_relevant_groups()))) + except ValueError: + # error reported by call_helper() return set(), packagelist # json doesn't understand sets or tuples, so we get back a # lists of lists (packages) and a list of unicode strings @@ -873,28 +877,39 @@ class YumCollection(Collection): ``bcfg2-yum-helper`` command. """ cmd = [self.helper, "-c", self.cfgfile] - verbose = self.debug_flag or self.setup['verbose'] - if verbose: + if self.setup['verbose']: + cmd.append("-v") + if self.debug_flag: + if not self.setup['verbose']: + # ensure that running in debug gets -vv, even if + # verbose is not enabled + cmd.append("-v") cmd.append("-v") cmd.append(command) - self.debug_log("Packages: running %s" % " ".join(cmd), flag=verbose) + self.debug_log("Packages: running %s" % " ".join(cmd)) if inputdata: result = self.cmd.run(cmd, inputdata=json.dumps(inputdata)) else: result = self.cmd.run(cmd) if not result.success: + errlines = result.error.splitlines() self.logger.error("Packages: error running bcfg2-yum-helper: %s" % - result.error) + errlines[0]) + for line in errlines[1:]: + self.logger.error("Packages: %s" % line) elif result.stderr: + errlines = result.stderr.splitlines() self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - result.stderr, flag=verbose) + errlines[0]) + for line in errlines[1:]: + self.debug_log("Packages: %s" % line) try: return json.loads(result.stdout) except ValueError: err = sys.exc_info()[1] self.logger.error("Packages: error reading bcfg2-yum-helper " "output: %s" % err) - return None + raise def setup_data(self, force_update=False): """ Do any collection-level data setup tasks. This is called @@ -920,13 +935,21 @@ class YumCollection(Collection): if force_update: # we call this twice: one to clean up data from the old # config, and once to clean up data from the new config - self.call_helper("clean") + try: + self.call_helper("clean") + except ValueError: + # error reported by call_helper + pass os.unlink(self.cfgfile) self.write_config() if force_update: - self.call_helper("clean") + try: + self.call_helper("clean") + except ValueError: + # error reported by call_helper + pass class YumSource(Source): diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index e97607093..6827c3d1f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -14,6 +14,7 @@ from Bcfg2.Server.Statistics import track_statistics try: from django.db import models + from django.core.exceptions import MultipleObjectsReturned HAS_DJANGO = True class ProbesDataModel(models.Model, @@ -254,12 +255,15 @@ class Probes(Bcfg2.Server.Plugin.Probing, for group in self.cgroups[client.hostname]: try: - ProbesGroupsModel.objects.get(hostname=client.hostname, - group=group) - except ProbesGroupsModel.DoesNotExist: - grp = ProbesGroupsModel(hostname=client.hostname, - group=group) - grp.save() + ProbesGroupsModel.objects.get_or_create( + hostname=client.hostname, + group=group) + except MultipleObjectsReturned: + ProbesGroupsModel.objects.filter(hostname=client.hostname, + group=group).delete() + ProbesGroupsModel.objects.get_or_create( + hostname=client.hostname, + group=group) ProbesGroupsModel.objects.filter( hostname=client.hostname).exclude( group__in=self.cgroups[client.hostname]).delete() diff --git a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py index c3a2221f6..41e6bf8b5 100644 --- a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py @@ -6,7 +6,9 @@ import Bcfg2.Server.Plugin class ServiceCompat(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.GoalValidator): """ Use old-style service modes for older clients """ - name = 'ServiceCompat' + + create = False + __author__ = 'bcfg-dev@mcs.anl.gov' mode_map = {('true', 'true'): 'default', ('interactive', 'true'): 'interactive_only', diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index 34a6e89e0..dfe864d48 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -60,9 +60,48 @@ class Svn(Bcfg2.Server.Plugin.Version): self.client.callback_conflict_resolver = \ self.get_conflict_resolver(choice) + try: + if self.core.setup.cfp.get( + "svn", + "always_trust").lower() == "true": + self.client.callback_ssl_server_trust_prompt = \ + self.ssl_server_trust_prompt + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self.logger.debug("Svn: Using subversion cache for SSL " + "certificate trust") + + try: + if (self.core.setup.cfp.get("svn", "user") and + self.core.setup.cfp.get("svn", "password")): + self.client.callback_get_login = \ + self.get_login + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self.logger.info("Svn: Using subversion cache for " + "password-based authetication") + self.logger.debug("Svn: Initialized svn plugin with SVN directory %s" % self.vcs_path) + # pylint: disable=W0613 + def get_login(self, realm, username, may_save): + """ PySvn callback to get credentials for HTTP basic authentication """ + self.logger.debug("Svn: Logging in with username: %s" % + self.core.setup.cfp.get("svn", "user")) + return True, \ + self.core.setup.cfp.get("svn", "user"), \ + self.core.setup.cfp.get("svn", "password"), \ + False + # pylint: enable=W0613 + + def ssl_server_trust_prompt(self, trust_dict): + """ PySvn callback to always trust SSL certificates from SVN server """ + self.logger.debug("Svn: Trusting SSL certificate from %s, " + "issued by %s for realm %s" % + (trust_dict['hostname'], + trust_dict['issuer_dname'], + trust_dict['realm'])) + return True, trust_dict['failures'], False + def get_conflict_resolver(self, choice): """ Get a PySvn conflict resolution callback """ def callback(conflict_description): diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index 050ba3b3e..77bdd6576 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -114,7 +114,7 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): def Run(self): for helper in self.core.plugins['TemplateHelper'].entries.values(): - if self.HandlesFile(helper): + if self.HandlesFile(helper.name): self.check_helper(helper.name) def check_helper(self, helper): diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 69c3264f9..d087f4f87 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -81,9 +81,6 @@ class PackedDigitRange(object): # pylint: disable=E0012,R0924 def __str__(self): return "[%s]" % self.str - def __len__(self): - return sum(r[1] - r[0] + 1 for r in self.ranges) + len(self.ints) - def locked(fd): """ Acquire a lock on a file. diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index f3697d66f..c06074845 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -32,6 +32,8 @@ TIME_ZONE = None DEBUG = False TEMPLATE_DEBUG = DEBUG +ALLOWED_HOSTS = ['*'] + MEDIA_URL = '/site_media/' |