diff options
author | Sol Jerome <sol.jerome@gmail.com> | 2014-04-16 10:16:29 -0500 |
---|---|---|
committer | Sol Jerome <sol.jerome@gmail.com> | 2014-04-16 10:16:29 -0500 |
commit | d510e918e41b7b2b7b0b9351a40eab2794b49c83 (patch) | |
tree | 250715ab112c10612ee131925ad07b68591c09f3 /src | |
parent | 9ebdcdb2f7718ae9203b20dafea4bca9f310ed75 (diff) | |
parent | 24a261f842a4bc1d4dc125fad0f43343d5d4c9d8 (diff) | |
download | bcfg2-d510e918e41b7b2b7b0b9351a40eab2794b49c83.tar.gz bcfg2-d510e918e41b7b2b7b0b9351a40eab2794b49c83.tar.bz2 bcfg2-d510e918e41b7b2b7b0b9351a40eab2794b49c83.zip |
Merge branch 'maint' into master
Signed-off-by: Sol Jerome <sol.jerome@gmail.com>
Conflicts:
doc/appendix/guides/import-existing-ssh-keys.txt
misc/bcfg2.spec
src/lib/Bcfg2/Client/Tools/VCS.py
src/lib/Bcfg2/Client/Tools/YUM.py
src/lib/Bcfg2/Encryption.py
src/lib/Bcfg2/Reporting/Collector.py
src/lib/Bcfg2/Reporting/Storage/DjangoORM.py
src/lib/Bcfg2/Server/Core.py
src/lib/Bcfg2/Server/FileMonitor/__init__.py
src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
src/lib/Bcfg2/Server/Plugin/helpers.py
src/lib/Bcfg2/Server/Plugins/Metadata.py
src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
src/lib/Bcfg2/settings.py
src/sbin/bcfg2-crypt
src/sbin/bcfg2-reports
src/sbin/bcfg2-yum-helper
testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestAugeas.py
Diffstat (limited to 'src')
25 files changed, 375 insertions, 108 deletions
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py index 4f6953b2a..dc5fc6755 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py @@ -4,6 +4,7 @@ import sys import Bcfg2.Client.XML from augeas import Augeas from Bcfg2.Client.Tools.POSIX.base import POSIXTool +from Bcfg2.Client.Tools.POSIX.File import POSIXFile class AugeasCommand(object): @@ -187,13 +188,14 @@ class Insert(AugeasCommand): class POSIXAugeas(POSIXTool): """ Handle <Path type='augeas'...> entries. See :ref:`client-tools-augeas`. """ - - __handles__ = [('Path', 'augeas')] - __req__ = {'Path': ['type', 'name', 'setting', 'value']} + __req__ = ['name', 'mode', 'owner', 'group'] def __init__(self, config): POSIXTool.__init__(self, config) self._augeas = dict() + # file tool for setting initial values of files that don't + # exist + self.filetool = POSIXFile(logger, setup, config) def get_augeas(self, entry): """ Get an augeas object for the given entry. """ @@ -214,9 +216,9 @@ class POSIXAugeas(POSIXTool): return self._augeas[entry.get("name")] def fully_specified(self, entry): - return entry.text is not None + return len(entry.getchildren()) != 0 - def get_commands(self, entry, unverified=False): + def get_commands(self, entry): """ Get a list of commands to verify or install. @param entry: The entry to get commands from. @@ -229,7 +231,7 @@ class POSIXAugeas(POSIXTool): """ rv = [] for cmd in entry.iterchildren(): - if unverified and cmd.get("verified", "false") != "false": + if cmd.tag == "Initial": continue if cmd.tag in globals(): rv.append(globals()[cmd.tag](cmd, self.get_augeas(entry), @@ -266,7 +268,18 @@ class POSIXAugeas(POSIXTool): def install(self, entry): rv = True - for cmd in self.get_commands(entry, unverified=True): + if entry.get("current_exists", "true") == "false": + initial = entry.find("Initial") + if initial is not None: + self.logger.debug("Augeas: Setting initial data for %s" % + entry.get("name")) + file_entry = Bcfg2.Client.XML.Element("Path", + **dict(entry.attrib)) + file_entry.text = initial.text + self.filetool.install(file_entry) + # re-parse the file + self.get_augeas(entry).load() + for cmd in self.get_commands(entry): try: cmd.install() except: # pylint: disable=W0702 @@ -277,8 +290,7 @@ class POSIXAugeas(POSIXTool): try: self.get_augeas(entry).save() except: # pylint: disable=W0702 - self.logger.error( - "Failure saving Augeas changes to %s: %s" % - (entry.get("name"), sys.exc_info()[1])) + self.logger.error("Failure saving Augeas changes to %s: %s" % + (entry.get("name"), sys.exc_info()[1])) rv = False return POSIXTool.install(self, entry) and rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 712620206..8895eaae1 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -217,18 +217,13 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): acl.delete_entry(aclentry) if os.path.isdir(path): defacl = posix1e.ACL(filedef=path) - if not defacl.valid(): - # when a default ACL is queried on a directory that - # has no default ACL entries at all, you get an empty - # ACL, which is not valid. in this circumstance, we - # just copy the access ACL to get a base valid ACL - # that we can add things to. - defacl = posix1e.ACL(acl=acl) - else: - for aclentry in defacl: - if aclentry.tag_type in [posix1e.ACL_USER, - posix1e.ACL_GROUP]: - defacl.delete_entry(aclentry) + for aclentry in defacl: + if aclentry.tag_type in [posix1e.ACL_USER, + posix1e.ACL_USER_OBJ, + posix1e.ACL_GROUP, + posix1e.ACL_GROUP_OBJ, + posix1e.ACL_OTHER]: + defacl.delete_entry(aclentry) else: defacl = None @@ -254,10 +249,16 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): try: if scope == posix1e.ACL_USER: scopename = "user" - aclentry.qualifier = self._norm_uid(qualifier) + if qualifier: + aclentry.qualifier = self._norm_uid(qualifier) + else: + aclentry.tag_type = posix1e.ACL_USER_OBJ elif scope == posix1e.ACL_GROUP: scopename = "group" - aclentry.qualifier = self._norm_gid(qualifier) + if qualifier: + aclentry.qualifier = self._norm_gid(qualifier) + else: + aclentry.tag_type = posix1e.ACL_GROUP_OBJ except (OSError, KeyError): err = sys.exc_info()[1] self.logger.error("POSIX: Could not resolve %s %s: %s" % @@ -358,7 +359,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): try: # single octal digit rv = int(perms) - if rv > 0 and rv < 8: + if rv >= 0 and rv < 8: return rv else: self.logger.error("POSIX: Permissions digit out of range in " @@ -388,13 +389,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Get a string representation of the given ACL. aclkey must be a tuple of (<acl type>, <acl scope>, <qualifier>) """ atype, scope, qualifier = aclkey + if not qualifier: + qualifier = '' acl_str = [] if atype == 'default': acl_str.append(atype) - if scope == posix1e.ACL_USER: + if scope == posix1e.ACL_USER or scope == posix1e.ACL_USER_OBJ: acl_str.append("user") - elif scope == posix1e.ACL_GROUP: + elif scope == posix1e.ACL_GROUP or scope == posix1e.ACL_GROUP_OBJ: acl_str.append("group") + elif scope == posix1e.ACL_OTHER: + acl_str.append("other") acl_str.append(qualifier) acl_str.append(self._acl_perm2string(perms)) return ":".join(acl_str) @@ -414,7 +419,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Get data on the existing state of <path> -- e.g., whether or not it exists, owner, group, permissions, etc. """ try: - ondisk = os.stat(path) + ondisk = os.lstat(path) except OSError: self.logger.debug("POSIX: %s does not exist" % path) return (False, None, None, None, None, None) @@ -451,7 +456,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if HAS_SELINUX: try: - secontext = selinux.getfilecon(path)[1].split(":")[2] + secontext = selinux.lgetfilecon(path)[1].split(":")[2] except (OSError, KeyError): err = sys.exc_info()[1] self.logger.debug("POSIX: Could not get current SELinux " @@ -460,7 +465,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: secontext = None - if HAS_ACLS: + if HAS_ACLS and not stat.S_ISLNK(ondisk[stat.ST_MODE]): acls = self._list_file_acls(path) else: acls = None @@ -562,9 +567,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): wanted = dict() for acl in entry.findall("ACL"): if acl.get("scope") == "user": - scope = posix1e.ACL_USER + if acl.get("user"): + scope = posix1e.ACL_USER + else: + scope = posix1e.ACL_USER_OBJ elif acl.get("scope") == "group": - scope = posix1e.ACL_GROUP + if acl.get("group"): + scope = posix1e.ACL_GROUP + else: + scope = posix1e.ACL_GROUP_OBJ + elif acl.get("scope") == "other": + scope = posix1e.ACL_OTHER else: self.logger.error("POSIX: Unknown ACL scope %s" % acl.get("scope")) @@ -573,7 +586,10 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): self.logger.error("POSIX: No permissions set for ACL: %s" % Bcfg2.Client.XML.tostring(acl)) continue - wanted[(acl.get("type"), scope, acl.get(acl.get("scope")))] = \ + qual = acl.get(acl.get("scope")) + if not qual: + qual = '' + wanted[(acl.get("type"), scope, qual)] = \ self._norm_acl_perms(acl.get('perms')) return wanted @@ -587,11 +603,12 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ Given an ACL object, process it appropriately and add it to the return value """ try: + qual = '' if acl.tag_type == posix1e.ACL_USER: qual = pwd.getpwuid(acl.qualifier)[0] elif acl.tag_type == posix1e.ACL_GROUP: qual = grp.getgrgid(acl.qualifier)[0] - else: + elif atype == "access" or acl.tag_type == posix1e.ACL_MASK: return except (OSError, KeyError): err = sys.exc_info()[1] @@ -621,9 +638,38 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): _process_acl(acl, "default") return existing - def _verify_acls(self, entry, path=None): + def _verify_acls(self, entry, path=None): # pylint: disable=R0912 """ verify POSIX ACLs on the given entry. return True if all ACLS are correct, false otherwise """ + def _verify_acl(aclkey, perms): + """ Given ACL data, process it appropriately and add it to + missing or wrong lists if appropriate """ + if aclkey not in existing: + missing.append(self._acl2string(aclkey, perms)) + elif existing[aclkey] != perms: + wrong.append((self._acl2string(aclkey, perms), + self._acl2string(aclkey, existing[aclkey]))) + if path == entry.get("name"): + atype, scope, qual = aclkey + aclentry = Bcfg2.Client.XML.Element("ACL", type=atype, + perms=str(perms)) + if (scope == posix1e.ACL_USER or + scope == posix1e.ACL_USER_OBJ): + aclentry.set("scope", "user") + elif (scope == posix1e.ACL_GROUP or + scope == posix1e.ACL_GROUP_OBJ): + aclentry.set("scope", "group") + elif scope == posix1e.ACL_OTHER: + aclentry.set("scope", "other") + else: + self.logger.debug("POSIX: Unknown ACL scope %s on %s" % + (scope, path)) + return + + if scope != posix1e.ACL_OTHER: + aclentry.set(aclentry.get("scope"), qual) + entry.append(aclentry) + if not HAS_ACLS: if entry.findall("ACL"): self.logger.debug("POSIX: ACLs listed for %s but no pylibacl " @@ -644,25 +690,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): extra = [] wrong = [] for aclkey, perms in wanted.items(): - if aclkey not in existing: - missing.append(self._acl2string(aclkey, perms)) - elif existing[aclkey] != perms: - wrong.append((self._acl2string(aclkey, perms), - self._acl2string(aclkey, existing[aclkey]))) - if path == entry.get("name"): - atype, scope, qual = aclkey - aclentry = Bcfg2.Client.XML.Element("ACL", type=atype, - perms=str(perms)) - if scope == posix1e.ACL_USER: - aclentry.set("scope", "user") - elif scope == posix1e.ACL_GROUP: - aclentry.set("scope", "group") - else: - self.logger.debug("POSIX: Unknown ACL scope %s on %s" % - (scope, path)) - continue - aclentry.set(aclentry.get("scope"), qual) - entry.append(aclentry) + _verify_acl(aclkey, perms) for aclkey, perms in existing.items(): if aclkey not in wanted: diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 58a3bbdfc..1a3b22506 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -160,7 +160,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] and g[0] != entry.get("group")] + if entry.get("name") in g[3] and g[0] != entry.get("group") + and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): """ Verify a POSIXUser entry """ diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index 20a172d3d..027d91c71 100644 --- a/src/lib/Bcfg2/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -13,8 +13,6 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'systemd')] __req__ = {'Service': ['name', 'status']} - conflicts = ['Chkconfig'] - def get_svc_command(self, service, action): return "/bin/systemctl %s %s.service" % (action, service.get('name')) diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 4e8ac76a4..449503b55 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -165,12 +165,13 @@ class VCS(Bcfg2.Client.Tools.Tool): def Verifysvn(self, entry, _): """Verify svn repositories""" + # pylint: disable=E1101 headrev = pysvn.Revision(pysvn.opt_revision_kind.head) + # pylint: enable=E1101 client = pysvn.Client() try: cur_rev = str(client.info(entry.get('name')).revision.number) - server = client.info2(entry.get('sourceurl'), - headrev, + server = client.info2(entry.get('sourceurl'), headrev, recurse=False) if server: server_rev = str(server[0][1].rev.number) diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 8bb87540c..7782581c1 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -612,34 +612,38 @@ class YUM(Bcfg2.Client.Tools.PkgTool): package_fail = True stat['version_fail'] = True # Just chose the first pkg for the error message + current_pkg = all_pkg_objs[0] if virt_pkg: provides = \ - [p for p in all_pkg_objs[0].provides + [p for p in current_pkg.provides if p[0] == entry.get("name")][0] - entry.set('current_version', "%s:%s-%s" % provides[2]) + current_evr = provides[2] self.logger.info( " %s: Wrong version installed. " "Want %s, but %s provides %s" % (entry.get("name"), nevra2string(nevra), - nevra2string(all_pkg_objs[0]), + nevra2string(current_pkg), yum.misc.prco_tuple_to_string(provides))) else: - entry.set('current_version', "%s:%s-%s.%s" % - (all_pkg_objs[0].epoch, - all_pkg_objs[0].version, - all_pkg_objs[0].release, - all_pkg_objs[0].arch)) + current_evr = (current_pkg.epoch, + current_pkg.version, + current_pkg.release) self.logger.info(" %s: Wrong version installed. " "Want %s, but have %s" % (entry.get("name"), nevra2string(nevra), - nevra2string(all_pkg_objs[0]))) - entry.set('version', "%s:%s-%s.%s" % - (nevra.get('epoch', 'any'), - nevra.get('version', 'any'), - nevra.get('release', 'any'), - nevra.get('arch', 'any'))) + nevra2string(current_pkg))) + wanted_evr = (nevra.get('epoch', 'any'), + nevra.get('version', 'any'), + nevra.get('release', 'any')) + entry.set('current_version', "%s:%s-%s" % current_evr) + entry.set('version', "%s:%s-%s" % wanted_evr) + if yum.compareEVR(current_evr, wanted_evr) == 1: + entry.set("package_fail_action", "downgrade") + else: + entry.set("package_fail_action", "update") + qtext_versions.append("U(%s)" % str(all_pkg_objs[0])) continue @@ -912,6 +916,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): install_pkgs = [] gpg_keys = [] upgrade_pkgs = [] + downgrade_pkgs = [] reinstall_pkgs = [] def queue_pkg(pkg, inst, queue): @@ -953,11 +958,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if (not status.get('installed', False) and Bcfg2.Options.setup.yum_install_missing): queue_pkg(pkg, inst, install_pkgs) - elif (status.get('version_fail', False) and - Bcfg2.Options.setup.yum_fix_version): - queue_pkg(pkg, inst, upgrade_pkgs) - elif (status.get('verify_fail', False) and - Bcfg2.Options.setup.yum_reinstall_broken): + elif status.get('version_fail', False) and self.do_upgrade: + if pkg.get("package_fail_action") == "downgrade": + queue_pkg(pkg, inst, downgrade_pkgs) + else: + queue_pkg(pkg, inst, upgrade_pkgs) + elif status.get('verify_fail', False) and self.do_reinst: queue_pkg(pkg, inst, reinstall_pkgs) else: # Either there was no Install/Version/Verify @@ -1019,6 +1025,19 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self.logger.error("Error upgrading package %s: %s" % (pkg_arg, yume)) + if len(downgrade_pkgs) > 0: + self.logger.info("Attempting to downgrade packages") + + for inst in downgrade_pkgs: + pkg_arg = self.instance_status[inst].get('pkg').get('name') + self.logger.debug("Downgrading %s" % pkg_arg) + try: + self.yumbase.downgrade(**build_yname(pkg_arg, inst)) + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] + self.logger.error("Error downgrading package %s: %s" % + (pkg_arg, yume)) + if len(reinstall_pkgs) > 0: self.logger.info("Attempting to reinstall packages") for inst in reinstall_pkgs: diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index 91d4ac5c6..4ba06abae 100644 --- a/src/lib/Bcfg2/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -5,9 +5,29 @@ # pylint: disable=E0611,W0611,W0613,C0103 try: - from lxml.etree import Element, SubElement, XML, tostring + from lxml.etree import Element, SubElement, tostring, XMLParser from lxml.etree import XMLSyntaxError as ParseError + from lxml.etree import XML as _XML + from Bcfg2.Compat import wraps driver = 'lxml' + + # libxml2 2.9.0+ doesn't parse 10M+ documents by default: + # https://mail.gnome.org/archives/commits-list/2012-August/msg00645.html + try: + _parser = XMLParser(huge_tree=True) + except TypeError: + _parser = XMLParser() + + @wraps(_XML) + def XML(val, **kwargs): + """ unicode strings w/encoding declaration are not supported in + recent lxml.etree, so we try to read XML, and if it fails we try + encoding the string. """ + kwargs.setdefault('parser', _parser) + try: + return _XML(val, **kwargs) + except ValueError: + return _XML(val.encode(), **kwargs) except ImportError: # lxml not available from xml.parsers.expat import ExpatError as ParseError diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 6c1dfdccb..12c9cdaa8 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -6,6 +6,7 @@ import time import threading # pylint: disable=E0611 +from lockfile import LockFailed, LockTimeout try: from lockfile.pidlockfile import PIDLockFile from lockfile import Error as PIDFileError @@ -63,6 +64,8 @@ class ReportingCollector(object): bcfg2-admin""" self.terminate = None self.context = None + self.children = [] + self.cleanup_threshold = 25 if Bcfg2.Options.setup.debug: level = logging.DEBUG @@ -106,12 +109,24 @@ class ReportingCollector(object): self.terminate = threading.Event() atexit.register(self.shutdown) self.context = daemon.DaemonContext(detach_process=True) + iter = 0 if Bcfg2.Options.setup.daemon: self.logger.debug("Daemonizing") try: self.context.pidfile = PIDLockFile(Bcfg2.Options.setup.daemon) self.context.open() + except LockFailed: + self.logger.error("Failed to daemonize: %s" % + sys.exc_info()[1]) + self.shutdown() + return + except LockTimeout: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + self.setup['daemon']) + self.shutdown() + return except PIDFileError: self.logger.error("Error writing pid file: %s" % sys.exc_info()[1]) @@ -128,6 +143,13 @@ class ReportingCollector(object): continue store_thread = ReportingStoreThread(interaction, self.storage) store_thread.start() + self.children.append(store_thread) + + iter += 1 + if iter >= self.cleanup_threshold: + self.reap_children() + iter = 0 + except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") self.shutdown() @@ -147,3 +169,16 @@ class ReportingCollector(object): pass if self.storage: self.storage.shutdown() + + def reap_children(self): + """Join any non-live threads""" + newlist = [] + + self.logger.debug("Starting reap_children") + for child in self.children: + if child.isAlive(): + newlist.append(child) + else: + child.join() + self.logger.debug("Joined child thread %s" % child.getName()) + self.children = newlist diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 992687a85..b2d26d190 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -14,6 +14,7 @@ from django.core import management from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db.models import FieldDoesNotExist from django.core.cache import cache +from django import db #Used by GetCurrentEntry import difflib @@ -368,7 +369,12 @@ class DjangoORM(StorageBase): self._import_interaction(interaction) except: self.logger.error("Failed to import interaction: %s" % - sys.exc_info()[1]) + traceback.format_exc().splitlines()[-1]) + finally: + self.logger.info("%s: Closing database connection" % + self.__class__.__name__) + db.close_connection() + def validate(self): """Validate backend storage. Should be called once when loaded""" diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 7edf3a949..ef6799c2b 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5 <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.3.3</span> + <span>Bcfg2 Version 1.3.4</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 4f51ebe87..9c22d17ac 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -19,7 +19,7 @@ import Bcfg2.Server.Statistics import Bcfg2.Server.FileMonitor from itertools import chain from Bcfg2.Server.Cache import Cache -from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 +from Bcfg2.Compat import xmlrpclib, wraps # pylint: disable=W0622 from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin import track_statistics @@ -74,6 +74,24 @@ def sort_xml(node, key=None): node[:] = sorted_children +def close_db_connection(func): + """ Decorator that closes the Django database connection at the end of + the function. This should decorate any exposed function that + might open a database connection. """ + @wraps(func) + def inner(self, *args, **kwargs): + """ The decorated function """ + rv = func(self, *args, **kwargs) + if self._database_available: # pylint: disable=W0212 + from django import db + self.logger.debug("%s: Closing database connection" % + threading.current_thread().name) + db.close_connection() + return rv + + return inner + + class CoreInitError(Exception): """ Raised when the server core cannot be initialized. """ pass @@ -196,6 +214,12 @@ class Core(object): self.revision = '-1' atexit.register(self.shutdown) + #: if :func:`Bcfg2.Server.Core.shutdown` is called explicitly, + #: then :mod:`atexit` calls it *again*, so it gets called + #: twice. This is potentially bad, so we use + #: :attr:`Bcfg2.Server.Core._running` as a flag to determine + #: if the core needs to be shutdown, and only do it once. + self._running = True #: Threading event to signal worker threads (e.g., #: :attr:`fam_thread`) to shutdown @@ -403,14 +427,22 @@ class Core(object): def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ - self.logger.info("Shutting down core...") + if not self._running: + self.logger.debug("%s: Core already shut down" % self.name) + return + self.logger.info("%s: Shutting down core..." % self.name) if not self.terminate.isSet(): self.terminate.set() - self.fam.shutdown() - self.logger.info("FAM shut down") - for plugin in list(self.plugins.values()): - plugin.shutdown() - self.logger.info("All plugins shut down") + self._running = False + self.fam.shutdown() + self.logger.info("%s: FAM shut down" % self.name) + for plugin in list(self.plugins.values()): + plugin.shutdown() + self.logger.info("%s: All plugins shut down" % self.name) + if self._database_available: + from django import db + self.logger.info("%s: Closing database connection" % self.name) + db.close_connection() @property def metadata_cache_mode(self): @@ -601,9 +633,10 @@ class Core(object): del entry.attrib['realname'] return ret except: - self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, entry.get('realname'), - entry.get('name'))) + self.logger.error( + "Failed binding entry %s:%s with altsrc %s: %s" % + (entry.tag, entry.get('realname'), entry.get('name'), + sys.exc_info()[1])) entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) @@ -1052,6 +1085,7 @@ class Core(object): @exposed @track_statistics() + @close_db_connection def DeclareVersion(self, address, version): """ Declare the client version. @@ -1074,6 +1108,7 @@ class Core(object): return True @exposed + @close_db_connection def GetProbes(self, address): """ Fetch probes for the client. @@ -1099,6 +1134,7 @@ class Core(object): (client, err)) @exposed + @close_db_connection def RecvProbeData(self, address, probedata): """ Receive probe data from clients. @@ -1146,6 +1182,7 @@ class Core(object): return True @exposed + @close_db_connection def AssertProfile(self, address, profile): """ Set profile for a client. @@ -1165,6 +1202,7 @@ class Core(object): return True @exposed + @close_db_connection def GetConfig(self, address): """ Build config for a client by calling :func:`BuildConfiguration`. @@ -1184,6 +1222,7 @@ class Core(object): self.critical_error("Metadata consistency failure for %s" % client) @exposed + @close_db_connection def RecvStats(self, address, stats): """ Act on statistics upload with :func:`process_statistics`. @@ -1199,6 +1238,7 @@ class Core(object): return True @exposed + @close_db_connection def GetDecisionList(self, address, mode): """ Get the decision list for the client with :func:`GetDecisions`. diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index b8eb06aa1..c4b34a469 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -212,7 +212,7 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): AddMonitor.__doc__ = Pseudo.AddMonitor.__doc__ def shutdown(self): - if self.notifier: + if self.started and self.notifier: self.notifier.stop() shutdown.__doc__ = Pseudo.shutdown.__doc__ diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 109ace61f..22c97a0fe 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -123,12 +123,30 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): @classmethod def Errors(cls): - return {"unknown-entry-type": "error", + return {"missing-elements": "error", + "unknown-entry-type": "error", "unknown-entry-tag": "error", "required-attrs-missing": "error", "required-attr-format": "error", "extra-attrs": "warning"} + def check_default_acl(self, path): + """ Check that a default ACL contains either no entries or minimum + required entries """ + defaults = 0 + if path.xpath("ACL[@type='default' and @scope='user' and @user='']"): + defaults += 1 + if path.xpath("ACL[@type='default' and @scope='group' and @group='']"): + defaults += 1 + if path.xpath("ACL[@type='default' and @scope='other']"): + defaults += 1 + if defaults > 0 and defaults < 3: + self.LintError( + "missing-elements", + "A Path must have either no default ACLs or at" + " least default:user::, default:group:: and" + " default:other::") + def check_packages(self): """ Check Packages sources for Source entries with missing attributes. """ @@ -172,7 +190,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): rules.name)) def check_bundles(self): - """ Check bundles for BoundPath entries with missing + """ Check bundles for BoundPath and BoundPackage entries with missing attrs. """ if 'Bundler' not in self.core.plugins: return @@ -192,6 +210,15 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): "required-attrs-missing", "Path tags require either a 'name' or 'glob' " "attribute: \n%s" % self.RenderXML(path)) + # ensure that abstract Package tags have either name + # or group specified + for package in xdata.xpath("//Package"): + if ('name' not in package.attrib and + 'group' not in package.attrib): + self.LintError( + "required-attrs-missing", + "Package tags require either a 'name' or 'group' " + "attribute: \n%s" % self.RenderXML(package)) def check_entry(self, entry, filename): """ Generic entry check. @@ -231,6 +258,9 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): required_attrs['major'] = is_device_mode required_attrs['minor'] = is_device_mode + if tag == 'Path': + self.check_default_acl(entry) + if tag == 'ACL' and 'scope' in required_attrs: required_attrs[entry.get('scope')] = is_username diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 3ad78ade4..0b3f1e24d 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -90,6 +90,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "xml-failed-to-parse": "error", "xml-failed-to-read": "error", "xml-failed-to-verify": "error", + "xinclude-does-not-exist": "error", "input-output-error": "error"} def check_properties(self): @@ -115,6 +116,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): try: xdata = lxml.etree.parse(filename) if self.files is None: + self._expand_wildcard_xincludes(xdata) xdata.xinclude() return xdata except (lxml.etree.XIncludeError, SyntaxError): @@ -132,6 +134,33 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Failed to open file %s" % filename) return False + def _expand_wildcard_xincludes(self, xdata): + """ a lightweight version of + :func:`Bcfg2.Server.Plugin.helpers.XMLFileBacked._follow_xincludes` """ + xinclude = '%sinclude' % Bcfg2.Server.XI_NAMESPACE + for el in xdata.findall('//' + xinclude): + name = el.get("href") + if name.startswith("/"): + fpath = name + else: + fpath = os.path.join(os.path.dirname(xdata.docinfo.URL), name) + + # expand globs in xinclude, a bcfg2-specific extension + extras = glob.glob(fpath) + if not extras: + msg = "%s: %s does not exist, skipping: %s" % \ + (xdata.docinfo.URL, name, self.RenderXML(el)) + if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): + self.logger.debug(msg) + else: + self.LintError("xinclude-does-not-exist", msg) + + parent = el.getparent() + parent.remove(el) + for extra in extras: + if extra != xdata.docinfo.URL: + lxml.etree.SubElement(parent, xinclude, href=extra) + def validate(self, filename, schemafile, schema=None): """ Validate a file against the given schema. diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py index 04151d764..6383a3c99 100644 --- a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py +++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py @@ -10,7 +10,9 @@ import Bcfg2.Server.Lint try: import json -except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): import simplejson as json diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 294963669..724b34d8d 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -275,6 +275,7 @@ class ChildCore(Core): @exposed def GetConfig(self, client): """ Render the configuration for a client """ + self.metadata.update_client_list() self.logger.debug("%s: Building configuration for %s" % (self.name, client)) return lxml.etree.tostring(self.BuildConfiguration(client)) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index aa8db2bc0..407e9df46 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -275,7 +275,8 @@ class PluginDatabaseModel(object): inherit from. This is just a mixin; models must also inherit from django.db.models.Model to be valid Django models.""" - class Meta: # pylint: disable=C0111,W0232 + class Meta(object): # pylint: disable=W0232 + """ Model metadata options """ app_label = "Server" diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 622b69c79..c45d6fa84 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -216,6 +216,10 @@ class Metadata(object): """ raise NotImplementedError + def update_client_list(self): + """ Re-read the cached list of clients """ + raise NotImplementedError + class Connector(object): """ Connector plugins augment client metadata instances with diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 6ff256147..bf51ff678 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -686,7 +686,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, client = MetadataClientModel(hostname=client_name) # pylint: enable=E1102 client.save() - self.clients = self.list_clients() + self.update_client_list() return client else: try: @@ -739,7 +739,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, attribs, alias=True) def list_clients(self): - """ List all clients in client database """ + """ List all clients in client database. + + Making ``self.clients`` a property and reading the client list + dynamically from the database on every call to + ``self.clients`` can result in very high rates of database + reads, so we cache the ``list_clients()`` results to reduce + the database load. When the database is in use, the client + list is reread periodically with + :func:`Bcfg2.Server.Plugins.Metadata.update_client_list`. """ if self._use_db: return set([c.hostname for c in MetadataClientModel.objects.all()]) else: @@ -790,7 +798,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.warning(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) client.delete() - self.clients = self.list_clients() + self.update_client_list() else: return self._remove_xdata(self.clients_xml, "Client", client_name) @@ -859,8 +867,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, except KeyError: self.clientgroups[clname] = [profile] self.states['clients.xml'] = True - if self._use_db: - self.clients = self.list_clients() + self.update_client_list() def _get_condition(self, element): """ Return a predicate that returns True if a client meets @@ -1452,6 +1459,30 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return True # pylint: enable=R0911,R0912 + def update_client_list(self): + """ Re-read the client list from the database (if the database is in + use) """ + if self._use_db: + self.logger.debug("Metadata: Re-reading client list from database") + old = set(self.clients) + self.clients = self.list_clients() + new = set(self.clients) + added = new - old + removed = old - new + self.logger.debug("Metadata: Added %s clients: %s" % + (len(added), added)) + self.logger.debug("Metadata: Removed %s clients: %s" % + (len(removed), removed)) + # we could do this with set.symmetric_difference(), but we + # want detailed numbers of added/removed clients for + # logging + for client in added.union(removed): + self.expire_cache(client) + + def start_client_run(self, metadata): + """ Hook to reread client list if the database is in use """ + self.update_client_list() + def end_statistics(self, metadata): """ Hook to toggle clients in bootstrap mode """ if self.auth.get(metadata.hostname, diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index ba7baab11..c5fb46c97 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -10,7 +10,9 @@ import Bcfg2.Server.Plugin try: import json -except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 +except (ImportError, AttributeError): import simplejson as json PROBECODE = """#!/bin/sh diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index dba56eed2..c1d53a78e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -72,6 +72,7 @@ class AptSource(Source): def read_files(self): bdeps = dict() bprov = dict() + self.essentialpkgs = set() depfnames = ['Depends', 'Pre-Depends'] if self.recommended: depfnames.append('Recommends') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 6a493c19d..3cfda9e9c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -90,7 +90,9 @@ try: import yum try: import json - except ImportError: + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 + except (ImportError, AttributeError): import simplejson as json HAS_YUM = True except ImportError: @@ -354,8 +356,8 @@ class YumCollection(Collection): self.__class__._helper = find_executable('bcfg2-yum-helper') if not self.__class__._helper: self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" - # pylint: enable=W0212 - return self._helper + return self.__class__._helper + # pylint: enable=W0212 @property def use_yum(self): diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 553c16202..21d50ace6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -51,8 +51,10 @@ def load_django_models(): try: import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 87cee7029..04314218c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -13,8 +13,10 @@ from Bcfg2.Server.Plugin import PluginExecutionError try: import json + # py2.4 json library is structured differently + json.loads # pylint: disable=W0104 HAS_JSON = True -except ImportError: +except (ImportError, AttributeError): try: import simplejson as json HAS_JSON = True diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 35d4cfa0a..ae82724f3 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.3" +__version__ = "1.3.4" class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 |