diff options
Diffstat (limited to 'src/lib')
36 files changed, 437 insertions, 387 deletions
diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py index 0400e3ff7..f197a9074 100644 --- a/src/lib/Bcfg2/Client/Client.py +++ b/src/lib/Bcfg2/Client/Client.py @@ -106,7 +106,9 @@ class Client(object): self.logger.info(ret.text) finally: os.unlink(scriptname) - except: # pylint: disable=W0702 + except SystemExit: + raise + except: self._probe_failure(name, sys.exc_info()[1]) return ret @@ -258,8 +260,7 @@ class Client(object): except Bcfg2.Client.XML.ParseError: syntax_error = sys.exc_info()[1] self.fatal_error("The configuration could not be parsed: %s" % - (syntax_error)) - return(1) + syntax_error) times['config_parse'] = time.time() @@ -292,14 +293,17 @@ class Client(object): fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: # otherwise exit and give a warning to the user - self.fatal_error("An other instance of Bcfg2 is running. " - "If you what to bypass the check, run " - "with %s option" % + self.fatal_error("Another instance of Bcfg2 is running. " + "If you want to bypass the check, run " + "with the %s option" % Bcfg2.Options.OMIT_LOCK_CHECK.cmd) - except: # pylint: disable=W0702 + except SystemExit: + raise + except: lockfile = None self.logger.error("Failed to open lockfile") - # execute the said configuration + + # execute the configuration self.tools.Execute() if not self.setup['omit_lock_check']: diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 64460ea66..53180ab68 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -1,8 +1,10 @@ """ Frame is the Client Framework that verifies and installs entries, and generates statistics. """ +import os import sys import time +import select import fnmatch import logging import Bcfg2.Client.Tools @@ -160,6 +162,9 @@ class Frame(object): iprompt = entry.get('qtext') else: iprompt = prompt % (entry.tag, entry.get('name')) + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) try: ans = input(iprompt.encode(sys.stdout.encoding, 'replace')) if ans in ['y', 'Y']: diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index 7726da94c..b1a897c81 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -1,5 +1,8 @@ """Action driver""" +import os +import sys +import select import Bcfg2.Client.Tools from Bcfg2.Client.Frame import matches_white_list, passes_black_list from Bcfg2.Compat import input # pylint: disable=W0622 @@ -33,6 +36,10 @@ class Action(Bcfg2.Client.Tools.Tool): if self.setup['interactive']: prompt = ('Run Action %s, %s: (y/N): ' % (entry.get('name'), entry.get('command'))) + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], + 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) ans = input(prompt) if ans not in ['y', 'Y']: return False diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py index 9b0b998bb..9d0fe05e0 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py @@ -3,7 +3,6 @@ import os import sys import stat -import shutil import Bcfg2.Client.XML from Bcfg2.Client.Tools.POSIX.base import POSIXTool @@ -67,25 +66,14 @@ class POSIXDirectory(POSIXTool): rv &= self._makedirs(entry) if entry.get('prune', 'false') == 'true': - ulfailed = False for pent in entry.findall('Prune'): pname = pent.get('path') - ulfailed = False - if os.path.isdir(pname): - remove = shutil.rmtree - else: - remove = os.unlink try: self.logger.debug("POSIX: Removing %s" % pname) - remove(pname) + self._remove(pent) except OSError: err = sys.exc_info()[1] self.logger.error("POSIX: Failed to unlink %s: %s" % (pname, err)) - ulfailed = True - if ulfailed: - # even if prune failed, we still want to install the - # entry to make sure that we get permissions and - # whatnot set - rv = False + rv = False return POSIXTool.install(self, entry) and rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py index 5f1fbbe7c..f7251ca50 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Nonexistent.py @@ -2,7 +2,6 @@ import os import sys -import shutil from Bcfg2.Client.Tools.POSIX.base import POSIXTool @@ -19,25 +18,21 @@ class POSIXNonexistent(POSIXTool): def install(self, entry): ename = entry.get('name') - if entry.get('recursive', '').lower() == 'true': + recursive = entry.get('recursive', '').lower() == 'true' + if recursive: # ensure that configuration spec is consistent first for struct in self.config.getchildren(): - for entry in struct.getchildren(): - if (entry.tag == 'Path' and - entry.get('type') != 'nonexistent' and - entry.get('name').startswith(ename)): + for el in struct.getchildren(): + if (el.tag == 'Path' and + el.get('type') != 'nonexistent' and + el.get('name').startswith(ename)): self.logger.error('POSIX: Not removing %s. One or ' 'more files in this directory are ' 'specified in your configuration.' % ename) return False - remove = shutil.rmtree - elif os.path.isdir(ename): - remove = os.rmdir - else: - remove = os.remove try: - remove(ename) + self._remove(entry, recursive=recursive) return True except OSError: err = sys.exc_info()[1] diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index a9566b698..6388f6731 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -66,18 +66,26 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): rv &= self._set_perms(entry, path=os.path.join(root, path)) return rv + def _remove(self, entry, recursive=True): + """ Remove a Path entry, whatever that takes """ + if os.path.islink(entry.get('name')): + os.unlink(entry.get('name')) + elif os.path.isdir(entry.get('name')): + if recursive: + shutil.rmtree(entry.get('name')) + else: + os.rmdir(entry.get('name')) + else: + os.unlink(entry.get('name')) + def _exists(self, entry, remove=False): """ check for existing paths and optionally remove them. if the path exists, return the lstat of it """ try: ondisk = os.lstat(entry.get('name')) if remove: - if os.path.isdir(entry.get('name')): - remove = shutil.rmtree - else: - remove = os.unlink try: - remove(entry.get('name')) + self._remove(entry) return None except OSError: err = sys.exc_info()[1] diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index 5ac96f999..fc47883c9 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -41,7 +41,7 @@ def netmask_itoa(netmask, proto="ipv4"): size = 128 family = socket.AF_INET6 try: - int(netmask) + netmask = int(netmask) except ValueError: return netmask diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index 4022692be..927b25ba8 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -1,7 +1,9 @@ """This contains all Bcfg2 Tool modules""" import os +import sys import stat +import select from subprocess import Popen, PIPE import Bcfg2.Client.XML from Bcfg2.Compat import input, walk_packages # pylint: disable=W0622 @@ -373,6 +375,10 @@ class SvcTool(Tool): if self.setup['interactive']: prompt = ('Restart service %s?: (y/N): ' % entry.get('name')) + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], + 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) ans = input(prompt) if ans not in ['y', 'Y']: continue diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index f3765a5ec..b418d57b0 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -577,6 +577,11 @@ SERVER_VCS_ROOT = \ default=None, odesc='<VCS repository root>', cf=('server', 'vcs_root')) +SERVER_UMASK = \ + Option('Server umask', + default='0077', + odesc='<Server umask>', + cf=('server', 'umask')) # database options DB_ENGINE = \ @@ -1068,6 +1073,7 @@ CLI_COMMON_OPTIONS = dict(configfile=CFILE, syslog=LOGGING_SYSLOG) DAEMON_COMMON_OPTIONS = dict(daemon=DAEMON, + umask=SERVER_UMASK, listen_all=SERVER_LISTEN_ALL, daemon_uid=SERVER_DAEMON_USER, daemon_gid=SERVER_DAEMON_GROUP) diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index 869dc1aca..14065980d 100644 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -1,11 +1,13 @@ """ Interactively initialize a new repository. """ -import getpass + import os +import sys +import stat +import select import random import socket -import stat import string -import sys +import getpass import subprocess import Bcfg2.Server.Admin @@ -85,6 +87,14 @@ OS_LIST = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), ('Arch', 'arch')] +def safe_input(prompt): + """ input() that flushes the input buffer before accepting input """ + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) + return input(prompt) + + def gen_password(length): """Generates a random alphanumeric password with length characters.""" chars = string.letters + string.digits @@ -116,8 +126,8 @@ def create_conf(confpath, confdata): """ create the config file """ # Don't overwrite existing bcfg2.conf file if os.path.exists(confpath): - result = input("\nWarning: %s already exists. " - "Overwrite? [y/N]: " % confpath) + result = safe_input("\nWarning: %s already exists. " + "Overwrite? [y/N]: " % confpath) if result not in ['Y', 'y']: print("Leaving %s unchanged" % confpath) return @@ -186,8 +196,8 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_hostname(self): """Ask for the server hostname.""" - data = input("What is the server's hostname [%s]: " % - socket.getfqdn()) + data = safe_input("What is the server's hostname [%s]: " % + socket.getfqdn()) if data != '': self.data['shostname'] = data else: @@ -195,21 +205,21 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_config(self): """Ask for the configuration file path.""" - newconfig = input("Store Bcfg2 configuration in [%s]: " % - self.configfile) + newconfig = safe_input("Store Bcfg2 configuration in [%s]: " % + self.configfile) if newconfig != '': self.data['configfile'] = os.path.abspath(newconfig) def _prompt_repopath(self): """Ask for the repository path.""" while True: - newrepo = input("Location of Bcfg2 repository [%s]: " % + newrepo = safe_input("Location of Bcfg2 repository [%s]: " % self.data['repopath']) if newrepo != '': self.data['repopath'] = os.path.abspath(newrepo) if os.path.isdir(self.data['repopath']): - response = input("Directory %s exists. Overwrite? [y/N]:" % - self.data['repopath']) + response = safe_input("Directory %s exists. Overwrite? [y/N]:" + % self.data['repopath']) if response.lower().strip() == 'y': break else: @@ -225,8 +235,8 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_server(self): """Ask for the server name.""" - newserver = input("Input the server location [%s]: " % - self.data['server_uri']) + newserver = safe_input("Input the server location [%s]: " % + self.data['server_uri']) if newserver != '': self.data['server_uri'] = newserver @@ -238,7 +248,7 @@ class Init(Bcfg2.Server.Admin.Mode): prompt += ': ' while True: try: - osidx = int(input(prompt)) + osidx = int(safe_input(prompt)) self.data['os_sel'] = OS_LIST[osidx - 1][1] break except ValueError: @@ -248,27 +258,28 @@ class Init(Bcfg2.Server.Admin.Mode): """Ask for the key details (country, state, and location).""" print("The following questions affect SSL certificate generation.") print("If no data is provided, the default values are used.") - newcountry = input("Country name (2 letter code) for certificate: ") + newcountry = safe_input("Country name (2 letter code) for " + "certificate: ") if newcountry != '': if len(newcountry) == 2: self.data['country'] = newcountry else: while len(newcountry) != 2: - newcountry = input("2 letter country code (eg. US): ") + newcountry = safe_input("2 letter country code (eg. US): ") if len(newcountry) == 2: self.data['country'] = newcountry break else: self.data['country'] = 'US' - newstate = input("State or Province Name (full name) for " - "certificate: ") + newstate = safe_input("State or Province Name (full name) for " + "certificate: ") if newstate != '': self.data['state'] = newstate else: self.data['state'] = 'Illinois' - newlocation = input("Locality Name (eg, city) for certificate: ") + newlocation = safe_input("Locality Name (eg, city) for certificate: ") if newlocation != '': self.data['location'] = newlocation else: @@ -277,12 +288,12 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_keypath(self): """ Ask for the key pair location. Try to use sensible defaults depending on the OS """ - keypath = input("Path where Bcfg2 server private key will be created " - "[%s]: " % self.data['keypath']) + keypath = safe_input("Path where Bcfg2 server private key will be " + "created [%s]: " % self.data['keypath']) if keypath: self.data['keypath'] = keypath - certpath = input("Path where Bcfg2 server cert will be created" - "[%s]: " % self.data['certpath']) + certpath = safe_input("Path where Bcfg2 server cert will be created" + "[%s]: " % self.data['certpath']) if certpath: self.data['certpath'] = certpath diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index e41652205..130e85b67 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -1,8 +1,10 @@ """ Retrieves entries from clients and integrates the information into the repository """ -import getopt +import os import sys +import getopt +import select import Bcfg2.Server.Admin from Bcfg2.Compat import input # pylint: disable=W0622 @@ -99,6 +101,10 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): else: print(" => host entry: %s" % (choice.hostname)) + # flush input buffer + while len(select.select([sys.stdin.fileno()], [], [], + 0.0)[0]) > 0: + os.read(sys.stdin.fileno(), 4096) ans = input("Use this entry? [yN]: ") in ['y', 'Y'] if ans: return choice diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index 15ff79a35..6e313e84b 100644 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -74,7 +74,7 @@ class Reports(Bcfg2.Server.Admin.Mode): try: import south except ImportError: - print "Django south is required for Reporting" + print("Django south is required for Reporting") raise SystemExit(-3) def __call__(self, args): diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 69fb8d0cb..63149c15e 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -28,17 +28,15 @@ class Core(BaseCore): #: this server core self.server = None + daemon_args = dict(uid=self.setup['daemon_uid'], + gid=self.setup['daemon_gid'], + umask=int(self.setup['umask'], 8)) if self.setup['daemon']: - #: The :class:`daemon.DaemonContext` used to drop - #: privileges, write the PID file (with :class:`PidFile`), - #: and daemonize this core. - self.context = \ - daemon.DaemonContext(uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid'], - pidfile=PIDLockFile(self.setup['daemon'])) - else: - self.context = daemon.DaemonContext(uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid']) + daemon_args['pidfile'] = PIDLockFile(self.setup['daemon']) + #: The :class:`daemon.DaemonContext` used to drop + #: privileges, write the PID file (with :class:`PidFile`), + #: and daemonize this core. + self.context = daemon.DaemonContext(**daemon_args) __init__.__doc__ = BaseCore.__init__.__doc__.split('.. -----')[0] def _dispatch(self, method, args, dispatch_dict): diff --git a/src/lib/Bcfg2/Server/CherryPyCore.py b/src/lib/Bcfg2/Server/CherryPyCore.py index 4ddcd7bdf..d097fd08f 100644 --- a/src/lib/Bcfg2/Server/CherryPyCore.py +++ b/src/lib/Bcfg2/Server/CherryPyCore.py @@ -107,8 +107,10 @@ class Core(BaseCore): :class:`cherrypy.process.plugins.DropPrivileges`, daemonize with :class:`cherrypy.process.plugins.Daemonizer`, and write a PID file with :class:`cherrypy.process.plugins.PIDFile`. """ - DropPrivileges(cherrypy.engine, uid=self.setup['daemon_uid'], - gid=self.setup['daemon_gid']).subscribe() + DropPrivileges(cherrypy.engine, + uid=self.setup['daemon_uid'], + gid=self.setup['daemon_gid'], + umask=int(self.setup['umask'], 8)).subscribe() Daemonizer(cherrypy.engine).subscribe() PIDFile(cherrypy.engine, self.setup['daemon']).subscribe() return True diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index a5fda6f0d..ee875a7e8 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -516,7 +516,7 @@ class BaseCore(object): except PluginExecutionError: exc = sys.exc_info()[1] if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % format_exc()) + entry.set('failure', 'bind error: %s' % exc) self.logger.error("Failed to bind entry %s:%s: %s" % (entry.tag, entry.get('name'), exc)) except Exception: @@ -665,6 +665,8 @@ class BaseCore(object): os.chmod(piddir, 420) # 0644 if not self._daemonize(): return False + else: + os.umask(int(self.setup['umask'], 8)) if not self._run(): self.shutdown() @@ -1065,5 +1067,59 @@ class BaseCore(object): @exposed def get_statistics(self, _): """ Get current statistics about component execution from - :attr:`Bcfg2.Statistics.stats`. """ + :attr:`Bcfg2.Statistics.stats`. + + :returns: dict - The statistics data as returned by + :func:`Bcfg2.Statistics.Statistics.display` """ return Bcfg2.Statistics.stats.display() + + @exposed + def toggle_debug(self, address): + """ Toggle debug status of the FAM and all plugins + + :param address: Client (address, hostname) pair + :type address: tuple + :returns: bool - The new debug state of the FAM + """ + for plugin in self.plugins.values(): + plugin.toggle_debug() + return self.toggle_fam_debug(address) + + @exposed + def toggle_fam_debug(self, _): + """ Toggle debug status of the FAM + + :returns: bool - The new debug state of the FAM + """ + return self.fam.toggle_debug() + + @exposed + def set_debug(self, address, debug): + """ Explicitly set debug status of the FAM and all plugins + + :param debug: The new debug status. This can either be a + boolean, or a string describing the state (e.g., + "true" or "false"; case-insensitive) + :type debug: bool or string + :returns: bool - The new debug state + """ + if debug not in [True, False]: + debug = debug.lower() == "true" + for plugin in self.plugins.values(): + plugin.set_debug(debug) + return self.set_fam_debug(address, debug) + + @exposed + def set_fam_debug(self, _, debug): + """ Explicitly set debug status of the FAM + + :param debug: The new debug status of the FAM. This can + either be a boolean, or a string describing the + state (e.g., "true" or "false"; + case-insensitive) + :type debug: bool or string + :returns: bool - The new debug state of the FAM + """ + if debug not in [True, False]: + debug = debug.lower() == "true" + return self.fam.set_debug(debug) diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index d5aa8e4ad..178a47b1a 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -110,11 +110,18 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): if ievent.mask & amask: action = aname break + else: + # event action is not in the mask, and thus is not + # something we care about + self.debug_log("Ignoring event %s for %s" % (action, + ievent.pathname)) + return + try: watch = self.watchmgr.watches[ievent.wd] except KeyError: - LOGGER.error("Error handling event for %s: Watch %s not found" % - (ievent.pathname, ievent.wd)) + LOGGER.error("Error handling event %s for %s: Watch %s not found" % + (action, ievent.pathname, ievent.wd)) return # FAM-style file monitors return the full path to the parent # directory that is being watched, relative paths to anything diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index 72b1d2dd7..42ad4c041 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -50,6 +50,7 @@ import sys import fnmatch import logging from time import sleep, time +from Bcfg2.Server.Plugin import Debuggable LOGGER = logging.getLogger(__name__) @@ -104,7 +105,7 @@ class Event(object): return "%s (request ID %s)" % (str(self), self.requestID) -class FileMonitor(object): +class FileMonitor(Debuggable): """ The base class that all FAM implementions must inherit. The simplest instance of a FileMonitor subclass needs only to add @@ -128,8 +129,8 @@ class FileMonitor(object): .. ----- .. autoattribute:: __priority__ """ - #: Whether or not to produce debug logging - self.debug = debug + Debuggable.__init__(self) + self.debug_flag = debug #: A dict that records which objects handle which events. #: Keys are monitor handle IDs and values are objects whose @@ -168,13 +169,6 @@ class FileMonitor(object): example of this. """ self.started = True - def debug_log(self, msg): - """ Log a debug message. - - :param msg: The message to log iff :attr:`debug` is set.""" - if self.debug: - LOGGER.info(msg) - def should_ignore(self, event): """ Returns True if an event should be ignored, False otherwise. For events that include the full path, both the diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index 8d288f835..e74909ee9 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -9,7 +9,7 @@ class Debuggable(object): via XML-RPC on :class:`Bcfg2.Server.Plugin.base.Plugin` objects """ #: List of names of methods to be exposed as XML-RPC functions - __rmi__ = ['toggle_debug'] + __rmi__ = ['toggle_debug', 'set_debug'] def __init__(self, name=None): """ @@ -26,17 +26,25 @@ class Debuggable(object): self.debug_flag = False self.logger = logging.getLogger(name) - def toggle_debug(self): - """ Turn debugging output on or off. This method is exposed + def set_debug(self, debug): + """ Explicitly enable or disable debugging. This method is exposed via XML-RPC. :returns: bool - The new value of the debug flag """ - self.debug_flag = not self.debug_flag + self.debug_flag = debug self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__, self.debug_flag), flag=True) - return self.debug_flag + return debug + + def toggle_debug(self): + """ Turn debugging output on or off. This method is exposed + via XML-RPC. + + :returns: bool - The new value of the debug flag + """ + return self.set_debug(not self.debug_flag) def debug_log(self, message, flag=None): """ Log a message at the debug level. diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 894ed9851..318bf03f1 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -1526,12 +1526,12 @@ class GroupSpool(Plugin, Generator): else: return self.handles[event.requestID].rstrip("/") - def toggle_debug(self): + def set_debug(self, debug): for entry in self.entries.values(): - if hasattr(entry, "toggle_debug"): - entry.toggle_debug() - return Plugin.toggle_debug(self) - toggle_debug.__doc__ = Plugin.toggle_debug.__doc__ + if hasattr(entry, "set_debug"): + entry.set_debug(debug) + return Plugin.set_debug(self, debug) + set_debug.__doc__ = Plugin.set_debug.__doc__ def HandleEvent(self, event): """ HandleEvent is the event dispatcher for GroupSpool diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py index 73c70901b..8ebd8d921 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -2,12 +2,9 @@ <http://www.cheetahtemplate.org/>`_ templating system to generate :ref:`server-plugins-generators-cfg` files. """ -import logging -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator -LOGGER = logging.getLogger(__name__) - try: from Cheetah.Template import Template HAS_CHEETAH = True @@ -33,9 +30,7 @@ class CfgCheetahGenerator(CfgGenerator): def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) if not HAS_CHEETAH: - msg = "Cfg: Cheetah is not available: %s" % self.name - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Cheetah is not available") __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py index 00b95c970..da506a195 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py @@ -1,14 +1,11 @@ """ Handle .diff files, which apply diffs to plaintext files """ import os -import logging import tempfile -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError from subprocess import Popen, PIPE from Bcfg2.Server.Plugins.Cfg import CfgFilter -LOGGER = logging.getLogger(__name__) - class CfgDiffFilter(CfgFilter): """ CfgDiffFilter applies diffs to plaintext @@ -32,8 +29,7 @@ class CfgDiffFilter(CfgFilter): output = open(basename, 'r').read() os.unlink(basename) if ret != 0: - msg = "Error applying diff %s: %s" % (self.name, stderr) - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Error applying diff %s: %s" % + (self.name, stderr)) return output modify_data.__doc__ = CfgFilter.modify_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py index 26faf6e2c..3b4703ddb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -1,7 +1,6 @@ """ CfgEncryptedGenerator lets you encrypt your plaintext :ref:`server-plugins-generators-cfg` files on the server. """ -import logging from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP try: @@ -11,8 +10,6 @@ try: except ImportError: HAS_CRYPTO = False -LOGGER = logging.getLogger(__name__) - class CfgEncryptedGenerator(CfgGenerator): """ CfgEncryptedGenerator lets you encrypt your plaintext @@ -28,9 +25,7 @@ class CfgEncryptedGenerator(CfgGenerator): def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) if not HAS_CRYPTO: - msg = "Cfg: M2Crypto is not available" - LOGGER.error(msg) - raise PluginExecutionError(msg) + raise PluginExecutionError("M2Crypto is not available") __init__.__doc__ = CfgGenerator.__init__.__doc__ def handle_event(self, event): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py index feedbdb1b..130652aef 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -1,7 +1,6 @@ """ Handle encrypted Genshi templates (.crypt.genshi or .genshi.crypt files) """ -import logging from Bcfg2.Compat import StringIO from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import SETUP @@ -19,10 +18,6 @@ except ImportError: # CfgGenshiGenerator will raise errors if genshi doesn't exist TemplateLoader = object # pylint: disable=C0103 -LOGGER = logging.getLogger(__name__) - -LOGGER = logging.getLogger(__name__) - class EncryptedTemplateLoader(TemplateLoader): """ Subclass :class:`genshi.template.TemplateLoader` to decrypt @@ -53,6 +48,4 @@ class CfgEncryptedGenshiGenerator(CfgGenshiGenerator): def __init__(self, fname, spec, encoding): CfgGenshiGenerator.__init__(self, fname, spec, encoding) if not HAS_CRYPTO: - msg = "Cfg: M2Crypto is not available" - LOGGER.error(msg) - raise PluginExecutionError(msg) + raise PluginExecutionError("M2Crypto is not available") diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py index 023af7d4e..b702ac899 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -3,13 +3,10 @@ import os import sys import shlex -import logging -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError from subprocess import Popen, PIPE from Bcfg2.Server.Plugins.Cfg import CfgVerifier, CfgVerificationError -LOGGER = logging.getLogger(__name__) - class CfgExternalCommandVerifier(CfgVerifier): """ Invoke an external script to verify @@ -46,9 +43,6 @@ class CfgExternalCommandVerifier(CfgVerifier): if bangpath.startswith("#!"): self.cmd.extend(shlex.split(bangpath[2:].strip())) else: - msg = "%s: Cannot execute %s" % (self.__class__.__name__, - self.name) - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Cannot execute %s" % self.name) self.cmd.append(self.name) handle_event.__doc__ = CfgVerifier.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index ce77717da..df0c30c09 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -4,13 +4,10 @@ import re import sys -import logging import traceback -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgGenerator -LOGGER = logging.getLogger(__name__) - try: import genshi.core from genshi.template import TemplateLoader, NewTextTemplate @@ -67,11 +64,9 @@ class CfgGenshiGenerator(CfgGenerator): def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) if not HAS_GENSHI: - msg = "Cfg: Genshi is not available: %s" % fname - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - self.loader = self.__loader_cls__() + raise PluginExecutionError("Genshi is not available") self.template = None + self.loader = self.__loader_cls__(max_cache_size=0) __init__.__doc__ = CfgGenerator.__init__.__doc__ def get_data(self, entry, metadata): @@ -92,15 +87,15 @@ class CfgGenshiGenerator(CfgGenerator): stack = traceback.extract_tb(sys.exc_info()[2]) for quad in stack: if quad[0] == self.name: - LOGGER.error("Cfg: Error rendering %s at '%s': %s: %s" % - (fname, quad[2], err.__class__.__name__, err)) - break + raise PluginExecutionError("%s: %s at '%s'" % + (err.__class__.__name__, err, + quad[2])) raise except: - self._handle_genshi_exception(fname, sys.exc_info()) + self._handle_genshi_exception(sys.exc_info()) get_data.__doc__ = CfgGenerator.get_data.__doc__ - def _handle_genshi_exception(self, fname, exc): + def _handle_genshi_exception(self, exc): """ this is horrible, and I deeply apologize to whoever gets to maintain this after I go to the Great Beer Garden in the Sky. genshi is incredibly opaque about what's being executed, @@ -140,21 +135,16 @@ class CfgGenshiGenerator(CfgGenerator): # single line break) real_lineno = lineno - contents.code.co_firstlineno src = re.sub(r'\n\n+', '\n', contents.source).splitlines() - LOGGER.error("Cfg: Error rendering %s at '%s': %s: %s" % - (fname, src[real_lineno], err.__class__.__name__, - err)) + raise PluginExecutionError("%s: %s at '%s'" % + (err.__class__.__name__, err, + src[real_lineno])) raise def handle_event(self, event): - CfgGenerator.handle_event(self, event) - if self.data is None: - return try: self.template = self.loader.load(self.name, cls=NewTextTemplate, encoding=self.encoding) except: - msg = "Cfg: Could not load template %s: %s" % (self.name, - sys.exc_info()[1]) - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Failed to load template: %s" % + sys.exc_info()[1]) handle_event.__doc__ = CfgGenerator.handle_event.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py index e5ba0a51b..3b6fc8fa0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py @@ -1,11 +1,8 @@ """ Handle info.xml files """ -import logging -import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError, InfoXML from Bcfg2.Server.Plugins.Cfg import CfgInfo -LOGGER = logging.getLogger(__name__) - class CfgInfoXML(CfgInfo): """ CfgInfoXML handles :file:`info.xml` files for @@ -16,16 +13,15 @@ class CfgInfoXML(CfgInfo): def __init__(self, path): CfgInfo.__init__(self, path) - self.infoxml = Bcfg2.Server.Plugin.InfoXML(path) + self.infoxml = InfoXML(path) __init__.__doc__ = CfgInfo.__init__.__doc__ def bind_info_to_entry(self, entry, metadata): mdata = dict() self.infoxml.pnode.Match(metadata, mdata, entry=entry) if 'Info' not in mdata: - msg = "Failed to set metadata for file %s" % entry.get('name') - LOGGER.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + raise PluginExecutionError("Failed to set metadata for file %s" % + entry.get('name')) self._set_info(entry, mdata['Info'][None]) bind_info_to_entry.__doc__ = CfgInfo.bind_info_to_entry.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 58f6e1e42..db6810e7c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -542,8 +542,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): try: return generator.get_data(entry, metadata) except: - msg = "Cfg: exception rendering %s with %s: %s" % \ - (entry.get("name"), generator, sys.exc_info()[1]) + msg = "Cfg: Error rendering %s: %s" % (entry.get("name"), + sys.exc_info()[1]) LOGGER.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index 5faa6c018..8cc63a46f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -1,119 +1,15 @@ """ The Git plugin provides a revision interface for Bcfg2 repos using git. """ -import os import sys import Bcfg2.Server.Plugin - - -class GitAPIBase(object): - """ Base class for the various Git APIs (dulwich, GitPython, - subprocesses) """ - def __init__(self, path): - self.path = path - - def revision(self): - """ Get the current revision of the git repo as a string """ - raise NotImplementedError - - def pull(self): - """ Pull the latest version of the upstream git repo and - rebase against it. """ - raise NotImplementedError - +from subprocess import Popen, PIPE try: - from dulwich.client import get_transport_and_path - from dulwich.repo import Repo - from dulwich.file import GitFile, ensure_dir_exists - - class GitAPI(GitAPIBase): - """ API for :class:`Git` using :mod:`dulwich` """ - def __init__(self, path): - GitAPIBase.__init__(self, path) - self.repo = Repo(self.path) - self.client, self.origin_path = get_transport_and_path( - self.repo.get_config().get(("remote", "origin"), - "url")) - - def revision(self): - return self.repo.head() - - def pull(self): - try: - remote_refs = self.client.fetch( - self.origin_path, self.repo, - determine_wants=self.repo.object_store.determine_wants_all) - except KeyError: - etype, err = sys.exc_info()[:2] - # try to work around bug - # https://bugs.launchpad.net/dulwich/+bug/1025886 - try: - # pylint: disable=W0212 - self.client._fetch_capabilities.remove('thin-pack') - # pylint: enable=W0212 - except KeyError: - raise etype(err) - remote_refs = self.client.fetch( - self.origin_path, self.repo, - determine_wants=self.repo.object_store.determine_wants_all) - - tree_id = self.repo[remote_refs['HEAD']].tree - # iterate over tree content, giving path and blob sha. - for entry in self.repo.object_store.iter_tree_contents(tree_id): - entry_in_path = entry.in_path(self.repo.path) - ensure_dir_exists(os.path.split(entry_in_path.path)[0]) - GitFile(entry_in_path.path, - 'wb').write(self.repo[entry.sha].data) - + import git + HAS_GITPYTHON = True except ImportError: - try: - import git - - class GitAPI(GitAPIBase): - """ API for :class:`Git` using :mod:`git` (GitPython) """ - def __init__(self, path): - GitAPIBase.__init__(self, path) - self.repo = git.Repo(path) - - def revision(self): - return self.repo.head.commit.hexsha - - def pull(self): - self.repo.git.pull("--rebase") - - except ImportError: - from subprocess import Popen, PIPE - - try: - Popen(["git"], stdout=PIPE, stderr=PIPE).wait() - - class GitAPI(GitAPIBase): - """ API for :class:`Git` using subprocess to run git - commands """ - def revision(self): - proc = Popen(["git", "--work-tree", - os.path.join(self.path, ".git"), - "rev-parse", "HEAD"], stdout=PIPE, - stderr=PIPE) - rv, err = proc.communicate() - if proc.wait(): - raise Exception("Git: Error getting revision from %s: " - "%s" % (self.path, err)) - return rv.strip() # pylint: disable=E1103 - - def pull(self): - proc = Popen(["git", "--work-tree", - os.path.join(self.path, ".git"), - "pull", "--rebase"], stdout=PIPE, - stderr=PIPE) - err = proc.communicate()[1].strip() - if proc.wait(): - raise Exception("Git: Error pulling: %s" % err) - - except OSError: - raise ImportError("Could not import dulwich or GitPython " - "libraries, and no 'git' command found in PATH") + HAS_GITPYTHON = False class Git(Bcfg2.Server.Plugin.Version): @@ -121,35 +17,83 @@ class Git(Bcfg2.Server.Plugin.Version): using git. """ __author__ = 'bcfg-dev@mcs.anl.gov' __vcs_metadata_path__ = ".git" - __rmi__ = Bcfg2.Server.Plugin.Version.__rmi__ + ['Update'] + if HAS_GITPYTHON: + __rmi__ = Bcfg2.Server.Plugin.Version.__rmi__ + ['Update'] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Version.__init__(self, core, datastore) - self.repo = GitAPI(self.vcs_root) + if HAS_GITPYTHON: + self.repo = git.Repo(self.vcs_root) + else: + self.logger.debug("Git: GitPython not found, using CLI interface " + "to Git") + self.repo = None self.logger.debug("Initialized git plugin with git directory %s" % self.vcs_path) def get_revision(self): """Read git revision information for the Bcfg2 repository.""" try: - return self.repo.revision() + if HAS_GITPYTHON: + return self.repo.head.commit.hexsha + else: + cmd = ["git", "--git-dir", self.vcs_path, + "--work-tree", self.vcs_root, "rev-parse", "HEAD"] + self.debug_log("Git: Running cmd") + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + rv, err = proc.communicate() + if proc.wait(): + raise Exception(err) + return rv except: err = sys.exc_info()[1] - msg = "Failed to read git repository: %s" % err + msg = "Git: Error getting revision from %s: %s" % (self.vcs_root, + err) self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - def Update(self): + def Update(self, ref=None): """ Git.Update() => True|False Update the working copy against the upstream repository """ + self.logger.info("Git: Git.Update(ref='%s')" % ref) + self.debug_log("Git: Performing garbage collection on repo at %s" % + self.vcs_root) try: - self.repo.pull() - self.logger.info("Git repo at %s updated to %s" % - (self.vcs_root, self.get_revision())) - return True - except: # pylint: disable=W0702 - err = sys.exc_info()[1] - msg = "Failed to pull from git repository: %s" % err - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.repo.git.gc('--auto') + except git.GitCommandError: + self.logger.warning("Git: Failed to perform garbage collection: %s" + % sys.exc_info()[1]) + + if ref: + self.debug_log("Git: Checking out %s" % ref) + try: + self.repo.git.checkout('-f', ref) + except git.GitCommandError: + err = sys.exc_info()[1] + msg = "Git: Failed to checkout %s: %s" % (ref, err) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + # determine if we should try to pull to get the latest commit + # on this head + tracking = None + if not self.repo.head.is_detached: + self.debug_log("Git: Determining if %s is a tracking branch" % + self.repo.head.ref.name) + tracking = self.repo.head.ref.tracking_branch() + + if tracking is not None: + self.debug_log("Git: %s is a tracking branch, pulling from %s" % + (self.repo.head.ref.name, tracking)) + try: + self.repo.git.pull("--rebase") + except: # pylint: disable=W0702 + err = sys.exc_info()[1] + msg = "Git: Failed to pull from upstream: %s" % err + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + self.logger.info("Git: Repo at %s updated to %s" % + (self.vcs_root, self.get_revision())) + return True diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 8b0fc16ce..0ab72f2c5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -967,9 +967,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return self.aliases[cname] return cname except socket.herror: - warning = "address resolution error for %s" % address - self.logger.warning(warning) - raise Bcfg2.Server.Plugin.MetadataConsistencyError(warning) + err = "Address resolution error for %s: %s" % (address, + sys.exc_info()[1]) + self.logger.error(err) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(err) def _merge_groups(self, client, groups, categories=None): """ set group membership based on the contents of groups.xml diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index fbad0a37b..023547b7e 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -81,7 +81,7 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, if xtra: host_config.extend([self.line_fmt % (opt, val) for opt, val in list(xtra.items())]) - else: + if 'use' not in xtra: host_config.append(self.line_fmt % ('use', 'default')) host_config.append('}') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 94dc6d2fd..2735e389a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -75,11 +75,11 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, #: should be told to reload its data. self.parsed = set() - def toggle_debug(self): - Bcfg2.Server.Plugin.Debuggable.toggle_debug(self) + def set_debug(self, debug): + Bcfg2.Server.Plugin.Debuggable.set_debug(self, debug) for source in self.entries: - source.toggle_debug() - toggle_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.toggle_debug.__doc__ + source.set_debug(debug) + set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ def HandleEvent(self, event=None): """ HandleEvent is called whenever the FAM registers an event. @@ -121,8 +121,8 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, self.entries.append(source) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """ -``Index`` is responsible for calling :func:`source_from_xml` for each -``Source`` tag in each file. """ + ``Index`` is responsible for calling :func:`source_from_xml` + for each ``Source`` tag in each file. """ @Bcfg2.Server.Plugin.track_statistics() def source_from_xml(self, xsource): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 59e7a206e..220146100 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -102,6 +102,9 @@ FL = '{http://linux.duke.edu/metadata/filelists}' PULPSERVER = None PULPCONFIG = None +#: The path to bcfg2-yum-helper +HELPER = None + def _setup_pulp(setup): """ Connect to a Pulp server and pass authentication credentials. @@ -308,8 +311,6 @@ class YumCollection(Collection): (certdir, err)) self.pulp_cert_set = PulpCertificateSet(certdir, self.fam) - self._helper = None - @property def __package_groups__(self): """ YumCollections support package groups only if @@ -324,20 +325,20 @@ class YumCollection(Collection): a call to it; I wish there was a way to do this without forking, but apparently not); finally we check in /usr/sbin, the default location. """ - try: - return self.setup.cfp.get("packages:yum", "helper") - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - pass - - if not self._helper: - # first see if bcfg2-yum-helper is in PATH + global HELPER + if not HELPER: try: - Popen(['bcfg2-yum-helper'], - stdin=PIPE, stdout=PIPE, stderr=PIPE).wait() - self._helper = 'bcfg2-yum-helper' - except OSError: - self._helper = "/usr/sbin/bcfg2-yum-helper" - return self._helper + HELPER = self.setup.cfp.get("packages:yum", "helper") + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + # first see if bcfg2-yum-helper is in PATH + try: + self.debug_log("Checking for bcfg2-yum-helper in $PATH") + Popen(['bcfg2-yum-helper'], + stdin=PIPE, stdout=PIPE, stderr=PIPE).wait() + HELPER = 'bcfg2-yum-helper' + except OSError: + HELPER = "/usr/sbin/bcfg2-yum-helper" + return HELPER @property def use_yum(self): @@ -357,7 +358,7 @@ class YumCollection(Collection): def cachefiles(self): """ A list of the full path to all cachefiles used by this collection.""" - cachefiles = set(Collection.cachefiles(self)) + cachefiles = set(Collection.cachefiles.fget(self)) if self.cachefile: cachefiles.add(self.cachefile) return list(cachefiles) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 5a193219c..f30e060bd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -113,13 +113,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin, __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ - def toggle_debug(self): - rv = Bcfg2.Server.Plugin.Plugin.toggle_debug(self) - self.sources.toggle_debug() + def set_debug(self, debug): + rv = Bcfg2.Server.Plugin.Plugin.set_debug(self, debug) + self.sources.set_debug(debug) for collection in self.collections.values(): - collection.toggle_debug() + collection.set_debug(debug) return rv - toggle_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.toggle_debug.__doc__ + set_debug.__doc__ = Bcfg2.Server.Plugin.Plugin.set_debug.__doc__ @property def disableResolver(self): diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index bab7c4a4a..feb76aa57 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -9,7 +9,8 @@ import logging import tempfile from subprocess import Popen, PIPE import Bcfg2.Server.Plugin -from Bcfg2.Compat import u_str, reduce, b64encode # pylint: disable=W0622 +from Bcfg2.Server.Plugin import PluginExecutionError +from Bcfg2.Compat import any, u_str, reduce, b64encode # pylint: disable=W0622 LOGGER = logging.getLogger(__name__) @@ -111,9 +112,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, is regenerated each time a new key is generated. """ - name = 'SSHbase' __author__ = 'bcfg-dev@mcs.anl.gov' - keypatterns = ["ssh_host_dsa_key", "ssh_host_ecdsa_key", "ssh_host_rsa_key", @@ -250,9 +249,11 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for entry in list(self.entries.values()): if entry.specific.match(event.filename): entry.handle_event(event) - if event.filename.endswith(".pub"): - self.logger.info("New public key %s; invalidating " - "ssh_known_hosts cache" % event.filename) + if any(event.filename.startswith(kp) + for kp in self.keypatterns + if kp.endswith(".pub")): + self.debug_log("New public key %s; invalidating " + "ssh_known_hosts cache" % event.filename) self.skn = False return @@ -365,8 +366,9 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, is_bound = False while not is_bound: if tries >= 10: - self.logger.error("%s still not registered" % filename) - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "%s still not registered" % filename + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) self.core.fam.handle_events_in_interval(1) tries += 1 try: @@ -385,26 +387,30 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: keytype = 'rsa1' else: - self.logger.error("Unknown key filename: %s" % filename) - return + raise PluginExecutionError("Unknown key filename: %s" % filename) - fileloc = "%s/%s" % (self.data, hostkey) - publoc = self.data + '/' + ".".join([hostkey.split('.')[0], 'pub', - "H_%s" % client]) + fileloc = os.path.join(self.data, hostkey) + publoc = os.path.join(self.data, + ".".join([hostkey.split('.')[0], 'pub', + "H_%s" % client])) tempdir = tempfile.mkdtemp() - temploc = "%s/%s" % (tempdir, hostkey) + temploc = os.path.join(tempdir, hostkey) cmd = ["ssh-keygen", "-q", "-f", temploc, "-N", "", "-t", keytype, "-C", "root@%s" % client] + self.debug_log("SSHbase: Running: %s" % " ".join(cmd)) proc = Popen(cmd, stdout=PIPE, stdin=PIPE) - proc.communicate() - proc.wait() + err = proc.communicate()[1] + if proc.wait(): + raise PluginExecutionError("SSHbase: Error running ssh-keygen: %s" + % err) try: shutil.copy(temploc, fileloc) shutil.copy("%s.pub" % temploc, publoc) except IOError: err = sys.exc_info()[1] - self.logger.error("Temporary SSH keys not found: %s" % err) + raise PluginExecutionError("Temporary SSH keys not found: %s" % + err) try: os.unlink(temploc) @@ -412,7 +418,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, os.rmdir(tempdir) except OSError: err = sys.exc_info()[1] - self.logger.error("Failed to unlink temporary ssh keys: %s" % err) + raise PluginExecutionError("Failed to unlink temporary ssh keys: " + "%s" % err) def AcceptChoices(self, _, metadata): return [Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)] @@ -420,8 +427,9 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def AcceptPullData(self, specific, entry, log): """Per-plugin bcfg2-admin pull support.""" # specific will always be host specific - filename = "%s/%s.H_%s" % (self.data, entry['name'].split('/')[-1], - specific.hostname) + filename = os.path.join(self.data, + "%s.H_%s" % (entry['name'].split('/')[-1], + specific.hostname)) try: open(filename, 'w').write(entry['text']) if log: diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index ab55425a6..62396f860 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -1,13 +1,15 @@ """ The SSLCA generator handles the creation and management of ssl certificates and their keys. """ +import os +import sys import Bcfg2.Server.Plugin import Bcfg2.Options import lxml.etree import tempfile -import os from subprocess import Popen, PIPE, STDOUT from Bcfg2.Compat import ConfigParser, md5 +from Bcfg2.Server.Plugin import PluginExecutionError class SSLCA(Bcfg2.Server.Plugin.GroupSpool): @@ -107,6 +109,7 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): filename = os.path.join(path, "%s.H_%s" % (os.path.basename(path), metadata.hostname)) if filename not in list(self.entries.keys()): + self.logger.info("SSLCA: Generating new key %s" % filename) key = self.build_key(entry) open(self.data + filename, 'w').write(key) entry.text = key @@ -130,6 +133,7 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): cmd = ["openssl", "genrsa", bits] elif ktype == 'dsa': cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] + self.debug_log("SSLCA: Generating new key: %s" % " ".join(cmd)) return Popen(cmd, stdout=PIPE).stdout.read() def get_cert(self, entry, metadata): @@ -151,10 +155,11 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): self.core.Bind(el, metadata) # check if we have a valid hostfile - if (filename in list(self.entries.keys()) and + if (filename in self.entries.keys() and self.verify_cert(filename, key_filename, entry)): entry.text = self.entries[filename].data else: + self.logger.info("SSLCA: Generating new cert %s" % filename) cert = self.build_cert(key_filename, entry, metadata) open(self.data + filename, 'w').write(cert) self.entries[filename] = self.__child__(self.data + filename) @@ -231,22 +236,37 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): """ creates a new certificate according to the specification """ - req_config = self.build_req_config(entry, metadata) - req = self.build_request(key_filename, req_config, entry) - ca = self.cert_specs[entry.get('name')]['ca'] - ca_config = self.CAs[ca]['config'] - days = self.cert_specs[entry.get('name')]['days'] - passphrase = self.CAs[ca].get('passphrase') - cmd = ["openssl", "ca", "-config", ca_config, "-in", req, - "-days", days, "-batch"] - if passphrase: - cmd.extend(["-passin", "pass:%s" % passphrase]) - cert = Popen(cmd, stdout=PIPE).stdout.read() + req_config = None + req = None try: - os.unlink(req_config) - os.unlink(req) - except OSError: - self.logger.error("Failed to unlink temporary files") + req_config = self.build_req_config(entry, metadata) + req = self.build_request(key_filename, req_config, entry) + ca = self.cert_specs[entry.get('name')]['ca'] + ca_config = self.CAs[ca]['config'] + days = self.cert_specs[entry.get('name')]['days'] + passphrase = self.CAs[ca].get('passphrase') + cmd = ["openssl", "ca", "-config", ca_config, "-in", req, + "-days", days, "-batch"] + if passphrase: + cmd.extend(["-passin", "pass:%s" % passphrase]) + self.debug_log("SSLCA: Generating new certificate: %s" % + " ".join(cmd)) + proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) + (cert, err) = proc.communicate() + if proc.wait(): + # pylint: disable=E1103 + raise PluginExecutionError("SSLCA: Failed to generate cert: %s" + % err.splitlines()[-1]) + # pylint: enable=E1103 + finally: + try: + if req_config and os.path.exists(req_config): + os.unlink(req_config) + if req and os.path.exists(req): + os.unlink(req) + except OSError: + self.logger.error("SSLCA: Failed to unlink temporary files: %s" + % sys.exc_info()[1]) if (self.cert_specs[entry.get('name')]['append_chain'] and self.CAs[ca]['chaincert']): cert += open(self.CAs[ca]['chaincert']).read() @@ -258,7 +278,7 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): used to generate the required certificate request """ # create temp request config file - conffile = open(tempfile.mkstemp()[1], 'w') + fd, fname = tempfile.mkstemp() cfp = ConfigParser.ConfigParser({}) cfp.optionxform = str defaults = { @@ -290,18 +310,28 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): cfp.set('req_distinguished_name', item, self.cert_specs[entry.get('name')][item]) cfp.set('req_distinguished_name', 'CN', metadata.hostname) - cfp.write(conffile) - conffile.close() - return conffile.name + self.debug_log("SSLCA: Writing temporary request config to %s" % fname) + try: + cfp.write(os.fdopen(fd, 'w')) + except IOError: + raise PluginExecutionError("SSLCA: Failed to write temporary CSR " + "config file: %s" % sys.exc_info()[1]) + return fname def build_request(self, key_filename, req_config, entry): """ creates the certificate request """ - req = tempfile.mkstemp()[1] + fd, req = tempfile.mkstemp() + os.close(fd) days = self.cert_specs[entry.get('name')]['days'] key = self.data + key_filename cmd = ["openssl", "req", "-new", "-config", req_config, "-days", days, "-key", key, "-text", "-out", req] - Popen(cmd, stdout=PIPE).wait() + self.debug_log("SSLCA: Generating new CSR: %s" % " ".join(cmd)) + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) + err = proc.communicate()[1] + if proc.wait(): + raise PluginExecutionError("SSLCA: Failed to generate CSR: %s" % + err) return req diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index 627c82f25..f09d4839e 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -13,15 +13,25 @@ LOGGER = logging.getLogger(__name__) MODULE_RE = re.compile(r'(?P<filename>(?P<module>[^\/]+)\.py)$') -class HelperModule(Bcfg2.Server.Plugin.FileBacked): +class HelperModule(object): """ Representation of a TemplateHelper module """ def __init__(self, name, fam=None): - Bcfg2.Server.Plugin.FileBacked.__init__(self, name, fam=fam) + self.name = name + self.fam = fam self._module_name = MODULE_RE.search(self.name).group('module') self._attrs = [] - def Index(self): + def HandleEvent(self, event=None): + """ HandleEvent is called whenever the FAM registers an event. + + :param event: The event object + :type event: Bcfg2.Server.FileMonitor.Event + :returns: None + """ + if event and event.code2str() not in ['exists', 'changed', 'created']: + return + try: module = imp.load_source(self._module_name, self.name) except: # pylint: disable=W0702 @@ -54,27 +64,23 @@ class HelperModule(Bcfg2.Server.Plugin.FileBacked): self._attrs = newattrs -class HelperSet(Bcfg2.Server.Plugin.DirectoryBacked): - """ A set of template helper modules """ - ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$") - patterns = MODULE_RE - __child__ = HelperModule - - class TemplateHelper(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): + Bcfg2.Server.Plugin.Connector, + Bcfg2.Server.Plugin.DirectoryBacked): """ A plugin to provide helper classes and functions to templates """ - name = 'TemplateHelper' __author__ = 'chris.a.st.pierre@gmail.com' + ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$") + patterns = MODULE_RE + __child__ = HelperModule def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) - self.helpers = HelperSet(self.data, core.fam) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, core.fam) def get_additional_data(self, _): return dict([(h._module_name, h) # pylint: disable=W0212 - for h in self.helpers.entries.values()]) + for h in self.entries.values()]) class TemplateHelperLint(Bcfg2.Server.Lint.ServerlessPlugin): @@ -130,9 +136,9 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerlessPlugin): @classmethod def Errors(cls): - return {"templatehelper-import-error":"error", - "templatehelper-no-export":"error", - "templatehelper-nonlist-export":"error", - "templatehelper-nonexistent-export":"error", - "templatehelper-reserved-export":"error", - "templatehelper-underscore-export":"warning"} + return {"templatehelper-import-error": "error", + "templatehelper-no-export": "error", + "templatehelper-nonlist-export": "error", + "templatehelper-nonexistent-export": "error", + "templatehelper-reserved-export": "error", + "templatehelper-underscore-export": "warning"} |