From 432f448983ff27452d82d62314d91c942f31bce5 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 25 Mar 2013 11:19:11 -0400 Subject: Packages: properly implemented deepcopy() for PackagesSources objects --- src/lib/Bcfg2/Server/Plugin/helpers.py | 33 ++++++++++++++++++++++ .../Server/Plugins/Packages/PackagesSources.py | 12 ++++++-- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 7 ++++- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 11 +++++++- 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 0b81077a3..93f690e18 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -1601,3 +1601,36 @@ class GroupSpool(Plugin, Generator): return reqid = self.core.fam.AddMonitor(name, self) self.handles[reqid] = relative + + +class DeepcopyMixin(object): + """ Mixin that adds a __deepcopy__() function that tries to do the + right thing by: + + #. Creating a new object of the same type by calling the + constructor with the arguments returned by + :func:`Bcfg2.Server.Plugin.helpers.DeepcopyMixin._deepcopy_constructor_args`; + #. Individually copying each attribute of the original object that + a) is not magical (``__whatever__``); b) is not callable; c) is + not a property; and d) is not in + :attr:`Bcfg2.Server.Plugin.helpers.DeepcopyMixin._deepcopy_exclude`. + """ + + #: Exclude the named attributes from the copy. + _deepcopy_exclude = ["fam", "logger", "setup"] + + def _deepcopy_constructor_args(self): + """ Get a tuple of arguments that will be passed to the + constructor of this class to create a base object before the + individual attributes are deepcopied. """ + raise NotImplementedError + + def __deepcopy__(self, memo): + rv = self.__class__(*self._deepcopy_constructor_args()) + for attr in dir(self): + if (not callable(getattr(self, attr)) and + (not attr.startswith("__") or not attr.endswith("__")) and + attr not in self._deepcopy_exclude and + not isinstance(getattr(self.__class__, attr, None), property)): + setattr(rv, attr, copy.deepcopy(getattr(self, attr), memo)) + return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 2735e389a..3069e4068 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -8,13 +8,17 @@ from Bcfg2.Server.Plugins.Packages.Source import SourceInitError class PackagesSources(Bcfg2.Server.Plugin.StructFile, - Bcfg2.Server.Plugin.Debuggable): + Bcfg2.Server.Plugin.Debuggable, + Bcfg2.Server.Plugin.DeepcopyMixin): """ PackagesSources handles parsing of the :mod:`Bcfg2.Server.Plugins.Packages` ``sources.xml`` file, and the creation of the appropriate :class:`Bcfg2.Server.Plugins.Packages.Source.Source` object for each ``Source`` tag. """ + _deepcopy_exclude = Bcfg2.Server.Plugin.DeepcopyMixin._deepcopy_exclude + \ + ['pkg_obj'] + __identifier__ = None def __init__(self, filename, cachepath, fam, packages, setup): @@ -39,6 +43,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, If ``sources.xml`` cannot be read """ Bcfg2.Server.Plugin.Debuggable.__init__(self) + Bcfg2.Server.Plugin.DeepcopyMixin.__init__(self) try: Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, should_monitor=True) @@ -129,7 +134,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, """ Create a :class:`Bcfg2.Server.Plugins.Packages.Source.Source` subclass object from XML representation of a source in ``sources.xml``. - ``source_from-xml`` determines the appropriate subclass of + ``source_from_xml`` determines the appropriate subclass of ``Source`` to instantiate according to the ``type`` attribute of the ``Source`` tag. @@ -176,3 +181,6 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, def __len__(self): return len(self.entries) + + def _deepcopy_constructor_args(self): + return (self.name, self.cachepath, None, None, dict(self.setup)) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 985405e65..40c1f6676 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -92,7 +92,8 @@ class SourceInitError(Exception): REPO_RE = re.compile(r'(?:pulp/repos/|/RPMS\.|/)([^/]+)/?$') -class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 +class Source(Bcfg2.Server.Plugin.Debuggable, # pylint: disable=R0902 + Bcfg2.Server.Plugin.DeepcopyMixin): """ ``Source`` objects represent a single tag in ``sources.xml``. Note that a single Source tag can itself describe multiple repositories (if it uses the "url" attribute @@ -128,6 +129,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :raises: :class:`Bcfg2.Server.Plugins.Packages.Source.SourceInitError` """ Bcfg2.Server.Plugin.Debuggable.__init__(self) + Bcfg2.Server.Plugin.DeepcopyMixin.__init__(self) #: The base filesystem path under which cache data for this #: source should be stored @@ -738,3 +740,6 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 if group in metadata.groups: return True return False + + def _deepcopy_constructor_args(self): + return (self.basepath, self.xsource, dict(self.setup)) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 308a0efc4..cc78497f1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -185,6 +185,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for (key, value) in list(attrib.items()): entry.attrib.__setitem__(key, value) + def get_config(self, metadata): + """ Get yum/apt config, as a string, for the specified client. + + :param metadata: The client to create the config for. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ + return self.get_collection(metadata).get_config() + def HandleEntry(self, entry, metadata): """ Bind configuration entries. ``HandleEntry`` handles entries two different ways: @@ -538,7 +546,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ collection = self.get_collection(metadata) return dict(sources=collection.get_additional_data(), - allsources=copy.deepcopy(self.sources)) + allsources=copy.deepcopy(self.sources), + get_config=self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in -- cgit v1.2.3-1-g7c22 From 70cc6bb182245d20c54afba79a265ef40f1ed080 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 25 Mar 2013 11:39:50 -0400 Subject: Revert "Packages: properly implemented deepcopy() for PackagesSources objects" This reverts commit 432f448983ff27452d82d62314d91c942f31bce5. --- src/lib/Bcfg2/Server/Plugin/helpers.py | 33 ---------------------- .../Server/Plugins/Packages/PackagesSources.py | 12 ++------ src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 7 +---- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 11 +------- 4 files changed, 4 insertions(+), 59 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 93f690e18..0b81077a3 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -1601,36 +1601,3 @@ class GroupSpool(Plugin, Generator): return reqid = self.core.fam.AddMonitor(name, self) self.handles[reqid] = relative - - -class DeepcopyMixin(object): - """ Mixin that adds a __deepcopy__() function that tries to do the - right thing by: - - #. Creating a new object of the same type by calling the - constructor with the arguments returned by - :func:`Bcfg2.Server.Plugin.helpers.DeepcopyMixin._deepcopy_constructor_args`; - #. Individually copying each attribute of the original object that - a) is not magical (``__whatever__``); b) is not callable; c) is - not a property; and d) is not in - :attr:`Bcfg2.Server.Plugin.helpers.DeepcopyMixin._deepcopy_exclude`. - """ - - #: Exclude the named attributes from the copy. - _deepcopy_exclude = ["fam", "logger", "setup"] - - def _deepcopy_constructor_args(self): - """ Get a tuple of arguments that will be passed to the - constructor of this class to create a base object before the - individual attributes are deepcopied. """ - raise NotImplementedError - - def __deepcopy__(self, memo): - rv = self.__class__(*self._deepcopy_constructor_args()) - for attr in dir(self): - if (not callable(getattr(self, attr)) and - (not attr.startswith("__") or not attr.endswith("__")) and - attr not in self._deepcopy_exclude and - not isinstance(getattr(self.__class__, attr, None), property)): - setattr(rv, attr, copy.deepcopy(getattr(self, attr), memo)) - return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 3069e4068..2735e389a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -8,17 +8,13 @@ from Bcfg2.Server.Plugins.Packages.Source import SourceInitError class PackagesSources(Bcfg2.Server.Plugin.StructFile, - Bcfg2.Server.Plugin.Debuggable, - Bcfg2.Server.Plugin.DeepcopyMixin): + Bcfg2.Server.Plugin.Debuggable): """ PackagesSources handles parsing of the :mod:`Bcfg2.Server.Plugins.Packages` ``sources.xml`` file, and the creation of the appropriate :class:`Bcfg2.Server.Plugins.Packages.Source.Source` object for each ``Source`` tag. """ - _deepcopy_exclude = Bcfg2.Server.Plugin.DeepcopyMixin._deepcopy_exclude + \ - ['pkg_obj'] - __identifier__ = None def __init__(self, filename, cachepath, fam, packages, setup): @@ -43,7 +39,6 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, If ``sources.xml`` cannot be read """ Bcfg2.Server.Plugin.Debuggable.__init__(self) - Bcfg2.Server.Plugin.DeepcopyMixin.__init__(self) try: Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, should_monitor=True) @@ -134,7 +129,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, """ Create a :class:`Bcfg2.Server.Plugins.Packages.Source.Source` subclass object from XML representation of a source in ``sources.xml``. - ``source_from_xml`` determines the appropriate subclass of + ``source_from-xml`` determines the appropriate subclass of ``Source`` to instantiate according to the ``type`` attribute of the ``Source`` tag. @@ -181,6 +176,3 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, def __len__(self): return len(self.entries) - - def _deepcopy_constructor_args(self): - return (self.name, self.cachepath, None, None, dict(self.setup)) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 40c1f6676..985405e65 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -92,8 +92,7 @@ class SourceInitError(Exception): REPO_RE = re.compile(r'(?:pulp/repos/|/RPMS\.|/)([^/]+)/?$') -class Source(Bcfg2.Server.Plugin.Debuggable, # pylint: disable=R0902 - Bcfg2.Server.Plugin.DeepcopyMixin): +class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 """ ``Source`` objects represent a single tag in ``sources.xml``. Note that a single Source tag can itself describe multiple repositories (if it uses the "url" attribute @@ -129,7 +128,6 @@ class Source(Bcfg2.Server.Plugin.Debuggable, # pylint: disable=R0902 :raises: :class:`Bcfg2.Server.Plugins.Packages.Source.SourceInitError` """ Bcfg2.Server.Plugin.Debuggable.__init__(self) - Bcfg2.Server.Plugin.DeepcopyMixin.__init__(self) #: The base filesystem path under which cache data for this #: source should be stored @@ -740,6 +738,3 @@ class Source(Bcfg2.Server.Plugin.Debuggable, # pylint: disable=R0902 if group in metadata.groups: return True return False - - def _deepcopy_constructor_args(self): - return (self.basepath, self.xsource, dict(self.setup)) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index cc78497f1..308a0efc4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -185,14 +185,6 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for (key, value) in list(attrib.items()): entry.attrib.__setitem__(key, value) - def get_config(self, metadata): - """ Get yum/apt config, as a string, for the specified client. - - :param metadata: The client to create the config for. - :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - """ - return self.get_collection(metadata).get_config() - def HandleEntry(self, entry, metadata): """ Bind configuration entries. ``HandleEntry`` handles entries two different ways: @@ -546,8 +538,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ collection = self.get_collection(metadata) return dict(sources=collection.get_additional_data(), - allsources=copy.deepcopy(self.sources), - get_config=self.get_config) + allsources=copy.deepcopy(self.sources)) def end_client_run(self, metadata): """ Hook to clear the cache for this client in -- cgit v1.2.3-1-g7c22 From de3df96fe533b3a01de4ca85658fff5bc2f8b7b0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 25 Mar 2013 11:39:52 -0400 Subject: Revert "Packages: expose full source list via Connector interface" This reverts commit fe7a3e9c7a7d66cccbd825465cefcf88165a0c3a. --- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 308a0efc4..c3eadc6bb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -5,7 +5,6 @@ determine the completeness of the client configuration. """ import os import sys import glob -import copy import shutil import lxml.etree import Bcfg2.Logger @@ -537,8 +536,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :return: dict of lists of ``url_map`` data """ collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data(), - allsources=copy.deepcopy(self.sources)) + return dict(sources=collection.get_additional_data()) def end_client_run(self, metadata): """ Hook to clear the cache for this client in -- cgit v1.2.3-1-g7c22 From 3dc289678812238c2fcc54098b1d8de9bf64f900 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 25 Mar 2013 11:42:23 -0400 Subject: Packages: add and expose get_config() function to get configs for other hosts --- src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py | 2 +- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 2735e389a..739320cb0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -129,7 +129,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, """ Create a :class:`Bcfg2.Server.Plugins.Packages.Source.Source` subclass object from XML representation of a source in ``sources.xml``. - ``source_from-xml`` determines the appropriate subclass of + ``source_from_xml`` determines the appropriate subclass of ``Source`` to instantiate according to the ``type`` attribute of the ``Source`` tag. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index c3eadc6bb..0b6889b09 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -184,6 +184,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for (key, value) in list(attrib.items()): entry.attrib.__setitem__(key, value) + def get_config(self, metadata): + """ Get yum/apt config, as a string, for the specified client. + + :param metadata: The client to create the config for. + :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata + """ + return self.get_collection(metadata).get_config() + def HandleEntry(self, entry, metadata): """ Bind configuration entries. ``HandleEntry`` handles entries two different ways: @@ -536,7 +544,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, :return: dict of lists of ``url_map`` data """ collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data()) + return dict(sources=collection.get_additional_data(), + get_config=self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in -- cgit v1.2.3-1-g7c22 From 9c603d8267c0a511968a8a553d7fa0b2d5bf9b73 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 25 Mar 2013 13:31:21 -0400 Subject: Handle FAM monitor failures more gracefully: * Where possible, create the file or directory that is about to be monitored. This ensures that content can be added later without need to restart Bcfg2. (Otherwise, adding the monitor would fail, and so when you did create the file in question, bcfg2-server would never be notified of it.) * When not possible, give better error messages. --- src/lib/Bcfg2/Server/Core.py | 4 ++ src/lib/Bcfg2/Server/Plugin/base.py | 16 +++++++- src/lib/Bcfg2/Server/Plugin/helpers.py | 47 +++++++++++++++++----- src/lib/Bcfg2/Server/Plugins/Base.py | 9 +---- src/lib/Bcfg2/Server/Plugins/Bundler.py | 12 +----- src/lib/Bcfg2/Server/Plugins/Decisions.py | 13 +----- src/lib/Bcfg2/Server/Plugins/FileProbes.py | 3 +- src/lib/Bcfg2/Server/Plugins/GroupPatterns.py | 3 +- src/lib/Bcfg2/Server/Plugins/Metadata.py | 4 +- src/lib/Bcfg2/Server/Plugins/NagiosGen.py | 24 +++-------- src/lib/Bcfg2/Server/Plugins/Ohai.py | 4 -- .../Server/Plugins/Packages/PackagesSources.py | 11 ++--- src/lib/Bcfg2/Server/Plugins/Properties.py | 42 +++++++------------ .../Testlib/TestServer/TestPlugin/Testbase.py | 22 +++++++++- .../Testlib/TestServer/TestPlugin/Testhelpers.py | 38 ++++++++++++++--- .../TestServer/TestPlugin/Testinterfaces.py | 11 ----- .../TestServer/TestPlugins/TestGroupPatterns.py | 15 ++++++- .../Testlib/TestServer/TestPlugins/TestMetadata.py | 20 +++++++-- .../Testlib/TestServer/TestPlugins/TestProbes.py | 9 ++--- .../TestServer/TestPlugins/TestProperties.py | 28 ++++++------- 20 files changed, 190 insertions(+), 145 deletions(-) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 382f11e50..8ceb8cfc1 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -420,6 +420,10 @@ class BaseCore(object): except PluginInitError: self.logger.error("Failed to instantiate plugin %s" % plugin, exc_info=1) + except OSError: + err = sys.exc_info()[1] + self.logger.error("Failed to add a file monitor while " + "instantiating plugin %s: %s" % (plugin, err)) except: self.logger.error("Unexpected instantiation failure for plugin %s" % plugin, exc_info=1) diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index f7bc08717..ecd970b54 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -97,15 +97,21 @@ class Plugin(Debuggable): :param datastore: The path to the Bcfg2 repository on the filesystem :type datastore: string - :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` + :raises: :exc:`OSError` if adding a file monitor failed; + :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError` + on other errors .. autoattribute:: Bcfg2.Server.Plugin.base.Debuggable.__rmi__ """ + Debuggable.__init__(self, name=self.name) self.Entries = {} self.core = core self.data = os.path.join(datastore, self.name) + if not os.path.exists(self.data): + self.logger.warning("%s: %s does not exist, creating" % + (self.name, self.data)) + os.makedirs(self.data) self.running = True - Debuggable.__init__(self, name=self.name) @classmethod def init_repo(cls, repo): @@ -125,5 +131,11 @@ class Plugin(Debuggable): self.debug_log("Shutting down %s plugin" % self.name) self.running = False + def set_debug(self, debug): + for entry in self.Entries.values(): + if isinstance(entry, Debuggable): + entry.set_debug(debug) + return Debuggable.set_debug(self, debug) + def __str__(self): return "%s Plugin" % self.__class__.__name__ diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 0b81077a3..448d8b3b6 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -193,7 +193,7 @@ class PluginDatabaseModel(object): app_label = "Server" -class FileBacked(object): +class FileBacked(Debuggable): """ This object caches file data in memory. FileBacked objects are principally meant to be used as a part of :class:`Bcfg2.Server.Plugin.helpers.DirectoryBacked`. """ @@ -206,7 +206,7 @@ class FileBacked(object): changes :type fam: Bcfg2.Server.FileMonitor.FileMonitor """ - object.__init__(self) + Debuggable.__init__(self) #: A string containing the raw data in this file self.data = '' @@ -246,7 +246,7 @@ class FileBacked(object): return "%s: %s" % (self.__class__.__name__, self.name) -class DirectoryBacked(object): +class DirectoryBacked(Debuggable): """ DirectoryBacked objects represent a directory that contains files, represented by objects of the type listed in :attr:`__child__`, and other directories recursively. It monitors @@ -280,7 +280,7 @@ class DirectoryBacked(object): .. ----- .. autoattribute:: __child__ """ - object.__init__(self) + Debuggable.__init__(self) self.data = os.path.normpath(data) self.fam = fam @@ -299,8 +299,17 @@ class DirectoryBacked(object): self.handles = {} # Monitor everything in the plugin's directory + if not os.path.exists(self.data): + self.logger.warning("%s does not exist, creating" % self.data) + os.makedirs(self.data) self.add_directory_monitor('') + def set_debug(self, debug): + for entry in self.entries.values(): + if isinstance(entry, Debuggable): + entry.set_debug(debug) + return Debuggable.set_debug(self, debug) + def __getitem__(self, key): return self.entries[key] @@ -459,7 +468,11 @@ class XMLFileBacked(FileBacked): #: behavior, set ``__identifier__`` to ``None``. __identifier__ = 'name' - def __init__(self, filename, fam=None, should_monitor=False): + #: If ``create`` is set, then it overrides the ``create`` argument + #: to the constructor. + create = None + + def __init__(self, filename, fam=None, should_monitor=False, create=None): """ :param filename: The full path to the file to cache and monitor :type filename: string @@ -474,6 +487,13 @@ class XMLFileBacked(FileBacked): :class:`Bcfg2.Server.Plugin.helpers.XMLDirectoryBacked` object). :type should_monitor: bool + :param create: Create the file if it doesn't exist. + ``create`` can be either an + :class:`lxml.etree._Element` object, which will + be used as initial content, or a string, which + will be used as the name of the (empty) tag + that will be the initial content of the file. + :type create: lxml.etree._Element or string .. ----- .. autoattribute:: __identifier__ @@ -497,6 +517,17 @@ class XMLFileBacked(FileBacked): #: "Extra" files included in this file by XInclude. self.extras = [] + if ((create or (self.create is not None and self.create)) + and not os.path.exists(self.name)): + toptag = create or self.create + self.logger.warning("%s does not exist, creating" % self.name) + if hasattr(toptag, "getroottree"): + el = toptag + else: + el = lxml.etree.Element(toptag) + el.getroottree().write(self.name, xml_declaration=False, + pretty_print=True) + #: Whether or not to monitor this file for changes. self.should_monitor = should_monitor if fam and should_monitor: @@ -776,8 +807,8 @@ class XMLSrc(XMLFileBacked): __cacheobj__ = dict __priority_required__ = True - def __init__(self, filename, fam=None, should_monitor=False): - XMLFileBacked.__init__(self, filename, fam, should_monitor) + def __init__(self, filename, fam=None, should_monitor=False, create=None): + XMLFileBacked.__init__(self, filename, fam, should_monitor, create) self.items = {} self.cache = None self.pnode = None @@ -1450,8 +1481,6 @@ class GroupSpool(Plugin, Generator): def __init__(self, core, datastore): Plugin.__init__(self, core, datastore) Generator.__init__(self) - if self.data[-1] == '/': - self.data = self.data[:-1] #: See :class:`Bcfg2.Server.Plugins.interfaces.Generator` for #: details on the Entries attribute. diff --git a/src/lib/Bcfg2/Server/Plugins/Base.py b/src/lib/Bcfg2/Server/Plugins/Base.py index d662da60a..a18204d60 100644 --- a/src/lib/Bcfg2/Server/Plugins/Base.py +++ b/src/lib/Bcfg2/Server/Plugins/Base.py @@ -20,13 +20,8 @@ class Base(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Structure.__init__(self) - try: - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, - self.data, - self.core.fam) - except OSError: - self.logger.error("Failed to load Base repository") - raise Bcfg2.Server.Plugin.PluginInitError + Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data, + self.core.fam) def BuildStructures(self, metadata): """Build structures for client described by metadata.""" diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 7030c1574..24301155d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -92,16 +92,8 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Structure.__init__(self) self.encoding = core.setup['encoding'] self.__child__ = self.template_dispatch - try: - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, - self.data, - self.core.fam) - except OSError: - err = sys.exc_info()[1] - msg = "Failed to load Bundle repository %s: %s" % (self.data, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - + Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, self.data, + self.core.fam) global SETUP SETUP = core.setup diff --git a/src/lib/Bcfg2/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py index eae18fdfe..66f299bc9 100644 --- a/src/lib/Bcfg2/Server/Plugins/Decisions.py +++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py @@ -2,7 +2,6 @@ blacklist certain entries. """ import os -import sys import lxml.etree import Bcfg2.Server.Plugin @@ -40,18 +39,10 @@ class Decisions(Bcfg2.Server.Plugin.EntrySet, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Decision.__init__(self) - Bcfg2.Server.Plugin.EntrySet.__init__(self, '(white|black)list', - self.data, - DecisionFile, + self.data, DecisionFile, core.setup['encoding']) - try: - core.fam.AddMonitor(self.data, self) - except OSError: - err = sys.exc_info()[1] - msg = 'Adding filemonitor for %s failed: %s' % (self.data, err) - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) + core.fam.AddMonitor(self.data, self) def HandleEvent(self, event): """ Handle events on Decision files by passing them off to diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index 5ec0d7280..882c22c49 100644 --- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -67,7 +67,8 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, 'config.xml'), fam=core.fam, - should_monitor=True) + should_monitor=True, + create=self.name) self.entries = dict() self.probes = dict() diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 5716a134f..8ce3fcd3a 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -3,7 +3,6 @@ import os import re import sys -import logging import Bcfg2.Server.Lint import Bcfg2.Server.Plugin from Bcfg2.Utils import PackedDigitRange @@ -67,6 +66,7 @@ class PatternMap(object): class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): """ representation of GroupPatterns config.xml """ __identifier__ = None + create = 'GroupPatterns' def __init__(self, filename, core=None): try: @@ -77,7 +77,6 @@ class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): should_monitor=True) self.core = core self.patterns = [] - self.logger = logging.getLogger(self.__class__.__name__) def Index(self): Bcfg2.Server.Plugin.XMLFileBacked.Index(self) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 8fb3a0998..9adc8a442 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -94,9 +94,11 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): # then we immediately set should_monitor to the proper value, # so that XInclude'd files get properly watched fpath = os.path.join(metadata.data, basefile) + toptag = os.path.splitext(basefile)[0].title() Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath, fam=metadata.core.fam, - should_monitor=False) + should_monitor=False, + create=toptag) self.should_monitor = watch_clients self.metadata = metadata self.basefile = basefile diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index c39bd4c42..466665382 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -5,26 +5,9 @@ import re import sys import glob import socket -import logging import Bcfg2.Server import Bcfg2.Server.Plugin -LOGGER = logging.getLogger(__name__) - - -class NagiosGenConfig(Bcfg2.Server.Plugin.StructFile): - """ NagiosGen config file handler """ - - def __init__(self, filename, fam): - # create config.xml if missing - if not os.path.exists(filename): - LOGGER.warning("NagiosGen: %s missing. " - "Creating empty one for you." % filename) - open(filename, "w").write("") - - Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, - should_monitor=True) - class NagiosGen(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Generator): @@ -36,8 +19,11 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Generator.__init__(self) - self.config = NagiosGenConfig(os.path.join(self.data, 'config.xml'), - core.fam) + self.config = \ + Bcfg2.Server.Plugin.StructFile(os.path.join(self.data, + 'config.xml'), + core.fam, should_monitor=True, + create=self.name) self.Entries = {'Path': {'/etc/nagiosgen.status': self.createhostconfig, '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index ebc03197e..07b04f3f0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -69,10 +69,6 @@ class Ohai(Bcfg2.Server.Plugin.Plugin, self.probe = lxml.etree.Element('probe', name='Ohai', source='Ohai', interpreter='/bin/sh') self.probe.text = PROBECODE - try: - os.stat(self.data) - except OSError: - os.makedirs(self.data) self.cache = OhaiCache(self.data) def GetProbes(self, _): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 739320cb0..ff57d57e8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -16,6 +16,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, each ``Source`` tag. """ __identifier__ = None + create = "Sources" def __init__(self, filename, cachepath, fam, packages, setup): """ @@ -39,14 +40,8 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, If ``sources.xml`` cannot be read """ Bcfg2.Server.Plugin.Debuggable.__init__(self) - try: - Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, - should_monitor=True) - except OSError: - err = sys.exc_info()[1] - msg = "Packages: Failed to read configuration file: %s" % err - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) + Bcfg2.Server.Plugin.StructFile.__init__(self, filename, fam=fam, + should_monitor=True) #: The full path to the directory where #: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` data diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 3ebad40e3..e97f66675 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -266,8 +266,13 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): return repr(self.xdata) -class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): - """ A collection of properties files. """ +class Properties(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector, + Bcfg2.Server.Plugin.DirectoryBacked): + """ The properties plugin maps property files into client metadata + instances. """ + + #: Extensions that are understood by Properties. extensions = ["xml"] if HAS_JSON: extensions.append("json") @@ -284,14 +289,18 @@ class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): #: Ignore XML schema (``.xsd``) files ignore = re.compile(r'.*\.xsd$') - def __init__(self, data, fam): - Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, data, fam) + def __init__(self, core, datastore): + global SETUP # pylint: disable=W0603 + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Connector.__init__(self) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, core.fam) + SETUP = core.setup #: Instead of creating children of this object with a static #: object, we use :func:`property_dispatcher` to create a #: child of the appropriate subclass of :class:`PropertyFile` self.__child__ = self.property_dispatcher - __init__.__doc__ = Bcfg2.Server.Plugin.DirectoryBacked.__init__.__doc__ + __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def property_dispatcher(self, fname, fam): """ Dispatch an event on a Properties file to the @@ -314,30 +323,9 @@ class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): raise Bcfg2.Server.Plugin.PluginExecutionError( "Properties: Unknown extension %s" % fname) - -class Properties(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): - """ The properties plugin maps property files into client metadata - instances. """ - - def __init__(self, core, datastore): - global SETUP # pylint: disable=W0603 - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - SETUP = core.setup - try: - self.store = PropDirectoryBacked(self.data, core.fam) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Error while creating Properties store: %s" % - err) - raise Bcfg2.Server.Plugin.PluginInitError - - __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ - def get_additional_data(self, metadata): rv = dict() - for fname, pfile in self.store.entries.items(): + for fname, pfile in self.entries.items(): rv[fname] = pfile.get_additional_data(metadata) return rv get_additional_data.__doc__ = \ diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py index a1e624824..318f5ceaa 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py @@ -72,14 +72,32 @@ class TestPlugin(TestDebuggable): if core is None: core = Mock() core.setup = MagicMock() - return self.test_obj(core, datastore) + @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock()) + def inner(): + return self.test_obj(core, datastore) + return inner() - def test__init(self): + @patch("os.makedirs") + @patch("os.path.exists") + def test__init(self, mock_exists, mock_makedirs): core = Mock() core.setup = MagicMock() + + mock_exists.return_value = True + p = self.get_obj(core=core) + self.assertEqual(p.data, os.path.join(datastore, p.name)) + self.assertEqual(p.core, core) + mock_exists.assert_any_call(p.data) + self.assertFalse(mock_makedirs.called) + + mock_exists.reset_mock() + mock_makedirs.reset_mock() + mock_exists.return_value = False p = self.get_obj(core=core) self.assertEqual(p.data, os.path.join(datastore, p.name)) self.assertEqual(p.core, core) + mock_exists.assert_any_call(p.data) + mock_makedirs.assert_any_call(p.data) @patch("os.makedirs") def test_init_repo(self, mock_makedirs): diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index fb51eb1fe..fceddcc69 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -158,6 +158,7 @@ class TestDirectoryBacked(Bcfg2TestCase): """ ensure that the child object has the correct interface """ self.assertTrue(hasattr(self.test_obj.__child__, "HandleEvent")) + @patch("os.makedirs", Mock()) def get_obj(self, fam=None): if fam is None: fam = Mock() @@ -171,12 +172,26 @@ class TestDirectoryBacked(Bcfg2TestCase): fam) return inner() - def test__init(self): + @patch("os.makedirs") + @patch("os.path.exists") + def test__init(self, mock_exists, mock_makedirs): @patch("%s.%s.add_directory_monitor" % (self.test_obj.__module__, self.test_obj.__name__)) def inner(mock_add_monitor): + mock_exists.return_value = True + db = self.test_obj(datastore, Mock()) + mock_add_monitor.assert_called_with('') + mock_exists.assert_called_with(db.data) + self.assertFalse(mock_makedirs.called) + + mock_add_monitor.reset_mock() + mock_exists.reset_mock() + mock_makedirs.reset_mock() + mock_exists.return_value = False db = self.test_obj(datastore, Mock()) mock_add_monitor.assert_called_with('') + mock_exists.assert_called_with(db.data) + mock_makedirs.assert_called_with(db.data) inner() @@ -220,6 +235,7 @@ class TestDirectoryBacked(Bcfg2TestCase): mock_isdir.return_value = True for path in self.testpaths.values(): reset() + print "testing %s" % path db.add_directory_monitor(path) db.fam.AddMonitor.assert_called_with(os.path.join(db.data, path), db) @@ -395,10 +411,14 @@ class TestXMLFileBacked(TestFileBacked): should_monitor = None path = os.path.join(datastore, "test", "test1.xml") + @patch("os.makedirs", Mock()) def get_obj(self, path=None, fam=None, should_monitor=False): if path is None: path = self.path - return self.test_obj(path, fam=fam, should_monitor=should_monitor) + @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock()) + def inner(): + return self.test_obj(path, fam=fam, should_monitor=should_monitor) + return inner() def test__init(self): fam = Mock() @@ -1186,13 +1206,18 @@ class TestXMLDirectoryBacked(TestDirectoryBacked): class TestPrioDir(TestPlugin, TestGenerator, TestXMLDirectoryBacked): test_obj = PrioDir - @patch("Bcfg2.Server.Plugin.helpers.%s.add_directory_monitor" % - test_obj.__name__, - Mock()) def get_obj(self, core=None): if core is None: core = Mock() - return self.test_obj(core, datastore) + + @patch("%s.%s.add_directory_monitor" % + (self.test_obj.__module__, self.test_obj.__name__), + Mock()) + @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock()) + def inner(): + return self.test_obj(core, datastore) + + return inner() def test_HandleEvent(self): TestXMLDirectoryBacked.test_HandleEvent(self) @@ -1816,6 +1841,7 @@ class TestGroupSpool(TestPlugin, TestGenerator): return inner() def test__init(self): + @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock()) @patch("%s.%s.AddDirectoryMonitor" % (self.test_obj.__module__, self.test_obj.__name__)) def inner(mock_Add): diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py index 35f4e0700..1f5c4790b 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py @@ -97,11 +97,6 @@ class TestProbing(Bcfg2TestCase): class TestStatistics(TestPlugin): test_obj = Statistics - def get_obj(self, core=None): - if core is None: - core = Mock() - return self.test_obj(core, datastore) - def test_process_statistics(self): s = self.get_obj() self.assertRaises(NotImplementedError, @@ -354,12 +349,6 @@ class TestGoalValidator(Bcfg2TestCase): class TestVersion(TestPlugin): test_obj = Version - def get_obj(self, core=None): - if core is None: - core = Mock() - core.setup = MagicMock() - return self.test_obj(core, datastore) - def test_get_revision(self): d = self.get_obj() self.assertRaises(NotImplementedError, d.get_revision) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py index a9346156c..c6e6f5ef7 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py @@ -92,7 +92,12 @@ class TestPatternFile(TestXMLFileBacked): core.fam = fam elif not core: core = Mock() - return self.test_obj(path, core=core) + + @patchIf(not isinstance(lxml.etree.Element, Mock), + "lxml.etree.Element", Mock()) + def inner(): + return self.test_obj(path, core=core) + return inner() @patch("Bcfg2.Server.Plugins.GroupPatterns.PatternMap") def test_Index(self, mock_PatternMap): @@ -135,6 +140,14 @@ class TestPatternFile(TestXMLFileBacked): class TestGroupPatterns(TestPlugin, TestConnector): test_obj = GroupPatterns + def get_obj(self, core=None): + @patchIf(not isinstance(lxml.etree.Element, Mock), + "lxml.etree.Element", Mock()) + def inner(): + return TestPlugin.get_obj(self, core=core) + return inner() + + def test_get_additional_groups(self): gp = self.get_obj() gp.config = Mock() diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py index 69ea45de6..742946c42 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py @@ -94,7 +94,13 @@ def get_metadata_object(core=None, watch_clients=False, use_db=False): core.setup = MagicMock() core.metadata_cache = MagicMock() core.setup.cfp.getboolean = Mock(return_value=use_db) - return Metadata(core, datastore, watch_clients=watch_clients) + + @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock()) + @patchIf(not isinstance(lxml.etree.Element, Mock), + "lxml.etree.Element", Mock()) + def inner(): + return Metadata(core, datastore, watch_clients=watch_clients) + return inner() class TestMetadataDB(DBModelTestCase): @@ -203,7 +209,11 @@ class TestXMLMetadataConfig(TestXMLFileBacked): def get_obj(self, basefile="clients.xml", core=None, watch_clients=False): self.metadata = get_metadata_object(core=core, watch_clients=watch_clients) - return XMLMetadataConfig(self.metadata, watch_clients, basefile) + @patchIf(not isinstance(lxml.etree.Element, Mock), + "lxml.etree.Element", Mock()) + def inner(): + return XMLMetadataConfig(self.metadata, watch_clients, basefile) + return inner() def test__init(self): xmc = self.get_obj() @@ -1521,7 +1531,11 @@ class TestMetadata_ClientsXML(TestMetadataBase): if metadata is None: metadata = self.get_obj() metadata.core.fam = Mock() - metadata.clients_xml = metadata._handle_file("clients.xml") + @patchIf(not isinstance(lxml.etree.Element, Mock), + "lxml.etree.Element", Mock()) + def inner(): + metadata.clients_xml = metadata._handle_file("clients.xml") + inner() metadata = TestMetadata.load_clients_data(self, metadata=metadata, xdata=xdata) return TestMetadataBase.load_clients_data(self, metadata=metadata, diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index 899fb24a0..2163aa037 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -201,9 +201,7 @@ class TestProbes(TestProbing, TestConnector, TestDatabaseBacked): test_obj = Probes def get_obj(self, core=None): - if core is None: - core = MagicMock() - return self.test_obj(core, datastore) + return TestDatabaseBacked.get_obj(self, core=core) def get_test_probedata(self): test_xdata = lxml.etree.Element("test") @@ -247,9 +245,10 @@ text # test__init(), which relies on being able to check the calls # of load_data(), and thus on load_data() being consistently # mocked) - @patch("Bcfg2.Server.Plugins.Probes.Probes.load_data", new=load_data) + @patch("%s.%s.load_data" % (self.test_obj.__module__, + self.test_obj.__name__), new=load_data) def inner(): - return Probes(core, datastore) + return self.get_obj(core) return inner() diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py index 93e2fff51..896f5861e 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py @@ -418,8 +418,8 @@ class TestXMLPropertyFile(TestPropertyFile, TestStructFile): self.assertFalse(mock_copy.called) -class TestPropDirectoryBacked(TestDirectoryBacked): - test_obj = PropDirectoryBacked +class TestProperties(TestPlugin, TestConnector, TestDirectoryBacked): + test_obj = Properties testfiles = ['foo.xml', 'bar.baz.xml'] if HAS_JSON: testfiles.extend(["foo.json", "foo.xml.json"]) @@ -428,17 +428,13 @@ class TestPropDirectoryBacked(TestDirectoryBacked): ignore = ['foo.xsd', 'bar.baz.xsd', 'quux.xml.xsd'] badevents = ['bogus.txt'] - -class TestProperties(TestPlugin, TestConnector): - test_obj = Properties - - def test__init(self): - TestPlugin.test__init(self) - - core = Mock() - p = self.get_obj(core=core) - self.assertIsInstance(p.store, PropDirectoryBacked) - self.assertEqual(Bcfg2.Server.Plugins.Properties.SETUP, core.setup) + def get_obj(self, core=None): + @patch("%s.%s.add_directory_monitor" % (self.test_obj.__module__, + self.test_obj.__name__), + Mock()) + def inner(): + return TestPlugin.get_obj(self, core=core) + return inner() @patch("copy.copy") def test_get_additional_data(self, mock_copy): @@ -446,11 +442,11 @@ class TestProperties(TestPlugin, TestConnector): p = self.get_obj() metadata = Mock() - p.store.entries = {"foo.xml": Mock(), - "foo.yml": Mock()} + p.entries = {"foo.xml": Mock(), + "foo.yml": Mock()} rv = p.get_additional_data(metadata) expected = dict() - for name, entry in p.store.entries.items(): + for name, entry in p.entries.items(): entry.get_additional_data.assert_called_with(metadata) expected[name] = entry.get_additional_data.return_value self.assertItemsEqual(rv, expected) -- cgit v1.2.3-1-g7c22 From a02800b75bf6d6279260c1be3b6b4ab8929bd431 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 25 Mar 2013 16:22:58 -0400 Subject: travis-ci: start running py3k tests with some optional deps --- .travis.yml | 6 ------ testsuite/before_install.sh | 9 ++++++--- testsuite/install.sh | 8 +++++--- testsuite/requirements.txt | 2 ++ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index f5aade735..655d9fad5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,6 @@ python: env: - WITH_OPTIONAL_DEPS=yes - WITH_OPTIONAL_DEPS=no -matrix: - exclude: - - python: "3.2" - env: WITH_OPTIONAL_DEPS=yes - - python: "3.3" - env: WITH_OPTIONAL_DEPS=yes before_install: - testsuite/before_install.sh install: diff --git a/testsuite/before_install.sh b/testsuite/before_install.sh index 884971e45..5f1a59aaf 100755 --- a/testsuite/before_install.sh +++ b/testsuite/before_install.sh @@ -2,9 +2,12 @@ # before_install script for Travis-CI +PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') + sudo apt-get update -qq -sudo apt-get install -qq swig pylint libxml2-utils +sudo apt-get install -qq swig libxml2-utils if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then - sudo apt-get install -qq python-selinux python-pylibacl python-pyinotify \ - python-yaml yum + if [[ ${PYVER:0:1} == "2" ]]; then + sudo apt-get install -qq python-selinux python-pylibacl yum + fi fi diff --git a/testsuite/install.sh b/testsuite/install.sh index c1685f831..535e594b5 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -7,12 +7,14 @@ pip install -r testsuite/requirements.txt --use-mirrors PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then + pip install --use-mirrors genshi PyYAML pyinotify if [[ $PYVER == "2.5" ]]; then - # markdown 2.2.0 is broken on py2.5, so until 2.2.1 is released use 2.1 - pip install --use-mirrors 'markdown<2.2' pip install --use-mirrors simplejson + if [[ ${PYVER:0:1} == "2" ]]; then + # django supports py3k, but South doesn't, and the django bits + # in bcfg2 require South + pip install cheetah django South M2Crypto fi - pip install --use-mirrors genshi cheetah 'django<1.4' South M2Crypto else # python < 2.6 requires M2Crypto for SSL communication, not just # for encryption support diff --git a/testsuite/requirements.txt b/testsuite/requirements.txt index 8529b247f..c59810a99 100644 --- a/testsuite/requirements.txt +++ b/testsuite/requirements.txt @@ -3,3 +3,5 @@ nose mock sphinx daemon +pylint +pep8 -- cgit v1.2.3-1-g7c22 From 71769cd76d2fede1d858e8d35c5d5c62da8bde35 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 10:25:20 -0400 Subject: Metadata: better stringification of ClientMetadata objects for debugging --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 9adc8a442..fc5924bc4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -328,6 +328,11 @@ class ClientMetadata(object): return grp return '' + def __repr__(self): + return "%s(%s, profile=%s, groups=%s)" % (self.__class__.__name__, + self.hostname, + self.profile, self.groups) + class MetadataQuery(object): """ This class provides query methods for the metadata of all -- cgit v1.2.3-1-g7c22 From d92f67d2f8148afd191c292903e3fb6db9abddbc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 10:25:46 -0400 Subject: bcfg2-lint: added check to ensure that default group is a profile group --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index fc5924bc4..b2c68c1a7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1482,6 +1482,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): self.duplicate_groups() self.duplicate_default_groups() self.duplicate_clients() + self.default_is_profile() @classmethod def Errors(cls): @@ -1491,7 +1492,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "non-profile-set-as-profile": "error", "duplicate-group": "error", "duplicate-client": "error", - "multiple-default-groups": "error"} + "multiple-default-groups": "error", + "default-is-not-profile": "error"} def deprecated_options(self): """ check for the location='floating' option, which has been @@ -1581,3 +1583,13 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): self.LintError("duplicate-%s" % etype, "%s %s is defined multiple times:\n%s" % (etype.title(), ename, "\n".join(els))) + + def default_is_profile(self): + if (self.metadata.default and + not self.metadata.groups[self.metadata.default].is_profile): + xdata = \ + self.metadata.groups_xml.xdata.xpath("//Group[@name='%s']" % + self.metadata.default)[0] + self.LintError("default-is-not-profile", + "Default group is not a profile group:\n%s" % + self.RenderXML(xdata)) -- cgit v1.2.3-1-g7c22 From f22ad439539c85ab32daf04bdfa88fca6adaf6da Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 10:26:10 -0400 Subject: bcfg2-lint: only check clients.xml things if clients.xml is in use --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index b2c68c1a7..41d65a7bd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1498,6 +1498,9 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): def deprecated_options(self): """ check for the location='floating' option, which has been deprecated in favor of floating='true' """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return clientdata = self.metadata.clients_xml.xdata for el in clientdata.xpath("//Client"): loc = el.get("location") @@ -1523,6 +1526,9 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): def bogus_profiles(self): """ check for clients that have profiles that are either not flagged as public groups in groups.xml, or don't exist """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return for client in self.metadata.clients_xml.xdata.findall('.//Client'): profile = client.get("profile") if profile not in self.metadata.groups: @@ -1563,6 +1569,9 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): def duplicate_clients(self): """ check for clients that are defined twice. """ + if not hasattr(self.metadata, "clients_xml"): + # using metadata database + return self.duplicate_entries( self.metadata.clients_xml.xdata.xpath("//Client"), "client") -- cgit v1.2.3-1-g7c22 From 00317c9a97694ec5cf610e1c319260d63abc8eba Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 10:26:34 -0400 Subject: Metadata: apply default group consistently on first and subsequent client runs --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 41d65a7bd..2cb72286e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1144,6 +1144,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if not profile and self.groups[cgroup].is_profile: profile = cgroup + if not len(groups) and self.default: + # no initial groups; add the default profile + profile = self.default + groups.add(self.default) + groups, categories = self._merge_groups(client, groups, categories=categories) -- cgit v1.2.3-1-g7c22 From 7477be321de8b0653b75fc424d2fc4c6a943cd29 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 10:45:30 -0400 Subject: travis-ci: fixed dangling if in install.sh --- testsuite/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/testsuite/install.sh b/testsuite/install.sh index 535e594b5..599b24ac7 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -10,6 +10,7 @@ if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then pip install --use-mirrors genshi PyYAML pyinotify if [[ $PYVER == "2.5" ]]; then pip install --use-mirrors simplejson + fi if [[ ${PYVER:0:1} == "2" ]]; then # django supports py3k, but South doesn't, and the django bits # in bcfg2 require South -- cgit v1.2.3-1-g7c22 From d557da2330e1e6749e0fdd5464d49902ec26f1df Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 12:22:59 -0400 Subject: travis-ci: install django < 1.5 for tests --- testsuite/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/install.sh b/testsuite/install.sh index 599b24ac7..c6249002d 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -14,7 +14,7 @@ if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then if [[ ${PYVER:0:1} == "2" ]]; then # django supports py3k, but South doesn't, and the django bits # in bcfg2 require South - pip install cheetah django South M2Crypto + pip install cheetah 'django<1.5' South M2Crypto fi else # python < 2.6 requires M2Crypto for SSL communication, not just -- cgit v1.2.3-1-g7c22 From 9673ba716310f50855cba74195e53878316af886 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 12:23:16 -0400 Subject: Metadata: fixed setting default group when appropriate --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 68 +++++++++++++++++--------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 2cb72286e..4af4df666 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1092,7 +1092,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " "been read yet") client = client.lower() - if client in self.core.metadata_cache: return self.core.metadata_cache[client] @@ -1103,6 +1102,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, categories = dict() profile = None + def _add_group(grpname): + """ Add a group to the set of groups for this client. + Handles setting categories and category suppression. + Returns the new profile for the client (which might be + unchanged). """ + groups.add(grpname) + if grpname in self.groups: + group = self.groups[grpname] + category = group.category + if category: + if category in categories: + self.logger.warning("%s: Group %s suppressed by " + "category %s; %s already a member " + "of %s" % + (self.name, grpname, category, + client, categories[category])) + return + categories[category] = grpname + if not profile and group.is_profile: + return grpname + else: + return profile + if client not in self.clients: pgroup = None if client in self.clientgroups: @@ -1112,46 +1134,28 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if pgroup: self.set_profile(client, pgroup, (None, None)) - groups.add(pgroup) - category = self.groups[pgroup].category - if category: - categories[category] = pgroup - if (pgroup in self.groups and self.groups[pgroup].is_profile): - profile = pgroup + profile = _add_group(pgroup) else: msg = "Cannot add new client %s; no default group set" % client self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) - if client in self.clientgroups: - for cgroup in self.clientgroups[client]: - if cgroup in groups: - continue - if cgroup not in self.groups: - self.groups[cgroup] = MetadataGroup(cgroup) - category = self.groups[cgroup].category - if category and category in categories: - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, cgroup, category, - client, categories[category])) - continue - if category: - categories[category] = cgroup - groups.add(cgroup) - # favor client groups for setting profile - if not profile and self.groups[cgroup].is_profile: - profile = cgroup - - if not len(groups) and self.default: - # no initial groups; add the default profile - profile = self.default - groups.add(self.default) + for cgroup in self.clientgroups.get(client, []): + if cgroup in groups: + continue + if cgroup not in self.groups: + self.groups[cgroup] = MetadataGroup(cgroup) + profile = _add_group(cgroup) groups, categories = self._merge_groups(client, groups, categories=categories) + if len(groups) == 0 and self.default: + # no initial groups; add the default profile + profile = _add_group(self.default) + groups, categories = self._merge_groups(client, groups, + categories=categories) + bundles = set() for group in groups: try: -- cgit v1.2.3-1-g7c22 From 64eec5fe6a9b640bb77dd65f10f3fac5a586347c Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 12:47:14 -0400 Subject: testsuite: fixed issues found by latest version of pep8 --- src/lib/Bcfg2/Client/Client.py | 6 ++-- src/lib/Bcfg2/Client/Frame.py | 28 +++++++-------- src/lib/Bcfg2/Client/Tools/APK.py | 2 +- src/lib/Bcfg2/Client/Tools/Action.py | 4 +-- src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py | 2 +- src/lib/Bcfg2/Client/Tools/IPS.py | 6 ++-- src/lib/Bcfg2/Client/Tools/MacPorts.py | 2 +- src/lib/Bcfg2/Client/Tools/POSIX/Device.py | 4 +-- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 4 +-- src/lib/Bcfg2/Client/Tools/Pacman.py | 6 ++-- src/lib/Bcfg2/Client/Tools/Portage.py | 4 +-- src/lib/Bcfg2/Client/Tools/SELinux.py | 16 ++++----- src/lib/Bcfg2/Client/Tools/SMF.py | 21 ++++++----- src/lib/Bcfg2/Client/Tools/SYSV.py | 4 +-- src/lib/Bcfg2/Client/Tools/Upstart.py | 2 +- src/lib/Bcfg2/Logger.py | 2 +- src/lib/Bcfg2/Server/Admin/Init.py | 4 +-- src/lib/Bcfg2/Server/Admin/Minestruct.py | 2 +- src/lib/Bcfg2/Server/Admin/Perf.py | 11 +++--- src/lib/Bcfg2/Server/Admin/Pull.py | 2 +- src/lib/Bcfg2/Server/Admin/Xcmd.py | 5 ++- src/lib/Bcfg2/Server/Admin/__init__.py | 42 +++++++++++----------- src/lib/Bcfg2/Server/Core.py | 4 +-- src/lib/Bcfg2/Server/FileMonitor/Fam.py | 2 +- src/lib/Bcfg2/Server/FileMonitor/Pseudo.py | 2 +- src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 13 ++++--- src/lib/Bcfg2/Server/Plugin/helpers.py | 26 +++++++------- src/lib/Bcfg2/Server/Plugin/interfaces.py | 15 ++++---- src/lib/Bcfg2/Server/Plugins/Bundler.py | 14 ++++---- .../Server/Plugins/Cfg/CfgPrivateKeyCreator.py | 5 ++- src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 6 ++-- src/lib/Bcfg2/Server/Plugins/Cvs.py | 6 ++-- src/lib/Bcfg2/Server/Plugins/Darcs.py | 6 ++-- src/lib/Bcfg2/Server/Plugins/FileProbes.py | 7 ++-- src/lib/Bcfg2/Server/Plugins/Fossil.py | 6 ++-- src/lib/Bcfg2/Server/Plugins/GroupPatterns.py | 4 +-- src/lib/Bcfg2/Server/Plugins/Metadata.py | 12 +++---- src/lib/Bcfg2/Server/Plugins/Ohai.py | 2 +- src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 4 +-- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 4 +-- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 12 +++---- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 16 ++++----- src/lib/Bcfg2/Server/Plugins/Probes.py | 11 +++--- src/lib/Bcfg2/Server/Plugins/Reporting.py | 5 +-- src/lib/Bcfg2/Server/Plugins/SSHbase.py | 5 +-- src/lib/Bcfg2/Server/Plugins/__init__.py | 6 ++-- src/lib/Bcfg2/Utils.py | 2 +- src/sbin/bcfg2-crypt | 6 ++-- src/sbin/bcfg2-info | 29 ++++++++------- src/sbin/bcfg2-report-collector | 10 +++--- 50 files changed, 202 insertions(+), 217 deletions(-) diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py index 2df0b11cd..5633764a8 100644 --- a/src/lib/Bcfg2/Client/Client.py +++ b/src/lib/Bcfg2/Client/Client.py @@ -164,7 +164,8 @@ class Client(object): if len(probes.findall(".//probe")) > 0: try: # upload probe responses - self.proxy.RecvProbeData(Bcfg2.Client.XML.tostring( + self.proxy.RecvProbeData( + Bcfg2.Client.XML.tostring( probedata, xml_declaration=False).decode('UTF-8')) except Bcfg2.Proxy.ProxyError: @@ -320,7 +321,8 @@ class Client(object): feedback = self.tools.GenerateStats() try: - self.proxy.RecvStats(Bcfg2.Client.XML.tostring( + self.proxy.RecvStats( + Bcfg2.Client.XML.tostring( feedback, xml_declaration=False).decode('UTF-8')) except Bcfg2.Proxy.ProxyError: diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index bc6bd4d4c..ad5447736 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -61,8 +61,8 @@ class Frame(object): self.removal = [] self.logger = logging.getLogger(__name__) for driver in drivers[:]: - if driver not in Bcfg2.Client.Tools.drivers and \ - isinstance(driver, str): + if (driver not in Bcfg2.Client.Tools.drivers and + isinstance(driver, str)): self.logger.error("Tool driver %s is not available" % driver) drivers.remove(driver) @@ -128,7 +128,7 @@ class Frame(object): if entry.tag == 'Package'] if pkgs: self.logger.debug("The following packages are specified in bcfg2:") - self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == None]) + self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] is None]) self.logger.debug("The following packages are prereqs added by " "Packages:") self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages']) @@ -187,19 +187,19 @@ class Frame(object): """ # Need to process decision stuff early so that dryrun mode # works with it - self.whitelist = [entry for entry in self.states \ + self.whitelist = [entry for entry in self.states if not self.states[entry]] if not self.setup['file']: if self.setup['decision'] == 'whitelist': dwl = self.setup['decision_list'] - w_to_rem = [e for e in self.whitelist \ + w_to_rem = [e for e in self.whitelist if not matches_white_list(e, dwl)] if w_to_rem: self.logger.info("In whitelist mode: " "suppressing installation of:") self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in w_to_rem]) - self.whitelist = [x for x in self.whitelist \ + self.whitelist = [x for x in self.whitelist if x not in w_to_rem] elif self.setup['decision'] == 'blacklist': b_to_rem = \ @@ -230,7 +230,7 @@ class Frame(object): cfile not in self.whitelist): continue tools = [t for t in self.tools - if t.handlesEntry(cfile) and t.canVerify(cfile)] + if t.handlesEntry(cfile) and t.canVerify(cfile)] if not tools: continue if (self.setup['interactive'] and not @@ -321,7 +321,7 @@ class Frame(object): if bundle not in all_bundle_names: self.logger.info("Warning: Bundle %s not found" % bundle) - bundles = filter(lambda b: + bundles = filter(lambda b: \ b.get('name') not in self.setup['skipbundle'], bundles) if self.setup['skipindep']: @@ -387,8 +387,8 @@ class Frame(object): """Install all entries.""" self.DispatchInstallCalls(self.whitelist) mods = self.modified - mbundles = [struct for struct in self.config.findall('Bundle') if \ - [mod for mod in mods if mod in struct]] + mbundles = [struct for struct in self.config.findall('Bundle') + if any(True for mod in mods if mod in struct)] if self.modified: # Handle Bundle interdeps @@ -403,19 +403,19 @@ class Frame(object): self.logger.error("%s.Inventory() call failed:" % tool.name, exc_info=1) - clobbered = [entry for bundle in mbundles for entry in bundle \ + clobbered = [entry for bundle in mbundles for entry in bundle if (not self.states[entry] and entry not in self.blacklist)] if clobbered: self.logger.debug("Found clobbered entries:") - self.logger.debug(["%s:%s" % (entry.tag, entry.get('name')) \ + self.logger.debug(["%s:%s" % (entry.tag, entry.get('name')) for entry in clobbered]) if not self.setup['interactive']: self.DispatchInstallCalls(clobbered) for bundle in self.config.findall('.//Bundle'): - if self.setup['bundle'] and \ - bundle.get('name') not in self.setup['bundle']: + if (self.setup['bundle'] and + bundle.get('name') not in self.setup['bundle']): # prune out unspecified bundles when running with -b continue for tool in self.tools: diff --git a/src/lib/Bcfg2/Client/Tools/APK.py b/src/lib/Bcfg2/Client/Tools/APK.py index 8a02b7d6d..58641ed37 100644 --- a/src/lib/Bcfg2/Client/Tools/APK.py +++ b/src/lib/Bcfg2/Client/Tools/APK.py @@ -32,7 +32,7 @@ class APK(Bcfg2.Client.Tools.PkgTool): """Verify Package status for entry.""" if not 'version' in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % - (entry.attrib['name'])) + entry.attrib['name']) return False if entry.attrib['name'] in self.installed: diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index 87565d96d..da4412b1d 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -20,12 +20,12 @@ class Action(Bcfg2.Client.Tools.Tool): the whitelist or blacklist """ if self.setup['decision'] == 'whitelist' and \ not matches_white_list(action, self.setup['decision_list']): - self.logger.info("In whitelist mode: suppressing Action:" + \ + self.logger.info("In whitelist mode: suppressing Action: %s" % action.get('name')) return False if self.setup['decision'] == 'blacklist' and \ not passes_black_list(action, self.setup['decision_list']): - self.logger.info("In blacklist mode: suppressing Action:" + \ + self.logger.info("In blacklist mode: suppressing Action: %s" % action.get('name')) return False return True diff --git a/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py index 635318805..395f56f6a 100644 --- a/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py @@ -31,7 +31,7 @@ class FreeBSDPackage(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, _): if not 'version' in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % - (entry.attrib['name'])) + entry.attrib['name']) return False if entry.attrib['name'] in self.installed: if self.installed[entry.attrib['name']] == entry.attrib['version']: diff --git a/src/lib/Bcfg2/Client/Tools/IPS.py b/src/lib/Bcfg2/Client/Tools/IPS.py index dc4d48235..aff276c3a 100644 --- a/src/lib/Bcfg2/Client/Tools/IPS.py +++ b/src/lib/Bcfg2/Client/Tools/IPS.py @@ -51,9 +51,9 @@ class IPS(Bcfg2.Client.Tools.PkgTool): pass else: if entry.get('version') != self.installed[pname]: - self.logger.debug("IPS: Package %s: have %s want %s" \ - % (pname, self.installed[pname], - entry.get('version'))) + self.logger.debug("IPS: Package %s: have %s want %s" % + (pname, self.installed[pname], + entry.get('version'))) return False # need to implement pkg chksum validation diff --git a/src/lib/Bcfg2/Client/Tools/MacPorts.py b/src/lib/Bcfg2/Client/Tools/MacPorts.py index bc3765ec6..bd9d24df3 100644 --- a/src/lib/Bcfg2/Client/Tools/MacPorts.py +++ b/src/lib/Bcfg2/Client/Tools/MacPorts.py @@ -38,7 +38,7 @@ class MacPorts(Bcfg2.Client.Tools.PkgTool): """Verify Package status for entry.""" if not 'version' in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % - (entry.attrib['name'])) + entry.attrib['name']) return False if entry.attrib['name'] in self.installed: diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py index d5aaf069d..9b84adad0 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py @@ -12,8 +12,8 @@ class POSIXDevice(POSIXTool): def fully_specified(self, entry): if entry.get('dev_type') in ['block', 'char']: # check if major/minor are properly specified - if (entry.get('major') == None or - entry.get('minor') == None): + if (entry.get('major') is None or + entry.get('minor') is None): return False return True diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index f46875743..11f331ddb 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -712,8 +712,8 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): tmpentry.set('mode', oct_mode(newmode)) for acl in tmpentry.findall('ACL'): acl.set('perms', - oct_mode(self._norm_acl_perms(acl.get('perms')) | \ - ACL_MAP['x'])) + oct_mode(self._norm_acl_perms(acl.get('perms')) | + ACL_MAP['x'])) for cpath in created: rv &= self._set_perms(tmpentry, path=cpath) return rv diff --git a/src/lib/Bcfg2/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index 12785afee..be6698e18 100644 --- a/src/lib/Bcfg2/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py @@ -30,12 +30,12 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): def VerifyPackage(self, entry, _): '''Verify Package status for entry''' - self.logger.info("VerifyPackage : %s : %s" % entry.get('name'), - entry.get('version')) + self.logger.info("VerifyPackage: %s : %s" % (entry.get('name'), + entry.get('version'))) if not 'version' in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % - (entry.attrib['name'])) + entry.attrib['name']) return False if entry.attrib['name'] in self.installed: diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py index 6b38d7dec..69d52b052 100644 --- a/src/lib/Bcfg2/Client/Tools/Portage.py +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -13,8 +13,8 @@ class Portage(Bcfg2.Client.Tools.PkgTool): __req__ = {'Package': ['name', 'version']} pkgtype = 'ebuild' # requires a working PORTAGE_BINHOST in make.conf - _binpkgtool = ('emerge --getbinpkgonly %s', ('=%s-%s', \ - ['name', 'version'])) + _binpkgtool = ('emerge --getbinpkgonly %s', ('=%s-%s', ['name', + 'version'])) pkgtool = ('emerge %s', ('=%s-%s', ['name', 'version'])) def __init__(self, logger, cfg, setup): diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index 451495be2..f277cb0c8 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -512,14 +512,14 @@ class SELinuxSefcontextHandler(SELinuxEntryHandler): char="-c", door="-D") filetypenames = dict(all="all files", - regular="regular file", - directory="directory", - symlink="symbolic link", - pipe="named pipe", - socket="socket", - block="block device", - char="character device", - door="door") + regular="regular file", + directory="directory", + symlink="symbolic link", + pipe="named pipe", + socket="socket", + block="block device", + char="character device", + door="door") filetypeattrs = dict([v, k] for k, v in filetypenames.iteritems()) custom_re = re.compile(r'-f \'(?P[a-z ]+)\'.*? \'(?P.*)\'') diff --git a/src/lib/Bcfg2/Client/Tools/SMF.py b/src/lib/Bcfg2/Client/Tools/SMF.py index 68d8b2965..32a3df2f5 100644 --- a/src/lib/Bcfg2/Client/Tools/SMF.py +++ b/src/lib/Bcfg2/Client/Tools/SMF.py @@ -48,12 +48,12 @@ class SMF(Bcfg2.Client.Tools.SvcTool): gname = "/etc/rc*.d/%s" % filename files = glob.glob(gname.replace('_', '.')) if files: - self.logger.debug("Matched %s with %s" % \ + self.logger.debug("Matched %s with %s" % (entry.get("FMRI"), ":".join(files))) return entry.get('status') == 'on' else: - self.logger.debug("No service matching %s" % \ - (entry.get("FMRI"))) + self.logger.debug("No service matching %s" % + entry.get("FMRI")) return entry.get('status') == 'off' try: srvdata = \ @@ -81,8 +81,7 @@ class SMF(Bcfg2.Client.Tools.SvcTool): os.rename(loc, loc.replace('/S', '/DISABLED.S')) return True except OSError: - self.logger.error("Failed to rename init script %s" % \ - (loc)) + self.logger.error("Failed to rename init script %s" % loc) return False else: return self.cmd.run("/usr/sbin/svcadm disable %s" % @@ -118,12 +117,12 @@ class SMF(Bcfg2.Client.Tools.SvcTool): def FindExtra(self): """Find Extra SMF Services.""" - allsrv = [name for name, version in \ - [srvc.split() - for srvc in self.cmd.run([ - "/usr/bin/svcs", "-a", "-H", - "-o", "FMRI,STATE"]).stdout.splitlines()] - if version != 'disabled'] + allsrv = [] + for srvc in self.cmd.run(["/usr/bin/svcs", "-a", "-H", + "-o", "FMRI,STATE"]).stdout.splitlines(): + name, version = srvc.split() + if version != 'disabled': + allsrv.append(name) for svc in self.getSupportedEntries(): if svc.get("FMRI") in allsrv: diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 38072c52e..aca7d593c 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -41,7 +41,7 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.noaskfile.write(noask) # flush admin file contents to disk self.noaskfile.flush() - self.pkgtool = (self.pkgtool[0] % ("-a %s" % (self.noaskname)), \ + self.pkgtool = (self.pkgtool[0] % ("-a %s" % (self.noaskname)), self.pkgtool[1]) except: # pylint: disable=W0702 self.pkgtool = (self.pkgtool[0] % "", self.pkgtool[1]) @@ -66,7 +66,7 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): desired_version = entry.get('version') if desired_version == 'any': desired_version = self.installed.get(entry.get('name'), - desired_version) + desired_version) if not self.cmd.run(["/usr/bin/pkginfo", "-q", "-v", desired_version, entry.get('name')]): diff --git a/src/lib/Bcfg2/Client/Tools/Upstart.py b/src/lib/Bcfg2/Client/Tools/Upstart.py index cd1c4a2bc..921ca849f 100644 --- a/src/lib/Bcfg2/Client/Tools/Upstart.py +++ b/src/lib/Bcfg2/Client/Tools/Upstart.py @@ -48,7 +48,7 @@ class Upstart(Bcfg2.Client.Tools.SvcTool): match = re.compile("%s( \(.*\))? (start|stop)/(running|waiting)" % entry.get('name')).match(output) - if match == None: + if match is None: # service does not exist entry.set('current_status', 'off') status = False diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index 5bbc9ff96..1f7348207 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -197,7 +197,7 @@ def setup_logging(procname, to_console=True, to_syslog=True, params = [] if to_console: - if to_console == True: + if to_console is True: to_console = logging.WARNING if level == 0: clvl = to_console diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index 4b8d65597..6175d8ed0 100644 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -227,8 +227,8 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_password(self): """Ask for a password or generate one if none is provided.""" newpassword = getpass.getpass( - "Input password used for communication verification " - "(without echoing; leave blank for a random): ").strip() + "Input password used for communication verification " + "(without echoing; leave blank for a random): ").strip() if len(newpassword) != 0: self.data['password'] = newpassword diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index 6d0dab106..13c0563ec 100644 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -44,7 +44,7 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): extra.add(item) except: self.log.error("Failed to find extra entry info for client %s" % - client) + client) raise SystemExit(1) root = lxml.etree.Element("Base") self.log.info("Found %d extra entries" % (len(extra))) diff --git a/src/lib/Bcfg2/Server/Admin/Perf.py b/src/lib/Bcfg2/Server/Admin/Perf.py index 86eb6810d..f6bc22959 100644 --- a/src/lib/Bcfg2/Server/Admin/Perf.py +++ b/src/lib/Bcfg2/Server/Admin/Perf.py @@ -18,8 +18,7 @@ class Perf(Bcfg2.Server.Admin.Mode): 'password': Bcfg2.Options.SERVER_PASSWORD, 'server': Bcfg2.Options.SERVER_LOCATION, 'user': Bcfg2.Options.CLIENT_USER, - 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, - } + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT} setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], @@ -31,8 +30,8 @@ class Perf(Bcfg2.Server.Admin.Mode): timeout=setup['timeout']) data = proxy.get_statistics() for key in sorted(data.keys()): - output.append((key, ) + - tuple(["%.06f" % item - for item in data[key][:-1]] + \ - [data[key][-1]])) + output.append( + (key, ) + + tuple(["%.06f" % item + for item in data[key][:-1]] + [data[key][-1]])) self.print_table(output) diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index 130e85b67..9f1b3d138 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -65,7 +65,7 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): for plugin in self.bcore.pull_sources: try: (owner, group, mode, contents) = \ - plugin.GetCurrentEntry(client, etype, ename) + plugin.GetCurrentEntry(client, etype, ename) break except Bcfg2.Server.Plugin.PluginExecutionError: if plugin == self.bcore.pull_sources[-1]: diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py index 79eeebc7c..be556bed4 100644 --- a/src/lib/Bcfg2/Server/Admin/Xcmd.py +++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py @@ -19,8 +19,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode): 'key': Bcfg2.Options.SERVER_KEY, 'certificate': Bcfg2.Options.CLIENT_CERT, 'ca': Bcfg2.Options.CLIENT_CA, - 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, - } + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT} setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(args) Bcfg2.Proxy.RetryMethod.max_retries = 1 @@ -54,5 +53,5 @@ class Xcmd(Bcfg2.Server.Admin.Mode): print("Proxy Error: %s" % err) return - if data != None: + if data is not None: print(data) diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py index 19175533f..a3c45b8c3 100644 --- a/src/lib/Bcfg2/Server/Admin/__init__.py +++ b/src/lib/Bcfg2/Server/Admin/__init__.py @@ -1,23 +1,21 @@ """ Base classes for admin modes """ -__all__ = [ - 'Backup', - 'Bundle', - 'Client', - 'Compare', - 'Group', - 'Init', - 'Minestruct', - 'Perf', - 'Pull', - 'Query', - 'Reports', - 'Snapshots', - 'Syncdb', - 'Tidy', - 'Viz', - 'Xcmd' - ] +__all__ = ['Backup', + 'Bundle', + 'Client', + 'Compare', + 'Group', + 'Init', + 'Minestruct', + 'Perf', + 'Pull', + 'Query', + 'Reports', + 'Snapshots', + 'Syncdb', + 'Tidy', + 'Viz', + 'Xcmd'] import re import sys @@ -105,15 +103,15 @@ class Mode(object): # Calculate column widths (longest item in each column # plus padding on both sides) cols = list(zip(*rows)) - col_widths = [max([len(str(item)) + 2 * padding for \ - item in col]) for col in cols] + col_widths = [max([len(str(item)) + 2 * padding + for item in col]) for col in cols] borderline = vdelim.join([w * hdelim for w in col_widths]) # Print out the table print(borderline) for row in rows: - print(vdelim.join([justify(str(item), width) for \ - (item, width) in zip(row, col_widths)])) + print(vdelim.join([justify(str(item), width) + for (item, width) in zip(row, col_widths)])) if hdr: print(borderline) hdr = False diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 8ceb8cfc1..deb9065a5 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -397,7 +397,7 @@ class BaseCore(object): self.logger.debug("Loading plugin %s" % plugin) try: mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % - (plugin)).Server.Plugins, plugin) + (plugin)).Server.Plugins, plugin) except ImportError: try: mod = __import__(plugin, globals(), locals(), @@ -1084,7 +1084,7 @@ class BaseCore(object): Bcfg2.Server.Plugin.MetadataRuntimeError): err = sys.exc_info()[1] self.critical_error("Unable to assert profile for %s: %s" % - (client, err)) + (client, err)) return True @exposed diff --git a/src/lib/Bcfg2/Server/FileMonitor/Fam.py b/src/lib/Bcfg2/Server/FileMonitor/Fam.py index 253bb2801..09d41038e 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Fam.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Fam.py @@ -51,7 +51,7 @@ class Fam(FileMonitor): else: handle = self.filemonitor.monitorFile(path, None) self.handles[handle.requestID()] = handle - if obj != None: + if obj is not None: self.users[handle.requestID()] = obj return handle.requestID() AddMonitor.__doc__ = FileMonitor.AddMonitor.__doc__ diff --git a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py index 24cd099d0..b1e1adab7 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py @@ -24,6 +24,6 @@ class Pseudo(FileMonitor): self.events.append(Event(handleID, fname, 'exists')) self.events.append(Event(handleID, path, 'endExist')) - if obj != None: + if obj is not None: self.handles[handleID] = obj return handleID diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 2a10da417..be4264b07 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -61,7 +61,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): Path=dict( device=dict(name=is_filename, owner=is_username, group=is_username, - dev_type=lambda v: \ + dev_type=lambda v: v in Bcfg2.Client.Tools.POSIX.base.device_map), directory=dict(name=is_filename, owner=is_username, group=is_username, mode=is_octal_mode), @@ -98,10 +98,10 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): SEBoolean={None: dict(name=None, value=lambda v: v in ['on', 'off'])}, SEModule={None: dict(name=None, __text__=None)}, - SEPort={None: - dict(name=lambda v: re.match(r'^\d+(-\d+)?/(tcp|udp)', - v), - selinuxtype=is_selinux_type)}, + SEPort={ + None: dict(name=lambda v: re.match(r'^\d+(-\d+)?/(tcp|udp)', + v), + selinuxtype=is_selinux_type)}, SEFcontext={None: dict(name=None, selinuxtype=is_selinux_type)}, SENode={None: dict(name=lambda v: "/" in v, selinuxtype=is_selinux_type, @@ -116,8 +116,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): SEPermissive={None: dict(name=is_selinux_type)}, POSIXGroup={None: dict(name=is_username)}, POSIXUser={None: dict(name=is_username)}, - MemberOf={None: dict(__text__=is_username)}, - ) + MemberOf={None: dict(__text__=is_username)}) def Run(self): self.check_packages() diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 448d8b3b6..bddd0ba20 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -25,15 +25,15 @@ except ImportError: HAS_DJANGO = False #: A dict containing default metadata for Path entries from bcfg2.conf -DEFAULT_FILE_METADATA = Bcfg2.Options.OptionParser(dict( - configfile=Bcfg2.Options.CFILE, - owner=Bcfg2.Options.MDATA_OWNER, - group=Bcfg2.Options.MDATA_GROUP, - mode=Bcfg2.Options.MDATA_MODE, - secontext=Bcfg2.Options.MDATA_SECONTEXT, - important=Bcfg2.Options.MDATA_IMPORTANT, - paranoid=Bcfg2.Options.MDATA_PARANOID, - sensitive=Bcfg2.Options.MDATA_SENSITIVE)) +DEFAULT_FILE_METADATA = Bcfg2.Options.OptionParser( + dict(configfile=Bcfg2.Options.CFILE, + owner=Bcfg2.Options.MDATA_OWNER, + group=Bcfg2.Options.MDATA_GROUP, + mode=Bcfg2.Options.MDATA_MODE, + secontext=Bcfg2.Options.MDATA_SECONTEXT, + important=Bcfg2.Options.MDATA_IMPORTANT, + paranoid=Bcfg2.Options.MDATA_PARANOID, + sensitive=Bcfg2.Options.MDATA_SENSITIVE)) DEFAULT_FILE_METADATA.parse([Bcfg2.Options.CFILE.cmd, Bcfg2.Options.CFILE]) del DEFAULT_FILE_METADATA['args'] del DEFAULT_FILE_METADATA['configfile'] @@ -786,14 +786,14 @@ class InfoNode (INode): Client="lambda m, e: '%(name)s' == m.hostname and predicate(m, e)", Group="lambda m, e: '%(name)s' in m.groups and predicate(m, e)", Path="lambda m, e: ('%(name)s' == e.get('name') or " + - "'%(name)s' == e.get('realname')) and " + - "predicate(m, e)") + "'%(name)s' == e.get('realname')) and " + + "predicate(m, e)") nraw = dict( Client="lambda m, e: '%(name)s' != m.hostname and predicate(m, e)", Group="lambda m, e: '%(name)s' not in m.groups and predicate(m, e)", Path="lambda m, e: '%(name)s' != e.get('name') and " + - "'%(name)s' != e.get('realname') and " + - "predicate(m, e)") + "'%(name)s' != e.get('realname') and " + + "predicate(m, e)") containers = ['Group', 'Client', 'Path'] diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index cb996b1ca..0fd711be9 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -337,12 +337,11 @@ class ThreadedStatistics(Statistics, Threaded, threading.Thread): pending_data = [] try: while not self.work_queue.empty(): - (metadata, data) = self.work_queue.get_nowait() - pending_data.append( - (metadata.hostname, - lxml.etree.tostring( - data, - xml_declaration=False).decode("UTF-8"))) + (metadata, xdata) = self.work_queue.get_nowait() + data = \ + lxml.etree.tostring(xdata, + xml_declaration=False).decode("UTF-8") + pending_data.append((metadata.hostname, data)) except Empty: pass @@ -409,7 +408,7 @@ class ThreadedStatistics(Statistics, Threaded, threading.Thread): def run(self): if not self._load(): return - while not self.terminate.isSet() and self.work_queue != None: + while not self.terminate.isSet() and self.work_queue is not None: try: (client, xdata) = self.work_queue.get(block=True, timeout=2) except Empty: @@ -419,7 +418,7 @@ class ThreadedStatistics(Statistics, Threaded, threading.Thread): self.logger.error("ThreadedStatistics: %s" % err) continue self.handle_statistic(client, xdata) - if self.work_queue != None and not self.work_queue.empty(): + if self.work_queue is not None and not self.work_queue.empty(): self._save() def process_statistics(self, metadata, data): diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 24301155d..a9b9bf8c1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -13,7 +13,7 @@ import Bcfg2.Server.Lint try: import genshi.template.base - import Bcfg2.Server.Plugins.TGenshi + from Bcfg2.Server.Plugins.TGenshi import removecomment, TemplateFile HAS_GENSHI = True except ImportError: HAS_GENSHI = False @@ -34,14 +34,12 @@ class BundleFile(Bcfg2.Server.Plugin.StructFile): if HAS_GENSHI: - class BundleTemplateFile(Bcfg2.Server.Plugins.TGenshi.TemplateFile, + class BundleTemplateFile(TemplateFile, Bcfg2.Server.Plugin.StructFile): """ Representation of a Genshi-templated bundle XML file """ def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugins.TGenshi.TemplateFile.__init__(self, name, - specific, - encoding) + TemplateFile.__init__(self, name, specific, encoding) Bcfg2.Server.Plugin.StructFile.__init__(self, name) self.logger = logging.getLogger(name) @@ -52,9 +50,9 @@ if HAS_GENSHI: msg = "No parsed template information for %s" % self.name self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - stream = self.template.generate(metadata=metadata, - repo=SETUP['repo']).filter( - Bcfg2.Server.Plugins.TGenshi.removecomment) + stream = self.template.generate( + metadata=metadata, + repo=SETUP['repo']).filter(removecomment) data = lxml.etree.XML(stream.render('xml', strip_whitespace=False), parser=Bcfg2.Server.XMLParser) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index aaeb65cd6..581a997d8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -48,9 +48,8 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if (HAS_CRYPTO and SETUP.cfp.has_section("sshkeys") and SETUP.cfp.has_option("sshkeys", "passphrase")): - return Bcfg2.Encryption.get_passphrases(SETUP)[SETUP.cfp.get( - "sshkeys", - "passphrase")] + return Bcfg2.Encryption.get_passphrases(SETUP)[ + SETUP.cfp.get("sshkeys", "passphrase")] return None def handle_event(self, event): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index ec3ba222c..f347b898c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -87,7 +87,7 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, Bcfg2.Server.Plugin.Debuggable.__init__(self) self.encoding = encoding __init__.__doc__ = Bcfg2.Server.Plugin.SpecificData.__init__.__doc__ + \ -""" + """ .. ----- .. autoattribute:: CfgBaseFileMatcher.__basenames__ .. autoattribute:: CfgBaseFileMatcher.__extensions__ @@ -758,8 +758,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, pass if not rv or not rv[0].hostname: - rv.append(Bcfg2.Server.Plugin.Specificity( - hostname=metadata.hostname)) + rv.append( + Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)) return rv def build_filename(self, specific): diff --git a/src/lib/Bcfg2/Server/Plugins/Cvs.py b/src/lib/Bcfg2/Server/Plugins/Cvs.py index ba1559a1a..22cacaa76 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cvs.py +++ b/src/lib/Bcfg2/Server/Plugins/Cvs.py @@ -20,9 +20,9 @@ class Cvs(Bcfg2.Server.Plugin.Version): """Read cvs revision information for the Bcfg2 repository.""" try: data = Popen("env LC_ALL=C cvs log", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() + shell=True, + cwd=self.vcs_root, + stdout=PIPE).stdout.readlines() return data[3].strip('\n') except IndexError: msg = "Failed to read CVS log" diff --git a/src/lib/Bcfg2/Server/Plugins/Darcs.py b/src/lib/Bcfg2/Server/Plugins/Darcs.py index 0033e00f3..b4abafb0e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Darcs.py +++ b/src/lib/Bcfg2/Server/Plugins/Darcs.py @@ -20,9 +20,9 @@ class Darcs(Bcfg2.Server.Plugin.Version): """Read Darcs changeset information for the Bcfg2 repository.""" try: data = Popen("env LC_ALL=C darcs changes", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() + shell=True, + cwd=self.vcs_root, + stdout=PIPE).stdout.readlines() revision = data[0].strip('\n') except: msg = "Failed to read darcs repository" diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index 882c22c49..d816192aa 100644 --- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -226,11 +226,8 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, root = lxml.etree.Element("FileInfo") root.append(info) try: - open(infoxml, - "w").write( - lxml.etree.tostring(root, - xml_declaration=False, - pretty_print=True).decode('UTF-8')) + root.getroottree().write(infoxml, xml_declaration=False, + pretty_print=True) except IOError: err = sys.exc_info()[1] self.logger.error("Could not write %s: %s" % (infoxml, err)) diff --git a/src/lib/Bcfg2/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py index f6735df12..d0c328b36 100644 --- a/src/lib/Bcfg2/Server/Plugins/Fossil.py +++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py @@ -20,9 +20,9 @@ class Fossil(Bcfg2.Server.Plugin.Version): """Read fossil revision information for the Bcfg2 repository.""" try: data = Popen("env LC_ALL=C fossil info", - shell=True, - cwd=self.vcs_root, - stdout=PIPE).stdout.readlines() + shell=True, + cwd=self.vcs_root, + stdout=PIPE).stdout.readlines() revline = [line.split(': ')[1].strip() for line in data if \ line.split(': ')[0].strip() == 'checkout'][-1] return revline.split(' ')[0] diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 8ce3fcd3a..fcfae2088 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -15,10 +15,10 @@ class PatternMap(object): self.pattern = pattern self.rangestr = rangestr self.groups = groups - if pattern != None: + if pattern is not None: self.re = re.compile(pattern) self.process = self.process_re - elif rangestr != None: + elif rangestr is not None: if '\\' in rangestr: raise Exception("Backslashes are not allowed in NameRanges") range_finder = r'\[\[[\d\-,]+\]\]' diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 4af4df666..10b09842a 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -602,7 +602,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _add_xdata(self, config, tag, name, attribs=None, alias=False): """ Generic method to add XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) - if node != None: + if node is not None: raise Bcfg2.Server.Plugin.MetadataConsistencyError("%s \"%s\" " "already exists" % (tag, name)) @@ -662,7 +662,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _update_xdata(self, config, tag, name, attribs, alias=False): """ Generic method to modify XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) - if node == None: + if node is None: self.logger.error("%s \"%s\" does not exist" % (tag, name)) raise Bcfg2.Server.Plugin.MetadataConsistencyError xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % @@ -679,7 +679,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """Update a groups attributes.""" if self._use_db: msg = "Metadata does not support updating groups with " + \ - "use_database enabled" + "use_database enabled" self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: @@ -707,7 +707,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def _remove_xdata(self, config, tag, name): """ Generic method to remove XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata) - if node == None: + if node is None: self.logger.error("%s \"%s\" does not exist" % (tag, name)) raise Bcfg2.Server.Plugin.MetadataConsistencyError xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % @@ -1557,8 +1557,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): tag as a definition if it a) has profile or public set; or b) has any children. """ self.duplicate_entries( - self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"), + self.metadata.groups_xml.xdata.xpath("//Groups/Group") + + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"), "group", include=lambda g: (g.get("profile") or g.get("public") or diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index 07b04f3f0..a88c245bd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -32,7 +32,7 @@ class OhaiCache(object): self.cache = dict() def __setitem__(self, item, value): - if value == None: + if value is None: # simply return if the client returned nothing return self.cache[item] = json.loads(value) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 27f493677..5ae160994 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -30,8 +30,8 @@ class AptCollection(Collection): """ Get an APT configuration file (i.e., ``sources.list``). :returns: string """ - lines = ["# This config was generated automatically by the Bcfg2 " \ - "Packages plugin", ''] + lines = ["# This config was generated automatically by the Bcfg2 " + "Packages plugin", ''] for source in self: if source.rawurl: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 985405e65..0c6ee0759 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -52,8 +52,8 @@ import re import sys import Bcfg2.Server.Plugin from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ - HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \ - urlopen, cPickle, md5 + HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ + cPickle, md5 def fetch_url(url): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 6b8ed1f7d..77186d2cb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -66,7 +66,7 @@ from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ # pylint: enable=W0622 from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ - fetch_url + fetch_url LOGGER = logging.getLogger(__name__) @@ -281,7 +281,7 @@ class YumCollection(Collection): #: Define a unique cache file for this collection to use #: for cached yum metadata self.cachefile = os.path.join(self.cachepath, - "cache-%s" % self.cachekey) + "cache-%s" % self.cachekey) if not os.path.exists(self.cachefile): os.mkdir(self.cachefile) @@ -1169,7 +1169,7 @@ class YumSource(Source): if entry.get('name').startswith('/'): self.needed_paths.add(entry.get('name')) pro = pdata.find(RP + 'provides') - if pro != None: + if pro is not None: for entry in pro.getchildren(): prov = entry.get('name') if prov not in self.provides[arch]: @@ -1185,9 +1185,9 @@ class YumSource(Source): try: groupid = group.xpath('id')[0].text self.yumgroups[groupid] = {'mandatory': list(), - 'default': list(), - 'optional': list(), - 'conditional': list()} + 'default': list(), + 'optional': list(), + 'conditional': list()} except IndexError: continue try: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 0b6889b09..4f163a1ab 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -247,14 +247,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return True elif entry.tag == 'Path': # managed entries for yum/apt configs - if (entry.get("name") == \ - self.core.setup.cfp.get("packages", - "yum_config", - default=YUM_CONFIG_DEFAULT) or - entry.get("name") == \ - self.core.setup.cfp.get("packages", - "apt_config", - default=APT_CONFIG_DEFAULT)): + if (entry.get("name") == + self.core.setup.cfp.get("packages", + "yum_config", + default=YUM_CONFIG_DEFAULT) or + entry.get("name") == + self.core.setup.cfp.get("packages", + "apt_config", + default=APT_CONFIG_DEFAULT)): return True return False diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index f106b75a4..634826f4b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -214,10 +214,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: - datafile = open(os.path.join(self.data, 'probed.xml'), 'w') - datafile.write(lxml.etree.tostring( - top, xml_declaration=False, - pretty_print='true').decode('UTF-8')) + top.getroottree().write(os.path.join(self.data, 'probed.xml'), + xml_declaration=False, + pretty_print='true') except IOError: err = sys.exc_info()[1] self.logger.error("Failed to write probed.xml: %s" % err) @@ -246,7 +245,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, grp.save() ProbesGroupsModel.objects.filter( hostname=client.hostname).exclude( - group__in=self.cgroups[client.hostname]).delete() + group__in=self.cgroups[client.hostname]).delete() def load_data(self): """ Load probe data from the appropriate backend (probed.xml @@ -320,7 +319,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, def ReceiveDataItem(self, client, data, cgroups, cprobedata): """Receive probe results pertaining to client.""" - if data.text == None: + if data.text is None: self.logger.info("Got null response to probe %s from %s" % (data.get('name'), client.hostname)) cprobedata[data.get('name')] = ProbeData('') diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index a6dc2c1ef..3bd6fd14f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -92,8 +92,9 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): # try 3 times to store the data for i in [1, 2, 3]: try: - self.transport.store(client.hostname, cdata, - lxml.etree.tostring( + self.transport.store( + client.hostname, cdata, + lxml.etree.tostring( stats, xml_declaration=False).decode('UTF-8')) self.debug_log("%s: Queued statistics data for %s" % diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index c7db67301..0c5644395 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -201,9 +201,10 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, if specific.hostname and specific.hostname in names: hostnames = names[specific.hostname] elif specific.group: - hostnames = list(chain( + hostnames = list( + chain( *[names[cmeta.hostname] - for cmeta in \ + for cmeta in mquery.by_groups([specific.group])])) elif specific.all: # a generic key for all hosts? really? diff --git a/src/lib/Bcfg2/Server/Plugins/__init__.py b/src/lib/Bcfg2/Server/Plugins/__init__.py index b33eeba28..063e14315 100644 --- a/src/lib/Bcfg2/Server/Plugins/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/__init__.py @@ -1,7 +1,6 @@ """Imports for Bcfg2.Server.Plugins.""" -__all__ = [ - 'Account', +__all__ = ['Account', 'Base', 'Bundler', 'Bzr', @@ -28,5 +27,4 @@ __all__ = [ 'Svn', 'TCheetah', 'Trigger', - 'TGenshi', - ] + 'TGenshi'] diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 33da8bd71..601217556 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -172,7 +172,7 @@ class Executor(object): :param proc: The process to kill upon timeout. :type proc: subprocess.Popen :returns: None """ - if proc.poll() == None: + if proc.poll() is None: try: proc.kill() self.logger.warning("Process exceeeded timeout, killing") diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index eae316da5..aad89882f 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -161,8 +161,8 @@ class Encryptor(object): continue except TypeError: pchunk = None - for pname, passphrase in \ - Bcfg2.Encryption.get_passphrases(self.setup).items(): + passphrases = Bcfg2.Encryption.get_passphrases(self.setup) + for pname, passphrase in passphrases.items(): self.logger.debug("Trying passphrase %s" % pname) try: pchunk = self._decrypt(chunk, passphrase) @@ -341,7 +341,7 @@ class PropertiesEncryptor(Encryptor): # actually need to unchunk anything xdata = data[0] # find root element - while xdata.getparent() != None: + while xdata.getparent() is not None: xdata = xdata.getparent() return lxml.etree.tostring(xdata, xml_declaration=False, diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index cfcc95be2..462d41398 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -380,7 +380,7 @@ Bcfg2 client itself.""") xml_declaration=False).decode('UTF-8') except Exception: print("Failed to build entry %s for host %s: %s" % - (fname, client, traceback.format_exc().splitlines()[-1])) + (fname, client, traceback.format_exc().splitlines()[-1])) raise try: if outfile: @@ -468,19 +468,18 @@ Bcfg2 client itself.""") def do_config(self, _): """ config - Print out the current configuration of Bcfg2""" output = [ - ('Description', 'Value'), - ('Path Bcfg2 repository', self.setup['repo']), - ('Plugins', self.setup['plugins']), - ('Password', self.setup['password']), - ('Server Metadata Connector', self.setup['mconnect']), - ('Filemonitor', self.setup['filemonitor']), - ('Server address', self.setup['location']), - ('Path to key', self.setup['key']), - ('Path to SSL certificate', self.setup['cert']), - ('Path to SSL CA certificate', self.setup['ca']), - ('Protocol', self.setup['protocol']), - ('Logging', self.setup['logging']) - ] + ('Description', 'Value'), + ('Path Bcfg2 repository', self.setup['repo']), + ('Plugins', self.setup['plugins']), + ('Password', self.setup['password']), + ('Server Metadata Connector', self.setup['mconnect']), + ('Filemonitor', self.setup['filemonitor']), + ('Server address', self.setup['location']), + ('Path to key', self.setup['key']), + ('Path to SSL certificate', self.setup['cert']), + ('Path to SSL CA certificate', self.setup['ca']), + ('Protocol', self.setup['protocol']), + ('Logging', self.setup['logging'])] print_tabular(output) def do_probes(self, args): @@ -739,7 +738,7 @@ def build_usage(): # shim for python 2.4, __func__ is im_func funcattr = getattr(attr, "__func__", getattr(attr, "im_func", None)) - if (funcattr != None and + if (funcattr is not None and funcattr.func_name not in cmd_blacklist and funcattr.func_name.startswith("do_") and funcattr.func_doc): diff --git a/src/sbin/bcfg2-report-collector b/src/sbin/bcfg2-report-collector index a0ee2259a..594be13bf 100755 --- a/src/sbin/bcfg2-report-collector +++ b/src/sbin/bcfg2-report-collector @@ -12,12 +12,10 @@ from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError def main(): logger = logging.getLogger('bcfg2-report-collector') - optinfo = dict( - daemon=Bcfg2.Options.DAEMON, - repo=Bcfg2.Options.SERVER_REPOSITORY, - filemonitor=Bcfg2.Options.SERVER_FILEMONITOR, - web_configfile=Bcfg2.Options.WEB_CFILE, - ) + optinfo = dict(daemon=Bcfg2.Options.DAEMON, + repo=Bcfg2.Options.SERVER_REPOSITORY, + filemonitor=Bcfg2.Options.SERVER_FILEMONITOR, + web_configfile=Bcfg2.Options.WEB_CFILE) optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.REPORTING_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) -- cgit v1.2.3-1-g7c22 From 6c996f42c53a36fc0406f836d64b8c1bec6f4bcc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 13:27:41 -0400 Subject: testsuite: fixed issues found by latest version of pylint --- src/lib/Bcfg2/Client/Frame.py | 15 +++++------ src/lib/Bcfg2/Client/Tools/DebInit.py | 2 +- src/lib/Bcfg2/Client/Tools/Encap.py | 2 +- src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py | 2 +- src/lib/Bcfg2/Client/Tools/Portage.py | 2 +- src/lib/Bcfg2/Client/Tools/SELinux.py | 2 +- src/lib/Bcfg2/Client/Tools/Upstart.py | 2 +- src/lib/Bcfg2/Client/__init__.py | 6 ++--- src/lib/Bcfg2/Logger.py | 12 ++++----- src/lib/Bcfg2/Options.py | 6 ++--- src/lib/Bcfg2/Server/Admin/__init__.py | 21 +++------------ src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 6 ++--- src/lib/Bcfg2/Server/Plugin/helpers.py | 26 +++++++++--------- src/lib/Bcfg2/Server/Plugins/Bundler.py | 2 +- .../Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py | 2 +- src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 10 +++---- src/lib/Bcfg2/Server/Plugins/GroupPatterns.py | 4 +-- src/lib/Bcfg2/Server/Plugins/Metadata.py | 1 + src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 6 ++--- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 2 +- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 2 +- src/lib/Bcfg2/Server/Plugins/Probes.py | 8 +++--- src/lib/Bcfg2/Server/Plugins/__init__.py | 31 +++------------------- src/lib/Bcfg2/Server/__init__.py | 5 ++-- src/lib/Bcfg2/__init__.py | 3 ++- testsuite/pylintrc.conf | 5 +++- 26 files changed, 74 insertions(+), 111 deletions(-) diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index ad5447736..fe4943fdf 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -310,10 +310,10 @@ class Frame(object): for bundle in self.setup['bundle']: if bundle not in all_bundle_names: self.logger.info("Warning: Bundle %s not found" % bundle) - bundles = filter(lambda b: b.get('name') in self.setup['bundle'], - bundles) + bundles = [b for b in bundles + if b.get('name') in self.setup['bundle']] elif self.setup['indep']: - bundles = filter(lambda b: b.tag != 'Bundle', bundles) + bundles = [b for b in bundles if b.tag != 'Bundle'] if self.setup['skipbundle']: # warn if non-existent bundle given if not self.setup['bundle_quick']: @@ -321,14 +321,13 @@ class Frame(object): if bundle not in all_bundle_names: self.logger.info("Warning: Bundle %s not found" % bundle) - bundles = filter(lambda b: \ - b.get('name') not in self.setup['skipbundle'], - bundles) + bundles = [b for b in bundles + if b.get('name') not in self.setup['skipbundle']] if self.setup['skipindep']: - bundles = filter(lambda b: b.tag == 'Bundle', bundles) + bundles = [b for b in bundles if b.tag == 'Bundle'] self.whitelist = [e for e in self.whitelist - if True in [e in b for b in bundles]] + if any(e in b for b in bundles)] # first process prereq actions for bundle in bundles[:]: diff --git a/src/lib/Bcfg2/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index ca556e98b..d916b1662 100644 --- a/src/lib/Bcfg2/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -16,7 +16,7 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'deb')] __req__ = {'Service': ['name', 'status']} svcre = \ - re.compile("/etc/.*/(?P[SK])(?P\d+)(?P\S+)") + re.compile(r'/etc/.*/(?P[SK])(?P\d+)(?P\S+)') # implement entry (Verify|Install) ops def VerifyService(self, entry, _): diff --git a/src/lib/Bcfg2/Client/Tools/Encap.py b/src/lib/Bcfg2/Client/Tools/Encap.py index 678e0f00c..270f0a5f2 100644 --- a/src/lib/Bcfg2/Client/Tools/Encap.py +++ b/src/lib/Bcfg2/Client/Tools/Encap.py @@ -13,7 +13,7 @@ class Encap(Bcfg2.Client.Tools.PkgTool): __req__ = {'Package': ['version', 'url']} pkgtype = 'encap' pkgtool = ("/usr/local/bin/epkg -l -f -q %s", ("%s", ["url"])) - splitter = re.compile('.*/(?P[\w-]+)\-(?P[\w\.+-]+)') + splitter = re.compile(r'.*/(?P[\w-]+)\-(?P[\w\.+-]+)') def RefreshPackages(self): """Try to find encap packages.""" diff --git a/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py index 395f56f6a..31925fa3c 100644 --- a/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py @@ -21,7 +21,7 @@ class FreeBSDPackage(Bcfg2.Client.Tools.PkgTool): def RefreshPackages(self): self.installed = {} packages = self.cmd.run("/usr/sbin/pkg_info -a -E").stdout.splitlines() - pattern = re.compile('(.*)-(\d.*)') + pattern = re.compile(r'(.*)-(\d.*)') for pkg in packages: if pattern.match(pkg): name = pattern.match(pkg).group(1) diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py index 69d52b052..17e7755a9 100644 --- a/src/lib/Bcfg2/Client/Tools/Portage.py +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -22,7 +22,7 @@ class Portage(Bcfg2.Client.Tools.PkgTool): Bcfg2.Client.Tools.PkgTool.__init__(self, logger, cfg, setup) self._initialised = True self.__important__ = self.__important__ + ['/etc/make.conf'] - self._pkg_pattern = re.compile('(.*)-(\d.*)') + self._pkg_pattern = re.compile(r'(.*)-(\d.*)') self._ebuild_pattern = re.compile('(ebuild|binary)') self.cfg = cfg self.installed = {} diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index f277cb0c8..617b7c2ef 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -170,7 +170,7 @@ class SELinuxEntryHandler(object): key_format = ("name",) value_format = () str_format = '%(name)s' - custom_re = re.compile(' (?P\S+)$') + custom_re = re.compile(r' (?P\S+)$') custom_format = None def __init__(self, tool, logger, setup, config): diff --git a/src/lib/Bcfg2/Client/Tools/Upstart.py b/src/lib/Bcfg2/Client/Tools/Upstart.py index 921ca849f..c96eab69d 100644 --- a/src/lib/Bcfg2/Client/Tools/Upstart.py +++ b/src/lib/Bcfg2/Client/Tools/Upstart.py @@ -46,7 +46,7 @@ class Upstart(Bcfg2.Client.Tools.SvcTool): entry.get('name')) return False - match = re.compile("%s( \(.*\))? (start|stop)/(running|waiting)" % + match = re.compile(r'%s( \(.*\))? (start|stop)/(running|waiting)' % entry.get('name')).match(output) if match is None: # service does not exist diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index dd5ae1e83..8c8c4fd94 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -1,11 +1,11 @@ """This contains all Bcfg2 Client modules""" -__all__ = ["Frame", "Tools", "XML", "Client"] - import os import sys import select -from Bcfg2.Compat import input # pylint: disable=W0622 +from Bcfg2.Compat import input, walk_packages # pylint: disable=W0622 + +__all__ = [m[1] for m in walk_packages(path=__path__)] def prompt(msg): diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index 1f7348207..e537b6148 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -144,9 +144,9 @@ def add_console_handler(level=logging.DEBUG): # tell the handler to use this format console.setFormatter(TermiosFormatter()) try: - console.set_name("console") + console.set_name("console") # pylint: disable=E1101 except AttributeError: - console.name = "console" + console.name = "console" # pylint: disable=W0201 logging.root.addHandler(console) @@ -162,9 +162,9 @@ def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG): ('localhost', 514), syslog_facility) try: - syslog.set_name("syslog") + syslog.set_name("syslog") # pylint: disable=E1101 except AttributeError: - syslog.name = "syslog" + syslog.name = "syslog" # pylint: disable=W0201 syslog.setLevel(level) syslog.setFormatter( logging.Formatter('%(name)s[%(process)d]: %(message)s')) @@ -179,9 +179,9 @@ def add_file_handler(to_file, level=logging.DEBUG): """Add a logging handler that logs to to_file.""" filelog = logging.FileHandler(to_file) try: - filelog.set_name("file") + filelog.set_name("file") # pylint: disable=E1101 except AttributeError: - filelog.name = "file" + filelog.name = "file" # pylint: disable=W0201 filelog.setLevel(level) filelog.setFormatter( logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(message)s')) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 7c91ca3cc..66e987b91 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -308,14 +308,14 @@ def list_split(c_string): """ split an option string on commas, optionally surrounded by whitespace, returning a list """ if c_string: - return re.split("\s*,\s*", c_string) + return re.split(r'\s*,\s*', c_string) return [] def colon_split(c_string): """ split an option string on colons, returning a list """ if c_string: - return c_string.split(':') + return c_string.split(r':') return [] @@ -355,7 +355,7 @@ def get_size(value): '512m', '2g'), get the absolute number of bytes as an integer """ if value == -1: return value - mat = re.match("(\d+)([KkMmGg])?", value) + mat = re.match(r'(\d+)([KkMmGg])?', value) if not mat: raise ValueError("Not a valid size", value) rvalue = int(mat.group(1)) diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py index a3c45b8c3..7bba05eb3 100644 --- a/src/lib/Bcfg2/Server/Admin/__init__.py +++ b/src/lib/Bcfg2/Server/Admin/__init__.py @@ -1,29 +1,14 @@ """ Base classes for admin modes """ -__all__ = ['Backup', - 'Bundle', - 'Client', - 'Compare', - 'Group', - 'Init', - 'Minestruct', - 'Perf', - 'Pull', - 'Query', - 'Reports', - 'Snapshots', - 'Syncdb', - 'Tidy', - 'Viz', - 'Xcmd'] - import re import sys import logging import lxml.etree import Bcfg2.Server.Core import Bcfg2.Options -from Bcfg2.Compat import ConfigParser +from Bcfg2.Compat import ConfigParser, walk_packages + +__all__ = [m[1] for m in walk_packages(path=__path__)] class Mode(object): diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index be4264b07..c709d639c 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -87,12 +87,12 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): command=None)}, ACL=dict( default=dict(scope=lambda v: v in ['user', 'group'], - perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}', + perms=lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}', v)), access=dict(scope=lambda v: v in ['user', 'group'], - perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}', + perms=lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}', v)), - mask=dict(perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}', + mask=dict(perms=lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}', v))), Package={"__any__": dict(name=None)}, SEBoolean={None: dict(name=None, diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index bddd0ba20..a7cdea879 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -41,15 +41,15 @@ del DEFAULT_FILE_METADATA['configfile'] LOGGER = logging.getLogger(__name__) #: a compiled regular expression for parsing info and :info files -INFO_REGEX = re.compile('owner:(\s)*(?P\S+)|' + - 'group:(\s)*(?P\S+)|' + - 'mode:(\s)*(?P\w+)|' + - 'secontext:(\s)*(?P\S+)|' + - 'paranoid:(\s)*(?P\S+)|' + - 'sensitive:(\s)*(?P\S+)|' + - 'encoding:(\s)*(?P\S+)|' + - 'important:(\s)*(?P\S+)|' + - 'mtime:(\s)*(?P\w+)|') +INFO_REGEX = re.compile(r'owner:(\s)*(?P\S+)|' + + r'group:(\s)*(?P\S+)|' + + r'mode:(\s)*(?P\w+)|' + + r'secontext:(\s)*(?P\S+)|' + + r'paranoid:(\s)*(?P\S+)|' + + r'sensitive:(\s)*(?P\S+)|' + + r'encoding:(\s)*(?P\S+)|' + + r'important:(\s)*(?P\S+)|' + + r'mtime:(\s)*(?P\w+)|') def bind_info(entry, metadata, infoxml=None, default=DEFAULT_FILE_METADATA): @@ -873,7 +873,7 @@ class XMLDirectoryBacked(DirectoryBacked): #: Only track and include files whose names (not paths) match this #: compiled regex. - patterns = re.compile('^.*\.xml$') + patterns = re.compile(r'^.*\.xml$') #: The type of child objects to create for files contained within #: the directory that is tracked. Default is @@ -1142,7 +1142,7 @@ class EntrySet(Debuggable): #: file is encountered that does not match the ``basename`` #: argument passed to the constructor or ``ignore``, then a #: warning will be produced. - ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\\.genshi_include)$") + ignore = re.compile(r'^(\.#.*|.*~|\\..*\\.(sw[px])|.*\\.genshi_include)$') # The ``basename`` argument passed to the constructor will be #: processed as a string that contains a regular expression (i.e., @@ -1205,8 +1205,8 @@ class EntrySet(Debuggable): base_pat = basename else: base_pat = re.escape(basename) - pattern = '(.*/)?%s(\.((H_(?P\S+))|' % base_pat - pattern += '(G(?P\d+)_(?P\S+))))?$' + pattern = r'(.*/)?%s(\.((H_(?P\S+))|' % base_pat + pattern += r'(G(?P\d+)_(?P\S+))))?$' #: ``specific`` is a regular expression that is used to #: determine the specificity of a file in this entry set. It diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index a9b9bf8c1..5c5e3da0c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -83,7 +83,7 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, """ The bundler creates dependent clauses based on the bundle/translation scheme from Bcfg1. """ __author__ = 'bcfg-dev@mcs.anl.gov' - patterns = re.compile('^(?P.*)\.(xml|genshi)$') + patterns = re.compile(r'^(?P.*)\.(xml|genshi)$') def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py index c2e5afbad..83a5c1165 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -90,7 +90,7 @@ class CfgGenshiGenerator(CfgGenerator): #: exception in a Genshi template so we can provide a decent error #: message that actually tells the end user where an error #: occurred. - pyerror_re = re.compile('<\w+ u?[\'"](.*?)\s*\.\.\.[\'"]>') + pyerror_re = re.compile(r'<\w+ u?[\'"](.*?)\s*\.\.\.[\'"]>') def __init__(self, fname, spec, encoding): CfgGenerator.__init__(self, fname, spec, encoding) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index f347b898c..ffc26713f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -111,12 +111,12 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, components = ['^(?P%s)' % '|'.join(re.escape(b) for b in basenames)] if cls.__specific__: - components.append('(|\\.H_(?P\S+?)|' + - '\.G(?P\d+)_(?P\S+?))') + components.append(r'(|\\.H_(?P\S+?)|' + + r'\.G(?P\d+)_(?P\S+?))') if cls.__extensions__: - components.append('\\.(?P%s)' % - '|'.join(cls.__extensions__)) - components.append('$') + components.append(r'\\.(?P%s)' % + r'|'.join(cls.__extensions__)) + components.append(r'$') return re.compile("".join(components)) @classmethod diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index fcfae2088..8d1e50526 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -23,8 +23,8 @@ class PatternMap(object): raise Exception("Backslashes are not allowed in NameRanges") range_finder = r'\[\[[\d\-,]+\]\]' self.process = self.process_range - self.re = re.compile('^' + re.sub(range_finder, '(\d+)', - rangestr)) + self.re = re.compile(r'^' + re.sub(range_finder, r'(\d+)', + rangestr)) dmatcher = re.compile(re.sub(range_finder, r'\[\[([\d\-,]+)\]\]', rangestr)) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 10b09842a..3611f2c6e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1603,6 +1603,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): (etype.title(), ename, "\n".join(els))) def default_is_profile(self): + """ ensure that the default group is a profile group """ if (self.metadata.default and not self.metadata.groups[self.metadata.default].is_profile): xdata = \ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 5ae160994..4eefd0722 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -104,8 +104,8 @@ class AptSource(Source): vindex = 0 for dep in words[1].split(','): if '|' in dep: - cdeps = [re.sub('\s+', '', - re.sub('\(.*\)', '', cdep)) + cdeps = [re.sub(r'\s+', '', + re.sub(r'\(.*\)', '', cdep)) for cdep in dep.split('|')] dyn_dname = "choice-%s-%s-%s" % (pkgname, barch, @@ -114,7 +114,7 @@ class AptSource(Source): bdeps[barch][pkgname].append(dyn_dname) bprov[barch][dyn_dname] = set(cdeps) else: - raw_dep = re.sub('\(.*\)', '', dep) + raw_dep = re.sub(r'\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() bdeps[barch][pkgname].append(raw_dep) elif words[0] == 'Provides': diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 0c6ee0759..b4d481459 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -65,7 +65,7 @@ def fetch_url(url): :raises: URLError - Failure fetching URL :returns: string - the content of the page at the given URL """ if '@' in url: - mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url) + mobj = re.match(r'(\w+://)([^:]+):([^@]+)@(.*)$', url) if not mobj: raise ValueError("Invalid URL") user = mobj.group(2) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 77186d2cb..7438c633b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -422,7 +422,7 @@ class YumCollection(Collection): config.add_section(reponame) added = True except ConfigParser.DuplicateSectionError: - match = re.search("-(\d+)", reponame) + match = re.search(r'-(\d+)', reponame) if match: rid = int(match.group(1)) + 1 else: diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 634826f4b..d57eb4228 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -111,15 +111,15 @@ class ProbeData(str): class ProbeSet(Bcfg2.Server.Plugin.EntrySet): """ Handle universal and group- and host-specific probe files """ - ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") + ignore = re.compile(r'^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$') probename = \ - re.compile("(.*/)?(?P\S+?)(\.(?P(?:G\d\d)|H)_\S+)?$") - bangline = re.compile('^#!\s*(?P.*)$') + re.compile(r'(.*/)?(?P\S+?)(\.(?P(?:G\d\d)|H)_\S+)?$') + bangline = re.compile(r'^#!\s*(?P.*)$') basename_is_regex = True def __init__(self, path, fam, encoding, plugin_name): self.plugin_name = plugin_name - Bcfg2.Server.Plugin.EntrySet.__init__(self, '[0-9A-Za-z_\-]+', path, + Bcfg2.Server.Plugin.EntrySet.__init__(self, r'[0-9A-Za-z_\-]+', path, Bcfg2.Server.Plugin.SpecificData, encoding) fam.AddMonitor(path, self) diff --git a/src/lib/Bcfg2/Server/Plugins/__init__.py b/src/lib/Bcfg2/Server/Plugins/__init__.py index 063e14315..ad51cf368 100644 --- a/src/lib/Bcfg2/Server/Plugins/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/__init__.py @@ -1,30 +1,5 @@ """Imports for Bcfg2.Server.Plugins.""" -__all__ = ['Account', - 'Base', - 'Bundler', - 'Bzr', - 'Cfg', - 'Cvs', - 'Darcs', - 'Decisions', - 'Fossil', - 'Git', - 'GroupPatterns', - 'Hg', - 'Hostbase', - 'Metadata', - 'NagiosGen', - 'Ohai', - 'Packages', - 'Properties', - 'Probes', - 'Pkgmgr', - 'Rules', - 'SSHbase', - 'Snapshots', - 'Statistics', - 'Svn', - 'TCheetah', - 'Trigger', - 'TGenshi'] +from Bcfg2.Compat import walk_packages + +__all__ = [m[1] for m in walk_packages(path=__path__)] diff --git a/src/lib/Bcfg2/Server/__init__.py b/src/lib/Bcfg2/Server/__init__.py index 3eb300a98..0678e4579 100644 --- a/src/lib/Bcfg2/Server/__init__.py +++ b/src/lib/Bcfg2/Server/__init__.py @@ -1,10 +1,9 @@ """This is the set of modules for Bcfg2.Server.""" import lxml.etree +from Bcfg2.Compat import walk_packages -__all__ = ["Admin", "Core", "FileMonitor", "Plugin", "Plugins", - "Hostbase", "Reports", "Snapshots", "XMLParser", - "XI", "XI_NAMESPACE"] +__all__ = [m[1] for m in walk_packages(path=__path__)] XI = 'http://www.w3.org/2001/XInclude' XI_NAMESPACE = '{%s}' % XI diff --git a/src/lib/Bcfg2/__init__.py b/src/lib/Bcfg2/__init__.py index 3fe2a0d75..41743d100 100644 --- a/src/lib/Bcfg2/__init__.py +++ b/src/lib/Bcfg2/__init__.py @@ -1,3 +1,4 @@ """Base modules definition.""" -__all__ = ['Server', 'Client', 'Logger', 'Options', 'Proxy', 'Statistics'] +from Bcfg2.Compat import walk_packages +__all__ = [m[1] for m in walk_packages(path=__path__)] diff --git a/testsuite/pylintrc.conf b/testsuite/pylintrc.conf index 14ccd1d23..15f8ae23c 100644 --- a/testsuite/pylintrc.conf +++ b/testsuite/pylintrc.conf @@ -33,7 +33,7 @@ load-plugins=ext.exception_messages # can either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). -disable=F0401,W0142,W0511,W0603,W1201,R0201,R0801,R0901,R0902,R0903,R0904,R0921,R0922,C0302,I0011,E0100,E0101,E0102,E0106 +disable=F0401,W0142,W0511,W0603,W1201,R0201,R0801,R0901,R0902,R0903,R0904,R0921,R0922,R0924,C0302,I0011,E0100,E0101,E0102,E0106 # Some of these are disabled because they warn about things we _want_: # # * W0142: Used * or ** magic @@ -41,9 +41,12 @@ disable=F0401,W0142,W0511,W0603,W1201,R0201,R0801,R0901,R0902,R0903,R0904,R0921, # * I0011: Locally disabling a pylint message # * R0921: Abstract class not referenced # * R0922: Abstract class is only referenced a small number of times +# * R0924: Badly implemented Container # # We have several modules, e.g., Bcfg2.Server.Plugin.interfaces, that # only declare abstract classes, which makes R0921 and R0922 useless. +# We also have several container objects that are immutable, which +# R0924 doesn't like. # Some of these are disabled because they just aren't that useful: # -- cgit v1.2.3-1-g7c22 From 7ad5c9c34a92080a5c426f9498ac4d09b615ec35 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 17:18:01 -0400 Subject: Plugin: use appropriate loggers for Debuggable helper objects --- src/lib/Bcfg2/Server/Plugin/helpers.py | 69 +++++++++++++++++----------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index a7cdea879..4bfef89f3 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -231,10 +231,10 @@ class FileBacked(Debuggable): self.Index() except IOError: err = sys.exc_info()[1] - LOGGER.error("Failed to read file %s: %s" % (self.name, err)) + self.logger.error("Failed to read file %s: %s" % (self.name, err)) except: err = sys.exc_info()[1] - LOGGER.error("Failed to parse file %s: %s" % (self.name, err)) + self.logger.error("Failed to parse file %s: %s" % (self.name, err)) def Index(self): """ Index() is called by :func:`HandleEvent` every time the @@ -329,7 +329,7 @@ class DirectoryBacked(Debuggable): dirpathname = os.path.join(self.data, relative) if relative not in self.handles.values(): if not os.path.isdir(dirpathname): - LOGGER.error("%s is not a directory" % dirpathname) + self.logger.error("%s is not a directory" % dirpathname) return reqid = self.fam.AddMonitor(dirpathname, self) self.handles[reqid] = relative @@ -374,8 +374,8 @@ class DirectoryBacked(Debuggable): return if event.requestID not in self.handles: - LOGGER.warn("Got %s event with unknown handle (%s) for %s" % - (action, event.requestID, event.filename)) + self.logger.warn("Got %s event with unknown handle (%s) for %s" % + (action, event.requestID, event.filename)) return # Clean up path names @@ -385,7 +385,7 @@ class DirectoryBacked(Debuggable): event.filename = event.filename[len(self.data) + 1:] if self.ignore and self.ignore.search(event.filename): - LOGGER.debug("Ignoring event %s" % event.filename) + self.logger.debug("Ignoring event %s" % event.filename) return # Calculate the absolute and relative paths this event refers to @@ -420,19 +420,20 @@ class DirectoryBacked(Debuggable): # class doesn't support canceling, so at least let # the user know that a restart might be a good # idea. - LOGGER.warn("Directory properties for %s changed, please " - " consider restarting the server" % abspath) + self.logger.warn("Directory properties for %s changed, " + "please consider restarting the server" % + abspath) else: # Got a "changed" event for a directory that we # didn't know about. Go ahead and treat it like a # "created" event, but log a warning, because this # is unexpected. - LOGGER.warn("Got %s event for unexpected dir %s" % - (action, abspath)) + self.logger.warn("Got %s event for unexpected dir %s" % + (action, abspath)) self.add_directory_monitor(relpath) else: - LOGGER.warn("Got unknown dir event %s %s %s" % - (event.requestID, event.code2str(), abspath)) + self.logger.warn("Got unknown dir event %s %s %s" % + (event.requestID, event.code2str(), abspath)) elif self.patterns.search(event.filename): if action in ['exists', 'created']: self.add_entry(relpath, event) @@ -444,16 +445,15 @@ class DirectoryBacked(Debuggable): # know about. Go ahead and treat it like a # "created" event, but log a warning, because this # is unexpected. - LOGGER.warn("Got %s event for unexpected file %s" % - (action, - abspath)) + self.logger.warn("Got %s event for unexpected file %s" % + (action, abspath)) self.add_entry(relpath, event) else: - LOGGER.warn("Got unknown file event %s %s %s" % - (event.requestID, event.code2str(), abspath)) + self.logger.warn("Got unknown file event %s %s %s" % + (event.requestID, event.code2str(), abspath)) else: - LOGGER.warn("Could not process filename %s; ignoring" % - event.filename) + self.logger.warn("Could not process filename %s; ignoring" % + event.filename) class XMLFileBacked(FileBacked): @@ -559,9 +559,9 @@ class XMLFileBacked(FileBacked): if not extras: msg = "%s: %s does not exist, skipping" % (self.name, name) if el.findall('./%sfallback' % Bcfg2.Server.XI_NAMESPACE): - LOGGER.debug(msg) + self.logger.debug(msg) else: - LOGGER.warning(msg) + self.logger.warning(msg) parent = el.getparent() parent.remove(el) @@ -581,7 +581,8 @@ class XMLFileBacked(FileBacked): self.xdata.getroottree().xinclude() except lxml.etree.XIncludeError: err = sys.exc_info()[1] - LOGGER.error("XInclude failed on %s: %s" % (self.name, err)) + self.logger.error("XInclude failed on %s: %s" % (self.name, + err)) self.entries = self.xdata.getchildren() if self.__identifier__ is not None: @@ -820,7 +821,7 @@ class XMLSrc(XMLFileBacked): data = open(self.name).read() except IOError: msg = "Failed to read file %s: %s" % (self.name, sys.exc_info()[1]) - LOGGER.error(msg) + self.logger.error(msg) raise PluginExecutionError(msg) self.items = {} try: @@ -828,7 +829,7 @@ class XMLSrc(XMLFileBacked): except lxml.etree.XMLSyntaxError: msg = "Failed to parse file %s: %s" % (self.name, sys.exc_info()[1]) - LOGGER.error(msg) + self.logger.error(msg) raise PluginExecutionError(msg) self.pnode = self.__node__(xdata, self.items) self.cache = None @@ -838,7 +839,7 @@ class XMLSrc(XMLFileBacked): if self.__priority_required__: msg = "Got bogus priority %s for file %s" % \ (xdata.get('priority'), self.name) - LOGGER.error(msg) + self.logger.error(msg) raise PluginExecutionError(msg) del xdata, data @@ -848,8 +849,8 @@ class XMLSrc(XMLFileBacked): if self.cache is None or self.cache[0] != metadata: cache = (metadata, self.__cacheobj__()) if self.pnode is None: - LOGGER.error("Cache method called early for %s; " - "forcing data load" % self.name) + self.logger.error("Cache method called early for %s; " + "forcing data load" % self.name) self.HandleEvent() return self.pnode.Match(metadata, cache[1]) @@ -1285,8 +1286,8 @@ class EntrySet(Debuggable): self.entry_init(event) else: if event.filename not in self.entries: - LOGGER.warning("Got %s event for unknown file %s" % - (action, event.filename)) + self.logger.warning("Got %s event for unknown file %s" % + (action, event.filename)) if action == 'changed': # received a bogus changed event; warn, but treat # it like a created event @@ -1322,7 +1323,7 @@ class EntrySet(Debuggable): entry_type = self.entry_type if event.filename in self.entries: - LOGGER.warn("Got duplicate add for %s" % event.filename) + self.logger.warn("Got duplicate add for %s" % event.filename) else: fpath = os.path.join(self.path, event.filename) try: @@ -1330,8 +1331,8 @@ class EntrySet(Debuggable): specific=specific) except SpecificityError: if not self.ignore.match(event.filename): - LOGGER.error("Could not process filename %s; ignoring" % - fpath) + self.logger.error("Could not process filename %s; ignoring" + % fpath) return self.entries[event.filename] = entry_type(fpath, spec, self.encoding) @@ -1396,8 +1397,8 @@ class EntrySet(Debuggable): for line in open(fpath).readlines(): match = INFO_REGEX.match(line) if not match: - LOGGER.warning("Failed to match line in %s: %s" % (fpath, - line)) + self.logger.warning("Failed to match line in %s: %s" % + (fpath, line)) continue else: mgd = match.groupdict() -- cgit v1.2.3-1-g7c22 From f6b458324f0be89f48229d4d1b5f3be9ae047497 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 17:18:31 -0400 Subject: testsuite: can't disable pylint R0924, since it doesn't exist on older pylint and pylint barfs --- src/lib/Bcfg2/Server/Plugin/helpers.py | 9 +++++++++ src/lib/Bcfg2/Utils.py | 15 ++++++++++++++- src/lib/Bcfg2/version.py | 2 +- testsuite/pylintrc.conf | 5 +---- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 4bfef89f3..0123d68a1 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -313,6 +313,15 @@ class DirectoryBacked(Debuggable): def __getitem__(self, key): return self.entries[key] + def __len__(self): + return len(self.entries) + + def __delitem__(self, key): + del self.entries[key] + + def __setitem__(self, key, val): + self.entries[key] = val + def __iter__(self): return iter(list(self.entries.items())) diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 601217556..ed5c0a377 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -22,7 +22,7 @@ class ClassName(object): return owner.__name__ -class PackedDigitRange(object): +class PackedDigitRange(object): # pylint: disable=E0012,R0924 """ Representation of a set of integer ranges. A range is described by a comma-delimited string of integers and ranges, e.g.:: @@ -145,6 +145,19 @@ class ExecutorResult(object): returned a tuple of (return value, stdout split by lines). """ return (self.retval, self.stdout.splitlines())[idx] + def __len__(self): + """ This provides compatibility with the old Executor, which + returned a tuple of (return value, stdout split by lines). """ + return 2 + + def __delitem__(self, _): + raise TypeError("'%s' object doesn't support item deletion" % + self.__class__.__name__) + + def __setitem__(self, idx, val): + raise TypeError("'%s' object does not support item assignment" % + self.__class__.__name__) + def __nonzero__(self): return self.__bool__() diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 6f3ba3e49..12fc584fe 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -5,7 +5,7 @@ import re __version__ = "1.3.1" -class Bcfg2VersionInfo(tuple): +class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 """ object to make granular version operations (particularly comparisons) easier """ diff --git a/testsuite/pylintrc.conf b/testsuite/pylintrc.conf index 15f8ae23c..14ccd1d23 100644 --- a/testsuite/pylintrc.conf +++ b/testsuite/pylintrc.conf @@ -33,7 +33,7 @@ load-plugins=ext.exception_messages # can either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). -disable=F0401,W0142,W0511,W0603,W1201,R0201,R0801,R0901,R0902,R0903,R0904,R0921,R0922,R0924,C0302,I0011,E0100,E0101,E0102,E0106 +disable=F0401,W0142,W0511,W0603,W1201,R0201,R0801,R0901,R0902,R0903,R0904,R0921,R0922,C0302,I0011,E0100,E0101,E0102,E0106 # Some of these are disabled because they warn about things we _want_: # # * W0142: Used * or ** magic @@ -41,12 +41,9 @@ disable=F0401,W0142,W0511,W0603,W1201,R0201,R0801,R0901,R0902,R0903,R0904,R0921, # * I0011: Locally disabling a pylint message # * R0921: Abstract class not referenced # * R0922: Abstract class is only referenced a small number of times -# * R0924: Badly implemented Container # # We have several modules, e.g., Bcfg2.Server.Plugin.interfaces, that # only declare abstract classes, which makes R0921 and R0922 useless. -# We also have several container objects that are immutable, which -# R0924 doesn't like. # Some of these are disabled because they just aren't that useful: # -- cgit v1.2.3-1-g7c22 From 1ab5df5b1ed6b6082ba453677450bb1b177fcfc0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 17:19:45 -0400 Subject: fixed regex errors introduced by 6c996f42c53a36fc0406f836d64b8c1bec6f4bcc --- src/lib/Bcfg2/Server/Plugin/helpers.py | 6 +++--- src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 4 ++-- src/lib/Bcfg2/Server/Plugins/Probes.py | 2 +- testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 0123d68a1..8cc1439e0 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -1152,7 +1152,7 @@ class EntrySet(Debuggable): #: file is encountered that does not match the ``basename`` #: argument passed to the constructor or ``ignore``, then a #: warning will be produced. - ignore = re.compile(r'^(\.#.*|.*~|\\..*\\.(sw[px])|.*\\.genshi_include)$') + ignore = re.compile(r'^(\.#.*|.*~|\..*\.(sw[px])|.*\.genshi_include)$') # The ``basename`` argument passed to the constructor will be #: processed as a string that contains a regular expression (i.e., @@ -1215,8 +1215,8 @@ class EntrySet(Debuggable): base_pat = basename else: base_pat = re.escape(basename) - pattern = r'(.*/)?%s(\.((H_(?P\S+))|' % base_pat - pattern += r'(G(?P\d+)_(?P\S+))))?$' + pattern = r'(.*/)?' + base_pat + \ + r'(\.((H_(?P\S+))|(G(?P\d+)_(?P\S+))))?$' #: ``specific`` is a regular expression that is used to #: determine the specificity of a file in this entry set. It diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index ffc26713f..926172e57 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -111,10 +111,10 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, components = ['^(?P%s)' % '|'.join(re.escape(b) for b in basenames)] if cls.__specific__: - components.append(r'(|\\.H_(?P\S+?)|' + + components.append(r'(|\.H_(?P\S+?)|' + r'\.G(?P\d+)_(?P\S+?))') if cls.__extensions__: - components.append(r'\\.(?P%s)' % + components.append(r'\.(?P%s)' % r'|'.join(cls.__extensions__)) components.append(r'$') return re.compile("".join(components)) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index d57eb4228..fa56564fa 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -111,7 +111,7 @@ class ProbeData(str): class ProbeSet(Bcfg2.Server.Plugin.EntrySet): """ Handle universal and group- and host-specific probe files """ - ignore = re.compile(r'^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$') + ignore = re.compile(r'^(\.#.*|.*~|\..*\.(tmp|sw[px])|probed\.xml)$') probename = \ re.compile(r'(.*/)?(?P\S+?)(\.(?P(?:G\d\d)|H)_\S+)?$') bangline = re.compile(r'^#!\s*(?P.*)$') diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index fceddcc69..b77e4b647 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -1477,7 +1477,8 @@ class TestEntrySet(TestDebuggable): bogus))) for ignore in self.ignore: - self.assertTrue(eset.ignore.match(ignore)) + self.assertTrue(eset.ignore.match(ignore), + "%s should be ignored but wasn't" % ignore) self.assertFalse(eset.ignore.match(basename)) self.assertFalse(eset.ignore.match(basename + ".G20_foo")) -- cgit v1.2.3-1-g7c22 From 0fae9849fd7047c299468fd6728db56d6861ee12 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 17:20:11 -0400 Subject: Probes: fixed unit tests for new use of lxml.etree._ElementTree.write instead of open().write() --- .../Testlib/TestServer/TestPlugins/TestProbes.py | 178 ++++++++++++++------- 1 file changed, 118 insertions(+), 60 deletions(-) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index 2163aa037..1022bdc5a 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -1,5 +1,6 @@ import os import sys +import copy import time import lxml.etree import Bcfg2.Server @@ -25,6 +26,47 @@ test_data = dict(a=1, b=[1, 2, 3], c="test", d=dict(a=1, b=dict(a=1), c=(1, "2", 3))) +class FakeElement(lxml.etree._Element): + getroottree = Mock() + + def __init__(self, el): + self._element = el + + def __getattribute__(self, attr): + el = lxml.etree._Element.__getattribute__(self, + '__dict__')['_element'] + if attr == "getroottree": + return FakeElement.getroottree + elif attr == "_element": + return el + else: + return getattr(el, attr) + + +class StoringElement(object): + OriginalElement = copy.copy(lxml.etree.Element) + + def __init__(self): + self.element = None + self.return_value = None + + def __call__(self, *args, **kwargs): + self.element = self.OriginalElement(*args, **kwargs) + self.return_value = FakeElement(self.element) + return self.return_value + + +class StoringSubElement(object): + OriginalSubElement = copy.copy(lxml.etree.SubElement) + + def __call__(self, parent, tag, **kwargs): + try: + return self.OriginalSubElement(parent._element, tag, + **kwargs) + except AttributeError: + return self.OriginalSubElement(parent, tag, **kwargs) + + class FakeList(list): pass @@ -288,61 +330,71 @@ text probes._write_data_db.assert_called_with("test") self.assertFalse(probes._write_data_xml.called) - @patch("%s.open" % builtins) - def test__write_data_xml(self, mock_open): + def test__write_data_xml(self): probes = self.get_probes_object(use_db=False) probes.probedata = self.get_test_probedata() probes.cgroups = self.get_test_cgroups() - probes._write_data_xml(None) - - mock_open.assert_called_with(os.path.join(datastore, probes.name, - "probed.xml"), "w") - data = lxml.etree.XML(mock_open.return_value.write.call_args[0][0]) - self.assertEqual(len(data.xpath("//Client")), 2) - - foodata = data.find("Client[@name='foo.example.com']") - self.assertIsNotNone(foodata) - self.assertIsNotNone(foodata.get("timestamp")) - self.assertEqual(len(foodata.findall("Probe")), - len(probes.probedata['foo.example.com'])) - self.assertEqual(len(foodata.findall("Group")), - len(probes.cgroups['foo.example.com'])) - xml = foodata.find("Probe[@name='xml']") - self.assertIsNotNone(xml) - self.assertIsNotNone(xml.get("value")) - xdata = lxml.etree.XML(xml.get("value")) - self.assertIsNotNone(xdata) - self.assertIsNotNone(xdata.find("test")) - self.assertEqual(xdata.find("test").get("foo"), "foo") - text = foodata.find("Probe[@name='text']") - self.assertIsNotNone(text) - self.assertIsNotNone(text.get("value")) - multiline = foodata.find("Probe[@name='multiline']") - self.assertIsNotNone(multiline) - self.assertIsNotNone(multiline.get("value")) - self.assertGreater(len(multiline.get("value").splitlines()), 1) - - bardata = data.find("Client[@name='bar.example.com']") - self.assertIsNotNone(bardata) - self.assertIsNotNone(bardata.get("timestamp")) - self.assertEqual(len(bardata.findall("Probe")), - len(probes.probedata['bar.example.com'])) - self.assertEqual(len(bardata.findall("Group")), - len(probes.cgroups['bar.example.com'])) - empty = bardata.find("Probe[@name='empty']") - self.assertIsNotNone(empty) - self.assertIsNotNone(empty.get("value")) - self.assertEqual(empty.get("value"), "") - if HAS_JSON: - jdata = bardata.find("Probe[@name='json']") - self.assertIsNotNone(jdata) - self.assertIsNotNone(jdata.get("value")) - self.assertItemsEqual(test_data, json.loads(jdata.get("value"))) - if HAS_YAML: - ydata = bardata.find("Probe[@name='yaml']") - self.assertIsNotNone(ydata) - self.assertIsNotNone(ydata.get("value")) - self.assertItemsEqual(test_data, yaml.load(ydata.get("value"))) + + @patch("lxml.etree.Element") + @patch("lxml.etree.SubElement", StoringSubElement()) + def inner(mock_Element): + mock_Element.side_effect = StoringElement() + probes._write_data_xml(None) + + top = mock_Element.side_effect.return_value + write = top.getroottree.return_value.write + self.assertEqual(write.call_args[0][0], + os.path.join(datastore, probes.name, + "probed.xml")) + + data = top._element + foodata = data.find("Client[@name='foo.example.com']") + self.assertIsNotNone(foodata) + self.assertIsNotNone(foodata.get("timestamp")) + self.assertEqual(len(foodata.findall("Probe")), + len(probes.probedata['foo.example.com'])) + self.assertEqual(len(foodata.findall("Group")), + len(probes.cgroups['foo.example.com'])) + xml = foodata.find("Probe[@name='xml']") + self.assertIsNotNone(xml) + self.assertIsNotNone(xml.get("value")) + xdata = lxml.etree.XML(xml.get("value")) + self.assertIsNotNone(xdata) + self.assertIsNotNone(xdata.find("test")) + self.assertEqual(xdata.find("test").get("foo"), "foo") + text = foodata.find("Probe[@name='text']") + self.assertIsNotNone(text) + self.assertIsNotNone(text.get("value")) + multiline = foodata.find("Probe[@name='multiline']") + self.assertIsNotNone(multiline) + self.assertIsNotNone(multiline.get("value")) + self.assertGreater(len(multiline.get("value").splitlines()), 1) + + bardata = data.find("Client[@name='bar.example.com']") + self.assertIsNotNone(bardata) + self.assertIsNotNone(bardata.get("timestamp")) + self.assertEqual(len(bardata.findall("Probe")), + len(probes.probedata['bar.example.com'])) + self.assertEqual(len(bardata.findall("Group")), + len(probes.cgroups['bar.example.com'])) + empty = bardata.find("Probe[@name='empty']") + self.assertIsNotNone(empty) + self.assertIsNotNone(empty.get("value")) + self.assertEqual(empty.get("value"), "") + if HAS_JSON: + jdata = bardata.find("Probe[@name='json']") + self.assertIsNotNone(jdata) + self.assertIsNotNone(jdata.get("value")) + self.assertItemsEqual(test_data, + json.loads(jdata.get("value"))) + if HAS_YAML: + ydata = bardata.find("Probe[@name='yaml']") + self.assertIsNotNone(ydata) + self.assertIsNotNone(ydata.get("value")) + self.assertItemsEqual(test_data, + yaml.load(ydata.get("value"))) + + inner() @skipUnless(HAS_DJANGO, "Django not found, skipping") def test__write_data_db(self): @@ -414,18 +466,24 @@ text probes._load_data_db.assert_any_call() self.assertFalse(probes._load_data_xml.called) - @patch("%s.open" % builtins) @patch("lxml.etree.parse") - def test__load_data_xml(self, mock_parse, mock_open): + def test__load_data_xml(self, mock_parse): probes = self.get_probes_object(use_db=False) - # to get the value for lxml.etree.parse to parse, we call - # _write_data_xml, mock the open() call, and grab the data - # that gets "written" to probed.xml probes.probedata = self.get_test_probedata() probes.cgroups = self.get_test_cgroups() - probes._write_data_xml(None) - xdata = \ - lxml.etree.XML(str(mock_open.return_value.write.call_args[0][0])) + + # to get the value for lxml.etree.parse to parse, we call + # _write_data_xml, mock the lxml.etree._ElementTree.write() + # call, and grab the data that gets "written" to probed.xml + @patch("lxml.etree.Element") + @patch("lxml.etree.SubElement", StoringSubElement()) + def inner(mock_Element): + mock_Element.side_effect = StoringElement() + probes._write_data_xml(None) + top = mock_Element.side_effect.return_value + return top._element + + xdata = inner() mock_parse.return_value = xdata.getroottree() probes.probedata = dict() probes.cgroups = dict() -- cgit v1.2.3-1-g7c22 From bc35aa70ab8794b73019d90a41eb252fbb80ff52 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 22:12:20 -0400 Subject: testsuite: fixed more unit test stuff --- src/lib/Bcfg2/Client/Tools/Pacman.py | 2 +- src/lib/Bcfg2/Client/Tools/SMF.py | 2 +- src/lib/Bcfg2/Server/Lint/RequiredAttrs.py | 8 ++++---- src/lib/Bcfg2/Server/Plugin/helpers.py | 8 ++++---- src/lib/Bcfg2/Server/Plugins/Fossil.py | 4 ++-- src/lib/Bcfg2/Server/Plugins/Metadata.py | 2 +- src/lib/Bcfg2/Server/Plugins/Ohai.py | 23 +++++++++++++++++++--- .../Server/Plugins/Packages/PackagesSources.py | 1 + src/lib/Bcfg2/Server/Plugins/Probes.py | 5 +++-- src/lib/Bcfg2/Server/Plugins/SSHbase.py | 2 +- src/lib/Bcfg2/Server/Plugins/TemplateHelper.py | 2 +- .../Testlib/TestServer/TestPlugin/Testhelpers.py | 1 - testsuite/install.sh | 3 ++- 13 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index be6698e18..a4cfd3315 100644 --- a/src/lib/Bcfg2/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py @@ -31,7 +31,7 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): '''Verify Package status for entry''' self.logger.info("VerifyPackage: %s : %s" % (entry.get('name'), - entry.get('version'))) + entry.get('version'))) if not 'version' in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % diff --git a/src/lib/Bcfg2/Client/Tools/SMF.py b/src/lib/Bcfg2/Client/Tools/SMF.py index 32a3df2f5..8b23a4a37 100644 --- a/src/lib/Bcfg2/Client/Tools/SMF.py +++ b/src/lib/Bcfg2/Client/Tools/SMF.py @@ -76,7 +76,7 @@ class SMF(Bcfg2.Client.Tools.SvcTool): if entry.get("FMRI").startswith('lrc'): try: loc = entry.get("FMRI")[4:].replace('_', '.') - self.logger.debug("Renaming file %s to %s" % \ + self.logger.debug("Renaming file %s to %s" % (loc, loc.replace('/S', '/DISABLED.S'))) os.rename(loc, loc.replace('/S', '/DISABLED.S')) return True diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index c709d639c..40ff71dbd 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -5,9 +5,9 @@ import os import re import lxml.etree import Bcfg2.Server.Lint -import Bcfg2.Client.Tools.POSIX import Bcfg2.Client.Tools.VCS from Bcfg2.Server.Plugins.Packages import Apt, Yum +from Bcfg2.Client.Tools.POSIX.base import device_map try: from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile HAS_GENSHI = True @@ -59,10 +59,10 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.required_attrs = dict( Path=dict( - device=dict(name=is_filename, owner=is_username, + device=dict(name=is_filename, + owner=is_username, group=is_username, - dev_type=lambda v: - v in Bcfg2.Client.Tools.POSIX.base.device_map), + dev_type=lambda v: v in device_map), directory=dict(name=is_filename, owner=is_username, group=is_username, mode=is_octal_mode), file=dict(name=is_filename, owner=is_username, diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 8cc1439e0..00b55a83b 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -796,14 +796,14 @@ class InfoNode (INode): Client="lambda m, e: '%(name)s' == m.hostname and predicate(m, e)", Group="lambda m, e: '%(name)s' in m.groups and predicate(m, e)", Path="lambda m, e: ('%(name)s' == e.get('name') or " + - "'%(name)s' == e.get('realname')) and " + - "predicate(m, e)") + "'%(name)s' == e.get('realname')) and " + + "predicate(m, e)") nraw = dict( Client="lambda m, e: '%(name)s' != m.hostname and predicate(m, e)", Group="lambda m, e: '%(name)s' not in m.groups and predicate(m, e)", Path="lambda m, e: '%(name)s' != e.get('name') and " + - "'%(name)s' != e.get('realname') and " + - "predicate(m, e)") + "'%(name)s' != e.get('realname') and " + + "predicate(m, e)") containers = ['Group', 'Client', 'Path'] diff --git a/src/lib/Bcfg2/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py index d0c328b36..6165ac651 100644 --- a/src/lib/Bcfg2/Server/Plugins/Fossil.py +++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py @@ -23,8 +23,8 @@ class Fossil(Bcfg2.Server.Plugin.Version): shell=True, cwd=self.vcs_root, stdout=PIPE).stdout.readlines() - revline = [line.split(': ')[1].strip() for line in data if \ - line.split(': ')[0].strip() == 'checkout'][-1] + revline = [line.split(': ')[1].strip() for line in data + if line.split(': ')[0].strip() == 'checkout'][-1] return revline.split(' ')[0] except IndexError: msg = "Failed to read fossil info" diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 3611f2c6e..b181c2237 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -446,7 +446,7 @@ class MetadataQuery(object): return [self.by_name(name) for name in self.all_clients()] -class MetadataGroup(tuple): +class MetadataGroup(tuple): # pylint: disable=E0012,R0924 """ representation of a metadata group. basically just a named tuple """ # pylint: disable=R0913,W0613 diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index a88c245bd..8a1b07ac8 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -2,8 +2,10 @@ operating system using ohai (http://wiki.opscode.com/display/chef/Ohai) """ -import lxml.etree import os +import sys +import glob +import lxml.etree import Bcfg2.Server.Plugin try: @@ -31,22 +33,37 @@ class OhaiCache(object): self.dirname = dirname self.cache = dict() + def itempath(self, item): + return os.path.join(self.dirname, "%s.json" % item) + def __setitem__(self, item, value): if value is None: # simply return if the client returned nothing return self.cache[item] = json.loads(value) - open("%s/%s.json" % (self.dirname, item), 'w').write(value) + open(self.itempath(item), 'w').write(value) def __getitem__(self, item): if item not in self.cache: try: - data = open("%s/%s.json" % (self.dirname, item)).read() + data = open(self.itempath(item)).read() except: raise KeyError(item) self.cache[item] = json.loads(data) return self.cache[item] + def __delitem__(self, item): + if item in self.cache: + del self.cache[item] + try: + os.unlink(self.itempath(item)) + except: + raise IndexError("Could not unlink %s: %s" % (self.itempath(item), + sys.exc_info()[1])) + + def __len__(self): + return len(glob.glob(self.itempath('*'))) + def __iter__(self): data = list(self.cache.keys()) data.extend([x[:-5] for x in os.listdir(self.dirname)]) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index ff57d57e8..332f0bbab 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -7,6 +7,7 @@ import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Packages.Source import SourceInitError +# pylint: disable=E0012,R0924 class PackagesSources(Bcfg2.Server.Plugin.StructFile, Bcfg2.Server.Plugin.Debuggable): """ PackagesSources handles parsing of the diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index fa56564fa..a8001d9af 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -58,7 +58,7 @@ class ClientProbeDataSet(dict): dict.__init__(self, *args, **kwargs) -class ProbeData(str): +class ProbeData(str): # pylint: disable=E0012,R0924 """ a ProbeData object emulates a str object, but also has .xdata, .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ @@ -231,9 +231,10 @@ class Probes(Bcfg2.Server.Plugin.Probing, if pdata.data != data: pdata.data = data pdata.save() + ProbesDataModel.objects.filter( hostname=client.hostname).exclude( - probe__in=self.probedata[client.hostname]).delete() + probe__in=self.probedata[client.hostname]).delete() for group in self.cgroups[client.hostname]: try: diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 0c5644395..fc07a90e9 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -205,7 +205,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, chain( *[names[cmeta.hostname] for cmeta in - mquery.by_groups([specific.group])])) + mquery.by_groups([specific.group])])) elif specific.all: # a generic key for all hosts? really? hostnames = list(chain(*list(names.values()))) diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index ea7454e11..7dd15f7b5 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -82,7 +82,7 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): """ A plugin to provide helper classes and functions to templates """ __author__ = 'chris.a.st.pierre@gmail.com' - ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$") + ignore = re.compile(r'^(\.#.*|.*~|\..*\.(sw[px])|.*\.py[co])$') patterns = MODULE_RE __child__ = HelperModule diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index b77e4b647..58e61e13b 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -235,7 +235,6 @@ class TestDirectoryBacked(Bcfg2TestCase): mock_isdir.return_value = True for path in self.testpaths.values(): reset() - print "testing %s" % path db.add_directory_monitor(path) db.fam.AddMonitor.assert_called_with(os.path.join(db.data, path), db) diff --git a/testsuite/install.sh b/testsuite/install.sh index c6249002d..817ed5911 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -9,7 +9,8 @@ PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0: if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then pip install --use-mirrors genshi PyYAML pyinotify if [[ $PYVER == "2.5" ]]; then - pip install --use-mirrors simplejson + # markdown 2.2+ doesn't work on py2.5 + pip install --use-mirrors simplejson 'markdown<2.2' fi if [[ ${PYVER:0:1} == "2" ]]; then # django supports py3k, but South doesn't, and the django bits -- cgit v1.2.3-1-g7c22 From 4a364848c6d0e64a38d5d481ff978c519389814c Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 23:12:51 -0400 Subject: testsuite: more text fixes --- doc/development/compat.txt | 2 ++ src/lib/Bcfg2/Client/Frame.py | 2 +- src/lib/Bcfg2/Client/Tools/SELinux.py | 3 ++- src/lib/Bcfg2/Compat.py | 10 +++++++++- src/lib/Bcfg2/Proxy.py | 2 ++ src/lib/Bcfg2/SSLServer.py | 3 ++- src/lib/Bcfg2/Server/Plugins/Metadata.py | 1 + src/lib/Bcfg2/Server/Plugins/Ohai.py | 16 +++++++++------- src/lib/Bcfg2/Utils.py | 2 +- src/sbin/bcfg2-build-reports | 2 +- .../TestPlugins/TestCfg/TestCfgGenshiGenerator.py | 1 + testsuite/requirements.txt | 2 +- 12 files changed, 32 insertions(+), 14 deletions(-) diff --git a/doc/development/compat.txt b/doc/development/compat.txt index 90df45676..f90274ce5 100644 --- a/doc/development/compat.txt +++ b/doc/development/compat.txt @@ -100,6 +100,8 @@ behavior (e.g., :func:`input`) do not cause unexpected side-effects. +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ | long | :func:`long` | :func:`int` | +---------------------------------+--------------------------------------------------+---------------------------------------------------------+ +| cmp | :func:`cmp` | Not implemented | ++---------------------------------+--------------------------------------------------+---------------------------------------------------------+ Python 2.4 compatibility ------------------------ diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index fe4943fdf..ada5320b8 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -6,7 +6,7 @@ import fnmatch import logging import Bcfg2.Client.Tools from Bcfg2.Client import prompt -from Bcfg2.Compat import any, all # pylint: disable=W0622 +from Bcfg2.Compat import any, all, cmp # pylint: disable=W0622 def cmpent(ent1, ent2): diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index 617b7c2ef..fe7d09ce9 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -12,6 +12,7 @@ import seobject import Bcfg2.Client.XML import Bcfg2.Client.Tools from Bcfg2.Client.Tools.POSIX.File import POSIXFile +from Bcfg2.Compat import long def pack128(int_val): @@ -47,7 +48,7 @@ def netmask_itoa(netmask, proto="ipv4"): if netmask > size: raise ValueError("Netmask too large: %s" % netmask) - res = 0L + res = long(0) for i in range(netmask): res |= 1 << (size - i - 1) netmask = socket.inet_ntop(family, pack128(res)) diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 44c76303c..47f62e26b 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -10,7 +10,7 @@ Python 2.4 and such-like """ import sys -# pylint: disable=E0611,W0611,W0622,C0103 +# pylint: disable=E0601,E0602,E0611,W0611,W0622,C0103 try: from email.Utils import formatdate @@ -270,3 +270,11 @@ try: except NameError: # longs are just ints in py3k long = int + + +try: + cmp +except NameError: + def cmp(a, b): + """ Py3k implementation of cmp() """ + return (a > b) - (a < b) diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index 62b83d0b4..b2b9fcc2e 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -326,6 +326,7 @@ class XMLRPCTransport(xmlrpclib.Transport): return self.parse_response(response) if sys.hexversion < 0x03000000: + # pylint: disable=E1101 def send_request(self, host, handler, request_body, debug): """ send_request() changed significantly in py3k.""" conn = self.make_connection(host) @@ -334,6 +335,7 @@ class XMLRPCTransport(xmlrpclib.Transport): self.send_user_agent(conn) self.send_content(conn, request_body) return conn + # pylint: enable=E1101 def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None, diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index a149b676f..2e1762ce4 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -52,10 +52,11 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): allow_none=self.allow_none, encoding=self.encoding) except: + err = sys.exc_info() self.logger.error("Unexpected handler error", exc_info=1) # report exception back to server raw_response = xmlrpclib.dumps( - xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value)), + xmlrpclib.Fault(1, "%s:%s" % (err[0].__name__, err[1]) allow_none=self.allow_none, encoding=self.encoding) return raw_response diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index b181c2237..7561aec1e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -75,6 +75,7 @@ if HAS_DJANGO: yield client.hostname def keys(self): + """ Get keys for the mapping """ return [c.hostname for c in MetadataClientModel.objects.all()] def __contains__(self, key): diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index 8a1b07ac8..1ec3cbd60 100644 --- a/src/lib/Bcfg2/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py @@ -33,20 +33,22 @@ class OhaiCache(object): self.dirname = dirname self.cache = dict() - def itempath(self, item): - return os.path.join(self.dirname, "%s.json" % item) + def hostpath(self, host): + """ Get the path to the file that contains Ohai data for the + given host """ + return os.path.join(self.dirname, "%s.json" % host) def __setitem__(self, item, value): if value is None: # simply return if the client returned nothing return self.cache[item] = json.loads(value) - open(self.itempath(item), 'w').write(value) + open(self.hostpath(item), 'w').write(value) def __getitem__(self, item): if item not in self.cache: try: - data = open(self.itempath(item)).read() + data = open(self.hostpath(item)).read() except: raise KeyError(item) self.cache[item] = json.loads(data) @@ -56,13 +58,13 @@ class OhaiCache(object): if item in self.cache: del self.cache[item] try: - os.unlink(self.itempath(item)) + os.unlink(self.hostpath(item)) except: - raise IndexError("Could not unlink %s: %s" % (self.itempath(item), + raise IndexError("Could not unlink %s: %s" % (self.hostpath(item), sys.exc_info()[1])) def __len__(self): - return len(glob.glob(self.itempath('*'))) + return len(glob.glob(self.hostpath('*'))) def __iter__(self): data = list(self.cache.keys()) diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index ed5c0a377..39cf5255e 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -210,7 +210,7 @@ class Executor(object): :type timeout: float :returns: :class:`Bcfg2.Utils.ExecutorResult` """ - if isinstance(command, basestring): + if isinstance(command, str): cmdstr = command else: cmdstr = " ".join(command) diff --git a/src/sbin/bcfg2-build-reports b/src/sbin/bcfg2-build-reports index 27e7c2475..1c9e9ad97 100755 --- a/src/sbin/bcfg2-build-reports +++ b/src/sbin/bcfg2-build-reports @@ -13,7 +13,7 @@ import sys from time import asctime, strptime from lxml.etree import XML, XSLT, parse, Element, ElementTree, SubElement, tostring, XMLSyntaxError # Compatibility imports -from Bcfg2.Compat import ConfigParser +from Bcfg2.Compat import ConfigParser, cmp def generatereport(rspec, nrpt): """ diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py index 385f8df77..2e8b7bfa5 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py @@ -2,6 +2,7 @@ import os import sys import lxml.etree from mock import Mock, MagicMock, patch +import Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import * from Bcfg2.Server.Plugin import PluginExecutionError diff --git a/testsuite/requirements.txt b/testsuite/requirements.txt index c59810a99..42862b4ef 100644 --- a/testsuite/requirements.txt +++ b/testsuite/requirements.txt @@ -2,6 +2,6 @@ lxml nose mock sphinx -daemon +python-daemon pylint pep8 -- cgit v1.2.3-1-g7c22 From 75a16795fc460c4f83fce06a1662c3c6b19a4cfc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 26 Mar 2013 23:33:19 -0400 Subject: testsuite: more test fixes --- src/lib/Bcfg2/Client/Tools/SELinux.py | 2 +- src/lib/Bcfg2/SSLServer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index fe7d09ce9..0041ce61a 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -12,7 +12,7 @@ import seobject import Bcfg2.Client.XML import Bcfg2.Client.Tools from Bcfg2.Client.Tools.POSIX.File import POSIXFile -from Bcfg2.Compat import long +from Bcfg2.Compat import long # pylint: disable=W0622 def pack128(int_val): diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index 2e1762ce4..7c557a4d7 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -56,7 +56,7 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher): self.logger.error("Unexpected handler error", exc_info=1) # report exception back to server raw_response = xmlrpclib.dumps( - xmlrpclib.Fault(1, "%s:%s" % (err[0].__name__, err[1]) + xmlrpclib.Fault(1, "%s:%s" % (err[0].__name__, err[1])), allow_none=self.allow_none, encoding=self.encoding) return raw_response -- cgit v1.2.3-1-g7c22 From 26ba8791756c180da86b6dc3eecb6ebf03ee23f5 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 27 Mar 2013 13:28:55 -0500 Subject: Compat: Fix traceback Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 47f62e26b..4e9239e26 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -273,7 +273,7 @@ except NameError: try: - cmp + cmp = cmp except NameError: def cmp(a, b): """ Py3k implementation of cmp() """ -- cgit v1.2.3-1-g7c22 From 810bed32be1edf6c79cf0952029d31339461eaf9 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 27 Mar 2013 13:39:00 -0400 Subject: Inotify: use correct logger --- src/lib/Bcfg2/Server/FileMonitor/Inotify.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py index cdd52dbb9..2cdf27ed8 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -3,14 +3,11 @@ support. """ import os import errno -import logging import pyinotify from Bcfg2.Compat import reduce # pylint: disable=W0622 from Bcfg2.Server.FileMonitor import Event from Bcfg2.Server.FileMonitor.Pseudo import Pseudo -LOGGER = logging.getLogger(__name__) - class Inotify(Pseudo, pyinotify.ProcessEvent): """ File monitor backend with `inotify @@ -123,8 +120,9 @@ class Inotify(Pseudo, pyinotify.ProcessEvent): try: watch = self.watchmgr.watches[ievent.wd] except KeyError: - LOGGER.error("Error handling event %s for %s: Watch %s not found" % - (action, ievent.pathname, ievent.wd)) + self.logger.error("Error handling event %s for %s: " + "Watch %s not found" % + (action, ievent.pathname, ievent.wd)) return # FAM-style file monitors return the full path to the parent # directory that is being watched, relative paths to anything -- cgit v1.2.3-1-g7c22 From b90d6dccd515551829f2661a5d855f92d1e23103 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 27 Mar 2013 13:40:46 -0400 Subject: XMLFileBacked: track FAM monitors separately from processed xincludes to avoid infinite loop with pseudo FAM --- src/lib/Bcfg2/Server/Plugin/helpers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 00b55a83b..2b878d7e2 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -526,6 +526,10 @@ class XMLFileBacked(FileBacked): #: "Extra" files included in this file by XInclude. self.extras = [] + #: Extra FAM monitors set by this object for files included by + #: XInclude. + self.extra_monitors = [] + if ((create or (self.create is not None and self.create)) and not os.path.exists(self.name)): toptag = create or self.create @@ -576,9 +580,11 @@ class XMLFileBacked(FileBacked): parent.remove(el) for extra in extras: if extra != self.name and extra not in self.extras: - self.add_monitor(extra) + self.extras.append(extra) lxml.etree.SubElement(parent, xinclude, href=extra) self._follow_xincludes(fname=extra) + if extra not in self.extra_monitors: + self.add_monitor(extra) def Index(self): self.xdata = lxml.etree.XML(self.data, base_url=self.name, @@ -607,7 +613,7 @@ class XMLFileBacked(FileBacked): :type fpath: string :returns: None """ - self.extras.append(fpath) + self.extra_monitors.append(fpath) if self.fam and self.should_monitor: self.fam.AddMonitor(fpath, self) -- cgit v1.2.3-1-g7c22 From e28a8f7c5764427f69a3335b974ab22c8429b600 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 27 Mar 2013 13:41:16 -0400 Subject: skip Compat plugins for anything >= 1.3.0, not > --- src/lib/Bcfg2/Server/Plugins/POSIXCompat.py | 2 +- src/lib/Bcfg2/Server/Plugins/ServiceCompat.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py index 490ee6f20..1736becc7 100644 --- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py @@ -15,7 +15,7 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin, def validate_goals(self, metadata, goals): """Verify that we are generating correct old POSIX entries.""" - if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0): + if metadata.version_info and metadata.version_info >= (1, 3, 0, '', 0): # do not care about a client that is _any_ 1.3.0 release # (including prereleases and RCs) return diff --git a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py index 0aea439f9..c3a2221f6 100644 --- a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py @@ -14,7 +14,7 @@ class ServiceCompat(Bcfg2.Server.Plugin.Plugin, def validate_goals(self, metadata, config): """ Apply defaults """ - if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0): + if metadata.version_info and metadata.version_info >= (1, 3, 0, '', 0): # do not care about a client that is _any_ 1.3.0 release # (including prereleases and RCs) return -- cgit v1.2.3-1-g7c22 From 919d3a664dc3c211fabc3298e8a3648d275c17e2 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 27 Mar 2013 16:44:58 -0400 Subject: Metadata: create clients when version is declared (if necessary) --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 34 +++++++++++++------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 7561aec1e..bdf3b87fe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -944,16 +944,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if group not in self.groups: self.debug_log("Client %s set as nonexistent group %s" % (client, group)) - for gname, ginfo in list(self.groups.items()): - for group in ginfo.groups: - if group not in self.groups: - self.debug_log("Group %s set as nonexistent group %s" % - (gname, group)) def set_profile(self, client, profile, addresspair): """Set group parameter for provided client.""" - self.logger.info("Asserting client %s profile to %s" % - (client, profile)) + self.logger.info("Asserting client %s profile to %s" % (client, + profile)) if False in list(self.states.values()): raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not " "been read yet") @@ -1004,19 +999,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.clients_xml.write() def set_version(self, client, version): - """Set group parameter for provided client.""" - if client in self.clients: - if client not in self.versions or version != self.versions[client]: - self.logger.info("Setting client %s version to %s" % - (client, version)) - if not self._use_db: - self.update_client(client, dict(version=version)) - self.clients_xml.write() - self.versions[client] = version - else: - msg = "Cannot set version on non-existent client %s" % client - self.logger.error(msg) - raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) + """Set version for provided client.""" + if client not in self.clients: + # this creates the client as a side effect + self.get_initial_metadata(client) + + if client not in self.versions or version != self.versions[client]: + self.logger.info("Setting client %s version to %s" % (client, + version)) + if not self._use_db: + self.update_client(client, dict(version=version)) + self.clients_xml.write() + self.versions[client] = version def resolve_client(self, addresspair, cleanup_cache=False): """Lookup address locally or in DNS to get a hostname.""" -- cgit v1.2.3-1-g7c22 From d6bb92cfb417f2e2786645246ac03ea2d4b1bece Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 27 Mar 2013 16:47:27 -0400 Subject: testsuite: install python-daemon last so pylint/pep8 tests are run on py3k --- testsuite/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/requirements.txt b/testsuite/requirements.txt index 42862b4ef..2d6dbc557 100644 --- a/testsuite/requirements.txt +++ b/testsuite/requirements.txt @@ -2,6 +2,6 @@ lxml nose mock sphinx -python-daemon pylint pep8 +python-daemon -- cgit v1.2.3-1-g7c22 From 3d60058e1361ece220964d6e25245b6cf3a1e6e1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 27 Mar 2013 16:57:13 -0400 Subject: testsuite: fix more tests --- src/lib/Bcfg2/SSLServer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index 7c557a4d7..990bcd512 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -198,8 +198,10 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): try: username, password = auth_content.split(":") except TypeError: + # pylint: disable=E0602 username, pw = auth_content.split(bytes(":", encoding='utf-8')) password = pw.decode('utf-8') + # pylint: enable=E0602 except ValueError: username = auth_content password = "" -- cgit v1.2.3-1-g7c22 From 52824a1cf8786c318269f71f0cc7c234d4a92f73 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 28 Mar 2013 10:22:53 -0400 Subject: bcfg2_local.py: remove duplicate -Q options --- tools/bcfg2_local.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index 2b9d39342..70e8fc0b2 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -64,6 +64,11 @@ class LocalClient(Client): def main(): optinfo = Bcfg2.Options.CLIENT_COMMON_OPTIONS optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) + if 'bundle_quick' in optinfo: + # CLIENT_BUNDLEQUICK option uses -Q, just like the server repo + # option. the server repo is more important for this + # application. + del optinfo['bundle_quick'] setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) -- cgit v1.2.3-1-g7c22 From b8b15234bcf3fc4edecf219e2c3331e9eee54b17 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Thu, 28 Mar 2013 09:36:21 -0400 Subject: Pass through retval from start, stop, status, etc. The various init scripts have the usual start, stop, and status functions which are called from a case statement. The functions even nicely return various values for success and failure. Unfortunately, those values were not passed all the way back to the calling shell. Previously, the init scripts would return 0 if any of start, stop, or status failed. This commit ensures that they they pass the return value back to the caller. Why does this matter? Well, beyond just being generally good citizens, bcfg2's own chkconfig client tool expects stopped services to return a non-zero exit code when their status function is called. Otherwise it flags the service state as incorrect and tries to stop it on every run. --- debian/bcfg2-report-collector.init | 9 +++++++-- debian/bcfg2-server.init | 9 +++++++-- debian/bcfg2.init | 9 ++++++--- redhat/scripts/bcfg2-report-collector.init | 9 +++++++-- redhat/scripts/bcfg2-server.init | 3 +++ redhat/scripts/bcfg2.init | 4 +++- 6 files changed, 33 insertions(+), 10 deletions(-) diff --git a/debian/bcfg2-report-collector.init b/debian/bcfg2-report-collector.init index 2d182385a..df7b751cb 100755 --- a/debian/bcfg2-report-collector.init +++ b/debian/bcfg2-report-collector.init @@ -32,6 +32,7 @@ test -x $DAEMON || exit 5 # Internal variables BINARY=$(basename $DAEMON) +RETVAL=0 start () { echo -n "Starting Configuration Report Collector: " @@ -85,22 +86,26 @@ status () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status + RETVAL=$? ;; restart|reload|force-reload) stop sleep 5 start + RETVAL=$? ;; *) log_success_msg "Usage: $0 {start|stop|status|reload|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/debian/bcfg2-server.init b/debian/bcfg2-server.init index 8de16b9b5..04774c063 100755 --- a/debian/bcfg2-server.init +++ b/debian/bcfg2-server.init @@ -41,6 +41,7 @@ test -x $DAEMON || exit 5 # Internal variables BINARY=$(basename $DAEMON) +RETVAL=0 start () { echo -n "Starting Configuration Management Server: " @@ -91,22 +92,26 @@ status () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status + RETVAL=$? ;; restart|reload|force-reload) stop sleep 5 start + RETVAL=$? ;; *) log_success_msg "Usage: $0 {start|stop|status|reload|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/debian/bcfg2.init b/debian/bcfg2.init index 4f83adbf6..b2e47b346 100755 --- a/debian/bcfg2.init +++ b/debian/bcfg2.init @@ -47,6 +47,7 @@ fi # Internal variables BINARY=$(basename $BCFG2) +RETVAL=0 # Include lsb functions . /lib/lsb/init-functions @@ -70,17 +71,19 @@ start () { case "$1" in start) start + RETVAL=$? ;; stop|status) - exit 0 + RETVAL=0 ;; restart|force-reload) start + RETVAL=$? ;; *) echo "Usage: $0 {start|stop|status|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/redhat/scripts/bcfg2-report-collector.init b/redhat/scripts/bcfg2-report-collector.init index a8e23f080..43e875a6b 100755 --- a/redhat/scripts/bcfg2-report-collector.init +++ b/redhat/scripts/bcfg2-report-collector.init @@ -32,6 +32,7 @@ test -x $DAEMON || exit 5 # Internal variables BINARY=$(basename $DAEMON) +RETVAL=0 start () { echo -n "Starting Configuration Report Collector: " @@ -79,22 +80,26 @@ status () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status + RETVAL=$? ;; restart|reload|force-reload) stop sleep 5 start + RETVAL=$? ;; *) echo "Usage: $0 {start|stop|status|reload|restart|force-reload}" - exit 1 + RETVAL=1 ;; esac -exit 0 +exit $RETVAL diff --git a/redhat/scripts/bcfg2-server.init b/redhat/scripts/bcfg2-server.init index ffac6ac3d..c4412d1c3 100755 --- a/redhat/scripts/bcfg2-server.init +++ b/redhat/scripts/bcfg2-server.init @@ -59,9 +59,11 @@ stop () { case "$1" in start) start + RETVAL=$? ;; stop) stop + RETVAL=$? ;; status) status $prog @@ -71,6 +73,7 @@ case "$1" in stop sleep 5 start + RETVAL=$? ;; *) echo $"Usage: $0 {start|stop|status|restart|reload|force-reload}" diff --git a/redhat/scripts/bcfg2.init b/redhat/scripts/bcfg2.init index 5cfdf47bc..9c26434ff 100755 --- a/redhat/scripts/bcfg2.init +++ b/redhat/scripts/bcfg2.init @@ -54,12 +54,14 @@ start () { case "$1" in start) start + RETVAL=$? ;; stop|status) - exit 0 + RETVAL=0 ;; restart|force-reload) start + RETVAL=$? ;; *) echo "Usage: $0 {start|stop|status|restart|force-reload}" -- cgit v1.2.3-1-g7c22 From 0987e2a52629607a9868201085e9ba5b995f8fb8 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 28 Mar 2013 13:25:23 -0400 Subject: bcfg2-lint: ensure all XML properties files are parseable --- src/lib/Bcfg2/Server/Lint/Validate.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 37bc230d1..a5f41c7af 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -83,17 +83,12 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): else: self.LintError("properties-schema-not-found", "No schema found for %s" % filename) + # ensure that it at least parses + self.parse(filename) - def validate(self, filename, schemafile, schema=None): - """validate a file against the given lxml.etree.Schema. - return True on success, False on failure """ - if schema is None: - # if no schema object was provided, instantiate one - schema = self._load_schema(schemafile) - if not schema: - return False + def parse(self, filename): try: - datafile = lxml.etree.parse(filename) + return lxml.etree.parse(filename) except SyntaxError: lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT) self.LintError("xml-failed-to-parse", @@ -106,6 +101,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Failed to open file %s" % filename) return False + def validate(self, filename, schemafile, schema=None): + """validate a file against the given lxml.etree.Schema. + return True on success, False on failure """ + if schema is None: + # if no schema object was provided, instantiate one + schema = self._load_schema(schemafile) + if not schema: + return False + datafile = self.parse(filename) if not schema.validate(datafile): cmd = ["xmllint"] if self.files is None: -- cgit v1.2.3-1-g7c22 From af98f300fdf7362eb792df1456bf9f4a2fbc90bb Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 28 Mar 2013 15:24:16 -0400 Subject: new GroupLogic plugin --- doc/server/plugins/connectors/grouplogic.txt | 122 +++++++++++++++++++++++++++ schemas/grouplogic.xsd | 110 ++++++++++++++++++++++++ src/lib/Bcfg2/Server/Lint/Validate.py | 3 +- src/lib/Bcfg2/Server/Plugins/GroupLogic.py | 39 +++++++++ 4 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 doc/server/plugins/connectors/grouplogic.txt create mode 100644 schemas/grouplogic.xsd create mode 100644 src/lib/Bcfg2/Server/Plugins/GroupLogic.py diff --git a/doc/server/plugins/connectors/grouplogic.txt b/doc/server/plugins/connectors/grouplogic.txt new file mode 100644 index 000000000..b9a5b00d6 --- /dev/null +++ b/doc/server/plugins/connectors/grouplogic.txt @@ -0,0 +1,122 @@ +.. -*- mode: rst -*- + +.. _server-plugins-connectors-grouplogic: + +========== +GroupLogic +========== + +.. versionadded:: 1.3.2 + +GroupLogic is a connector plugin that lets you use an XML Genshi +template to dynamically set additional groups for clients. + +Usage +===== + +To use the GroupLogic plugin, first do ``mkdir +/var/lib/bcfg2/GroupLogic``. Add ``GroupLogic`` to your ``plugins`` +line in ``/etc/bcfg2.conf``. Next, create +``/var/lib/bcfg2/GroupLogic/groups.xml``: + +.. code-block:: xml + + + + +``groups.xml`` is structured very similarly to the +:ref:`server-plugins-grouping-metadata` ``groups.xml``. A Group tag +that contains no children is a declaration of membership; a Group or +Client tag that does contain children is a conditional. + +Unlike ``Metadata/groups.xml``, GroupLogic supports genshi templating, +so you can dynamically create groups. ``GroupLogic/groups.xml`` is +rendered for each client, and the groups set in it are added to the +client metadata. + +.. note:: + + Also unlike ``Metadata/groups.xml``, GroupLogic can not be used to + associate bundles with clients directly, or to negate groups. But + you can use GroupLogic to assign a group that is associated with a + bundle in Metadata. + +Consider the case where you have four environments -- dev, test, +staging, and production -- and four components to a web application -- +the frontend, the API, the database server, and the caching proxy. In +order to make files specific to the component *and* to the +environment, you need groups to describe each combination: +webapp-frontend-dev, webapp-frontend-test, and so on. You *could* do +this in ``Metadata/groups.xml``: + +.. code-block:: xml + + + + + + + + + + ... + + + ... + + ... + + +Creating the sixteen groups this way is incredibly tedious, and this +is a quite *small* site. GroupLogic can automate this process. + +Assume that we've declared the groups thusly in +``Metadata/groups.xml``: + +.. code-block:: xml + + + + + + + + + + + + +One way to automate the creation of the groups would be to simply +generate the tedious config: + +.. code-block:: xml + + + + + + + + + + + + + +But, since ``GroupLogic/groups.xml`` is rendered for each client +individually, there's a more elegant way to accomplish the same thing: + +.. code-block:: xml + + + + + + + + +This gets only the component and environment for the current client, +and, if both are set, sets the single appropriate group. diff --git a/schemas/grouplogic.xsd b/schemas/grouplogic.xsd new file mode 100644 index 000000000..bf43bceb3 --- /dev/null +++ b/schemas/grouplogic.xsd @@ -0,0 +1,110 @@ + + + + + GroupLogic schema for bcfg2 + + + + + + + + + A **GroupLogicDeclarationType** declares a Group to be added + to a client. + + + + + + The group name + + + + + + + + + + The top-level tag of a GroupLogic configuration file. + + + + + + + + + Elements within Group tags only apply to clients that are + members of that group (or vice-versa; see #element_negate + below) + + + + + + + Elements within Client tags only apply to the named client + (or vice-versa; see #element_negate below) + + + + + + + Nesting GroupLogic tags is allowed in order to support + XInclude. + + + + + + + + + + + A **GroupLogicContainerType** is a tag used to provide logic. + Child entries of a GroupLogicContainerType tag only apply to + machines that match the condition specified -- either + membership in a group, or a matching client name. + :xml:attribute:`GroupLogicContainerType:negate` can be set to + negate the sense of the match. + + + + + + + + The group name + + + + + + + Negate the sense of this group or client; i.e., entries + within this tag are only used on clients that are not + members of the group, or that have hostnames that do not + match. + + + + + + + + + + + A GroupLogic file is a genshi file that can be used to + dynamically add additional groups to a client. + + + + diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index a5f41c7af..14d17a1e6 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -40,7 +40,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", "SSLCA/**/cert.xml": "sslca-cert.xsd", - "SSLCA/**/key.xml": "sslca-key.xsd" + "SSLCA/**/key.xml": "sslca-key.xsd", + "GroupLogic/groups.xml": "grouplogic.xsd" } self.filelists = {} diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py new file mode 100644 index 000000000..c572fee2b --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -0,0 +1,39 @@ +import os +import lxml.etree +import Bcfg2.Server.Plugin +try: + from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile +except ImportError: + # BundleTemplateFile missing means that genshi is missing. we + # import genshi to get the _real_ error + import genshi # pylint: disable=W0611 + + +class GroupLogicConfig(BundleTemplateFile): + create = lxml.etree.Element("GroupLogic", + nsmap=dict(py="http://genshi.edgewall.org/")) + + def __init__(self, name, fam): + BundleTemplateFile.__init__(self, name, + Bcfg2.Server.Plugin.Specificity(), None) + self.fam = fam + self.should_monitor = True + self.fam.AddMonitor(self.name, self) + + def _match(self, item, metadata): + if item.tag == 'Group' and not len(item.getchildren()): + return [item] + return BundleTemplateFile._match(self, item, metadata) + + +class GroupLogic(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Connector.__init__(self) + self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), + core.fam) + + def get_additional_groups(self, metadata): + return [el.get("name") + for el in self.config.get_xml_value(metadata).findall("Group")] -- cgit v1.2.3-1-g7c22 From 1d4d10fb8e003b0dac64ea50d61aaac006e9e5e1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 28 Mar 2013 15:24:28 -0400 Subject: doc: fixed typos --- doc/server/plugins/grouping/metadata.txt | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/doc/server/plugins/grouping/metadata.txt b/doc/server/plugins/grouping/metadata.txt index fe0d2683e..32834b458 100644 --- a/doc/server/plugins/grouping/metadata.txt +++ b/doc/server/plugins/grouping/metadata.txt @@ -119,20 +119,19 @@ a simple ``groups.xml`` file: - + -A Group or Client tag that does not contain any child tags is a -declaration of membership; a Group or Client tag that does contain -children is a conditional. So the example above does not assign -either the ``rhel5`` or ``rhel6`` groups to machines in the -``mail-server`` group, but conditionally assigns the -``sendmail-server`` or ``postfix-server`` groups depending on the OS -of the client. (Presumably in this example the OS groups are set by a -probe.) +A Group tag that does not contain any child tags is a declaration of +membership; a Group or Client tag that does contain children is a +conditional. So the example above does not assign either the +``rhel5`` or ``rhel6`` groups to machines in the ``mail-server`` +group, but conditionally assigns the ``sendmail-server`` or +``postfix-server`` groups depending on the OS of the client. +(Presumably in this example the OS groups are set by a probe.) Consequently, a client that is RHEL 5 and a member of the ``mail-server`` profile group would also be a member of the -- cgit v1.2.3-1-g7c22 From 9a1ea02d4adedb867e04dd3b9698c4819762777c Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 28 Mar 2013 15:24:50 -0400 Subject: XMLFileBacked: fixed deprecated logic when create is lxml.etree._Element object --- src/lib/Bcfg2/Server/Plugin/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 2b878d7e2..b14968d77 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -530,8 +530,8 @@ class XMLFileBacked(FileBacked): #: XInclude. self.extra_monitors = [] - if ((create or (self.create is not None and self.create)) - and not os.path.exists(self.name)): + if ((create is not None or self.create not in [None, False]) and + not os.path.exists(self.name)): toptag = create or self.create self.logger.warning("%s does not exist, creating" % self.name) if hasattr(toptag, "getroottree"): -- cgit v1.2.3-1-g7c22 From 76f996d12103f446b785fd727480d12b2c6a6b91 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 28 Mar 2013 15:50:52 -0400 Subject: testsuite: fixed unit tests --- src/lib/Bcfg2/Encryption.py | 6 ++++- .../Testlib/TestServer/TestPlugin/Testhelpers.py | 26 +++++++++------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/lib/Bcfg2/Encryption.py b/src/lib/Bcfg2/Encryption.py index 2b4ba6237..b4674d72f 100755 --- a/src/lib/Bcfg2/Encryption.py +++ b/src/lib/Bcfg2/Encryption.py @@ -27,7 +27,7 @@ ALGORITHM = "aes_256_cbc" #: Default initialization vector. For best security, you should use a #: unique IV for each message. :func:`ssl_encrypt` does this in an #: automated fashion. -IV = '\0' * 16 +IV = r'\0' * 16 #: The config file section encryption options and passphrases are #: stored in @@ -116,9 +116,11 @@ def ssl_decrypt(data, passwd, algorithm=ALGORITHM): # base64-decode the data data = b64decode(data) salt = data[8:16] + # pylint: disable=E1101 hashes = [md5(passwd + salt).digest()] for i in range(1, 3): hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + # pylint: enable=E1101 key = hashes[0] + hashes[1] iv = hashes[2] @@ -144,9 +146,11 @@ def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): if salt is None: salt = Rand.rand_bytes(8) + # pylint: disable=E1101 hashes = [md5(passwd + salt).digest()] for i in range(1, 3): hashes.append(md5(hashes[i - 1] + passwd + salt).digest()) + # pylint: enable=E1101 key = hashes[0] + hashes[1] iv = hashes[2] diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index 58e61e13b..94866cf39 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -410,11 +410,12 @@ class TestXMLFileBacked(TestFileBacked): should_monitor = None path = os.path.join(datastore, "test", "test1.xml") - @patch("os.makedirs", Mock()) def get_obj(self, path=None, fam=None, should_monitor=False): if path is None: path = self.path - @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock()) + + @patchIf(not isinstance(os.path.exists, Mock), + "os.path.exists", Mock()) def inner(): return self.test_obj(path, fam=fam, should_monitor=should_monitor) return inner() @@ -422,20 +423,16 @@ class TestXMLFileBacked(TestFileBacked): def test__init(self): fam = Mock() xfb = self.get_obj() - if self.should_monitor is True: + if self.should_monitor: self.assertIsNotNone(xfb.fam) + fam.reset_mock() + xfb = self.get_obj(fam=fam, should_monitor=True) + fam.AddMonitor.assert_called_with(self.path, xfb) else: self.assertIsNone(xfb.fam) - - if self.should_monitor is not True: xfb = self.get_obj(fam=fam) self.assertFalse(fam.AddMonitor.called) - if self.should_monitor is not False: - fam.reset_mock() - xfb = self.get_obj(fam=fam, should_monitor=True) - fam.AddMonitor.assert_called_with(self.path, xfb) - @patch("glob.glob") @patch("lxml.etree.parse") def test_follow_xincludes(self, mock_parse, mock_glob): @@ -623,7 +620,7 @@ class TestXMLFileBacked(TestFileBacked): def test_add_monitor(self): xfb = self.get_obj() xfb.add_monitor("/test/test2.xml") - self.assertIn("/test/test2.xml", xfb.extras) + self.assertIn("/test/test2.xml", xfb.extra_monitors) fam = Mock() if self.should_monitor is not True: @@ -632,14 +629,14 @@ class TestXMLFileBacked(TestFileBacked): fam.reset_mock() xfb.add_monitor("/test/test3.xml") self.assertFalse(fam.AddMonitor.called) - self.assertIn("/test/test3.xml", xfb.extras) + self.assertIn("/test/test3.xml", xfb.extra_monitors) if self.should_monitor is not False: fam.reset_mock() xfb = self.get_obj(fam=fam, should_monitor=True) xfb.add_monitor("/test/test4.xml") fam.AddMonitor.assert_called_with("/test/test4.xml", xfb) - self.assertIn("/test/test4.xml", xfb.extras) + self.assertIn("/test/test4.xml", xfb.extra_monitors) class TestStructFile(TestXMLFileBacked): @@ -2036,6 +2033,3 @@ class TestGroupSpool(TestPlugin, TestGenerator): gs.event_id.assert_called_with(event) self.assertNotIn("/baz/quux", gs.entries) self.assertNotIn("/baz/quux", gs.Entries[gs.entry_type]) - - - -- cgit v1.2.3-1-g7c22 From 4d29fe21b69795505ce27bccbdb59566bc20d7b5 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 28 Mar 2013 17:18:58 -0400 Subject: GroupLogic: added docstrings --- src/lib/Bcfg2/Server/Lint/Validate.py | 3 +++ src/lib/Bcfg2/Server/Plugins/GroupLogic.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 14d17a1e6..ae7c75804 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -88,6 +88,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.parse(filename) def parse(self, filename): + """ Parse an XML file, raising the appropriate LintErrors if + it can't be parsed or read. Return the + lxml.etree._ElementTree parsed from the file. """ try: return lxml.etree.parse(filename) except SyntaxError: diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index c572fee2b..810b273af 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -1,3 +1,6 @@ +""" GroupLogic is a connector plugin that lets you use an XML Genshi +template to dynamically set additional groups for clients. """ + import os import lxml.etree import Bcfg2.Server.Plugin @@ -10,6 +13,7 @@ except ImportError: class GroupLogicConfig(BundleTemplateFile): + """ Representation of the GroupLogic groups.xml file """ create = lxml.etree.Element("GroupLogic", nsmap=dict(py="http://genshi.edgewall.org/")) @@ -28,6 +32,10 @@ class GroupLogicConfig(BundleTemplateFile): class GroupLogic(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): + """ GroupLogic is a connector plugin that lets you use an XML + Genshi template to dynamically set additional groups for + clients. """ + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) -- cgit v1.2.3-1-g7c22 From 644da402180bfccfa3c033c88dabed6d8dfff6dd Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Thu, 28 Mar 2013 21:38:38 -0500 Subject: Change generated APT sources default to someplace APT will actually look, and avoid potential name conflicts there --- doc/server/plugins/generators/packages.txt | 40 +++++++++++------------ src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index 73145fd6b..606e1e128 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -434,7 +434,7 @@ configs. Simply add entries like these to the appropriate bundles: .. code-block:: xml - + If you want to change the path to either of those files, you can set ``yum_config`` or ``apt_config`` in ``bcfg2.conf`` to the path to the @@ -702,25 +702,25 @@ It understands the following directives: [packages] section ------------------ -+-------------+------------------------------------------------------+----------+-----------------------------+ -| Name | Description | Values | Default | -+=============+======================================================+==========+=============================+ -| resolver | Enable dependency resolution | Boolean | True | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| metadata | Enable metadata processing. Disabling ``metadata`` | Boolean | True | -| | implies disabling ``resolver`` as well. | | | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| yum_config | The path at which to generate Yum configs. | String | /etc/yum.repos.d/bcfg2.repo | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| apt_config | The path at which to generate APT configs. | String | /etc/apt/sources.d/bcfg2 | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| gpg_keypath | The path on the client RPM GPG keys will be copied | String | /etc/pki/rpm-gpg | -| | to before they are imported on the client. | | | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| version | Set the version attribute used when binding Packages | any|auto | auto | -+-------------+------------------------------------------------------+----------+-----------------------------+ -| cache | Path where Packages will store its cache | String | /Packages/cache | -+-------------+------------------------------------------------------+----------+-----------------------------+ ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| Name | Description | Values | Default | ++=============+======================================================+==========+===================================================================+ +| resolver | Enable dependency resolution | Boolean | True | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| metadata | Enable metadata processing. Disabling ``metadata`` | Boolean | True | +| | implies disabling ``resolver`` as well. | | | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| yum_config | The path at which to generate Yum configs. | String | /etc/yum.repos.d/bcfg2.repo | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| apt_config | The path at which to generate APT configs. | String | /etc/apt/sources.list.d/bcfg2-packages-generated-sources.list | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| gpg_keypath | The path on the client RPM GPG keys will be copied | String | /etc/pki/rpm-gpg | +| | to before they are imported on the client. | | | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| version | Set the version attribute used when binding Packages | any|auto | auto | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ +| cache | Path where Packages will store its cache | String | /Packages/cache | ++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+ [packages:yum] section diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 4f163a1ab..b75256318 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -18,7 +18,7 @@ from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" #: The default path for generated apt configs -APT_CONFIG_DEFAULT = "/etc/apt/sources.d/bcfg2" +APT_CONFIG_DEFAULT = "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" class Packages(Bcfg2.Server.Plugin.Plugin, -- cgit v1.2.3-1-g7c22 From 20d1a9597bb039d693978c2b02e62e65826a5436 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 29 Mar 2013 10:47:53 -0500 Subject: Probes: Handle unicode probes There is no good way of sending unicode probes to older clients which do not have support for them. This change will cause unicode probes to be skipped for unsupported clients and handled properly for new clients. Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Client/Client.py | 10 +++++----- src/lib/Bcfg2/Server/Plugins/Probes.py | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py index 5633764a8..613e10e75 100644 --- a/src/lib/Bcfg2/Client/Client.py +++ b/src/lib/Bcfg2/Client/Client.py @@ -91,7 +91,7 @@ class Client(object): try: script.write("#!%s\n" % (probe.attrib.get('interpreter', '/bin/sh'))) - script.write(probe.text) + script.write(probe.text.encode('utf-8')) script.close() os.chmod(scriptname, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | @@ -105,7 +105,7 @@ class Client(object): self._probe_failure(name, "Return value %s" % rv) self.logger.info("Probe %s has result:" % name) self.logger.info(rv.stdout) - ret.text = rv.stdout + ret.text = rv.stdout.decode('utf-8') finally: os.unlink(scriptname) except SystemExit: @@ -167,7 +167,7 @@ class Client(object): self.proxy.RecvProbeData( Bcfg2.Client.XML.tostring( probedata, - xml_declaration=False).decode('UTF-8')) + xml_declaration=False).decode('utf-8')) except Bcfg2.Proxy.ProxyError: err = sys.exc_info()[1] self.fatal_error("Failed to upload probe data: %s" % err) @@ -229,7 +229,7 @@ class Client(object): self.fatal_error("Failed to get decision list: %s" % err) try: - rawconfig = self.proxy.GetConfig().encode('UTF-8') + rawconfig = self.proxy.GetConfig().encode('utf-8') except Bcfg2.Proxy.ProxyError: err = sys.exc_info()[1] self.fatal_error("Failed to download configuration from " @@ -324,7 +324,7 @@ class Client(object): self.proxy.RecvStats( Bcfg2.Client.XML.tostring( feedback, - xml_declaration=False).decode('UTF-8')) + xml_declaration=False).decode('utf-8')) except Bcfg2.Proxy.ProxyError: err = sys.exc_info()[1] self.logger.error("Failed to upload configuration statistics: " diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index a8001d9af..09e648750 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -63,7 +63,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data) + return str.__new__(cls, data.encode('utf-8')) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -153,7 +153,17 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): probe = lxml.etree.Element('probe') probe.set('name', os.path.basename(name)) probe.set('source', self.plugin_name) - probe.text = entry.data + if metadata.version_info and metadata.version_info > (1, 3, 1, '', 0): + probe.text = entry.data.decode('utf-8') + else: + # msg = "Client unable to handle unicode probes." + # raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + try: + probe.text = entry.data + except: + self.logger.error("Client unable to handle unicode probes. " + "Skipping %s" % probe.get('name')) + continue match = self.bangline.match(entry.data.split('\n')[0]) if match: probe.set('interpreter', match.group('interpreter')) @@ -210,7 +220,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): lxml.etree.SubElement(ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe])) + value=str(self.probedata[client][probe]).decode('utf-8')) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: -- cgit v1.2.3-1-g7c22 From f22ce193d4d42298ebd4e7e2379f31d5a870690c Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 28 Mar 2013 22:31:57 -0400 Subject: Statistics: fixed calculation of average time --- src/lib/Bcfg2/Statistics.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Statistics.py b/src/lib/Bcfg2/Statistics.py index a869b03cd..01b0a80b1 100644 --- a/src/lib/Bcfg2/Statistics.py +++ b/src/lib/Bcfg2/Statistics.py @@ -30,8 +30,8 @@ class Statistic(object): """ self.min = min(self.min, value) self.max = max(self.max, value) - self.ave = (((self.ave * (self.count - 1)) + value) / self.count) self.count += 1 + self.ave = (((self.ave * (self.count - 1)) + value) / self.count) def get_value(self): """ Get a tuple of all the stats tracked on this named item. @@ -46,6 +46,11 @@ class Statistic(object): """ return (self.name, (self.min, self.max, self.ave, self.count)) + def __repr__(self): + return "%s(%s, (min=%s, avg=%s, max=%s, count=%s))" % ( + self.__class__.__name__, + self.name, self.min, self.ave, self.max, self.count) + class Statistics(object): """ A collection of named :class:`Statistic` objects. """ -- cgit v1.2.3-1-g7c22 From a80f9aa17c9af06ae6d7a739cc6888aa230f3a60 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 29 Mar 2013 10:41:44 -0400 Subject: fixed pep8 stuff in yum tool --- src/lib/Bcfg2/Client/Tools/YUM.py | 93 ++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 45 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index c9fae7fc7..c30c0a13a 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -131,10 +131,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool): def __init__(self, logger, setup, config): self.yumbase = self._loadYumBase(setup=setup, logger=logger) Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) - self.ignores = [entry.get('name') for struct in config \ - for entry in struct \ - if entry.tag == 'Path' and \ - entry.get('type') == 'ignore'] + self.ignores = [] + for struct in config: + self.ignores.extend([entry.get('name') + for entry in struct + if (entry.tag == 'Path' and + entry.get('type') == 'ignore')]) self.instance_status = {} self.extra_instances = [] self.modlists = {} @@ -293,8 +295,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): group. """ missing = Bcfg2.Client.Tools.PkgTool.missing_attrs(self, entry) - if entry.get('name', None) == None and \ - entry.get('group', None) == None: + if (entry.get('name', None) is None and + entry.get('group', None) is None): missing += ['name', 'group'] return missing @@ -422,10 +424,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if entry.get('group'): self.logger.debug("Verifying packages for group %s" % - entry.get('group')) + entry.get('group')) else: self.logger.debug("Verifying package instances for %s" % - entry.get('name')) + entry.get('name')) self.verify_cache = dict() # Used for checking multilib packages self.modlists[entry] = modlist @@ -434,10 +436,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool): package_fail = False qtext_versions = [] virt_pkg = False - pkg_checks = self.pkg_checks and \ - entry.get('pkg_checks', 'true').lower() == 'true' - pkg_verify = self.pkg_verify and \ - entry.get('pkg_verify', 'true').lower() == 'true' + pkg_checks = (self.pkg_checks and + entry.get('pkg_checks', 'true').lower() == 'true') + pkg_verify = (self.pkg_verify and + entry.get('pkg_verify', 'true').lower() == 'true') yum_group = False if entry.get('name') == 'gpg-pubkey': @@ -455,15 +457,13 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if d] group_type = entry.get('choose', 'default') if group_type in ['default', 'optional', 'all']: - group_packages += [p - for p, d in - group.default_packages.items() - if d] + group_packages += [ + p for p, d in group.default_packages.items() + if d] if group_type in ['optional', 'all']: - group_packages += [p - for p, d in - group.optional_packages.items() - if d] + group_packages += [ + p for p, d in group.optional_packages.items() + if d] if len(group_packages) == 0: self.logger.error("No packages found for group %s" % entry.get("group")) @@ -489,7 +489,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): else: all_pkg_objs = \ self.yumbase.rpmdb.searchNevra(name=entry.get('name')) - if len(all_pkg_objs) == 0 and yum_group != True: + if len(all_pkg_objs) == 0 and yum_group is not True: # Some sort of virtual capability? Try to resolve it all_pkg_objs = self.yumbase.rpmdb.searchProvides(entry.get('name')) if len(all_pkg_objs) > 0: @@ -567,9 +567,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool): pkg_objs = [po for po in all_pkg_objs] else: pkg_objs = [po for po in all_pkg_objs - if po.checkPrco('provides', - (nevra["name"], 'EQ', - tuple(vlist)))] + if po.checkPrco('provides', + (nevra["name"], 'EQ', + tuple(vlist)))] elif entry.get('name') == 'gpg-pubkey': if 'version' not in nevra: self.logger.warning("Skipping verify: gpg-pubkey without " @@ -622,7 +622,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if self.setup.get('quick', False): # Passed -q on the command line continue - if not (pkg_verify and \ + if not (pkg_verify and inst.get('pkg_verify', 'true').lower() == 'true'): continue @@ -648,8 +648,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool): # Now take out the Yum specific objects / modlists / unproblems ignores = [ig.get('name') for ig in entry.findall('Ignore')] + \ - [ig.get('name') for ig in inst.findall('Ignore')] + \ - self.ignores + [ig.get('name') for ig in inst.findall('Ignore')] + \ + self.ignores for fname, probs in list(vrfy_result.items()): if fname in modlist: self.logger.debug(" %s in modlist, skipping" % fname) @@ -737,8 +737,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool): for pkg in pkg_objs: self.logger.debug(" Extra Instance Found: %s" % str(pkg)) Bcfg2.Client.XML.SubElement(extra_entry, 'Instance', - epoch=pkg.epoch, name=pkg.name, version=pkg.version, - release=pkg.release, arch=pkg.arch) + epoch=pkg.epoch, name=pkg.name, + version=pkg.version, + release=pkg.release, arch=pkg.arch) if pkg_objs == []: return None @@ -782,7 +783,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): ver = yum.misc.keyIdToRPMVer(gpg['keyid']) rel = yum.misc.keyIdToRPMVer(gpg['timestamp']) if not (ver == inst.get('version') and rel == inst.get('release')): - self.logger.info("GPG key file %s does not match gpg-pubkey-%s-%s"\ + self.logger.info("GPG key file %s does not match gpg-pubkey-%s-%s" % (key_file, inst.get('version'), inst.get('release'))) return False @@ -791,20 +792,21 @@ class YUM(Bcfg2.Client.Tools.PkgTool): gpg['timestamp']) == 0: result = tset.pgpImportPubkey(yum.misc.procgpgkey(rawkey)) else: - self.logger.debug("gpg-pubkey-%s-%s already installed"\ - % (inst.get('version'), - inst.get('release'))) + self.logger.debug("gpg-pubkey-%s-%s already installed" % + (inst.get('version'), inst.get('release'))) return True if result != 0: - self.logger.debug("Unable to install %s-%s" % \ - (self.instance_status[inst].get('pkg').get('name'), - nevra2string(inst))) + self.logger.debug( + "Unable to install %s-%s" % + (self.instance_status[inst].get('pkg').get('name'), + nevra2string(inst))) return False else: - self.logger.debug("Installed %s-%s-%s" % \ - (self.instance_status[inst].get('pkg').get('name'), - inst.get('version'), inst.get('release'))) + self.logger.debug( + "Installed %s-%s-%s" % + (self.instance_status[inst].get('pkg').get('name'), + inst.get('version'), inst.get('release'))) return True def _runYumTransaction(self): @@ -898,7 +900,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): # Remove extra instances. # Can not reverify because we don't have a package entry. if self.extra_instances is not None and len(self.extra_instances) > 0: - if (self.setup.get('remove') == 'all' or \ + if (self.setup.get('remove') == 'all' or self.setup.get('remove') == 'packages'): self.Remove(self.extra_instances) else: @@ -913,7 +915,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool): # Figure out which instances of the packages actually need something # doing to them and place in the appropriate work 'queue'. for pkg in packages: - insts = [pinst for pinst in pkg \ + insts = [pinst for pinst in pkg if pinst.tag in ['Instance', 'Package']] if insts: for inst in insts: @@ -1006,10 +1008,11 @@ class YUM(Bcfg2.Client.Tools.PkgTool): if not self.setup['kevlar']: for pkg_entry in [p for p in packages if self.canVerify(p)]: - self.logger.debug("Reverifying Failed Package %s" \ - % (pkg_entry.get('name'))) - states[pkg_entry] = self.VerifyPackage(pkg_entry, - self.modlists.get(pkg_entry, [])) + self.logger.debug("Reverifying Failed Package %s" % + pkg_entry.get('name')) + states[pkg_entry] = \ + self.VerifyPackage(pkg_entry, + self.modlists.get(pkg_entry, [])) for entry in [ent for ent in packages if states[ent]]: self.modified.append(entry) -- cgit v1.2.3-1-g7c22 From a619bf537ceb33848012f334200a7e552d7b4bdd Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 29 Mar 2013 10:41:55 -0400 Subject: remove some unnecessary __all__ variables --- src/lib/Bcfg2/Client/__init__.py | 4 +--- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 3 ++- src/lib/Bcfg2/__init__.py | 3 --- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 8c8c4fd94..e40ef750b 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -3,9 +3,7 @@ import os import sys import select -from Bcfg2.Compat import input, walk_packages # pylint: disable=W0622 - -__all__ = [m[1] for m in walk_packages(path=__path__)] +from Bcfg2.Compat import input # pylint: disable=W0622 def prompt(msg): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index b75256318..efbca28cd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -18,7 +18,8 @@ from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo" #: The default path for generated apt configs -APT_CONFIG_DEFAULT = "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +APT_CONFIG_DEFAULT = \ + "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" class Packages(Bcfg2.Server.Plugin.Plugin, diff --git a/src/lib/Bcfg2/__init__.py b/src/lib/Bcfg2/__init__.py index 41743d100..74a871f2a 100644 --- a/src/lib/Bcfg2/__init__.py +++ b/src/lib/Bcfg2/__init__.py @@ -1,4 +1 @@ """Base modules definition.""" - -from Bcfg2.Compat import walk_packages -__all__ = [m[1] for m in walk_packages(path=__path__)] -- cgit v1.2.3-1-g7c22 From fa402689293283222915f372fa37f233c97f4ed6 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 29 Mar 2013 12:49:08 -0400 Subject: Probes: fixed long lines --- src/lib/Bcfg2/Server/Plugins/Probes.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 09e648750..55e5ddc47 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -153,16 +153,16 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): probe = lxml.etree.Element('probe') probe.set('name', os.path.basename(name)) probe.set('source', self.plugin_name) - if metadata.version_info and metadata.version_info > (1, 3, 1, '', 0): + if (metadata.version_info and + metadata.version_info > (1, 3, 1, '', 0)): probe.text = entry.data.decode('utf-8') else: - # msg = "Client unable to handle unicode probes." - # raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: probe.text = entry.data - except: - self.logger.error("Client unable to handle unicode probes. " - "Skipping %s" % probe.get('name')) + except: # pylint: disable=W0702 + self.logger.error("Client unable to handle unicode " + "probes. Skipping %s" % + probe.get('name')) continue match = self.bangline.match(entry.data.split('\n')[0]) if match: @@ -219,8 +219,9 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - lxml.etree.SubElement(ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe]).decode('utf-8')) + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str(self.probedata[client][probe]).decode('utf-8')) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: -- cgit v1.2.3-1-g7c22 From cbf60d32586bf848fed9d6b0a37451c6cf05fa99 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 29 Mar 2013 16:56:24 -0400 Subject: Statistics: wrote unit tests --- src/lib/Bcfg2/Statistics.py | 4 +-- testsuite/Testsrc/Testlib/TestStatistics.py | 44 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 testsuite/Testsrc/Testlib/TestStatistics.py diff --git a/src/lib/Bcfg2/Statistics.py b/src/lib/Bcfg2/Statistics.py index 01b0a80b1..3825941af 100644 --- a/src/lib/Bcfg2/Statistics.py +++ b/src/lib/Bcfg2/Statistics.py @@ -28,8 +28,8 @@ class Statistic(object): :param value: The value to add to this statistic :type value: int or float """ - self.min = min(self.min, value) - self.max = max(self.max, value) + self.min = min(self.min, float(value)) + self.max = max(self.max, float(value)) self.count += 1 self.ave = (((self.ave * (self.count - 1)) + value) / self.count) diff --git a/testsuite/Testsrc/Testlib/TestStatistics.py b/testsuite/Testsrc/Testlib/TestStatistics.py new file mode 100644 index 000000000..496cbac28 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestStatistics.py @@ -0,0 +1,44 @@ +import os +import sys +from mock import Mock, MagicMock, patch + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * + +from Bcfg2.Statistics import * + + +class TestStatistic(Bcfg2TestCase): + def test_stat(self): + stat = Statistic("test", 1) + self.assertEqual(stat.get_value(), ("test", (1.0, 1.0, 1.0, 1))) + stat.add_value(10) + self.assertEqual(stat.get_value(), ("test", (1.0, 10.0, 5.5, 2))) + stat.add_value(100) + self.assertEqual(stat.get_value(), ("test", (1.0, 100.0, 37.0, 3))) + stat.add_value(12.345) + self.assertEqual(stat.get_value(), ("test", (1.0, 100.0, 30.83625, 4))) + stat.add_value(0.655) + self.assertEqual(stat.get_value(), ("test", (0.655, 100.0, 24.8, 5))) + + +class TestStatistics(Bcfg2TestCase): + def test_stats(self): + stats = Statistics() + self.assertEqual(stats.display(), dict()) + stats.add_value("test1", 1) + self.assertEqual(stats.display(), dict(test1=(1.0, 1.0, 1.0, 1))) + stats.add_value("test2", 1.23) + self.assertEqual(stats.display(), dict(test1=(1.0, 1.0, 1.0, 1), + test2=(1.23, 1.23, 1.23, 1))) + stats.add_value("test1", 10) + self.assertEqual(stats.display(), dict(test1=(1.0, 10.0, 5.5, 2), + test2=(1.23, 1.23, 1.23, 1))) -- cgit v1.2.3-1-g7c22 From eea6e49ae2546c5fa1d0817be46d9a410946d2f1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 29 Mar 2013 17:02:08 -0400 Subject: Frame: log modified bundles, fix error message --- src/lib/Bcfg2/Client/Frame.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index ada5320b8..71c91284c 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -417,15 +417,18 @@ class Frame(object): bundle.get('name') not in self.setup['bundle']): # prune out unspecified bundles when running with -b continue + if bundle in mbundles: + self.logger.debug("Bundle %s was modified" % bundle) + func = "BundleUpdated" + else: + self.logger.debug("Bundle %s was not modified" % bundle) + func = "BundleNotUpdated" for tool in self.tools: try: - if bundle in mbundles: - tool.BundleUpdated(bundle, self.states) - else: - tool.BundleNotUpdated(bundle, self.states) + getattr(tool, func)(bundle, self.states) except: - self.logger.error("%s.BundleNotUpdated() call failed:" % - tool.name, exc_info=1) + self.logger.error("%s.%s() call failed:" % + (tool.name, func), exc_info=1) def Remove(self): """Remove extra entries.""" -- cgit v1.2.3-1-g7c22 From 748a6c81e4233d7b4c75d9a529be26ada4306c5b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 29 Mar 2013 19:05:07 -0400 Subject: Added option to periodically dump performance stats to logs --- src/lib/Bcfg2/Options.py | 13 ++++++++++++- src/lib/Bcfg2/Server/Core.py | 24 +++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 66e987b91..099709cbc 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -1082,6 +1082,15 @@ VERBOSE = \ cmd='-v', cook=get_bool, cf=('logging', 'verbose')) +LOG_PERFORMANCE = \ + Option("Periodically log performance statistics", + default=False, + cf=('logging', 'performance')) +PERFLOG_INTERVAL = \ + Option("Performance statistics logging interval in seconds", + default=300.0, + cook=get_timeout, + cf=('logging', 'performance_interval')) # Plugin-specific options CFG_VALIDATION = \ @@ -1164,7 +1173,9 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, web_configfile=WEB_CFILE, backend=SERVER_BACKEND, vcs_root=SERVER_VCS_ROOT, - authentication=SERVER_AUTHENTICATION) + authentication=SERVER_AUTHENTICATION, + perflog=LOG_PERFORMANCE, + perflog_interval=PERFLOG_INTERVAL) CRYPT_OPTIONS = dict(encrypt=ENCRYPT, decrypt=DECRYPT, diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index deb9065a5..a0044d561 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -96,6 +96,7 @@ class BaseCore(object): .. automethod:: _block .. ----- .. automethod:: _file_monitor_thread + .. automethod:: _perflog_thread """ #: The Bcfg2 repository directory self.datastore = setup['repo'] @@ -317,6 +318,12 @@ class BaseCore(object): threading.Thread(name="%sFAMThread" % setup['filemonitor'], target=self._file_monitor_thread) + self.perflog_thread = None + if self.setup['perflog']: + self.perflog_thread = \ + threading.Thread(name="PerformanceLoggingThread", + target=self._perflog_thread) + #: A :func:`threading.Lock` for use by #: :func:`Bcfg2.Server.FileMonitor.FileMonitor.handle_event_set` self.lock = threading.Lock() @@ -349,11 +356,23 @@ class BaseCore(object): if isinstance(plugin, base_cls)], key=lambda p: (p.sort_order, p.name)) + def _perflog_thread(self): + """ The thread that periodically logs performance statistics + to syslog. """ + self.logger.debug("Performance logging thread starting") + while not self.terminate.isSet(): + self.terminate.wait(self.setup['perflog_interval']) + for name, stats in self.get_statistics(None).items(): + self.logger.info("Performance statistics: " + "%s min=%.06f, max=%.06f, average=%.06f, " + "count=%d" % ((name, ) + stats)) + def _file_monitor_thread(self): """ The thread that runs the :class:`Bcfg2.Server.FileMonitor.FileMonitor`. This also queries :class:`Bcfg2.Server.Plugin.interfaces.Version` plugins for the current revision of the Bcfg2 repo. """ + self.logger.debug("File monitor thread starting") famfd = self.fam.fileno() terminate = self.terminate while not terminate.isSet(): @@ -718,7 +737,8 @@ class BaseCore(object): :type event: Bcfg2.Server.FileMonitor.Event """ if event.filename != self.cfile: - print("Got event for unknown file: %s" % event.filename) + self.logger.error("Got event for unknown file: %s" % + event.filename) return if event.code2str() == 'deleted': return @@ -758,6 +778,8 @@ class BaseCore(object): self.fam.start() self.fam_thread.start() self.fam.AddMonitor(self.cfile, self) + if self.perflog_thread is not None: + self.perflog_thread.start() for plug in self.plugins_by_type(Bcfg2.Server.Plugin.Threaded): plug.start_threads() -- cgit v1.2.3-1-g7c22 From bef04ca6ed60f51f04756df5a063379c0089321d Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Fri, 29 Mar 2013 11:09:50 -0400 Subject: Promote bcfg2-info to have it's own Options dict Turns out that CLIENT_COMMON_OPTIONS and SERVER_COMMON_OPTIONS conflict. Adding ppath and max_copies (turns out the latter is also necessary) to the bcfg2-info code directly seemed like a maintenance problem waiting to happen, so I factored that out into a new INFO_COMMON_OPTIONS dict. That will keep any options parsing special cases out of the bcfg2-info code hopefully be more maintainable going forward. --- src/lib/Bcfg2/Options.py | 4 ++++ src/sbin/bcfg2-info | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 099709cbc..b7ec9c089 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -1280,6 +1280,10 @@ TEST_COMMON_OPTIONS = dict(noseopts=TEST_NOSEOPTS, xunit=TEST_XUNIT, validate=CFG_VALIDATION) +INFO_COMMON_OPTIONS = dict(ppath=PARANOID_PATH, + max_copies=PARANOID_MAX_COPIES) +INFO_COMMON_OPTIONS.update(CLI_COMMON_OPTIONS) +INFO_COMMON_OPTIONS.update(SERVER_COMMON_OPTIONS) class OptionParser(OptionSet): """ diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 462d41398..eac6ba71d 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -753,8 +753,7 @@ def main(): optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, interactive=Bcfg2.Options.INTERACTIVE, interpreter=Bcfg2.Options.INTERPRETER) - optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) - optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.INFO_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) setup.hm = "\n".join([" bcfg2-info [options] [command ]", "Options:", -- cgit v1.2.3-1-g7c22 From e59ffe8f211e7bf2314402b96b44e0e72021c15f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 2 Apr 2013 14:19:09 -0400 Subject: fix bogus bundle_quick option handling in bcfg2_local.py --- tools/bcfg2_local.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index 70e8fc0b2..edb5a7101 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -68,7 +68,8 @@ def main(): # CLIENT_BUNDLEQUICK option uses -Q, just like the server repo # option. the server repo is more important for this # application. - del optinfo['bundle_quick'] + optinfo['bundle_quick'] = Bcfg2.Options.Option('bundlequick', + default=False) setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) -- cgit v1.2.3-1-g7c22 From 7658ac1c97d03ad233da0d0cfc786659dabd4cd4 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 2 Apr 2013 14:19:24 -0400 Subject: testsuite: fixed Probes test that uses version information --- testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index 1022bdc5a..0794db62e 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -3,6 +3,7 @@ import sys import copy import time import lxml.etree +import Bcfg2.version import Bcfg2.Server import Bcfg2.Server.Plugin from mock import Mock, MagicMock, patch @@ -217,6 +218,8 @@ group-specific""" ps.get_matching.return_value = matching metadata = Mock() + metadata.version_info = \ + Bcfg2.version.Bcfg2VersionInfo(Bcfg2.version.__version__) pdata = ps.get_probe_data(metadata) ps.get_matching.assert_called_with(metadata) # we can't create a matching operator.attrgetter object, and I @@ -621,5 +624,3 @@ text metadata.hostname = "nonexistent" self.assertEqual(probes.get_additional_data(metadata), ClientProbeDataSet()) - - -- cgit v1.2.3-1-g7c22 From 21aaa0db5a408a83f874816f7d7c3ebfb0d4d6fe Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 4 Apr 2013 14:51:57 -0500 Subject: selinux: Fix selinux_baseline.py Signed-off-by: Sol Jerome --- tools/selinux_baseline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/selinux_baseline.py b/tools/selinux_baseline.py index b6997bb29..06f6e6b98 100755 --- a/tools/selinux_baseline.py +++ b/tools/selinux_baseline.py @@ -42,7 +42,7 @@ def main(): baseline.append(lxml.etree.Comment("%s entries" % etype)) extra = handler.FindExtra() for entry in extra: - entry.tag = "BoundSELinux" + entry.tag = "Bound%s" % etype baseline.extend(extra) print(lxml.etree.tostring(baseline, pretty_print=True)) -- cgit v1.2.3-1-g7c22 From 61e7e8c5aedfd3c1547776795075b24a201f5252 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 4 Apr 2013 15:25:49 -0500 Subject: Options: Fix PEP8 error Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Options.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index b7ec9c089..24079d9e5 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -1285,6 +1285,7 @@ INFO_COMMON_OPTIONS = dict(ppath=PARANOID_PATH, INFO_COMMON_OPTIONS.update(CLI_COMMON_OPTIONS) INFO_COMMON_OPTIONS.update(SERVER_COMMON_OPTIONS) + class OptionParser(OptionSet): """ OptionParser bootstraps option parsing, -- cgit v1.2.3-1-g7c22 From a86e58fe19dbb35f00bfb2bd57adce3db01533c3 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 4 Apr 2013 17:45:06 -0500 Subject: Frame: Sort incorrect entries Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Client/Frame.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 71c91284c..be5c37004 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -450,7 +450,8 @@ class Frame(object): self.logger.info('Incorrect entries: %d' % list(self.states.values()).count(False)) if phase == 'final' and list(self.states.values()).count(False): - for entry in self.states.keys(): + for entry in sorted(self.states.keys(), key=lambda e: e.tag + ":" + + e.get('name')): if not self.states[entry]: etype = entry.get('type') if etype: -- cgit v1.2.3-1-g7c22 From b92f31c2768850f67149b92514c10c881aeb6d56 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 4 Apr 2013 18:30:43 -0500 Subject: schema: Update gecos field description Signed-off-by: Sol Jerome --- schemas/types.xsd | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/schemas/types.xsd b/schemas/types.xsd index 524b327c5..31fea26a2 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -422,8 +422,10 @@ - Human-readable user name or comment. If this is not set, - the GECOS will be the same as the username. + This field is typically used to record general information + about the account or its user(s) such as their real name + and phone number. If this is not set, the GECOS will be + the same as the username. -- cgit v1.2.3-1-g7c22 From 9a10880166445bafcc80e8c89057e48876359e5a Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 5 Apr 2013 11:30:02 -0400 Subject: File: handle Path type="file" entries with no text content even if empty is not set --- src/lib/Bcfg2/Client/Tools/POSIX/File.py | 10 ++++------ .../Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py | 8 ++++++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 9b95d2234..168c35c98 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -34,13 +34,11 @@ class POSIXFile(POSIXTool): def _get_data(self, entry): """ Get a tuple of (, ) for the given entry """ - is_binary = False - if entry.get('encoding', 'ascii') == 'base64': - tempdata = b64decode(entry.text) - is_binary = True - - elif entry.get('empty', 'false') == 'true': + is_binary = entry.get('encoding', 'ascii') == 'base64' + if entry.get('empty', 'false') == 'true' or not entry.text: tempdata = '' + elif is_binary: + tempdata = b64decode(entry.text) else: tempdata = entry.text if isinstance(tempdata, unicode) and unicode != str: diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py index 662e0e1b6..8f933e08f 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py @@ -63,10 +63,18 @@ class TestPOSIXFile(TestPOSIXTool): entry.set("encoding", "base64") self.assertEqual(ptool._get_data(entry), ("test", True)) + entry = copy.deepcopy(orig_entry) + entry.set("encoding", "base64") + entry.set("empty", "true") + self.assertEqual(ptool._get_data(entry), ("", True)) + entry = copy.deepcopy(orig_entry) entry.set("empty", "true") self.assertEqual(ptool._get_data(entry), ("", False)) + entry = copy.deepcopy(orig_entry) + self.assertEqual(ptool._get_data(entry), ("", False)) + entry = copy.deepcopy(orig_entry) entry.text = "test" self.assertEqual(ptool._get_data(entry), ("test", False)) -- cgit v1.2.3-1-g7c22 From eca67b6890156a6d297e6e0b00ec84c1d900a622 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 5 Apr 2013 13:12:04 -0400 Subject: disable tests on py3k --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 655d9fad5..73b8a9594 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ python: - "2.5" - "2.6" - "2.7" - - "3.2" env: - WITH_OPTIONAL_DEPS=yes - WITH_OPTIONAL_DEPS=no -- cgit v1.2.3-1-g7c22 From ac028179e16549ccb318e98091dce2e6de12bd84 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 8 Apr 2013 17:15:50 -0500 Subject: LocalFilesystem: Use binary data format The pickle class returns 'bytes' objects in python 3 instead of 'str' objects which were returned in python2. Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py index 0a0f032e5..c7d5c512a 100644 --- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py +++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py @@ -87,7 +87,7 @@ class LocalFilesystem(TransportBase): # using a tmpfile to hopefully avoid the file monitor from grabbing too # soon - saved = open(tmp_file, 'w') + saved = open(tmp_file, 'wb') try: saved.write(payload) except IOError: @@ -123,7 +123,7 @@ class LocalFilesystem(TransportBase): self.debug_log("Handling event %s" % event.filename) payload = os.path.join(self.work_path, event.filename) try: - payloadfd = open(payload, "r") + payloadfd = open(payload, "rb") interaction = cPickle.load(payloadfd) payloadfd.close() os.unlink(payload) -- cgit v1.2.3-1-g7c22 From f3d9898cf7040132bda3936844050dd98ba599f6 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 9 Apr 2013 08:13:56 -0400 Subject: SSLCA: fixed parsing of subjectaltnames from cert.xml --- src/lib/Bcfg2/Server/Plugins/SSLCA.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index 7d00201da..f111ffc60 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -68,7 +68,7 @@ class SSLCACertSpec(SSLCAXMLSpec): def get_spec(self, metadata): rv = SSLCAXMLSpec.get_spec(self, metadata) rv['subjectaltname'] = [e.text for e in self.Match(metadata) - if e.tag == "SubjectAltName"] + if e.tag == "subjectAltName"] return rv -- cgit v1.2.3-1-g7c22 From d0389f6744b748a3b87b4a5d49d3132269780741 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 9 Apr 2013 10:44:27 -0400 Subject: fixed another SubjectAltName/subjectAltName mixup --- doc/server/plugins/generators/sslca.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/server/plugins/generators/sslca.txt b/doc/server/plugins/generators/sslca.txt index cab7eb233..7ef358a31 100644 --- a/doc/server/plugins/generators/sslca.txt +++ b/doc/server/plugins/generators/sslca.txt @@ -156,7 +156,7 @@ Example .. code-block:: xml - test.example.com + test.example.com -- cgit v1.2.3-1-g7c22 From 662be00a13e0071d9bb09f487143a66f3f072261 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 9 Apr 2013 09:10:09 -0500 Subject: Reporting: Add failures entry type Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Reporting/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 4be509f53..e63c180a8 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -139,9 +139,11 @@ class Interaction(models.Model): posixgroups = models.ManyToManyField("POSIXGroupEntry") failures = models.ManyToManyField("FailureEntry") - entry_types = ('actions', 'packages', 'paths', 'services', 'sebooleans', - 'seports', 'sefcontexts', 'senodes', 'selogins', 'seusers', - 'seinterfaces', 'sepermissives', 'semodules', 'posixusers', + entry_types = ('actions', 'failures', 'packages', + 'paths', 'services', 'sebooleans', + 'seports', 'sefcontexts', 'senodes', + 'selogins', 'seusers', 'seinterfaces', + 'sepermissives', 'semodules', 'posixusers', 'posixgroups') # Formerly InteractionMetadata -- cgit v1.2.3-1-g7c22 From e5e2dba171d9b0a9c14d87ae539b019e3335f1b6 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 9 Apr 2013 12:38:37 -0500 Subject: doc: Clarify reporting installation steps Signed-off-by: Sol Jerome --- doc/reports/dynamic.txt | 57 +++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index b3028e9e1..1e77ed757 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -53,40 +53,41 @@ Prerequisites Install ------- -Be sure to include the specified fields included in the example -``bcfg2.conf`` file. These can be specified in either ``/etc/bcfg2.conf``, -if it is readable by the webserver user, or ``/etc/bcfg2-web.conf``. Any -database supported by `Django `_ can be used. -As of version 1.3, `South `_ is used to control -schema changes. If your database is not supported by South, any updates -will need to be applied manually. Sqlite is configured by default. -Please see the :ref:`reporting-databases` section to configure alternative -databases. -.. warning:: +1. Be sure to include the specified fields included in the example + ``bcfg2.conf`` file. These can be specified in either + ``/etc/bcfg2.conf``, if it is readable by the webserver user, + or ``/etc/bcfg2-web.conf``. Any database supported by `Django + `_ can be used. As of version 1.3, + `South `_ is used to control schema changes. + If your database is not supported by South, any updates will need to + be applied manually. Sqlite is configured by default. Please see the + :ref:`reporting-databases` section to configure alternative databases. - If you are using an sqlite database, the directory containing the - database file will need to be writable by the web server. The reason - for this is that sqlite will create another file for its journal - when it tries to update the database file. + .. warning:: -.. note:: + If you are using an sqlite database, the directory containing the + database file will need to be writable by the web server. The reason + for this is that sqlite will create another file for its journal + when it tries to update the database file. + + .. note:: - Distributed environments can share a single remote database for - reporting. + Distributed environments can share a single remote database for + reporting. -After configuring your database be sure to run `bcfg2-admin reports init` -to create the schema. +2. After configuring your database be sure to run ``bcfg2-admin reports + init`` to create the schema. -To enable statistics collection in the bcfg2-server, add -:ref:`server-plugins-statistics-reporting` to the **plugins** -line in your ``bcfg2.conf`` and restart the bcfg2-server. A report collecting -daemon should be run to import the collected statistics into the backend. -Please see the section :ref:`Report Collector ` for more -information. +3. To enable statistics collection in the bcfg2-server, add + :ref:`server-plugins-statistics-reporting` to the **plugins** + line in your ``bcfg2.conf`` and restart the bcfg2-server. A report + collecting daemon should be run to import the collected statistics + into the backend. Please see the section :ref:`Report Collector + ` for more information. -Detailed installation instructions can be found :ref:`here -`. + Detailed installation instructions can be found :ref:`here + `. .. _dynamic-http-install: @@ -175,7 +176,7 @@ Upgrading .. note:: After the database is upgraded all of the old tables are left - intact. To remove them any table starting with reports_ can + intact. To remove them any table starting with **reports\_** can be dropped. 4. `(Optional)` Run the :ref:`Report Collector ` -- cgit v1.2.3-1-g7c22 From 2d861fb3c2ef62bfbf15ced8bca4e86dba0f439b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 10 Apr 2013 09:41:14 -0500 Subject: doc: Fix old SELinux tag Signed-off-by: Sol Jerome --- doc/server/selinux.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/server/selinux.txt b/doc/server/selinux.txt index 9f54b0d68..79384970a 100644 --- a/doc/server/selinux.txt +++ b/doc/server/selinux.txt @@ -142,13 +142,13 @@ necessary. Duplicate Entries ----------------- -It may be necessary to use `BoundSELinux` tags if a single fcontext +It may be necessary to use `BoundSEFcontext` tags if a single fcontext needs two different SELinux types depending on whether it's a symlink or a plain file. For instance: .. code-block:: xml - - + + -- cgit v1.2.3-1-g7c22 From 2cb245f4ebf5dbd37f11f02a7d1598b050799515 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 10 Apr 2013 09:46:12 -0500 Subject: RcUpdate: Fix detection of running services Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Client/Tools/RcUpdate.py | 24 ++++++++++++++++-------- src/lib/Bcfg2/Utils.py | 17 +++++++++++++++-- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index 552b27842..4b78581f7 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -12,6 +12,15 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): __handles__ = [('Service', 'rc-update')] __req__ = {'Service': ['name', 'status']} + def get_enabled_svcs(self): + """ + Return a list of all enabled services. + """ + return [line.split()[0] + for line in self.cmd.run(['/bin/rc-status', + '-s']).stdout.splitlines() + if 'started' in line] + def VerifyService(self, entry, _): """ Verify Service status for entry. @@ -21,9 +30,12 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): if entry.get('status') == 'ignore': return True + # get a list of all started services + allsrv = self.get_enabled_svcs() + # check if service is enabled - result = self.cmd.run(["/sbin/rc-update", "show", "default"]) - is_enabled = entry.get("name") in result.stdout + result = self.cmd.run(["/sbin/rc-update", "show", "default"]).stdout + is_enabled = entry.get("name") in result # check if init script exists try: @@ -34,8 +46,7 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): return False # check if service is enabled - result = self.cmd.run(self.get_svc_command(entry, "status")) - is_running = "started" in result.stdout + is_running = entry.get('name') in allsrv if entry.get('status') == 'on' and not (is_enabled and is_running): entry.set('current_status', 'off') @@ -70,10 +81,7 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): def FindExtra(self): """Locate extra rc-update services.""" - allsrv = [line.split()[0] - for line in self.cmd.run(['/bin/rc-status', - '-s']).stdout.splitlines() - if 'started' in line] + allsrv = self.get_enabled_svcs() self.logger.debug('Found active services:') self.logger.debug(allsrv) specified = [srv.get('name') for srv in self.getSupportedEntries()] diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 39cf5255e..581445bf4 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -108,10 +108,16 @@ class ExecutorResult(object): def __init__(self, stdout, stderr, retval): #: The output of the command - self.stdout = stdout + if isinstance(stdout, str): + self.stdout = stdout + else: + self.stdout = stdout.decode('utf-8') #: The error produced by the command - self.stderr = stderr + if isinstance(stdout, str): + self.stderr = stderr + else: + self.stderr = stderr.decode('utf-8') #: The return value of the command. self.retval = retval @@ -234,6 +240,13 @@ class Executor(object): for line in inputdata.splitlines(): self.logger.debug('> %s' % line) (stdout, stderr) = proc.communicate(input=inputdata) + + # py3k fixes + if not isinstance(stdout, str): + stdout = stdout.decode('utf-8') + if not isinstance(stderr, str): + stderr = stderr.decode('utf-8') + for line in stdout.splitlines(): # pylint: disable=E1103 self.logger.debug('< %s' % line) for line in stderr.splitlines(): # pylint: disable=E1103 -- cgit v1.2.3-1-g7c22 From c313bf1d75ae4fa885cd099f299177d84dd8a95d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 10 Apr 2013 10:29:35 -0500 Subject: PY3K: Fix client/server to work with python 3 Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Client/Client.py | 14 ++++++++++---- src/lib/Bcfg2/Server/BuiltinCore.py | 2 +- src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 2 ++ src/lib/Bcfg2/Server/Plugins/Probes.py | 16 ++++++++++++---- src/lib/Bcfg2/Server/Plugins/Reporting.py | 2 +- 5 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py index 613e10e75..fad2af575 100644 --- a/src/lib/Bcfg2/Client/Client.py +++ b/src/lib/Bcfg2/Client/Client.py @@ -15,7 +15,7 @@ import Bcfg2.Client.XML import Bcfg2.Client.Frame import Bcfg2.Client.Tools from Bcfg2.Utils import locked, Executor -from Bcfg2.Compat import xmlrpclib +from Bcfg2.Compat import xmlrpclib, u_str from Bcfg2.version import __version__ @@ -91,7 +91,10 @@ class Client(object): try: script.write("#!%s\n" % (probe.attrib.get('interpreter', '/bin/sh'))) - script.write(probe.text.encode('utf-8')) + if sys.hexversion >= 0x03000000: + script.write(probe.text) + else: + script.write(probe.text.encode('utf-8')) script.close() os.chmod(scriptname, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | @@ -105,7 +108,10 @@ class Client(object): self._probe_failure(name, "Return value %s" % rv) self.logger.info("Probe %s has result:" % name) self.logger.info(rv.stdout) - ret.text = rv.stdout.decode('utf-8') + if sys.hexversion >= 0x03000000: + ret.text = rv.stdout + else: + ret.text = rv.stdout.decode('utf-8') finally: os.unlink(scriptname) except SystemExit: @@ -247,7 +253,7 @@ class Client(object): self.logger.info("Starting Bcfg2 client run at %s" % times['start']) - rawconfig = self.get_config(times=times) + rawconfig = self.get_config(times=times).decode('utf-8') if self.setup['cache']: try: diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 4d7453840..35ebc9fe6 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -12,7 +12,7 @@ from Bcfg2.SSLServer import XMLRPCServer from lockfile import LockFailed # pylint: disable=E0611 try: - from daemon.pidfile import PIDLockFile + from lockfile.pidlockfile import PIDLockFile except ImportError: from daemon.pidlockfile import PIDLockFile # pylint: enable=E0611 diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 926172e57..ffe93c25b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -599,6 +599,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, else: try: if not isinstance(data, unicode): + if not isinstance(data, str): + data = data.decode('utf-8') data = u_str(data, self.encoding) except UnicodeDecodeError: msg = "Failed to decode %s: %s" % (entry.get('name'), diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 55e5ddc47..201b4ec70 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -155,7 +155,10 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): probe.set('source', self.plugin_name) if (metadata.version_info and metadata.version_info > (1, 3, 1, '', 0)): - probe.text = entry.data.decode('utf-8') + try: + probe.text = entry.data.decode('utf-8') + except AttributeError: + probe.text = entry.data else: try: probe.text = entry.data @@ -219,9 +222,14 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe]).decode('utf-8')) + try: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str(self.probedata[client][probe]).decode('utf-8')) + except AttributeError: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 3bd6fd14f..3354763d4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -96,7 +96,7 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable): client.hostname, cdata, lxml.etree.tostring( stats, - xml_declaration=False).decode('UTF-8')) + xml_declaration=False)) self.debug_log("%s: Queued statistics data for %s" % (self.__class__.__name__, client.hostname)) return -- cgit v1.2.3-1-g7c22 From 5ba85c684f52b6fbab7233126c2ba5ea18c51ec0 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 10 Apr 2013 10:50:18 -0500 Subject: Client: Remove unused import Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Client/Client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py index fad2af575..1676ee717 100644 --- a/src/lib/Bcfg2/Client/Client.py +++ b/src/lib/Bcfg2/Client/Client.py @@ -15,7 +15,7 @@ import Bcfg2.Client.XML import Bcfg2.Client.Frame import Bcfg2.Client.Tools from Bcfg2.Utils import locked, Executor -from Bcfg2.Compat import xmlrpclib, u_str +from Bcfg2.Compat import xmlrpclib from Bcfg2.version import __version__ -- cgit v1.2.3-1-g7c22 From 910c1148691f4c16c2b4ef8d600561ec7054784d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 10 Apr 2013 13:52:24 -0400 Subject: SELinux: when the policy is upgraded, local changes can be lost (although the fact that there are local changes may be retained). cope with this bug. --- src/lib/Bcfg2/Client/Tools/SELinux.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py index 0041ce61a..19d3fa6fc 100644 --- a/src/lib/Bcfg2/Client/Tools/SELinux.py +++ b/src/lib/Bcfg2/Client/Tools/SELinux.py @@ -204,7 +204,16 @@ class SELinuxEntryHandler(object): type, if the records object supports the customized() method """ if hasattr(self.records, "customized") and self.custom_re: - return dict([(k, self.all_records[k]) for k in self.custom_keys]) + rv = dict() + for key in self.custom_keys: + if key in self.all_records: + rv[key] = self.all_records[key] + else: + self.logger.warning("SELinux %s %s customized, but no " + "record found. This may indicate an " + "error in your SELinux policy." % + (self.etype, key)) + return rv else: # ValueError is really a pretty dumb exception to raise, # but that's what the seobject customized() method raises -- cgit v1.2.3-1-g7c22 From fecdf966af5873e2346d3b695e1dd611dc9cb00d Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 10 Apr 2013 13:00:20 -0500 Subject: Probes: Fix pylint error Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Probes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 201b4ec70..6025d5455 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -225,7 +225,8 @@ class Probes(Bcfg2.Server.Plugin.Probing, try: lxml.etree.SubElement( ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe]).decode('utf-8')) + value=str(self.probedata[client][probe]) + .decode('utf-8')) except AttributeError: lxml.etree.SubElement( ctag, 'Probe', name=probe, -- cgit v1.2.3-1-g7c22 From 245aa971b9a8ec31fd1e5a766ad3940702ff66b7 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 11 Apr 2013 14:47:46 -0400 Subject: POSIX: SELinux context fixes for Path entries * Fixed Path entries with secontext='__default__' where no fcontext rule applied to the path. * Permitted setting secontext='' when no SELinux context should be applied to a Path entry --- doc/server/plugins/generators/rules.txt | 9 +++++++-- src/lib/Bcfg2/Client/Tools/POSIX/base.py | 16 +++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index 2789411e7..845006115 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -117,8 +117,13 @@ describe the attributes available for various Path types. Note that ``secontext`` below expects a full context, not just the type. For instance, "``system_u:object_r:etc_t:s0``", not just ``etc_t``. You can also specify "``__default__``", which will restore -the context of the file to the default set by policy. See -:ref:`server-selinux` for more information. +the context of the file to the default set by policy. If a file has +no default context rule, and you don't wish to set one, you can +specify ``secontext=''`` (i.e., an empty ``secontext``), in which case +the client will not try to manage the SELinux context of the file at +all. + +See :ref:`server-selinux` for more information. Attributes common to all Path tags: diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 11f331ddb..16fe0acb5 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -275,7 +275,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if path is None: path = entry.get("name") context = entry.get("secontext") - if context is None: + if not context: # no context listed return True @@ -520,13 +520,19 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): "Current mtime is %s but should be %s" % (path, mtime, entry.get('mtime'))) - if HAS_SELINUX and entry.get("secontext"): + if HAS_SELINUX: + wanted_secontext = None if entry.get("secontext") == "__default__": - wanted_secontext = \ - selinux.matchpathcon(path, 0)[1].split(":")[2] + try: + wanted_secontext = \ + selinux.matchpathcon(path, 0)[1].split(":")[2] + except OSError: + errors.append("%s has no default SELinux context" % + entry.get("name")) else: wanted_secontext = entry.get("secontext") - if attrib['current_secontext'] != wanted_secontext: + if (wanted_secontext and + attrib['current_secontext'] != wanted_secontext): errors.append("SELinux context for path %s is incorrect. " "Current context is %s but should be %s" % (path, attrib['current_secontext'], -- cgit v1.2.3-1-g7c22 From e686de540cdc4c7ee7c802b2ddfd364c12e995e7 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 11 Apr 2013 17:32:06 -0500 Subject: Packages: Write Packages cache in binary format Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index b4d481459..852049736 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -615,7 +615,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 self.logger.info("Packages: Updating %s" % url) fname = self.escape_url(url) try: - open(fname, 'w').write(fetch_url(url)) + open(fname, 'wb').write(fetch_url(url)) except ValueError: self.logger.error("Packages: Bad url string %s" % url) raise -- cgit v1.2.3-1-g7c22 From 140e07dcfabe997161bd1ca957713b619c40b6cf Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 11 Apr 2013 17:41:07 -0500 Subject: Packages: Read cache files using binary format Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 852049736..7ba374dd3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -315,7 +315,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :raises: OSError - If the saved data cannot be read :raises: cPickle.UnpicklingError - If the saved data is corrupt """ - data = open(self.cachefile) + data = open(self.cachefile, 'rb') (self.pkgnames, self.deps, self.provides, self.essentialpkgs) = cPickle.load(data) -- cgit v1.2.3-1-g7c22 From c85ecc6f65517e904602f69e0c21d002058c6044 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 12 Apr 2013 09:46:50 -0500 Subject: Packages: Fix Apt dependency resolution for py3k Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 4eefd0722..bc2928fa6 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -93,6 +93,8 @@ class AptSource(Source): self.logger.error("Packages: Failed to read file %s" % fname) raise for line in reader.readlines(): + if not isinstance(line, str): + line = line.decode('utf-8') words = str(line.strip()).split(':', 1) if words[0] == 'Package': pkgname = words[1].strip().rstrip() -- cgit v1.2.3-1-g7c22 From 84850ccbceb0836ac3851066e745d77d5bbbf889 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 12 Apr 2013 11:54:37 -0500 Subject: bcfg2-reports: Fix exception handling Signed-off-by: Sol Jerome --- src/sbin/bcfg2-reports | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 2c4a918be..bb45e0009 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -233,7 +233,8 @@ def main(): try: entries = [l.strip().split(":") for l in open(options.file)] - except IOError, err: + except IOError: + err = sys.exc_info()[1] print("Cannot read entries from %s: %s" % (options.file, err)) return 2 -- cgit v1.2.3-1-g7c22 From aa8a6f6d72eb551d6a8c650b9fbc14ba30a3d4bf Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 15 Apr 2013 13:32:12 -0400 Subject: cherrypy blocker bug fixed in cherrypy 3.3, update specfile accordingly --- misc/bcfg2.spec | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index e6b21d76c..d8eb8e5de 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -153,7 +153,7 @@ Requires: bcfg2-server = %{version} # cherrypy 3.2.3 actually doesn't exist yet, but 3.2.2 has bugs that # prevent it from working: # https://bitbucket.org/cherrypy/cherrypy/issue/1154/assertionerror-in-recv-when-ssl-is-enabled -Requires: python-cherrypy > 3.2.2 +Requires: python-cherrypy > 3.3 %description server-cherrypy Bcfg2 helps system administrators produce a consistent, reproducible, @@ -523,4 +523,3 @@ fi * Fri Sep 15 2006 Narayan Desai - 0.8.4-1 - Initial log - -- cgit v1.2.3-1-g7c22 From 2bda9eecd15f55274def7b6002106c2e3fa6f0f3 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 16 Apr 2013 07:33:06 -0400 Subject: bcfg2-yum-helper: use Bcfg2.Logger to set up logging --- src/sbin/bcfg2-yum-helper | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 7e5c03fd5..7dbdad16b 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -9,33 +9,13 @@ import os import sys import yum import logging +import Bcfg2.Logger from optparse import OptionParser try: import json except ImportError: import simplejson as json -LOGGER = None - - -def get_logger(verbose=0): - """ set up logging according to the verbose level given on the - command line """ - global LOGGER - if LOGGER is None: - LOGGER = logging.getLogger(sys.argv[0]) - stderr = logging.StreamHandler() - if verbose: - level = logging.DEBUG - else: - level = logging.WARNING - LOGGER.setLevel(level) - LOGGER.addHandler(stderr) - syslog = logging.handlers.SysLogHandler("/dev/log") - syslog.setFormatter(logging.Formatter("%(name)s: %(message)s")) - LOGGER.addHandler(syslog) - return LOGGER - def pkg_to_tuple(package): """ json doesn't distinguish between tuples and lists, but yum @@ -76,7 +56,7 @@ class DepSolver(object): except AttributeError: self.yumbase._getConfig(cfgfile, debuglevel=verbose) # pylint: enable=E1121,W0212 - self.logger = get_logger(verbose) + self.logger = logging.getLogger(self.__class__.__name__) self._groups = None def get_groups(self): @@ -220,7 +200,17 @@ def main(): parser.add_option("-v", "--verbose", help="Verbosity level", action="count") (options, args) = parser.parse_args() - logger = get_logger(options.verbose) + + if options.verbose: + level = logging.DEBUG + clevel = logging.DEBUG + else: + level = logging.WARNING + clevel = logging.INFO + Bcfg2.Logger.setup_logging('bcfg2-yum-helper', to_syslog=True, + to_console=clevel, level=level) + logger = logging.getLogger('bcfg2-yum-helper') + try: cmd = args[0] except IndexError: -- cgit v1.2.3-1-g7c22 From 8f00b76e2d1931611f84e5da99ffa204895896f9 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 16 Apr 2013 09:08:04 -0400 Subject: pep-8: fixed indentation? --- src/lib/Bcfg2/Server/Plugins/Probes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 6025d5455..309b96475 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -225,8 +225,8 @@ class Probes(Bcfg2.Server.Plugin.Probing, try: lxml.etree.SubElement( ctag, 'Probe', name=probe, - value=str(self.probedata[client][probe]) - .decode('utf-8')) + value=str( + self.probedata[client][probe]).decode('utf-8')) except AttributeError: lxml.etree.SubElement( ctag, 'Probe', name=probe, -- cgit v1.2.3-1-g7c22 From 0395e9b832e2f436d8b678ab18890f0593c52d53 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 16 Apr 2013 09:42:31 -0400 Subject: Core: load plugins after daemonization so files/dirs created at plugin init time have proper permissions --- src/lib/Bcfg2/Server/Admin/Minestruct.py | 3 +- src/lib/Bcfg2/Server/Admin/Pull.py | 10 +- src/lib/Bcfg2/Server/Core.py | 183 ++++++++++++++----------------- src/sbin/bcfg2-info | 2 +- 4 files changed, 92 insertions(+), 106 deletions(-) diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index 13c0563ec..93e42305c 100644 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -3,6 +3,7 @@ import getopt import lxml.etree import sys import Bcfg2.Server.Admin +from Bcfg2.Server.Plugin import PullSource class Minestruct(Bcfg2.Server.Admin.StructureMode): @@ -39,7 +40,7 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): try: extra = set() - for source in self.bcore.pull_sources: + for source in self.bcore.plugins_by_type(PullSource): for item in source.GetExtra(client): extra.add(item) except: diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index 9f1b3d138..8001425df 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -6,6 +6,7 @@ import sys import getopt import select import Bcfg2.Server.Admin +from Bcfg2.Server.Plugin import PullSource, Generator from Bcfg2.Compat import input # pylint: disable=W0622 @@ -62,13 +63,14 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): given client/entry from statistics. """ new_entry = {'type': etype, 'name': ename} - for plugin in self.bcore.pull_sources: + pull_sources = self.bcore.plugins_by_type(PullSource) + for plugin in pull_sources: try: (owner, group, mode, contents) = \ plugin.GetCurrentEntry(client, etype, ename) break except Bcfg2.Server.Plugin.PluginExecutionError: - if plugin == self.bcore.pull_sources[-1]: + if plugin == pull_sources[-1]: print("Pull Source failure; could not fetch current state") raise SystemExit(1) @@ -121,8 +123,8 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): meta = self.bcore.build_metadata(client) # Find appropriate plugin in bcore - glist = [gen for gen in self.bcore.generators if - ename in gen.Entries.get(etype, {})] + glist = [gen for gen in self.bcore.plugins_by_type(Generator) + if ename in gen.Entries.get(etype, {})] if len(glist) != 1: self.errExit("Got wrong numbers of matching generators for entry:" "%s" % ([g.name for g in glist])) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index a0044d561..15dd01c53 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -19,8 +19,9 @@ from Bcfg2.Cache import Cache import Bcfg2.Statistics from itertools import chain from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 -from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError, \ - track_statistics +from Bcfg2.Server.Plugin.exceptions import * +from Bcfg2.Server.Plugin.interfaces import * +from Bcfg2.Server.Plugin import track_statistics try: import psyco @@ -175,6 +176,9 @@ class BaseCore(object): #: the first one loaded wins. self.plugin_blacklist = {} + #: The Metadata plugin + self.metadata = None + #: Revision of the Bcfg2 specification. This will be sent to #: the client in the configuration, and can be set by a #: :class:`Bcfg2.Server.Plugin.interfaces.Version` plugin. @@ -236,71 +240,6 @@ class BaseCore(object): self.logger.error("Failed to set ownership of database " "at %s: %s" % (db_settings['NAME'], err)) - if '' in setup['plugins']: - setup['plugins'].remove('') - - for plugin in setup['plugins']: - if not plugin in self.plugins: - self.init_plugin(plugin) - # Remove blacklisted plugins - for plugin, blacklist in list(self.plugin_blacklist.items()): - if len(blacklist) > 0: - self.logger.error("The following plugins conflict with %s;" - "Unloading %s" % (plugin, blacklist)) - for plug in blacklist: - del self.plugins[plug] - - # Log experimental plugins - expl = [plug for plug in list(self.plugins.values()) - if plug.experimental] - if expl: - self.logger.info("Loading experimental plugin(s): %s" % - (" ".join([x.name for x in expl]))) - self.logger.info("NOTE: Interfaces subject to change") - - # Log deprecated plugins - depr = [plug for plug in list(self.plugins.values()) - if plug.deprecated] - if depr: - self.logger.info("Loading deprecated plugin(s): %s" % - (" ".join([x.name for x in depr]))) - - # Find the metadata plugin and set self.metadata - mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata) - if len(mlist) >= 1: - #: The Metadata plugin - self.metadata = mlist[0] - if len(mlist) > 1: - self.logger.error("Multiple Metadata plugins loaded; " - "using %s" % self.metadata) - else: - self.logger.error("No Metadata plugin loaded; " - "failed to instantiate Core") - raise CoreInitError("No Metadata Plugin") - - #: The list of plugins that handle - #: :class:`Bcfg2.Server.Plugin.interfaces.Statistics` - self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics) - - #: The list of plugins that implement the - #: :class:`Bcfg2.Server.Plugin.interfaces.PullSource` - #: interface - self.pull_sources = \ - self.plugins_by_type(Bcfg2.Server.Plugin.PullSource) - - #: The list of - #: :class:`Bcfg2.Server.Plugin.interfaces.Generator` plugins - self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator) - - #: The list of plugins that handle - #: :class:`Bcfg2.Server.Plugin.interfaces.Structure` - #: generation - self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure) - - #: The list of plugins that implement the - #: :class:`Bcfg2.Server.Plugin.interfaces.Connector` interface - self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector) - #: The CA that signed the server cert self.ca = setup['ca'] @@ -332,10 +271,6 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() - if self.debug_flag: - # enable debugging on everything else. - self.plugins[plugin].set_debug(self.debug_flag) - def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -391,7 +326,7 @@ class BaseCore(object): def _update_vcs_revision(self): """ Update the revision of the current configuration on-disk from the VCS plugin """ - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version): + for plugin in self.plugins_by_type(Version): try: newrev = plugin.get_revision() if newrev != self.revision: @@ -403,6 +338,53 @@ class BaseCore(object): (plugin.name, sys.exc_info()[1])) self.revision = '-1' + def load_plugins(self): + while '' in self.setup['plugins']: + self.setup['plugins'].remove('') + + for plugin in self.setup['plugins']: + if not plugin in self.plugins: + self.init_plugin(plugin) + + # Remove blacklisted plugins + for plugin, blacklist in list(self.plugin_blacklist.items()): + if len(blacklist) > 0: + self.logger.error("The following plugins conflict with %s;" + "Unloading %s" % (plugin, blacklist)) + for plug in blacklist: + del self.plugins[plug] + + # Log experimental plugins + expl = [plug for plug in list(self.plugins.values()) + if plug.experimental] + if expl: + self.logger.info("Loading experimental plugin(s): %s" % + (" ".join([x.name for x in expl]))) + self.logger.info("NOTE: Interfaces subject to change") + + # Log deprecated plugins + depr = [plug for plug in list(self.plugins.values()) + if plug.deprecated] + if depr: + self.logger.info("Loading deprecated plugin(s): %s" % + (" ".join([x.name for x in depr]))) + + # Find the metadata plugin and set self.metadata + mlist = self.plugins_by_type(Metadata) + if len(mlist) >= 1: + self.metadata = mlist[0] + if len(mlist) > 1: + self.logger.error("Multiple Metadata plugins loaded; using %s" + % self.metadata) + else: + self.logger.error("No Metadata plugin loaded; " + "failed to instantiate Core") + raise CoreInitError("No Metadata Plugin") + + if self.debug_flag: + # enable debugging on plugins + self.plugins[plugin].set_debug(self.debug_flag) + def init_plugin(self, plugin): """ Import and instantiate a single plugin. The plugin is stored to :attr:`plugins`. @@ -487,8 +469,7 @@ class BaseCore(object): metadata.hostname)) start = time.time() try: - for plugin in \ - self.plugins_by_type(Bcfg2.Server.Plugin.ClientRunHooks): + for plugin in self.plugins_by_type(ClientRunHooks): try: getattr(plugin, hook)(metadata) except AttributeError: @@ -519,11 +500,10 @@ class BaseCore(object): :type data: list of lxml.etree._Element objects """ self.logger.debug("Validating structures for %s" % metadata.hostname) - for plugin in \ - self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): + for plugin in self.plugins_by_type(StructureValidator): try: plugin.validate_structures(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: + except ValidationError: err = sys.exc_info()[1] self.logger.error("Plugin %s structure validation failed: %s" % (plugin.name, err)) @@ -546,10 +526,10 @@ class BaseCore(object): :type data: list of lxml.etree._Element objects """ self.logger.debug("Validating goals for %s" % metadata.hostname) - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator): + for plugin in self.plugins_by_type(GoalValidator): try: plugin.validate_goals(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: + except ValidationError: err = sys.exc_info()[1] self.logger.error("Plugin %s goal validation failed: %s" % (plugin.name, err.message)) @@ -567,8 +547,9 @@ class BaseCore(object): :returns: list of :class:`lxml.etree._Element` objects """ self.logger.debug("Getting structures for %s" % metadata.hostname) - structures = list(chain(*[struct.BuildStructures(metadata) - for struct in self.structures])) + structures = list( + chain(*[struct.BuildStructures(metadata) + for struct in self.plugins_by_type(Structure)])) sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] missing = [b for b in metadata.bundles if b not in sbundles] if missing: @@ -653,8 +634,9 @@ class BaseCore(object): self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) - glist = [gen for gen in self.generators if - entry.get('name') in gen.Entries.get(entry.tag, {})] + generators = self.plugins_by_type(Generator) + glist = [gen for gen in generators + if entry.get('name') in gen.Entries.get(entry.tag, {})] if len(glist) == 1: return glist[0].Entries[entry.tag][entry.get('name')](entry, metadata) @@ -662,8 +644,8 @@ class BaseCore(object): generators = ", ".join([gen.name for gen in glist]) self.logger.error("%s %s served by multiple generators: %s" % (entry.tag, entry.get('name'), generators)) - g2list = [gen for gen in self.generators if - gen.HandlesEntry(entry, metadata)] + g2list = [gen for gen in generators + if gen.HandlesEntry(entry, metadata)] try: if len(g2list) == 1: return g2list[0].HandleEntry(entry, metadata) @@ -690,7 +672,7 @@ class BaseCore(object): revision=self.revision) try: meta = self.build_metadata(client) - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: self.logger.error("Metadata consistency error for client %s" % client) return lxml.etree.Element("error", type='metadata error') @@ -775,13 +757,15 @@ class BaseCore(object): return False try: + self.load_plugins() + self.fam.start() self.fam_thread.start() self.fam.AddMonitor(self.cfile, self) if self.perflog_thread is not None: self.perflog_thread.start() - for plug in self.plugins_by_type(Bcfg2.Server.Plugin.Threaded): + for plug in self.plugins_by_type(Threaded): plug.start_threads() except: self.shutdown() @@ -818,7 +802,7 @@ class BaseCore(object): """ self.logger.debug("Getting decision list for %s" % metadata.hostname) result = [] - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision): + for plugin in self.plugins_by_type(Decision): try: result.extend(plugin.GetDecisions(metadata, mode)) except: @@ -837,7 +821,7 @@ class BaseCore(object): """ if not hasattr(self, 'metadata'): # some threads start before metadata is even loaded - raise Bcfg2.Server.Plugin.MetadataRuntimeError + raise MetadataRuntimeError("Metadata not loaded yet") if self.metadata_cache_mode == 'initial': # the Metadata plugin handles loading the cached data if # we're only caching the initial metadata object @@ -847,10 +831,11 @@ class BaseCore(object): if not imd: self.logger.debug("Building metadata for %s" % client_name) imd = self.metadata.get_initial_metadata(client_name) - for conn in self.connectors: + connectors = self.plugins_by_type(Connector) + for conn in connectors: grps = conn.get_additional_groups(imd) self.metadata.merge_additional_groups(imd, grps) - for conn in self.connectors: + for conn in connectors: data = conn.get_additional_data(imd) self.metadata.merge_additional_data(imd, conn.name, data) imd.query.by_name = self.build_metadata @@ -871,7 +856,7 @@ class BaseCore(object): meta = self.build_metadata(client_name) state = statistics.find(".//Statistics") if state.get('version') >= '2.0': - for plugin in self.statistics: + for plugin in self.plugins_by_type(Statistics): try: plugin.process_statistics(meta, statistics) except: @@ -913,11 +898,11 @@ class BaseCore(object): meta = self.build_metadata(client) else: meta = None - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: err = sys.exc_info()[1] self.critical_error("Client metadata resolution error for %s: %s" % (address[0], err)) - except Bcfg2.Server.Plugin.MetadataRuntimeError: + except MetadataRuntimeError: err = sys.exc_info()[1] self.critical_error('Metadata system runtime failure for %s: %s' % (address[0], err)) @@ -1011,8 +996,7 @@ class BaseCore(object): version)) try: self.metadata.set_version(client, version) - except (Bcfg2.Server.Plugin.MetadataConsistencyError, - Bcfg2.Server.Plugin.MetadataRuntimeError): + except (MetadataConsistencyError, MetadataRuntimeError): err = sys.exc_info()[1] self.critical_error("Unable to set version for %s: %s" % (client, err)) @@ -1032,7 +1016,7 @@ class BaseCore(object): client, metadata = self.resolve_client(address, cleanup_cache=True) self.logger.debug("Getting probes for %s" % client) try: - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): + for plugin in self.plugins_by_type(Probing): for probe in plugin.GetProbes(metadata): resp.append(probe) return lxml.etree.tostring(resp, @@ -1102,8 +1086,7 @@ class BaseCore(object): self.logger.debug("%s sets its profile to %s" % (client, profile)) try: self.metadata.set_profile(client, profile, address) - except (Bcfg2.Server.Plugin.MetadataConsistencyError, - Bcfg2.Server.Plugin.MetadataRuntimeError): + except (MetadataConsistencyError, MetadataRuntimeError): err = sys.exc_info()[1] self.critical_error("Unable to assert profile for %s: %s" % (client, err)) @@ -1125,7 +1108,7 @@ class BaseCore(object): config = self.BuildConfiguration(client) return lxml.etree.tostring(config, xml_declaration=False).decode('UTF-8') - except Bcfg2.Server.Plugin.MetadataConsistencyError: + except MetadataConsistencyError: self.critical_error("Metadata consistency failure for %s" % client) @exposed diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index eac6ba71d..e1b9428ca 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -606,7 +606,7 @@ Bcfg2 client itself.""") # Dump all mappings unless type specified data = [('Plugin', 'Type', 'Name')] arglen = len(args.split()) - for generator in self.generators: + for generator in self.plugins_by_type(Bcfg2.Server.Plugin.Generator): if arglen == 0: etypes = list(generator.Entries.keys()) else: -- cgit v1.2.3-1-g7c22 From 6aba4702f3432829f859244d4f04035867ca6d60 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 16 Apr 2013 09:47:40 -0500 Subject: doc: [statistics] is deprecated Signed-off-by: Sol Jerome --- doc/reports/dynamic.txt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 1e77ed757..9de3f868f 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -200,11 +200,6 @@ An example using the defaults is listed below:: host = port = - [statistics] - config = /etc/bcfg2-web.conf - time_zone = - web_debug = False - [reporting] transport = DirectStore web_prefix = @@ -242,6 +237,8 @@ section: statistics ^^^^^^^^^^ +.. deprecated: 1.3.0 + * config: The config file to be read for additional reporting data. This is used to restrict what can be read by the web server. -- cgit v1.2.3-1-g7c22 From 1b37ec798573d9d6b624bcf04e4c920a1a8ef624 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 16 Apr 2013 10:57:36 -0400 Subject: BuiltinCore: gracefully handle existing lock on pidfile --- src/lib/Bcfg2/Server/BuiltinCore.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 35ebc9fe6..c3302f1d0 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -9,12 +9,12 @@ from Bcfg2.Server.Core import BaseCore, NoExposedMethod from Bcfg2.Compat import xmlrpclib, urlparse from Bcfg2.SSLServer import XMLRPCServer -from lockfile import LockFailed +from lockfile import LockFailed, LockTimeout # pylint: disable=E0611 try: - from lockfile.pidlockfile import PIDLockFile + from daemon.pidfile import TimeoutPIDLockFile except ImportError: - from daemon.pidlockfile import PIDLockFile + from daemon.pidlockfile import TimeoutPIDLockFile # pylint: enable=E0611 @@ -33,7 +33,8 @@ class Core(BaseCore): gid=self.setup['daemon_gid'], umask=int(self.setup['umask'], 8)) if self.setup['daemon']: - daemon_args['pidfile'] = PIDLockFile(self.setup['daemon']) + daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'], + acquire_timeout=5) #: The :class:`daemon.DaemonContext` used to drop #: privileges, write the PID file (with :class:`PidFile`), #: and daemonize this core. @@ -89,6 +90,11 @@ class Core(BaseCore): err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) return False + except LockTimeout: + err = sys.exc_info()[1] + self.logger.error("Failed to daemonize %s: Failed to acquire lock " + "on %s" % (self.name, self.setup['daemon'])) + return False def _run(self): """ Create :attr:`server` to start the server listening. """ -- cgit v1.2.3-1-g7c22 From d990a3446734a9b8bca666f94bdbf8cb3592b856 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 16 Apr 2013 11:24:01 -0400 Subject: Git: fix command debug log without GitPython installed --- src/lib/Bcfg2/Server/Plugins/Git.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index c8362db41..44971aba7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -44,7 +44,7 @@ class Git(Version): else: cmd = ["git", "--git-dir", self.vcs_path, "--work-tree", self.vcs_root, "rev-parse", "HEAD"] - self.debug_log("Git: Running cmd") + self.debug_log("Git: Running %s" % cmd) proc = Popen(cmd, stdout=PIPE, stderr=PIPE) rv, err = proc.communicate() if proc.wait(): -- cgit v1.2.3-1-g7c22 From 2875c06a476f9e2b49ca1cccbbcbd494f735a3e3 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 16 Apr 2013 14:13:54 -0400 Subject: Core: fixed pylint tests for wildcard imports --- src/lib/Bcfg2/Server/Core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 15dd01c53..61b41b9bc 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -19,8 +19,8 @@ from Bcfg2.Cache import Cache import Bcfg2.Statistics from itertools import chain from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622 -from Bcfg2.Server.Plugin.exceptions import * -from Bcfg2.Server.Plugin.interfaces import * +from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614 +from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614 from Bcfg2.Server.Plugin import track_statistics try: -- cgit v1.2.3-1-g7c22 From cc6df36fbefe3ab6c3a218c0a95fd66ee47f68aa Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 16 Apr 2013 16:19:51 -0400 Subject: tools: write all changed files when migrating perms to mode --- tools/upgrade/1.3/migrate_perms_to_mode.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/upgrade/1.3/migrate_perms_to_mode.py b/tools/upgrade/1.3/migrate_perms_to_mode.py index e061558d3..18abffec2 100755 --- a/tools/upgrade/1.3/migrate_perms_to_mode.py +++ b/tools/upgrade/1.3/migrate_perms_to_mode.py @@ -13,6 +13,7 @@ def setmodeattr(elem): elem.set('mode', elem.get('perms')) del elem.attrib['perms'] return True + return False def writefile(f, xdata): @@ -32,7 +33,7 @@ def convertinfo(ifile): return found = False for i in xdata.findall('//Info'): - found = setmodeattr(i) + found |= setmodeattr(i) if found: writefile(ifile, xdata) @@ -47,7 +48,7 @@ def convertstructure(structfile): return found = False for path in xdata.xpath('//BoundPath|//Path'): - found = setmodeattr(path) + found |= setmodeattr(path) if found: writefile(structfile, xdata) -- cgit v1.2.3-1-g7c22 From 00db5c30f585ca679f377dec1968a9d0eea9f85f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 17 Apr 2013 07:06:53 -0400 Subject: Core: added docstring for load_plugins --- src/lib/Bcfg2/Server/Core.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 61b41b9bc..59d67e566 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -339,6 +339,11 @@ class BaseCore(object): self.revision = '-1' def load_plugins(self): + """ Load all plugins, setting + :attr:`Bcfg2.Server.Core.BaseCore.plugins` and + :attr:`Bcfg2.Server.Core.BaseCore.metadata` as side effects. + This does not start plugin threads; that is done later, in + :func:`Bcfg2.Server.Core.BaseCore.run` """ while '' in self.setup['plugins']: self.setup['plugins'].remove('') -- cgit v1.2.3-1-g7c22 From 40e4a36063a63a10c1887de68fcdc7bfa0ed1a2d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 17 Apr 2013 07:45:40 -0400 Subject: bcfg2-info: fixed for new load_plugins() routine --- src/sbin/bcfg2-info | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index e1b9428ca..b1e188d81 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -120,7 +120,6 @@ class InfoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore): Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) self.prompt = '> ' self.cont = True - self.fam.handle_events_in_interval(4) def _get_client_list(self, hostglobs): """ given a host glob, get a list of clients that match it """ @@ -712,6 +711,8 @@ Bcfg2 client itself.""") def run(self, args): # pylint: disable=W0221 try: + self.load_plugins() + self.fam.handle_events_in_interval(1) if args: self.onecmd(" ".join(args)) else: -- cgit v1.2.3-1-g7c22 From 717e0c60d2979de6d6e0ff04ea40e8e9c1654cbf Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 17 Apr 2013 15:24:45 -0400 Subject: pylint fix for Bcfg2.settings --- src/lib/Bcfg2/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 7d405f868..9393830a8 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -54,11 +54,11 @@ DEFAULT_CONFIG = _default_config() def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): """ read the config file and set django settings based on it """ - # pylint: disable=W0603 + # pylint: disable=W0602,W0603 global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \ DATABASE_HOST, DATABASE_PORT, DEBUG, TEMPLATE_DEBUG, TIME_ZONE, \ MEDIA_URL - # pylint: enable=W0603 + # pylint: enable=W0602,W0603 if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG): print("%s does not exist, using %s for database configuration" % -- cgit v1.2.3-1-g7c22 From 45b18fcaaf9daa76dbf99df3e6f324233b2457c8 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Apr 2013 14:18:32 -0400 Subject: bcfg2-info: fixed "bcfg2-info clients" when using clients database --- src/sbin/bcfg2-info | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index b1e188d81..ac4c3af13 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -457,9 +457,7 @@ Bcfg2 client itself.""") def do_clients(self, _): """ clients - Print out client/profile info """ data = [('Client', 'Profile')] - clist = self.metadata.clients - clist.sort() - for client in clist: + for client in sorted(self.metadata.list_clients()): imd = self.metadata.get_initial_metadata(client) data.append((client, imd.profile)) print_tabular(data) -- cgit v1.2.3-1-g7c22 From f152f8c33477933c4224ffa47e404c0df8d21ff9 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 19 Apr 2013 14:40:43 -0400 Subject: Options: add environment var for config file --- src/lib/Bcfg2/Options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 24079d9e5..55f799a29 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -401,7 +401,8 @@ CFILE = \ Option('Specify configuration file', default=DEFAULT_CONFIG_LOCATION, cmd='-C', - odesc='') + odesc='', + env="BCFG2_CONFIG") LOCKFILE = \ Option('Specify lockfile', default='/var/lock/bcfg2.run', -- cgit v1.2.3-1-g7c22 From 29399cbc599919fd9c88448bde692132c803e69b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 23 Apr 2013 11:45:07 -0400 Subject: Cfg: removed debugging --- src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index 581a997d8..c7b62f352 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -69,7 +69,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): the given client metadata, and may be obtained by doing ``self.XMLMatch(metadata)`` :type spec: lxml.etree._Element - :returns: None + :returns: string - The filename of the private key """ if spec is None: spec = self.XMLMatch(metadata) @@ -140,7 +140,6 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if spec is None: spec = self.XMLMatch(metadata) category = spec.get("category", self.category) - print("category=%s" % category) if category is None: per_host_default = "true" else: -- cgit v1.2.3-1-g7c22