summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--doc/development/plugins.txt1
-rw-r--r--doc/man/bcfg2.conf.txt22
-rw-r--r--doc/server/caching.txt3
-rw-r--r--doc/server/plugins/generators/cfg.txt85
-rw-r--r--doc/server/plugins/generators/rules.txt26
-rw-r--r--doc/server/plugins/structures/bundler/index.txt55
-rw-r--r--examples/TemplateHelper/include.py108
-rw-r--r--man/bcfg2.conf.533
-rw-r--r--redhat/bcfg2.spec.in3
-rw-r--r--src/lib/Bcfg2/Client/Tools/YUM.py8
-rw-r--r--src/lib/Bcfg2/Logger.py13
-rw-r--r--src/lib/Bcfg2/Reporting/Transport/DirectStore.py21
-rw-r--r--src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py22
-rw-r--r--src/lib/Bcfg2/Reporting/Transport/base.py22
-rw-r--r--src/lib/Bcfg2/Server/BuiltinCore.py12
-rw-r--r--src/lib/Bcfg2/Server/Core.py48
-rw-r--r--src/lib/Bcfg2/Server/Plugin/interfaces.py27
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Bundler.py15
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py4
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py16
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Yum.py35
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Probes.py3
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Reporting.py26
-rw-r--r--src/lib/Bcfg2/Server/Plugins/SSLCA.py5
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Snapshots.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Svn.py3
-rwxr-xr-xsrc/sbin/bcfg2-info44
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py2
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py32
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py11
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py30
31 files changed, 480 insertions, 257 deletions
diff --git a/doc/development/plugins.txt b/doc/development/plugins.txt
index a680e7d04..91a4e6868 100644
--- a/doc/development/plugins.txt
+++ b/doc/development/plugins.txt
@@ -67,6 +67,7 @@ provide interfaces that a given plugin may or must implement.
Interfaces
^^^^^^^^^^
+.. class:: Bcfg2.Server.Plugin.interfaces
.. automodule:: Bcfg2.Server.Plugin.interfaces
Exposing XML-RPC Functions
diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt
index b8e252cc4..d8f2bc3df 100644
--- a/doc/man/bcfg2.conf.txt
+++ b/doc/man/bcfg2.conf.txt
@@ -434,6 +434,28 @@ Trigger Plugin
The Trigger plugin provides a method for calling external scripts when
clients are configured.
+Caching options
+---------------
+
+These options are specified in the **[caching]** section.
+
+ client_metadata
+ The following four caching modes are available for client
+ metadata:
+
+ * off: No caching of client metadata objects is performed. This
+ is the default.
+ * initial: Only initial metadata objects are cached. Initial
+ metadata objects are created only from the data in the
+ Metadata plugin, before additional groups from other plugins
+ are merged in.
+ * cautious: Final metadata objects are cached, but each client’s
+ cache is cleared at the start of each client run, immediately
+ after probe data is received. Cache is also cleared as in
+ aggressive mode. *on* is a synonym for cautious.
+ * aggressive: Final metadata objects are cached. Each plugin is
+ responsible for clearing cache when appropriate.
+
Client options
--------------
diff --git a/doc/server/caching.txt b/doc/server/caching.txt
index ab98e9902..51245bd08 100644
--- a/doc/server/caching.txt
+++ b/doc/server/caching.txt
@@ -20,7 +20,8 @@ can be generated anywhere from 7 to dozens of times per client run
was made more complex and powerful in 1.3.0, caching these objects
provides the easiest performance gain.
-There are four caching modes available:
+To enable caching, add a ``[caching]`` section to bcfg2.conf with a
+client_metadata option containing one of the following modes:
* ``off``: No caching of client metadata objects is performed. This
is the default.
diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt
index 25986a413..7d0e0acff 100644
--- a/doc/server/plugins/generators/cfg.txt
+++ b/doc/server/plugins/generators/cfg.txt
@@ -106,26 +106,13 @@ Genshi templates allow you to use the `Genshi
the deprecated :ref:`server-plugins-generators-tgenshi-index` plugin.
Genshi templates should be named with a ``.genshi`` extension, e.g.::
- % ls Cfg/etc/motd
+ % ls Cfg/etc/motd
info.xml motd.genshi
See the genshi `documentation
<http://genshi.edgewall.org/wiki/Documentation>`_ for examples of
Genshi syntax.
-Inside Genshi Templates
-~~~~~~~~~~~~~~~~~~~~~~~
-
-Several variables are pre-defined inside templates:
-
-* **metadata** is the client's :ref:`metadata
- <server-plugins-grouping-metadata-clientmetadata>`
-* **name** is the path name specified in Bcfg2
-* **path** is the path to the TGenshi template. It starts with a
- leading slash, and is relative to the Bcfg2 specification root.
- E.g., ``/Cfg/etc/foo.conf/foo.conf.genshi`` or
- ``/TGenshi/etc/foo.conf/template.newtxt.H_foo.example.com``
-
Troubleshooting
~~~~~~~~~~~~~~~
@@ -138,7 +125,8 @@ E.g.::
bcfg2-info buildfile /etc/foo.conf foo.example.com
-To generate a file with an altsrc attribute, you can run::
+To generate a file with an :ref:`altsrc
+<server-plugins-structures-altsrc>` attribute, you can run::
bcfg2-info buildfile /etc/foo/foo.conf --altsrc=/etc/foo.conf \
foo.example.com
@@ -149,13 +137,13 @@ debug``, and, once in the Python interpreter, run::
metadata = self.build_metadata("<hostname>")
path = "<relative path to template (see note below)>"
-
+
``path`` should be set to the path to the template file with a leading
slash, relative to the Bcfg2 specification root. See `Inside Genshi
Templates`_ for examples.
Then, run::
-
+
import os, Bcfg2.Options
from genshi.template import TemplateLoader, NewTextTemplate
name = os.path.dirname(path[path.find('/', 1):])
@@ -230,19 +218,9 @@ Cheetah templates allow you to use the `cheetah templating system
the deprecated :ref:`server-plugins-generators-tcheetah` plugin.
Cheetah templates should be named with a ``.cheetah`` extension, e.g.::
- % ls Cfg/etc/motd
+ % ls Cfg/etc/motd
info.xml motd.cheetah
-Inside Cheetah Templates
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-Several variables are pre-defined inside templates:
-
-* **self.metadata** is the client's :ref:`metadata
- <server-plugins-grouping-metadata-clientmetadata>`
-* **self.path** is the path name specified in Bcfg2
-* **self.source_path** is the path to the Genshi template on the filesystem.
-
Examples
~~~~~~~~
@@ -265,12 +243,48 @@ comment to appear in the final config file.::
# This is a comment in my template which will be stripped when it's processed through Cheetah
\# This comment will appear in the generated config file.
+Inside Templates
+----------------
+
+Several variables are pre-defined inside templates:
+
++-------------+--------------------------------------------------------+
+| Name | Description |
++=============+========================================================+
+| metadata | :ref:`Client metadata |
+| | <server-plugins-grouping-metadata-clientmetadata>` |
++-------------+--------------------------------------------------------+
+| name | The value of the ``name`` attribute as specified in |
+| | the Path entry in Bcfg2. If an :ref:`altsrc |
+| | <server-plugins-structures-altsrc>` attribute is used, |
+| | then ``name`` will be the value of that attribute. |
++-------------+--------------------------------------------------------+
+| source_path | The path to the template file on the filesystem |
++-------------+--------------------------------------------------------+
+| repo | The path to the Bcfg2 repository on the filesystem |
++-------------+--------------------------------------------------------+
+| path | In Genshi templates, ``path`` is a synonym for |
+| | ``source_path``. In Cheetah templates, it's a synonym |
+| | for ``name``. For this reason, use of ``path`` is |
+| | discouraged, and it may be deprecated in a future |
+| | release. |
++-------------+--------------------------------------------------------+
+
+To access these variables in a Genshi template, you can simply use the
+name, e.g.::
+
+ Path to this file: ${name}
+
+In a Cheetah template, the variables are properties of ``self``,
+e.g.::
+
+ Path to this file: $self.name
Notes on Using Templates
------------------------
Templates can be host and group specific as well. Deltas will not be
-processed for any genshi or cheetah base file.
+processed for any Genshi or Cheetah base file.
.. note::
@@ -278,19 +292,22 @@ processed for any genshi or cheetah base file.
or group-specific files, you will need to ensure that the ``.genshi``
or ``.cheetah`` extension is at the **end** of the filename. Using the
examples from above for *host.example.com* and group *server* you would
- have the following (using genshi only)::
+ have the following::
Cfg/etc/fstab/fstab.H_host.example.com.genshi
- Cfg/etc/fstab/fstab.G50_server.genshi
+ Cfg/etc/fstab/fstab.G50_server.cheetah
Genshi templates take precence over cheetah templates. For example, if
-two files exist named
+two files exist named::
Cfg/etc/fstab/fstab.genshi
Cfg/etc/fstab/fstab.cheetah
-the cheetah template is ignored. But you can mix genshi and cheetah when
-using different host-specific or group-specific files. For example:
+The Cheetah template is ignored. Exploiting this fact is probably a
+pretty bad idea in practice.
+
+You can mix Genshi and Cheetah when using different host-specific or
+group-specific files. For example:
Cfg/etc/fstab/fstab.H_host.example.com.genshi
Cfg/etc/fstab/fstab.G50_server.cheetah
diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt
index 079911238..542b38f01 100644
--- a/doc/server/plugins/generators/rules.txt
+++ b/doc/server/plugins/generators/rules.txt
@@ -300,24 +300,26 @@ nonexistent
+===========+====================+=============+
| type | Type of file | nonexistent |
+-----------+--------------------+-------------+
-| recursive | Recursively remove | true |
+| recursive | Recursively remove | Boolean |
| | directory contents | |
+-----------+--------------------+-------------+
permissions
^^^^^^^^^^^
-+-----------+--------------------------+--------+
-| Name | Description | Values |
-+===========+==========================+========+
-| mode | Mode of the file. | String |
-+-----------+--------------------------+--------+
-| owner | Owner of the file. | String |
-+-----------+--------------------------+--------+
-| group | Group of the file. | String |
-+-----------+--------------------------+--------+
-| secontext | SELinux context | String |
-+-----------+--------------------------+--------+
++-----------+-----------------------------+---------+
+| Name | Description | Values |
++===========+=============================+=========+
+| mode | Mode of the file | String |
++-----------+-----------------------------+---------+
+| owner | Owner of the file | String |
++-----------+-----------------------------+---------+
+| group | Group of the file | String |
++-----------+-----------------------------+---------+
+| secontext | SELinux context | String |
++-----------+-----------------------------+---------+
+| recursive | Recursively set permissions | Boolean |
++-----------+-----------------------------+---------+
symlink
^^^^^^^
diff --git a/doc/server/plugins/structures/bundler/index.txt b/doc/server/plugins/structures/bundler/index.txt
index 528df79db..1cc287ebd 100644
--- a/doc/server/plugins/structures/bundler/index.txt
+++ b/doc/server/plugins/structures/bundler/index.txt
@@ -113,40 +113,31 @@ demonstrates how group entries can be used in bundles)
Genshi templates
================
-Genshi xml templates can be specified one of two ways:
+Genshi XML templates allow you to use the `Genshi
+<http://genshi.edgewall.org>`_ templating system to dynamically
+generate a bundle. Genshi templates can be specified one of two ways:
-1. Add an xml-style genshi template to the Bundler directory with a
+1. Add an XML-style genshi template to the Bundler directory with a
``.genshi`` and the associated namespace attribute.
-2. Simply add the appropriate namespace attribute to your existing xml
+2. Simply add the appropriate namespace attribute to your existing XML
bundle.
-The namespace attribute in this case should look like the following::
+The top-level Bundle tag should look like the following::
- xmlns:py="http://genshi.edgewall.org/"
+ <Bundle name="foo" xmlns:py="http://genshi.edgewall.org/">
-Motivation
-----------
+Several variables are pre-defined inside templates:
-Static Bundles have served us relatively well, but have a relatively
-weak set of interal per-client differentiation mechanisms. In static
-Bundles, the group hierarchy (from the perspective of the current
-client) is available for use via boolean constraints with negation. This
-notion does not mesh well with the use of Probes, which can result in
-freeform data. In the presence of probe results, clients can have a wide
-array of data, and rendering down to a boolean logic may not always
-be desirable. Moreover, while static Bundles allow entry inclusion or
-exclusion based on group memberships, they do not allow programatic entry
-rendering. Hence, Genshi templates not only provide more control options,
-but it also provide the ability to perform new entry rendering operations.
++-------------+--------------------------------------------------------+
+| Name | Description |
++=============+========================================================+
+| metadata | :ref:`Client metadata |
+| | <server-plugins-grouping-metadata-clientmetadata>` |
++-------------+--------------------------------------------------------+
+| repo | The path to the Bcfg2 repository on the filesystem |
++-------------+--------------------------------------------------------+
-The `Genshi templating system`_ is used internally.
-
-.. _Genshi templating system: http://genshi.edgewall.org/
-
-Use
----
-
-.. warning::
+.. note::
``<Group>`` and ``<Client>`` tags are allowed inside of Genshi
templates as of Bcfg2 1.2. However, they do not behave the same
@@ -154,7 +145,7 @@ Use
<py:if test="'groupname' in metadata.groups">
</py:if>
-
+
The conditional is evaluated when the template is rendered, so
code inside the conditional is not executed if the conditional
fails. A ``<Group>`` tag is evaluated *after* the template is
@@ -163,16 +154,6 @@ Use
groups, you *must* use a Genshi conditional, not a ``<Group>``
tag. The same caveats apply to ``<Client>`` tags.
-Bcfg2 uses the Genshi API for templates, and performs a XML format
-stream rendering of the template into an lxml entry, which is included
-in the client configuration. :ref:`Client metadata <client-metadata>`
-is available inside of the template using the 'metadata' name. Note that
-only the markup Genshi template format can be used, as the target output
-format is XML.
-
-A Genshi template looks much like a Bundler file, except the Bundle tag
-has an additional `xmlns:py` attribute. See the examples.
-
Troubleshooting
---------------
diff --git a/examples/TemplateHelper/include.py b/examples/TemplateHelper/include.py
index 5fba75558..be0034f52 100644
--- a/examples/TemplateHelper/include.py
+++ b/examples/TemplateHelper/include.py
@@ -1,17 +1,18 @@
-"""IncludeHelper makes it easier to include group- and host-specific files in a template.
+""" IncludeHelper makes it easier to include group- and host-specific
+files in a template.
Synopsis:
{% python
import os
- include = metadata.TemplateHelper['include'].IncludeHelper
- custom = include(metadata, path).files(os.path.basename(name))
+ include = metadata.TemplateHelper['include']
+ custom = include.IncludeHelper(metadata, path).files(os.path.basename(name))
%}\
{% for file in custom %}\
-
- ########## Start ${include.specificity(file)} ##########
+
+ ########## Start ${include.describe_specificity(file)} ##########
{% include ${file} %}
- ########## End ${include.specificity(file)} ##########
+ ########## End ${include.describe_specificity(file)} ##########
{% end %}\
This would let you include files with the same base name; e.g. in a
@@ -27,71 +28,86 @@ This would result in two different sets of custom files being used,
one drawn from ''bar.conf.G_<group>.genshi_include'' and the other
from ''baz.conf.G_<group>.genshi_include''.
-==== Methods ====
-
-
-=== files ===
-
-Usage:
-
-
-
"""
import os
import re
-import Bcfg2.Options
-__export__ = ["IncludeHelper"]
+__export__ = ["IncludeHelper", "get_specificity", "describe_specificity"]
+
-class IncludeHelper (object):
+class IncludeHelper(object):
def __init__(self, metadata, path):
""" Constructor.
- The template path can be found in the ''path'' variable that is set for all Genshi templates."""
+ The template path can be found in the ''path'' variable that
+ is set for all Genshi templates. """
self.metadata = metadata
self.path = path
-
- def _get_basedir(self):
- setup = Bcfg2.Options.OptionParser({'repo':
- Bcfg2.Options.SERVER_REPOSITORY})
- setup.parse('--')
- return os.path.join(setup['repo'], os.path.dirname(self.path))
-
- def files(self, fname):
+
+ def get_basedir(self):
+ return os.path.dirname(self.path)
+
+ def files(self, fname, groups=None):
""" Return a list of files to include for this host. Files
are found in the template directory based on the following
patterns:
* ''<prefix>.H_<hostname>.genshi_include'': Host-specific files
* ''<prefix>.G_<group>.genshi_include'': Group-specific files
+ * ''<prefix>.genshi_include'': Non-specific includes
Note that there is no numeric priority on the group-specific
- files. All matching files are returned by
- ''IncludeHelper.files()''. """
+ files; all matching files are returned by
+ ``IncludeHelper.files()``. If you wish to only include files
+ for a subset of groups, pass the ``groups`` keyword argument.
+ Host-specific files are always included in the return
+ value. """
files = []
- hostfile = os.path.join(self._get_basedir(),
+ hostfile = os.path.join(self.get_basedir(),
"%s.H_%s.genshi_include" %
(fname, self.metadata.hostname))
if os.path.isfile(hostfile):
files.append(hostfile)
-
- for group in self.metadata.groups:
- filename = os.path.join(self._get_basedir(),
+
+ allfile = os.path.join(self.get_basedir(), "%s.genshi_include" % fname)
+ if os.path.isfile(allfile):
+ files.append(allfile)
+
+ if groups is None:
+ groups = sorted(self.metadata.groups)
+
+ for group in groups:
+ filename = os.path.join(self.get_basedir(),
"%s.G_%s.genshi_include" % (fname, group))
if os.path.isfile(filename):
files.append(filename)
- return sorted(files)
-
- @staticmethod
- def specificity(fname):
- """ Get a string describing the specificity of the given file """
- match = re.search(r'(G|H)_(.*)\.genshi_include', fname)
- if match:
- if match.group(1) == "G":
- stype = "group"
- else:
- stype = "host"
- return "%s-specific configs for %s" % (stype, match.group(2))
- return "Unknown specificity"
+ return files
+
+
+SPECIFICITY_RE = re.compile(r'(G|H)_(.*)\.genshi_include')
+
+
+def get_specificity(fname):
+ """ Get a tuple of (<type>, <parameter>) describing the
+ specificity of the given file. Specificity types are "host",
+ "group", or "all". The parameter will be either a hostname, a
+ group name, or None (for "all"). """
+ match = SPECIFICITY_RE.search(fname)
+ if match:
+ if match.group(1) == "G":
+ stype = "group"
+ else:
+ stype = "host"
+ return (stype, match.group(2))
+ return ("all", None)
+
+
+def describe_specificity(fname):
+ """ Get a string describing the specificity of the given file """
+ (stype, param) = get_specificity(fname)
+ if stype != "all":
+ return "%s-specific configs for %s" % (stype, param)
+ else:
+ return "Generic configs for all clients"
diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5
index 49aa5369f..dfcb42a24 100644
--- a/man/bcfg2.conf.5
+++ b/man/bcfg2.conf.5
@@ -1,4 +1,4 @@
-.TH "BCFG2.CONF" "5" "November 14, 2012" "1.3" "Bcfg2"
+.TH "BCFG2.CONF" "5" "November 26, 2012" "1.3" "Bcfg2"
.SH NAME
bcfg2.conf \- Configuration parameters for Bcfg2
.
@@ -394,6 +394,37 @@ executed on the client in the created files.
.sp
The Trigger plugin provides a method for calling external scripts when
clients are configured.
+.SH CACHING OPTIONS
+.sp
+These options are specified in the \fB[caching]\fP section.
+.INDENT 0.0
+.INDENT 3.5
+.INDENT 0.0
+.TP
+.B client_metadata
+The following four caching modes are available for client
+metadata:
+.INDENT 7.0
+.IP \(bu 2
+off: No caching of client metadata objects is performed. This
+is the default.
+.IP \(bu 2
+initial: Only initial metadata objects are cached. Initial
+metadata objects are created only from the data in the
+Metadata plugin, before additional groups from other plugins
+are merged in.
+.IP \(bu 2
+cautious: Final metadata objects are cached, but each client’s
+cache is cleared at the start of each client run, immediately
+after probe data is received. Cache is also cleared as in
+aggressive mode. \fIon\fP is a synonym for cautious.
+.IP \(bu 2
+aggressive: Final metadata objects are cached. Each plugin is
+responsible for clearing cache when appropriate.
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.UNINDENT
.SH CLIENT OPTIONS
.sp
These options only affect client functionality. They can be specified in
diff --git a/redhat/bcfg2.spec.in b/redhat/bcfg2.spec.in
index ede45a05e..70396f900 100644
--- a/redhat/bcfg2.spec.in
+++ b/redhat/bcfg2.spec.in
@@ -111,9 +111,6 @@ Configuration management system documentation
%{__perl} -pi -e 's@/etc/default@%{_sysconfdir}/sysconfig@g' debian/bcfg2-server.init
%{__perl} -pi -e 's@/etc/default@%{_sysconfdir}/sysconfig@g' tools/bcfg2-cron
-%{__perl} -pi -e 's@/usr/lib/bcfg2@%{_libexecdir}@g' debian/bcfg2.cron.daily
-%{__perl} -pi -e 's@/usr/lib/bcfg2@%{_libexecdir}@g' debian/bcfg2.cron.hourly
-
# don't start servers by default
%{__perl} -pi -e 's@chkconfig: (\d+)@chkconfig: -@' debian/bcfg2.init
%{__perl} -pi -e 's@chkconfig: (\d+)@chkconfig: -@' debian/bcfg2-server.init
diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py
index 5d20c0462..928aba1e1 100644
--- a/src/lib/Bcfg2/Client/Tools/YUM.py
+++ b/src/lib/Bcfg2/Client/Tools/YUM.py
@@ -751,7 +751,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
rescode, restring = self.yumbase.buildTransaction()
except yum.Errors.YumBaseError:
err = sys.exc_info()[1]
- self.logger.error("Yum transaction error: %s" % err)
+ self.logger.error("Error building Yum transaction: %s" % err)
cleanup()
return
@@ -767,7 +767,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
self.logger.info("Single Pass for Install Succeeded")
except yum.Errors.YumBaseError:
err = sys.exc_info()[1]
- self.logger.error("Yum transaction error: %s" % err)
+ self.logger.error("Error processing Yum transaction: %s" % err)
cleanup()
return
else:
@@ -788,7 +788,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
self.logger.debug(" %s" % restring)
except yum.Errors.YumBaseError:
err = sys.exc_info()[1]
- self.logger.error("Yum transaction error: %s" % err)
+ self.logger.error("Error rerunning Yum transaction: %s" % err)
self.yumbase.conf.skip_broken = skip_broken
@@ -850,7 +850,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
if inst not in self.instance_status:
self.logger.warning(
" Asked to install/update package never "
- "verified: %s" %
+ "verified: %s" %
nevra2string(build_yname(pkg.get('name'), inst)))
continue
status = self.instance_status[inst]
diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py
index a06d6e79e..24028f71c 100644
--- a/src/lib/Bcfg2/Logger.py
+++ b/src/lib/Bcfg2/Logger.py
@@ -158,7 +158,7 @@ def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG):
logging.Formatter('%(name)s[%(process)d]: %(message)s'))
logging.root.addHandler(syslog)
except socket.error:
- logging.root.error("failed to activate syslogging")
+ logging.root.error("Failed to activate syslogging")
except:
print("Failed to activate syslogging")
@@ -178,17 +178,22 @@ def setup_logging(procname, to_console=True, to_syslog=True,
if hasattr(logging, 'already_setup'):
return
+ params = []
+
if to_console:
if to_console == True:
- clvl = min(logging.WARNING, level)
- else:
- clvl = min(to_console, level)
+ to_console = logging.WARNING
+ clvl = min(to_console, level)
+ params.append("%s to console" % logging.getLevelName(clvl))
add_console_handler(clvl)
if to_syslog:
slvl = min(level, logging.INFO)
+ params.append("%s to syslog" % logging.getLevelName(slvl))
add_syslog_handler(procname, syslog_facility, level=slvl)
if to_file is not None:
+ params.append("%s to %s" % (logging.getLevelName(level), to_file))
add_file_handler(to_file, level=level)
logging.root.setLevel(logging.DEBUG)
+ logging.root.debug("Configured logging: %s" % "; ".join(params))
logging.already_setup = True
diff --git a/src/lib/Bcfg2/Reporting/Transport/DirectStore.py b/src/lib/Bcfg2/Reporting/Transport/DirectStore.py
index 8677efb5f..79d1b5aba 100644
--- a/src/lib/Bcfg2/Reporting/Transport/DirectStore.py
+++ b/src/lib/Bcfg2/Reporting/Transport/DirectStore.py
@@ -15,9 +15,14 @@ class DirectStore(TransportBase, threading.Thread):
TransportBase.__init__(self, setup)
threading.Thread.__init__(self)
self.save_file = os.path.join(self.data, ".saved")
+
self.storage = load_storage_from_config(setup)
+ self.storage.validate()
+
self.queue = Queue(100000)
self.terminate = threading.Event()
+ self.debug_log("Reporting: Starting %s thread" %
+ self.__class__.__name__)
self.start()
def shutdown(self):
@@ -35,6 +40,8 @@ class DirectStore(TransportBase, threading.Thread):
def run(self):
if not self._load():
+ self.logger.warning("Reporting: Failed to load saved data, "
+ "DirectStore thread exiting")
return
while not self.terminate.isSet() and self.queue is not None:
try:
@@ -42,16 +49,19 @@ class DirectStore(TransportBase, threading.Thread):
timeout=self.timeout)
start = time.time()
self.storage.import_interaction(interaction)
- self.logger.info("Imported data for %s in %s seconds" \
- % (interaction.get('hostname', '<unknown>'), \
- time.time() - start))
+ self.logger.info("Imported data for %s in %s seconds" %
+ (interaction.get('hostname', '<unknown>'),
+ time.time() - start))
except Empty:
+ self.debug_log("Reporting: Queue is empty")
continue
except:
err = sys.exc_info()[1]
self.logger.error("Reporting: Could not import interaction: %s"
% err)
continue
+ self.debug_log("Reporting: Stopping %s thread" %
+ self.__class__.__name__)
if self.queue is not None and not self.queue.empty():
self._save()
@@ -74,6 +84,8 @@ class DirectStore(TransportBase, threading.Thread):
def _save(self):
""" Save any saved data to a file """
+ self.debug_log("Reporting: Saving pending data to %s" %
+ self.save_file)
saved_data = []
try:
while not self.queue.empty():
@@ -93,6 +105,7 @@ class DirectStore(TransportBase, threading.Thread):
def _load(self):
""" Load any saved data from a file """
if not os.path.exists(self.save_file):
+ self.debug_log("Reporting: No saved data to load")
return True
saved_data = []
try:
@@ -106,6 +119,8 @@ class DirectStore(TransportBase, threading.Thread):
for interaction in saved_data:
# check that shutdown wasnt called early
if self.terminate.isSet():
+ self.logger.warning("Reporting: Shutdown called while loading "
+ " saved data")
return False
try:
diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py
index 8ccb9ed56..30ea39263 100644
--- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py
+++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py
@@ -20,7 +20,7 @@ class LocalFilesystem(TransportBase):
super(LocalFilesystem, self).__init__(setup)
self.work_path = "%s/work" % self.data
- self.logger.debug("LocalFilesystem: work path %s" % self.work_path)
+ self.debug_log("LocalFilesystem: work path %s" % self.work_path)
self.fmon = None
self._phony_collector = None
@@ -34,6 +34,11 @@ class LocalFilesystem(TransportBase):
traceback.format_exc().splitlines()[-1]))
raise TransportError
+ def set_debug(self, debug):
+ rv = TransportBase.set_debug(self, debug)
+ self.fmon.set_debug(debug)
+ return rv
+
def start_monitor(self, collector):
"""Start the file monitor. Most of this comes from BaseCore"""
setup = self.setup
@@ -44,12 +49,13 @@ class LocalFilesystem(TransportBase):
"forcing to default" % setup['filemonitor'])
fmon = Bcfg2.Server.FileMonitor.available['default']
- fmdebug = setup.get('debug', False)
try:
- self.fmon = fmon(debug=fmdebug)
- self.logger.info("Using the %s file monitor" % self.fmon.__class__.__name__)
+ self.fmon = fmon(debug=self.debug_flag)
+ self.logger.info("Using the %s file monitor" %
+ self.fmon.__class__.__name__)
except IOError:
- msg = "Failed to instantiate file monitor %s" % setup['filemonitor']
+ msg = "Failed to instantiate file monitor %s" % \
+ setup['filemonitor']
self.logger.error(msg, exc_info=1)
raise TransportError(msg)
self.fmon.start()
@@ -108,11 +114,11 @@ class LocalFilesystem(TransportBase):
#deviate from the normal routines here we only want one event
etype = event.code2str()
- self.logger.debug("Recieved event %s for %s" % (etype, event.filename))
+ self.debug_log("Recieved event %s for %s" % (etype, event.filename))
if os.path.basename(event.filename)[0] == '.':
return None
if etype in ('created', 'exists'):
- self.logger.debug("Handling event %s" % event.filename)
+ self.debug_log("Handling event %s" % event.filename)
payload = os.path.join(self.work_path, event.filename)
try:
payloadfd = open(payload, "r")
@@ -150,7 +156,7 @@ class LocalFilesystem(TransportBase):
except ReportingError:
raise TransportError
except:
- self.logger.error("Failed to load collector: %s" %
+ self.logger.error("Failed to load collector: %s" %
traceback.format_exc().splitlines()[-1])
raise TransportError
diff --git a/src/lib/Bcfg2/Reporting/Transport/base.py b/src/lib/Bcfg2/Reporting/Transport/base.py
index cca7beda0..530011e47 100644
--- a/src/lib/Bcfg2/Reporting/Transport/base.py
+++ b/src/lib/Bcfg2/Reporting/Transport/base.py
@@ -2,26 +2,38 @@
The base for all server -> collector Transports
"""
-import os.path
-import logging
+import os
+import sys
+from Bcfg2.Server.Plugin import Debuggable
+
class TransportError(Exception):
"""Generic TransportError"""
pass
+
class TransportImportError(TransportError):
"""Raised when a transport fails to import"""
pass
-class TransportBase(object):
+
+class TransportBase(Debuggable):
"""The base for all transports"""
def __init__(self, setup):
"""Do something here"""
clsname = self.__class__.__name__
- self.logger = logging.getLogger(clsname)
- self.logger.debug("Loading %s transport" % clsname)
+ Debuggable.__init__(self, name=clsname)
+ self.debug_log("Loading %s transport" % clsname)
self.data = os.path.join(setup['repo'], 'Reporting', clsname)
+ if not os.path.exists(self.data):
+ self.logger.info("%s does not exist, creating" % self.data)
+ try:
+ os.makedirs(self.data)
+ except OSError:
+ self.logger.warning("Could not create %s: %s" %
+ (self.data, sys.exc_info()[1]))
+ self.logger.warning("The transport may not function properly")
self.setup = setup
self.timeout = 2
diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py
index 63149c15e..4d7453840 100644
--- a/src/lib/Bcfg2/Server/BuiltinCore.py
+++ b/src/lib/Bcfg2/Server/BuiltinCore.py
@@ -9,6 +9,7 @@ from Bcfg2.Server.Core import BaseCore, NoExposedMethod
from Bcfg2.Compat import xmlrpclib, urlparse
from Bcfg2.SSLServer import XMLRPCServer
+from lockfile import LockFailed
# pylint: disable=E0611
try:
from daemon.pidfile import PIDLockFile
@@ -80,9 +81,14 @@ class Core(BaseCore):
def _daemonize(self):
""" Open :attr:`context` to drop privileges, write the PID
file, and daemonize the server core. """
- self.context.open()
- self.logger.info("%s daemonized" % self.name)
- return True
+ try:
+ self.context.open()
+ self.logger.info("%s daemonized" % self.name)
+ return True
+ except LockFailed:
+ err = sys.exc_info()[1]
+ self.logger.error("Failed to daemonize %s: %s" % (self.name, err))
+ return False
def _run(self):
""" Create :attr:`server` to start the server listening. """
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index ee875a7e8..040036fb2 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -10,7 +10,6 @@ import threading
import time
import inspect
import lxml.etree
-from traceback import format_exc
import Bcfg2.settings
import Bcfg2.Server
import Bcfg2.Logger
@@ -181,6 +180,19 @@ class BaseCore(object):
#: backend for plugins that have that capability
self._database_available = False
if Bcfg2.settings.HAS_DJANGO:
+ db_settings = Bcfg2.settings.DATABASES['default']
+ if ('daemon' in self.setup and 'daemon_uid' in self.setup and
+ self.setup['daemon'] and self.setup['daemon_uid'] and
+ db_settings['ENGINE'].endswith(".sqlite3") and
+ not os.path.exists(db_settings['NAME'])):
+ # syncdb will create the sqlite database, and we're
+ # going to daemonize, dropping privs to a non-root
+ # user, so we need to chown the database after
+ # creating it
+ do_chown = True
+ else:
+ do_chown = False
+
from django.core.exceptions import ImproperlyConfigured
from django.core import management
try:
@@ -188,11 +200,21 @@ class BaseCore(object):
verbosity=0)
self._database_available = True
except ImproperlyConfigured:
- self.logger.error("Django configuration problem: %s" %
- format_exc().splitlines()[-1])
+ err = sys.exc_info()[1]
+ self.logger.error("Django configuration problem: %s" % err)
except:
- self.logger.error("Database update failed: %s" %
- format_exc().splitlines()[-1])
+ err = sys.exc_info()[1]
+ self.logger.error("Database update failed: %s" % err)
+
+ if do_chown and self._database_available:
+ try:
+ os.chown(db_settings['NAME'],
+ self.setup['daemon_uid'],
+ self.setup['daemon_gid'])
+ except OSError:
+ err = sys.exc_info()[1]
+ self.logger.error("Failed to set ownership of database "
+ "at %s: %s" % (db_settings['NAME'], err))
if '' in setup['plugins']:
setup['plugins'].remove('')
@@ -207,20 +229,23 @@ class BaseCore(object):
"Unloading %s" % (plugin, blacklist))
for plug in blacklist:
del self.plugins[plug]
- # This section logs the experimental plugins
+
+ # Log experimental plugins
expl = [plug for plug in list(self.plugins.values())
if plug.experimental]
if expl:
self.logger.info("Loading experimental plugin(s): %s" %
(" ".join([x.name for x in expl])))
self.logger.info("NOTE: Interfaces subject to change")
- # This section logs the deprecated plugins
+
+ # Log deprecated plugins
depr = [plug for plug in list(self.plugins.values())
if plug.deprecated]
if depr:
self.logger.info("Loading deprecated plugin(s): %s" %
(" ".join([x.name for x in depr])))
+ # Find the metadata plugin and set self.metadata
mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata)
if len(mlist) >= 1:
#: The Metadata plugin
@@ -522,7 +547,7 @@ class BaseCore(object):
except Exception:
exc = sys.exc_info()[1]
if 'failure' not in entry.attrib:
- entry.set('failure', 'bind error: %s' % format_exc())
+ entry.set('failure', 'bind error: %s' % exc)
self.logger.error("Unexpected failure in BindStructure: %s %s"
% (entry.tag, entry.get('name')), exc_info=1)
@@ -599,7 +624,7 @@ class BaseCore(object):
try:
structures = self.GetStructures(meta)
except:
- self.logger.error("error in GetStructures", exc_info=1)
+ self.logger.error("Error in GetStructures", exc_info=1)
return lxml.etree.Element("error", type='structure error')
self.validate_structures(meta, structures)
@@ -662,7 +687,7 @@ class BaseCore(object):
os.chown(piddir,
self.setup['daemon_uid'],
self.setup['daemon_gid'])
- os.chmod(piddir, 420) # 0644
+ os.chmod(piddir, 493) # 0775
if not self._daemonize():
return False
else:
@@ -676,6 +701,9 @@ class BaseCore(object):
self.fam.start()
self.fam_thread.start()
self.fam.AddMonitor(self.cfile, self)
+
+ for plug in self.plugins_by_type(Bcfg2.Server.Plugin.Threaded):
+ plug.start_threads()
except:
self.shutdown()
raise
diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py
index cba3e8145..f42ada773 100644
--- a/src/lib/Bcfg2/Server/Plugin/interfaces.py
+++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py
@@ -299,12 +299,27 @@ class Statistics(Plugin):
raise NotImplementedError
-class ThreadedStatistics(Statistics, threading.Thread):
+class Threaded(object):
+ """ Threaded plugins use threads in any way. The thread must be
+ started after daemonization, so this class implements a single
+ method, :func:`start_threads`, that can be used to start threads
+ after daemonization of the server core. """
+
+ def start_threads(self):
+ """ Start this plugin's threads after daemonization.
+
+ :return: None
+ :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError`
+ """
+ raise NotImplementedError
+
+class ThreadedStatistics(Statistics, Threaded, threading.Thread):
""" ThreadedStatistics plugins process client statistics in a
separate thread. """
def __init__(self, core, datastore):
Statistics.__init__(self, core, datastore)
+ Threaded.__init__(self)
threading.Thread.__init__(self)
# Event from the core signaling an exit
self.terminate = core.terminate
@@ -312,6 +327,8 @@ class ThreadedStatistics(Statistics, threading.Thread):
self.pending_file = os.path.join(datastore, "etc",
"%s.pending" % self.name)
self.daemon = False
+
+ def start_threads(self):
self.start()
def _save(self):
@@ -517,11 +534,11 @@ class Version(Plugin):
def __init__(self, core, datastore):
Plugin.__init__(self, core, datastore)
+ if core.setup['vcs_root']:
+ self.vcs_root = core.setup['vcs_root']
+ else:
+ self.vcs_root = datastore
if self.__vcs_metadata_path__:
- if core.setup['vcs_root']:
- self.vcs_root = core.setup['vcs_root']
- else:
- self.vcs_root = datastore
self.vcs_path = os.path.join(self.vcs_root,
self.__vcs_metadata_path__)
diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py
index 7933fe9be..b200346bc 100644
--- a/src/lib/Bcfg2/Server/Plugins/Bundler.py
+++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py
@@ -19,6 +19,9 @@ except ImportError:
HAS_GENSHI = False
+SETUP = None
+
+
class BundleFile(Bcfg2.Server.Plugin.StructFile):
""" Representation of a bundle XML file """
def get_xml_value(self, metadata):
@@ -49,7 +52,8 @@ if HAS_GENSHI:
msg = "No parsed template information for %s" % self.name
self.logger.error(msg)
raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
- stream = self.template.generate(metadata=metadata).filter(
+ stream = self.template.generate(metadata=metadata,
+ repo=SETUP['repo']).filter(
Bcfg2.Server.Plugins.TGenshi.removecomment)
data = lxml.etree.XML(stream.render('xml',
strip_whitespace=False),
@@ -93,8 +97,13 @@ class Bundler(Bcfg2.Server.Plugin.Plugin,
self.data,
self.core.fam)
except OSError:
- self.logger.error("Failed to load Bundle repository")
- raise Bcfg2.Server.Plugin.PluginInitError
+ err = sys.exc_info()[1]
+ msg = "Failed to load Bundle repository %s: %s" % (self.data, err)
+ self.logger.error(msg)
+ raise Bcfg2.Server.Plugin.PluginInitError(msg)
+
+ global SETUP
+ SETUP = core.setup
def template_dispatch(self, name, _):
""" Add the correct child entry type to Bundler depending on
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py
index 8ebd8d921..724164cf5 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py
@@ -3,7 +3,7 @@
:ref:`server-plugins-generators-cfg` files. """
from Bcfg2.Server.Plugin import PluginExecutionError
-from Bcfg2.Server.Plugins.Cfg import CfgGenerator
+from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP
try:
from Cheetah.Template import Template
@@ -37,7 +37,9 @@ class CfgCheetahGenerator(CfgGenerator):
template = Template(self.data.decode(self.encoding),
compilerSettings=self.settings)
template.metadata = metadata
+ template.name = entry.get('realname', entry.get('name'))
template.path = entry.get('realname', entry.get('name'))
template.source_path = self.name
+ template.repo = SETUP['repo']
return template.respond()
get_data.__doc__ = CfgGenerator.get_data.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
index df0c30c09..3a78b4847 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
@@ -6,7 +6,7 @@ import re
import sys
import traceback
from Bcfg2.Server.Plugin import PluginExecutionError
-from Bcfg2.Server.Plugins.Cfg import CfgGenerator
+from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP
try:
import genshi.core
@@ -74,7 +74,9 @@ class CfgGenshiGenerator(CfgGenerator):
stream = \
self.template.generate(name=fname,
metadata=metadata,
- path=self.name).filter(removecomment)
+ path=self.name,
+ source_path=self.name,
+ repo=SETUP['repo']).filter(removecomment)
try:
try:
return stream.render('text', encoding=self.encoding,
@@ -135,9 +137,13 @@ class CfgGenshiGenerator(CfgGenerator):
# single line break)
real_lineno = lineno - contents.code.co_firstlineno
src = re.sub(r'\n\n+', '\n', contents.source).splitlines()
- raise PluginExecutionError("%s: %s at '%s'" %
- (err.__class__.__name__, err,
- src[real_lineno]))
+ try:
+ raise PluginExecutionError("%s: %s at '%s'" %
+ (err.__class__.__name__, err,
+ src[real_lineno]))
+ except IndexError:
+ raise PluginExecutionError("%s: %s" %
+ (err.__class__.__name__, err))
raise
def handle_event(self, event):
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index 220146100..37171e1b1 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -536,22 +536,37 @@ class YumCollection(Collection):
consumerapi = ConsumerAPI()
consumer = self._get_pulp_consumer(consumerapi=consumerapi)
if consumer is None:
- consumer = consumerapi.create(self.metadata.hostname,
- self.metadata.hostname,
- capabilities=dict(bind=False))
- lxml.etree.SubElement(independent, "BoundAction",
- name="pulp-update", timing="pre",
- when="always", status="check",
- command="pulp-consumer consumer update")
- self.pulp_cert_set.write_data(consumer['certificate'],
- self.metadata)
+ try:
+ consumer = \
+ consumerapi.create(self.metadata.hostname,
+ self.metadata.hostname,
+ capabilities=dict(bind=False))
+ lxml.etree.SubElement(
+ independent, "BoundAction", name="pulp-update",
+ timing="pre", when="always", status="check",
+ command="pulp-consumer consumer update")
+ self.pulp_cert_set.write_data(consumer['certificate'],
+ self.metadata)
+ except server.ServerRequestError:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: Could not create Pulp "
+ "consumer %s: %s" %
+ (self.metadata.hostname, err))
for source in self:
# each pulp source can only have one arch, so we don't
# have to check the arch in url_map
if (source.pulp_id and
source.pulp_id not in consumer['repoids']):
- consumerapi.bind(self.metadata.hostname, source.pulp_id)
+ try:
+ consumerapi.bind(self.metadata.hostname,
+ source.pulp_id)
+ except server.ServerRequestError:
+ err = sys.exc_info()[1]
+ self.logger.error("Packages: Could not bind %s to "
+ "Pulp repo %s: %s" %
+ (self.metadata.hostname,
+ source.pulp_id, err))
crt = lxml.etree.SubElement(independent, "BoundPath",
name=self.pulp_cert_set.certpath)
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index 90dff4a66..f106b75a4 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -162,6 +162,9 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
ret.append(probe)
return ret
+ def __str__(self):
+ return "ProbeSet for %s" % self.plugin_name
+
class Probes(Bcfg2.Server.Plugin.Probing,
Bcfg2.Server.Plugin.Connector,
diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py
index 1a8c3d941..60f5b1e09 100644
--- a/src/lib/Bcfg2/Server/Plugins/Reporting.py
+++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py
@@ -7,8 +7,8 @@ import lxml.etree
from Bcfg2.Reporting.Transport import load_transport_from_config, \
TransportError
from Bcfg2.Options import REPORTING_COMMON_OPTIONS
-from Bcfg2.Server.Plugin import Statistics, PullSource, PluginInitError, \
- PluginExecutionError
+from Bcfg2.Server.Plugin import Statistics, PullSource, Threaded, \
+ Debuggable, PluginInitError, PluginExecutionError
# required for reporting
try:
@@ -31,9 +31,10 @@ def _rpc_call(method):
return _real_rpc_call
-class Reporting(Statistics, PullSource): # pylint: disable=W0223
+# pylint: disable=W0223
+class Reporting(Statistics, Threaded, PullSource, Debuggable):
""" Unified statistics and reporting plugin """
- __rmi__ = ['Ping', 'GetExtra', 'GetCurrentEntry']
+ __rmi__ = Debuggable.__rmi__ + ['Ping', 'GetExtra', 'GetCurrentEntry']
CLIENT_METADATA_FIELDS = ('profile', 'bundles', 'aliases', 'addresses',
'groups', 'categories', 'uuid', 'version')
@@ -41,7 +42,8 @@ class Reporting(Statistics, PullSource): # pylint: disable=W0223
def __init__(self, core, datastore):
Statistics.__init__(self, core, datastore)
PullSource.__init__(self)
- self.core = core
+ Threaded.__init__(self)
+ Debuggable.__init__(self)
self.whoami = platform.node()
self.transport = None
@@ -54,14 +56,20 @@ class Reporting(Statistics, PullSource): # pylint: disable=W0223
self.logger.error(msg)
raise PluginInitError(msg)
+ def start_threads(self):
try:
- self.transport = load_transport_from_config(core.setup)
+ self.transport = load_transport_from_config(self.core.setup)
except TransportError:
msg = "%s: Failed to load transport: %s" % \
(self.name, traceback.format_exc().splitlines()[-1])
self.logger.error(msg)
raise PluginInitError(msg)
+ def set_debug(self, debug):
+ rv = Debuggable.set_debug(self, debug)
+ self.transport.set_debug(debug)
+ return rv
+
def process_statistics(self, client, xdata):
stats = xdata.find("Statistics")
stats.set('time', time.asctime(time.localtime()))
@@ -84,8 +92,8 @@ class Reporting(Statistics, PullSource): # pylint: disable=W0223
lxml.etree.tostring(
stats,
xml_declaration=False).decode('UTF-8'))
- self.logger.debug("%s: Queued statistics data for %s" %
- (self.__class__.__name__, client.hostname))
+ self.debug_log("%s: Queued statistics data for %s" %
+ (self.__class__.__name__, client.hostname))
return
except TransportError:
continue
@@ -94,7 +102,7 @@ class Reporting(Statistics, PullSource): # pylint: disable=W0223
% (self.__class__.__name__, i,
traceback.format_exc().splitlines()[-1]))
self.logger.error("%s: Retry limit reached for %s" %
- (self.__class__.__name__, client.hostname))
+ (self.__class__.__name__, client.hostname))
def shutdown(self):
super(Reporting, self).shutdown()
diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py
index 62396f860..b3a49c047 100644
--- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py
+++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py
@@ -73,9 +73,8 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool):
cert_spec.get('append_chain',
'false').lower() == 'true',
}
- cfp = ConfigParser.ConfigParser()
- cfp.read(self.core.cfile)
- self.CAs[ca] = dict(cfp.items('sslca_' + ca))
+ self.CAs[ca] = dict(self.core.setup.cfp.items('sslca_%s' %
+ ca))
self.Entries['Path'][ident] = self.get_cert
elif event.filename.endswith("info.xml"):
self.infoxml[ident] = Bcfg2.Server.Plugin.InfoXML(epath)
diff --git a/src/lib/Bcfg2/Server/Plugins/Snapshots.py b/src/lib/Bcfg2/Server/Plugins/Snapshots.py
index 1956af4ad..cc5946bb2 100644
--- a/src/lib/Bcfg2/Server/Plugins/Snapshots.py
+++ b/src/lib/Bcfg2/Server/Plugins/Snapshots.py
@@ -65,6 +65,8 @@ class Snapshots(Bcfg2.Server.Plugin.Statistics):
self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile)
self.work_queue = Queue()
self.loader = threading.Thread(target=self.load_snapshot)
+
+ def start_threads(self):
self.loader.start()
def load_snapshot(self):
diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py
index 17a275340..bc585570d 100644
--- a/src/lib/Bcfg2/Server/Plugins/Svn.py
+++ b/src/lib/Bcfg2/Server/Plugins/Svn.py
@@ -17,10 +17,9 @@ except ImportError:
class Svn(Bcfg2.Server.Plugin.Version):
"""Svn is a version plugin for dealing with Bcfg2 repos."""
__author__ = 'bcfg-dev@mcs.anl.gov'
+ __vcs_metadata_path__ = ".svn"
if HAS_SVN:
__rmi__ = Bcfg2.Server.Plugin.Version.__rmi__ + ['Update', 'Commit']
- else:
- __vcs_metadata_path__ = ".svn"
def callback_conflict_resolver(self):
"""PySvn callback function to resolve conflicts"""
diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info
index acb9e4f44..fa8c89b46 100755
--- a/src/sbin/bcfg2-info
+++ b/src/sbin/bcfg2-info
@@ -27,12 +27,6 @@ try:
except ImportError:
HAS_PROFILE = False
-try:
- from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile
- HAS_GENSHI = True
-except ImportError:
- HAS_GENSHI = False
-
class MockLog(object):
""" Fake logger that just discards all messages in order to mask
@@ -401,28 +395,24 @@ Bcfg2 client itself.""")
def do_buildbundle(self, args):
""" buildbundle <bundle> <hostname> - Render a templated
bundle for hostname (not written to disk) """
- if len(args.split()) == 2:
- bname, client = args.split()
- try:
- metadata = self.build_metadata(client)
- if bname in self.plugins['Bundler'].entries:
- bundle = self.plugins['Bundler'].entries[bname]
- if (HAS_GENSHI and
- isinstance(bundle,
- BundleTemplateFile)):
- stream = bundle.template.generate(metadata=metadata)
- print(stream.render("xml"))
- else:
- print(bundle.data)
- else:
- print("No such bundle %s" % bname)
- except: # pylint: disable=W0702
- err = sys.exc_info()[1]
- print("Failed to render bundle %s for host %s: %s" % (bname,
- client,
- err))
- else:
+ if len(args.split()) != 2:
print(self._get_usage(self.do_buildbundle))
+ return 1
+
+ bname, client = args.split()
+ try:
+ metadata = self.build_metadata(client)
+ bundle = self.plugins['Bundler'].entries[bname]
+ print(lxml.etree.tostring(bundle.get_xml_value(metadata),
+ xml_declaration=False,
+ pretty_print=True).decode('UTF-8'))
+ except KeyError:
+ print("No such bundle %s" % bname)
+ except: # pylint: disable=W0702
+ err = sys.exc_info()[1]
+ print("Failed to render bundle %s for host %s: %s" % (bname,
+ client,
+ err))
def do_automatch(self, args):
""" automatch [-f] <propertyfile> <hostname> - Perform automatch on
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py
index 2eda38cdc..a1e624824 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py
@@ -71,10 +71,12 @@ class TestPlugin(TestDebuggable):
def get_obj(self, core=None):
if core is None:
core = Mock()
+ core.setup = MagicMock()
return self.test_obj(core, datastore)
def test__init(self):
core = Mock()
+ core.setup = MagicMock()
p = self.get_obj(core=core)
self.assertEqual(p.data, os.path.join(datastore, p.name))
self.assertEqual(p.core, core)
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py
index 9a064663e..343f088b3 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py
@@ -108,15 +108,30 @@ class TestStatistics(TestPlugin):
s.process_statistics, None, None)
-class TestThreadedStatistics(TestStatistics):
+class TestThreaded(Bcfg2TestCase):
+ test_obj = Threaded
+
+ def get_obj(self):
+ return self.test_obj()
+
+ def test_start_threads(self):
+ s = self.get_obj()
+ self.assertRaises(NotImplementedError,
+ s.start_threads)
+
+
+class TestThreadedStatistics(TestStatistics, TestThreaded):
test_obj = ThreadedStatistics
data = [("foo.example.com", "<foo/>"),
("bar.example.com", "<bar/>")]
+ def get_obj(self, core=None):
+ return TestStatistics.get_obj(self, core=core)
+
@patch("threading.Thread.start")
- def test__init(self, mock_start):
- core = Mock()
- ts = self.get_obj(core)
+ def test_start_threads(self, mock_start):
+ ts = self.get_obj()
+ ts.start_threads()
mock_start.assert_any_call()
@patch("%s.open" % builtins)
@@ -157,7 +172,7 @@ class TestThreadedStatistics(TestStatistics):
# verify this call in an ugly way
self.assertItemsEqual(mock_dump.call_args[0][0], self.data)
self.assertEqual(mock_dump.call_args[0][1], mock_open.return_value)
-
+
@patch("os.unlink")
@patch("os.path.exists")
@patch("%s.open" % builtins)
@@ -169,7 +184,7 @@ class TestThreadedStatistics(TestStatistics):
core = Mock()
core.terminate.isSet.return_value = False
ts = self.get_obj(core)
-
+
ts.work_queue = Mock()
ts.work_queue.data = []
def reset():
@@ -279,7 +294,7 @@ class TestThreadedStatistics(TestStatistics):
ts = self.get_obj()
self.assertRaises(NotImplementedError,
ts.handle_statistic, None, None)
-
+
class TestPullSource(Bcfg2TestCase):
def test_GetCurrentEntry(self):
@@ -302,7 +317,7 @@ class TestPullTarget(Bcfg2TestCase):
class TestDecision(Bcfg2TestCase):
test_obj = Decision
-
+
def get_obj(self):
return self.test_obj()
@@ -332,6 +347,7 @@ class TestVersion(TestPlugin):
def get_obj(self, core=None):
if core is None:
core = Mock()
+ core.setup = MagicMock()
return self.test_obj(core, datastore)
def test_get_revision(self):
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py
index 1832e5e03..fc5d5e53d 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgCheetahGenerator.py
@@ -31,9 +31,18 @@ if HAS_CHEETAH or can_skip:
ccg.data = "data"
entry = lxml.etree.Element("Path", name="/test.txt")
metadata = Mock()
+ Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.SETUP = MagicMock()
self.assertEqual(ccg.get_data(entry, metadata),
mock_Template.return_value.respond.return_value)
+ Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.SETUP.__getitem__.assert_called_with("repo")
mock_Template.assert_called_with("data".decode(ccg.encoding),
compilerSettings=ccg.settings)
- mock_Template.return_value.respond.assert_called_with()
+ tmpl = mock_Template.return_value
+ tmpl.respond.assert_called_with()
+ self.assertEqual(tmpl.metadata, metadata)
+ self.assertEqual(tmpl.name, entry.get("name"))
+ self.assertEqual(tmpl.path, entry.get("name"))
+ self.assertEqual(tmpl.source_path, ccg.name)
+ self.assertEqual(tmpl.repo,
+ Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator.SETUP.__getitem__.return_value)
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py
index 4a849c11a..385f8df77 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py
@@ -51,15 +51,24 @@ if can_skip or HAS_GENSHI:
entry = lxml.etree.Element("Path", name="/test.txt")
metadata = Mock()
+ Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP = MagicMock()
+
def reset():
cgg.template.reset_mock()
cgg._handle_genshi_exception.reset_mock()
+ Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.reset_mock()
+
+ template_vars = dict(
+ name=entry.get("name"),
+ metadata=metadata,
+ path=cgg.name,
+ source_path=cgg.name,
+ repo=Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.return_value)
self.assertEqual(cgg.get_data(entry, metadata),
stream.render.return_value)
- cgg.template.generate.assert_called_with(name=entry.get("name"),
- metadata=metadata,
- path=cgg.name)
+ cgg.template.generate.assert_called_with(**template_vars)
+ Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo")
fltr.filter.assert_called_with(removecomment)
stream.render.assert_called_with("text", encoding=cgg.encoding,
strip_whitespace=False)
@@ -71,9 +80,8 @@ if can_skip or HAS_GENSHI:
stream.render.side_effect = render
self.assertEqual(cgg.get_data(entry, metadata),
stream.render.return_value)
- cgg.template.generate.assert_called_with(name=entry.get("name"),
- metadata=metadata,
- path=cgg.name)
+ cgg.template.generate.assert_called_with(**template_vars)
+ Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo")
fltr.filter.assert_called_with(removecomment)
self.assertEqual(stream.render.call_args_list,
[call("text", encoding=cgg.encoding,
@@ -84,9 +92,8 @@ if can_skip or HAS_GENSHI:
stream.render.side_effect = UndefinedError("test")
self.assertRaises(UndefinedError,
cgg.get_data, entry, metadata)
- cgg.template.generate.assert_called_with(name=entry.get("name"),
- metadata=metadata,
- path=cgg.name)
+ cgg.template.generate.assert_called_with(**template_vars)
+ Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo")
fltr.filter.assert_called_with(removecomment)
stream.render.assert_called_with("text", encoding=cgg.encoding,
strip_whitespace=False)
@@ -96,9 +103,8 @@ if can_skip or HAS_GENSHI:
cgg._handle_genshi_exception.side_effect = ValueError
self.assertRaises(ValueError,
cgg.get_data, entry, metadata)
- cgg.template.generate.assert_called_with(name=entry.get("name"),
- metadata=metadata,
- path=cgg.name)
+ cgg.template.generate.assert_called_with(**template_vars)
+ Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator.SETUP.__getitem__.assert_called_with("repo")
fltr.filter.assert_called_with(removecomment)
stream.render.assert_called_with("text", encoding=cgg.encoding,
strip_whitespace=False)