summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.travis.yml1
-rw-r--r--COPYRIGHT4
-rw-r--r--doc/_templates/indexsidebar.html1
-rw-r--r--doc/development/documentation.txt27
-rw-r--r--doc/development/lint.txt167
-rw-r--r--doc/server/plugins/connectors/grouplogic.txt4
-rw-r--r--doc/server/plugins/generators/packages.txt5
-rw-r--r--doc/server/plugins/generators/rules.txt6
-rw-r--r--doc/server/plugins/grouping/grouppatterns.txt17
-rw-r--r--gentoo/bcfg2-1.3.0.ebuild10
-rw-r--r--misc/bcfg2.spec1
-rw-r--r--schemas/packages.xsd9
-rw-r--r--schemas/selinux.xsd42
-rw-r--r--schemas/types.xsd29
-rwxr-xr-xsetup.py28
-rw-r--r--src/lib/Bcfg2/Client/Frame.py16
-rw-r--r--src/lib/Bcfg2/Client/Proxy.py8
-rw-r--r--src/lib/Bcfg2/Client/Tools/Chkconfig.py9
-rw-r--r--src/lib/Bcfg2/Client/Tools/POSIXUsers.py6
-rw-r--r--src/lib/Bcfg2/Client/Tools/Portage.py8
-rw-r--r--src/lib/Bcfg2/Client/Tools/SELinux.py20
-rw-r--r--src/lib/Bcfg2/Client/Tools/Systemd.py2
-rw-r--r--src/lib/Bcfg2/Client/__init__.py2
-rw-r--r--src/lib/Bcfg2/Compat.py29
-rw-r--r--src/lib/Bcfg2/Reporting/Transport/__init__.py16
-rw-r--r--src/lib/Bcfg2/Reporting/models.py30
-rw-r--r--src/lib/Bcfg2/Server/Admin/__init__.py1
-rw-r--r--src/lib/Bcfg2/Server/BuiltinCore.py2
-rw-r--r--src/lib/Bcfg2/Server/Lint/Comments.py123
-rwxr-xr-xsrc/lib/Bcfg2/Server/Lint/Genshi.py9
-rw-r--r--src/lib/Bcfg2/Server/Lint/GroupNames.py28
-rw-r--r--src/lib/Bcfg2/Server/Lint/InfoXML.py14
-rw-r--r--src/lib/Bcfg2/Server/Lint/MergeFiles.py2
-rw-r--r--src/lib/Bcfg2/Server/Lint/RequiredAttrs.py28
-rw-r--r--src/lib/Bcfg2/Server/Lint/Validate.py55
-rw-r--r--src/lib/Bcfg2/Server/Lint/__init__.py180
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Bundler.py7
-rw-r--r--src/lib/Bcfg2/Server/Plugins/GroupPatterns.py7
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Metadata.py84
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Apt.py5
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Source.py4
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/__init__.py4
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Pkgmgr.py16
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Probes.py14
-rw-r--r--src/lib/Bcfg2/Server/Plugins/SSHbase.py7
-rw-r--r--src/lib/Bcfg2/Server/Plugins/TemplateHelper.py19
-rw-r--r--src/lib/Bcfg2/Server/SSLServer.py2
-rw-r--r--src/lib/Bcfg2/Utils.py2
-rwxr-xr-xsrc/sbin/bcfg2-info30
-rwxr-xr-xsrc/sbin/bcfg2-lint11
-rwxr-xr-xsrc/sbin/bcfg2-test1
-rw-r--r--testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py9
-rwxr-xr-xtools/bcfg2-profile-templates.py95
-rwxr-xr-xtools/bcfg2_local.py3
-rwxr-xr-xtools/export.py3
-rwxr-xr-xtools/posixusers_baseline.py4
-rwxr-xr-xtools/selinux_baseline.py5
-rwxr-xr-xtools/upgrade/1.3/migrate_dbstats.py19
-rwxr-xr-xtools/yum-listpkgs-xml.py2
59 files changed, 942 insertions, 350 deletions
diff --git a/.travis.yml b/.travis.yml
index 73b8a9594..655d9fad5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,6 +3,7 @@ python:
- "2.5"
- "2.6"
- "2.7"
+ - "3.2"
env:
- WITH_OPTIONAL_DEPS=yes
- WITH_OPTIONAL_DEPS=no
diff --git a/COPYRIGHT b/COPYRIGHT
index 571fa6034..347d9b236 100644
--- a/COPYRIGHT
+++ b/COPYRIGHT
@@ -1,3 +1,7 @@
+This file contains a list of copyright holders. Anyone who
+contributes more than trivial fixes (typos, etc.) to Bcfg2 should also
+add themselves to this file. See LICENSE for the full license.
+
- Narayan Desai <desai@mcs.anl.gov> has written most of Bcfg2,
including all parts not explicitly mentioned in this file.
diff --git a/doc/_templates/indexsidebar.html b/doc/_templates/indexsidebar.html
index 39916315d..2133cdcc5 100644
--- a/doc/_templates/indexsidebar.html
+++ b/doc/_templates/indexsidebar.html
@@ -7,5 +7,6 @@
<ul>
<li><a href="http://docs.bcfg2.org/1.1/">Bcfg2 1.1 (stable)</a></li>
<li><a href="http://docs.bcfg2.org/1.2/">Bcfg2 1.2 (stable)</a></li>
+ <li><a href="http://docs.bcfg2.org/1.3/">Bcfg2 1.3 (stable)</a></li>
<li><a href="http://docs.bcfg2.org/dev/">Bcfg2 development documentation</a></li>
</ul>
diff --git a/doc/development/documentation.txt b/doc/development/documentation.txt
index 2a3cf46d1..4d8a7c9f8 100644
--- a/doc/development/documentation.txt
+++ b/doc/development/documentation.txt
@@ -8,13 +8,14 @@
There are two parts of documentation in the Bcfg2 project:
-* The wiki
-* The manual
+* The Wiki_
+* The Manual_
The wiki
========
.. _Wiki: http://bcfg2.org
+.. _Manual: http://docs.bcfg2.org
.. _Trac: http://trac.edgewall.org/
.. _OpenID: https://openid.org/
.. _MCS: http://www.mcs.anl.gov/
@@ -31,9 +32,9 @@ The manual
==========
.. _rst: http://en.wikipedia.org/wiki/ReStructuredText
.. _Sphinx: http://sphinx.pocoo.org
-.. _Docutils:
+.. _Docutils: http://docutils.sourceforge.net
-The source for the manual is located in the ``doc/`` directory in the
+The source for the Manual_ is located in the ``doc/`` directory in the
git repository or in the source tarball. All files are written in
rst_ (ReStructuredText) format. Sphinx_ is used to build the
documentation from the restructured text sources.
@@ -49,11 +50,20 @@ Building the Manual
apt-get -t lenny-backports install python-sphinx
- * The needed tools for Fedora based systems are in the `Fedora
+ * The tools for Fedora based systems are in the `Fedora
Package Collection <https://admin.fedoraproject.org/pkgdb>`_;
installation can be done easily with Yum::
yum -y install python-sphinx python-docutils
+
+ * The tools for RHEL6-based systems are in the base distribution; you can install them with Yum::
+
+ yum -y install python-sphinx python-docutils
+
+ * The tools for RHEL5-based systems are in the `Extra Packages for Enterprise Linux(EPEL) <https://fedoraproject.org/wiki/EPEL>`_ repository; if your system is configured for EPEL, you can install them with Yum::
+
+ yum -y install python-sphinx python-docutils
+
* Additionally, to build the PDF version:
@@ -80,14 +90,14 @@ Documentation Style Guide for Bcfg2
===================================
This is a style guide to use when creating documentation for Bcfg2. It
-is meant to be helpful, not a hinderence.
+is meant to be helpful, not a hindrance.
Basics
------
**Bcfg2**
- When referring to project, Bcfg2 is the preferred use of cases.
+ When referring to project, Bcfg2 is the preferred use of case.
**Monospace fonts**
@@ -97,8 +107,7 @@ Basics
**Repository**
When used alone this refers to a Bcfg2 :term:`repository`. When there
- is a chance for confusion, for instance in documents also talking
- about :term:`VCS`, be sure to use the longer Bcfg2 :term:`repository`.
+ is a chance for confusion, for instance in documents that also discuss :term:`VCS`, be sure to use the longer phrase "Bcfg2 :term:`repository`".
Sections
--------
diff --git a/doc/development/lint.txt b/doc/development/lint.txt
new file mode 100644
index 000000000..6a4651f92
--- /dev/null
+++ b/doc/development/lint.txt
@@ -0,0 +1,167 @@
+.. -*- mode: rst -*-
+
+.. _development-lint:
+
+===============================
+ bcfg2-lint Plugin Development
+===============================
+
+``bcfg2-lint``, like most parts of Bcfg2, has a pluggable backend that
+lets you easily write your own plugins to verify various parts of your
+Bcfg2 specification.
+
+Plugins are loaded in one of two ways:
+
+* They may be included in a module of the same name as the plugin
+ class in :mod:`Bcfg2.Server.Lint`, e.g.,
+ :mod:`Bcfg2.Server.Lint.Validate`.
+* They may be included directly in a Bcfg2 server plugin, called
+ "<plugin>Lint", e.g.,
+ :class:`Bcfg2.Server.Plugins.Metadata.MetadataLint`.
+
+Plugin Types
+============
+
+There are two types of ``bcfg2-lint`` plugins:
+
+Serverless plugins
+------------------
+
+Serverless plugins are run before ``bcfg2-lint`` starts up a local
+Bcfg2 server, so the amount of introspection they can do is fairly
+limited. They can directly examine the Bcfg2 specification, of
+course, but they can't examine the entries handled by a given plugin
+or anything that requires a running server.
+
+If a serverless plugin raises a lint error, however, the server will
+not be started and no `Server plugins`_ will be run. This makes them
+useful to check for the sorts of errors that might prevent the Bcfg2
+server from starting properly.
+
+Serverless plugins must subclass
+:class:`Bcfg2.Server.Lint.ServerlessPlugin`.
+
+:mod:`Bcfg2.Server.Lint.Validate` is an example of a serverless
+plugin.
+
+Server plugins
+--------------
+
+Server plugins are run after a local Bcfg2 server has been started,
+and have full access to all of the parsed data and so on. Because of
+this, they tend to be easier to use than `Serverless plugins`_, and
+thus are more common.
+
+Server plugins are only run if all `Serverless plugins`_ run
+successfully (i.e., raise no errors).
+
+Server plugins must subclass :class:`Bcfg2.Server.Lint.ServerPlugin`.
+
+:mod:`Bcfg2.Server.Lint.Genshi` is an example of a server plugin.
+
+Error Handling
+==============
+
+The job of a ``bcfg2-lint`` plugin is to find errors. Each error that
+a plugin may produce must have a name, a short string that briefly
+describes the error and will be used to configure error levels in
+``bcfg2.conf``. It must also have a default reporting level.
+Possible reporting levels are "error", "warning", or "silent". All of
+the errors that may be produced by a plugin must be returned as a dict
+by :func:`Bcfg2.Server.Lint.Plugin.Errors`. For instance, consider
+:func:`Bcfg2.Server.Lint.InfoXML.InfoXML.Errors`:
+
+.. code-block:: python
+
+ @classmethod
+ def Errors(cls):
+ return {"no-infoxml": "warning",
+ "deprecated-info-file": "warning",
+ "paranoid-false": "warning",
+ "required-infoxml-attrs-missing": "error"}
+
+This means that the :class:`Bcfg2.Server.Lint.InfoXML.InfoXML` lint
+plugin can produce five lint errors, although four of them are just
+warnings by default.
+
+The errors returned by each plugin's ``Errors()`` method will be
+passed to :func:`Bcfg2.Server.Lint.ErrorHandler.RegisterErrors`, which
+will use that information and the information in the config file to
+determine how to display (or not display) each error to the end user.
+
+Errors are produced in a plugin with
+:func:`Bcfg2.Server.Lint.Plugin.LintError`, which takes two arguments:
+the name of the error, which must correspond to a key in the dict
+returned by :func:`Bcfg2.Server.Lint.Plugin.Errors`, and a freeform
+string that will be displayed to the end user. Note that the error
+name and its display are thus only tied together when the error is
+produced; that is, a single error (by name) can have two completely
+different outputs.
+
+Basics
+======
+
+.. automodule:: Bcfg2.Server.Lint
+
+Existing ``bcfg2-lint`` Plugins
+===============================
+
+BundlerLint
+-----------
+
+.. autoclass:: Bcfg2.Server.Plugins.Bundler.BundlerLint
+
+Comments
+--------
+
+.. automodule:: Bcfg2.Server.Lint.Comments
+
+Genshi
+------
+
+.. automodule:: Bcfg2.Server.Lint.Genshi
+
+GroupNames
+----------
+
+.. automodule:: Bcfg2.Server.Lint.GroupNames
+
+GroupPatternsLint
+-----------------
+
+.. autoclass:: Bcfg2.Server.Plugins.GroupPatterns.GroupPatternsLint
+
+InfoXML
+-------
+
+.. automodule:: Bcfg2.Server.Lint.InfoXML
+
+MergeFiles
+----------
+
+.. automodule:: Bcfg2.Server.Lint.MergeFiles
+
+MetadataLint
+------------
+
+.. autoclass:: Bcfg2.Server.Plugins.Metadata.MetadataLint
+
+PkgmgrLint
+----------
+
+.. autoclass:: Bcfg2.Server.Plugins.Pkgmgr.PkgmgrLint
+
+RequiredAttrs
+-------------
+
+.. automodule:: Bcfg2.Server.Lint.RequiredAttrs
+
+TemplateHelperLint
+------------------
+
+.. autoclass:: Bcfg2.Server.Plugins.TemplateHelper.TemplateHelperLint
+
+Validate
+--------
+
+.. automodule:: Bcfg2.Server.Lint.Validate
diff --git a/doc/server/plugins/connectors/grouplogic.txt b/doc/server/plugins/connectors/grouplogic.txt
index b9a5b00d6..abf425202 100644
--- a/doc/server/plugins/connectors/grouplogic.txt
+++ b/doc/server/plugins/connectors/grouplogic.txt
@@ -110,8 +110,8 @@ individually, there's a more elegant way to accomplish the same thing:
<GroupLogic xmlns:py="http://genshi.edgewall.org/">
<?python
-component = metadata.group_in_category("webapp-component")
-env = metadata.group_in_category("environment")
+ component = metadata.group_in_category("webapp-component")
+ env = metadata.group_in_category("environment")
?>
<py:if test="component and env">
<Group name="${component}-${env}"/>
diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt
index 092cff1ae..a7cdfad2d 100644
--- a/doc/server/plugins/generators/packages.txt
+++ b/doc/server/plugins/generators/packages.txt
@@ -190,7 +190,8 @@ something like this:
<Group name="ubuntu-intrepid">
<Source type="apt"
url="http://us.archive.ubuntu.com/ubuntu"
- version="intrepid">
+ version="intrepid"
+ debsrc="true">
<Component>main</Component>
<Component>universe</Component>
<Arch>i386</Arch>
@@ -218,7 +219,7 @@ something like this:
.. warning:: You must regenerate the Packages cache when adding or
removing the recommended attribute (``bcfg2-admin xcmd
- Packages.Refresh``).
+ Packages.Refresh``).
.. [#f1] Bcfg2 will by default add **Essential** packages to the
client specification. You can disable this behavior by
diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt
index ccde65eb0..a85cd3fc9 100644
--- a/doc/server/plugins/generators/rules.txt
+++ b/doc/server/plugins/generators/rules.txt
@@ -377,9 +377,9 @@ For example:
<POSIXUser name="daemon" home="/sbin" shell="/sbin/nologin"
gecos="daemon" uid="2" group="daemon">
- <MemberOf>lp</MemberOf>
- <MemberOf>adm</MemberOf>
- <MemberOf>bin</MemberOf>
+ <MemberOf group="lp"/>
+ <MemberOf group="adm"/>
+ <MemberOf group="bin/>
</POSIXUser>
The group specified will automatically be created if it does not
diff --git a/doc/server/plugins/grouping/grouppatterns.txt b/doc/server/plugins/grouping/grouppatterns.txt
index 39c632f82..44ffa5066 100644
--- a/doc/server/plugins/grouping/grouppatterns.txt
+++ b/doc/server/plugins/grouping/grouppatterns.txt
@@ -8,7 +8,7 @@ GroupPatterns
The GroupPatterns plugin is a connector that can assign clients
group membership pased on patterns in client hostnames. Two basic
-methods are supported:
+methods are supported:
- regular expressions (NamePatterns)
- ranges (NameRange)
@@ -20,7 +20,7 @@ Setup
=====
#. Enable the GroupPatterns plugin
-#. Create the GroupPatterns/config.xml file (similar to the example below).
+#. Create the ``GroupPatterns/config.xml`` file (similar to the example below).
#. Client groups will be augmented based on the specification
Pattern Types
@@ -52,7 +52,7 @@ Examples
</GroupPattern>
<GroupPattern>
<NamePattern>(.*)</NamePattern>
- <Group>group-$1'</Group>
+ <Group>group-$1</Group>
</GroupPattern>
<GroupPattern>
<NameRange>node[[1-32]]</NameRange>
@@ -82,15 +82,14 @@ GroupPatterns configuration:
<GroupPatterns>
<GroupPattern>
- <NamePattern>^x(\w[^\d|\.]+)\d*\..*</NamePattern>
+ <NamePattern>x(\w[^\d\.]+)\d*\.</NamePattern>
<Group>$1-server</Group>
</GroupPattern>
</GroupPatterns>
Regex explanation:
-#. !^x Match any hostname that begins with "x"
-#. (\w[!^\d|\.]+) followed by one or more word characters that are not a decimal digit or "." and save the string to $1
-#. \d* followed by 0 or more decimal digit(s)
-#. \..* followed by a "."
-#. .* followed by 1 or more of anything else.
+#. ``x`` Match any hostname that begins with "x"
+#. ``(\w[!^\d|\.]+)`` followed by one or more word characters that are not a decimal digit or "." and save the string to $1
+#. ``\d*`` followed by 0 or more decimal digit(s)
+#. ``\.`` followed by a literal "."
diff --git a/gentoo/bcfg2-1.3.0.ebuild b/gentoo/bcfg2-1.3.0.ebuild
index e600448d9..4d8530b02 100644
--- a/gentoo/bcfg2-1.3.0.ebuild
+++ b/gentoo/bcfg2-1.3.0.ebuild
@@ -1,13 +1,13 @@
-# Copyright 1999-2012 Gentoo Foundation
+# Copyright 1999-2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# $Header: $
-EAPI="4"
+EAPI=4
-PYTHON_DEPEND="2:2.6"
+PYTHON_DEPEND="*:2.6"
SUPPORT_PYTHON_ABIS="1"
# ssl module required.
-RESTRICT_PYTHON_ABIS="2.4 2.5 3.*"
+RESTRICT_PYTHON_ABIS="2.5"
inherit distutils eutils
@@ -15,7 +15,7 @@ DESCRIPTION="configuration management tool"
HOMEPAGE="http://bcfg2.org"
SRC_URI="ftp://ftp.mcs.anl.gov/pub/bcfg/${P}.tar.gz"
-LICENSE="BSD"
+LICENSE="BSD-2"
SLOT="0"
KEYWORDS="~amd64 ~x86 ~amd64-linux ~x86-linux ~x64-solaris"
IUSE="doc cheetah genshi server"
diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec
index 0b975cb37..5feef1d74 100644
--- a/misc/bcfg2.spec
+++ b/misc/bcfg2.spec
@@ -334,6 +334,7 @@ touch %{buildroot}%{_sysconfdir}/bcfg2.conf \
%{python_sitelib}/Bcfg2/Logger.py*
%{python_sitelib}/Bcfg2/Options.py*
%{python_sitelib}/Bcfg2/Proxy.py*
+%{python_sitelib}/Bcfg2/Utils.py*
%{python_sitelib}/Bcfg2/version.py*
%{python_sitelib}/Bcfg2/Client
%{_mandir}/man1/bcfg2.1*
diff --git a/schemas/packages.xsd b/schemas/packages.xsd
index 2645a8be0..e01093c56 100644
--- a/schemas/packages.xsd
+++ b/schemas/packages.xsd
@@ -172,6 +172,15 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
+ <xsd:attribute type="xsd:boolean" name="debsrc">
+ <xsd:annotation>
+ <xsd:documentation>
+ Include ``deb-src`` lines in the generated APT
+ configuration. This only applies to sources with
+ :xml:attribute:`SourceType:type` = ``apt``.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
<xsd:attribute type="xsd:string" name="url">
<xsd:annotation>
<xsd:documentation>
diff --git a/schemas/selinux.xsd b/schemas/selinux.xsd
index 760953e34..3651549f5 100644
--- a/schemas/selinux.xsd
+++ b/schemas/selinux.xsd
@@ -80,6 +80,13 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
+ <xsd:attribute type="xsd:token" name="mlsrange">
+ <xsd:annotation>
+ <xsd:documentation>
+ SELinux MLS range to apply to this port
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
<xsd:attributeGroup ref="py:genshiAttrs"/>
</xsd:complexType>
@@ -127,6 +134,13 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
+ <xsd:attribute type="xsd:token" name="mlsrange">
+ <xsd:annotation>
+ <xsd:documentation>
+ SELinux MLS range to apply to files matching this specification
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
<xsd:attributeGroup ref="py:genshiAttrs"/>
</xsd:complexType>
@@ -157,6 +171,13 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
+ <xsd:attribute type="xsd:token" name="mlsrange">
+ <xsd:annotation>
+ <xsd:documentation>
+ SELinux MLS range to apply to this node
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
<xsd:attributeGroup ref="py:genshiAttrs"/>
</xsd:complexType>
@@ -205,6 +226,13 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
+ <xsd:attribute type="xsd:token" name="mlsrange">
+ <xsd:annotation>
+ <xsd:documentation>
+ SELinux MLS range to apply to this user
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
<xsd:attributeGroup ref="py:genshiAttrs"/>
</xsd:complexType>
@@ -235,6 +263,13 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
+ <xsd:attribute type="xsd:token" name="mlsrange">
+ <xsd:annotation>
+ <xsd:documentation>
+ SELinux MLS range to apply to this user
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
<xsd:attributeGroup ref="py:genshiAttrs"/>
</xsd:complexType>
@@ -258,6 +293,13 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
+ <xsd:attribute type="xsd:token" name="mlsrange">
+ <xsd:annotation>
+ <xsd:documentation>
+ SELinux MLS range to apply to this interface
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
<xsd:attributeGroup ref="py:genshiAttrs"/>
</xsd:complexType>
diff --git a/schemas/types.xsd b/schemas/types.xsd
index 9c4a0a48e..fe9b5c7d4 100644
--- a/schemas/types.xsd
+++ b/schemas/types.xsd
@@ -389,6 +389,27 @@
</xsd:restriction>
</xsd:simpleType>
+ <xsd:complexType name="MemberOfType">
+ <xsd:annotation>
+ <xsd:documentation>
+ Specify additional supplementary groups for the POSIXUser
+ </xsd:documentation>
+ </xsd:annotation>
+ <xsd:simpleContent>
+ <xsd:extension base="xsd:token">
+ <xsd:attribute name="group" type="xsd:token">
+ <xsd:annotation>
+ <xsd:documentation>
+ The name of the supplementary group. This can also be
+ specified as content of the tag, although that is
+ deprecated.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
+ </xsd:extension>
+ </xsd:simpleContent>
+ </xsd:complexType>
+
<xsd:complexType name="POSIXUserType">
<xsd:annotation>
<xsd:documentation>
@@ -396,13 +417,7 @@
</xsd:documentation>
</xsd:annotation>
<xsd:choice minOccurs='0' maxOccurs='unbounded'>
- <xsd:element name='MemberOf' type='xsd:token'>
- <xsd:annotation>
- <xsd:documentation>
- Specify additional supplementary groups for the POSIXUser
- </xsd:documentation>
- </xsd:annotation>
- </xsd:element>
+ <xsd:element name='MemberOf' type='MemberOfType'/>
</xsd:choice>
<xsd:attribute type="xsd:token" name="name" use="required">
<xsd:annotation>
diff --git a/setup.py b/setup.py
index 3e75d3a02..8c93285e5 100755
--- a/setup.py
+++ b/setup.py
@@ -4,26 +4,26 @@ from setuptools import setup
from glob import glob
import sys
-vfile = 'src/lib/Bcfg2/version.py'
+version_file = 'src/lib/Bcfg2/version.py'
try:
# python 2
- execfile(vfile)
+ execfile(version_file)
except NameError:
# py3k
- exec(compile(open(vfile).read(), vfile, 'exec'))
+ exec(compile(open(version_file).read(), version_file, 'exec'))
-# we only need m2crypto on < python2.6
-need_m2crypto = False
-version = sys.version_info[:2]
-if version < (2, 6):
- need_m2crypto = True
+inst_reqs = [
+ 'lockfile',
+ 'lxml',
+ 'python-daemon',
+]
-inst_reqs = ['lxml', 'genshi']
-if need_m2crypto:
+# we only need m2crypto on < python2.6
+if sys.version_info[:2] < (2, 6):
inst_reqs.append('M2Crypto')
setup(name="Bcfg2",
- version="1.3.1",
+ version=__version__, # Defined in src/lib/Bcfg2/version.py
description="Bcfg2 Server",
author="Narayan Desai",
author_email="desai@mcs.anl.gov",
@@ -52,9 +52,9 @@ setup(name="Bcfg2",
install_requires=inst_reqs,
tests_require=['mock', 'nose', 'sqlalchemy'],
package_dir={'': 'src/lib', },
- package_data={'Bcfg2.Reporting': [ 'templates/*.html',
- 'templates/*/*.html',
- 'templates/*/*.inc']},
+ package_data={'Bcfg2.Reporting': ['templates/*.html',
+ 'templates/*/*.html',
+ 'templates/*/*.inc']},
scripts=glob('src/sbin/*'),
data_files=[('share/bcfg2/schemas',
glob('schemas/*.xsd')),
diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py
index 11a82fcc0..b24b46dbc 100644
--- a/src/lib/Bcfg2/Client/Frame.py
+++ b/src/lib/Bcfg2/Client/Frame.py
@@ -161,7 +161,7 @@ class Frame(object):
def promptFilter(self, msg, entries):
"""Filter a supplied list based on user input."""
ret = []
- entries.sort(cmpent)
+ entries.sort(key=lambda e: e.tag + ":" + e.get('name'))
for entry in entries[:]:
if entry in self.unhandled:
# don't prompt for entries that can't be installed
@@ -427,10 +427,12 @@ class Frame(object):
# prune out unspecified bundles when running with -b
continue
if bundle in mbundles:
- self.logger.debug("Bundle %s was modified" % bundle)
+ self.logger.debug("Bundle %s was modified" %
+ bundle.get('name'))
func = "BundleUpdated"
else:
- self.logger.debug("Bundle %s was not modified" % bundle)
+ self.logger.debug("Bundle %s was not modified" %
+ bundle.get('name'))
func = "BundleNotUpdated"
for tool in self.tools:
try:
@@ -477,8 +479,8 @@ class Frame(object):
self.logger.info("%s:%s:%s" % (entry.tag, etype,
entry.get('name')))
else:
- self.logger.info(" %s:%s" % (entry.tag,
- entry.get('name')))
+ self.logger.info("%s:%s" % (entry.tag,
+ entry.get('name')))
self.logger.info('Total managed entries: %d' %
len(list(self.states.values())))
self.logger.info('Unmanaged entries: %d' % len(self.extra))
@@ -490,8 +492,8 @@ class Frame(object):
self.logger.info("%s:%s:%s" % (entry.tag, etype,
entry.get('name')))
else:
- self.logger.info(" %s:%s" % (entry.tag,
- entry.get('name')))
+ self.logger.info("%s:%s" % (entry.tag,
+ entry.get('name')))
if ((list(self.states.values()).count(False) == 0) and not self.extra):
self.logger.info('All entries correct.')
diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py
index 57be34369..eda3a7fce 100644
--- a/src/lib/Bcfg2/Client/Proxy.py
+++ b/src/lib/Bcfg2/Client/Proxy.py
@@ -24,6 +24,7 @@ from Bcfg2.Compat import httplib, xmlrpclib, urlparse, quote_plus
version = sys.version_info[:2]
has_py26 = version >= (2, 6)
+has_py32 = version >= (3, 2)
__all__ = ["ComponentProxy",
"RetryMethod",
@@ -171,11 +172,14 @@ class SSLHTTPConnection(httplib.HTTPConnection):
"""
if not has_py26:
httplib.HTTPConnection.__init__(self, host, port, strict)
- else:
+ elif not has_py32:
httplib.HTTPConnection.__init__(self, host, port, strict, timeout)
+ else:
+ # the strict parameter is deprecated.
+ # HTTP 0.9-style "Simple Responses" are not supported anymore.
+ httplib.HTTPConnection.__init__(self, host, port, timeout=timeout)
self.logger = logging.getLogger("%s.%s" % (self.__class__.__module__,
self.__class__.__name__))
-
self.key = key
self.cert = cert
self.ca = ca
diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py
index ec7f462b3..c3dcf7796 100644
--- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py
+++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py
@@ -65,16 +65,19 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool):
self.cmd.run("/sbin/chkconfig --add %s" % (entry.attrib['name']))
self.logger.info("Installing Service %s" % (entry.get('name')))
rv = True
- if entry.get('status') == 'off':
+ if (entry.get('status') == 'off' or
+ self.setup["servicemode"] == "build"):
rv &= self.cmd.run((rcmd + " --level 0123456") %
(entry.get('name'),
entry.get('status'))).success
- if entry.get("current_status") == "on":
+ if entry.get("current_status") == "on" and \
+ self.setup["servicemode"] != "disabled":
rv &= self.stop_service(entry).success
else:
rv &= self.cmd.run(rcmd % (entry.get('name'),
entry.get('status'))).success
- if entry.get("current_status") == "off":
+ if entry.get("current_status") == "off" and \
+ self.setup["servicemode"] != "disabled":
rv &= self.start_service(entry).success
return rv
diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
index 8ba1944d8..8f6bc5f37 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py
@@ -154,7 +154,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool):
if entry.get("current_exists", "true") == "true":
# verify supplemental groups
actual = [g[0] for g in self.user_supplementary_groups(entry)]
- expected = [e.text for e in entry.findall("MemberOf")]
+ expected = [e.get("group", e.text).strip()
+ for e in entry.findall("MemberOf")]
if set(expected) != set(actual):
entry.set('qtext',
"\n".join([entry.get('qtext', '')] +
@@ -254,7 +255,8 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool):
if entry.get('uid'):
cmd.extend(['-u', entry.get('uid')])
cmd.extend(['-g', entry.get('group')])
- extras = [e.text for e in entry.findall("MemberOf")]
+ extras = [e.get("group", e.text).strip()
+ for e in entry.findall("MemberOf")]
if extras:
cmd.extend(['-G', ",".join(extras)])
cmd.extend(['-d', entry.get('home')])
diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py
index 5e319a414..e52da081b 100644
--- a/src/lib/Bcfg2/Client/Tools/Portage.py
+++ b/src/lib/Bcfg2/Client/Tools/Portage.py
@@ -73,10 +73,10 @@ class Portage(Bcfg2.Client.Tools.PkgTool):
self.logger.debug('Running equery check on %s' %
entry.get('name'))
- for line in self.cmd.run(["/usr/bin/equery", "-N", "check",
- '=%s-%s' %
- (entry.get('name'),
- version)]).stdout.splitlines():
+ for line in self.cmd.run(
+ ["/usr/bin/equery", "-N", "check",
+ '=%s-%s' % (entry.get('name'),
+ entry.get('version'))]).stdout.splitlines():
if '!!!' in line and line.split()[1] not in modlist:
return False
diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py
index f38615062..92572ef1d 100644
--- a/src/lib/Bcfg2/Client/Tools/SELinux.py
+++ b/src/lib/Bcfg2/Client/Tools/SELinux.py
@@ -492,7 +492,8 @@ class SELinuxSeportHandler(SELinuxEntryHandler):
def _defaultargs(self, entry):
""" argument list for adding and modifying entries """
(port, proto) = entry.get("name").split("/")
- return (port, proto, '', entry.get("selinuxtype"))
+ return (port, proto, entry.get("mlsrange", ""),
+ entry.get("selinuxtype"))
def _deleteargs(self, entry):
return tuple(entry.get("name").split("/"))
@@ -565,7 +566,7 @@ class SELinuxSefcontextHandler(SELinuxEntryHandler):
""" argument list for adding, modifying, and deleting entries """
return (entry.get("name"), entry.get("selinuxtype"),
self.filetypeargs[entry.get("filetype", "all")],
- '', '')
+ entry.get("mlsrange", ""), '')
def primarykey(self, entry):
return ":".join([entry.tag, entry.get("name"),
@@ -600,7 +601,7 @@ class SELinuxSenodeHandler(SELinuxEntryHandler):
def _defaultargs(self, entry):
""" argument list for adding, modifying, and deleting entries """
(addr, netmask) = entry.get("name").split("/")
- return (addr, netmask, entry.get("proto"), "",
+ return (addr, netmask, entry.get("proto"), entry.get("mlsrange", ""),
entry.get("selinuxtype"))
@@ -612,7 +613,8 @@ class SELinuxSeloginHandler(SELinuxEntryHandler):
def _defaultargs(self, entry):
""" argument list for adding, modifying, and deleting entries """
- return (entry.get("name"), entry.get("selinuxuser"), "")
+ return (entry.get("name"), entry.get("selinuxuser"),
+ entry.get("mlsrange", ""))
class SELinuxSeuserHandler(SELinuxEntryHandler):
@@ -652,15 +654,16 @@ class SELinuxSeuserHandler(SELinuxEntryHandler):
# prefix. see the comment in Install() above for more
# details.
rv = [entry.get("name"),
- entry.get("roles", "").replace(" ", ",").split(",")]
+ entry.get("roles", "").replace(" ", ",").split(","),
+ '', entry.get("mlsrange", "")]
if self.needs_prefix:
- rv.extend(['', '', entry.get("prefix")])
+ rv.append(entry.get("prefix"))
else:
key = self._key(entry)
if key in self.all_records:
attrs = self._key2attrs(key)
if attrs['prefix'] != entry.get("prefix"):
- rv.extend(['', '', entry.get("prefix")])
+ rv.append(entry.get("prefix"))
return tuple(rv)
@@ -672,7 +675,8 @@ class SELinuxSeinterfaceHandler(SELinuxEntryHandler):
def _defaultargs(self, entry):
""" argument list for adding, modifying, and deleting entries """
- return (entry.get("name"), '', entry.get("selinuxtype"))
+ return (entry.get("name"), entry.get("mlsrange", ""),
+ entry.get("selinuxtype"))
class SELinuxSepermissiveHandler(SELinuxEntryHandler):
diff --git a/src/lib/Bcfg2/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py
index 027d91c71..20a172d3d 100644
--- a/src/lib/Bcfg2/Client/Tools/Systemd.py
+++ b/src/lib/Bcfg2/Client/Tools/Systemd.py
@@ -13,6 +13,8 @@ class Systemd(Bcfg2.Client.Tools.SvcTool):
__handles__ = [('Service', 'systemd')]
__req__ = {'Service': ['name', 'status']}
+ conflicts = ['Chkconfig']
+
def get_svc_command(self, service, action):
return "/bin/systemctl %s %s.service" % (action, service.get('name'))
diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py
index e40ef750b..3bc261f2f 100644
--- a/src/lib/Bcfg2/Client/__init__.py
+++ b/src/lib/Bcfg2/Client/__init__.py
@@ -19,7 +19,7 @@ def prompt(msg):
while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0:
os.read(sys.stdin.fileno(), 4096)
try:
- ans = input(msg.encode(sys.stdout.encoding, 'replace'))
+ ans = input(msg)
return ans in ['y', 'Y']
except EOFError:
# python 2.4.3 on CentOS doesn't like ^C for some reason
diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py
index 4e9239e26..d034c0777 100644
--- a/src/lib/Bcfg2/Compat.py
+++ b/src/lib/Bcfg2/Compat.py
@@ -89,11 +89,28 @@ def u_str(string, encoding=None):
else:
return unicode(string)
+try:
+ from functools import wraps
+except ImportError:
+ def wraps(wrapped): # pylint: disable=W0613
+ """ implementation of functools.wraps() for python 2.4 """
+ return lambda f: f
+
+
# base64 compat
if sys.hexversion >= 0x03000000:
from base64 import b64encode as _b64encode, b64decode as _b64decode
- b64encode = lambda s: _b64encode(s.encode('UTF-8')).decode('UTF-8')
- b64decode = lambda s: _b64decode(s.encode('UTF-8')).decode('UTF-8')
+
+ @wraps(_b64encode)
+ def b64encode(val, **kwargs): # pylint: disable=C0111
+ try:
+ return _b64encode(val, **kwargs)
+ except TypeError:
+ return _b64encode(val.encode('UTF-8'), **kwargs).decode('UTF-8')
+
+ @wraps(_b64decode)
+ def b64decode(val, **kwargs): # pylint: disable=C0111
+ return _b64decode(val.encode('UTF-8'), **kwargs).decode('UTF-8')
else:
from base64 import b64encode, b64decode
@@ -242,14 +259,6 @@ except ImportError:
from md5 import md5
-try:
- from functools import wraps
-except ImportError:
- def wraps(wrapped): # pylint: disable=W0613
- """ implementation of functools.wraps() for python 2.4 """
- return lambda f: f
-
-
def oct_mode(mode):
""" Convert a decimal number describing a POSIX permissions mode
to a string giving the octal mode. In Python 2, this is a synonym
diff --git a/src/lib/Bcfg2/Reporting/Transport/__init__.py b/src/lib/Bcfg2/Reporting/Transport/__init__.py
index 5c51dad1e..73bdd0b3a 100644
--- a/src/lib/Bcfg2/Reporting/Transport/__init__.py
+++ b/src/lib/Bcfg2/Reporting/Transport/__init__.py
@@ -2,11 +2,11 @@
Public transport routines
"""
-import traceback
-
+import sys
from Bcfg2.Reporting.Transport.base import TransportError, \
TransportImportError
+
def load_transport(transport_name, setup):
"""
Try to load the transport. Raise TransportImportError on failure
@@ -18,13 +18,14 @@ def load_transport(transport_name, setup):
try:
mod = __import__(transport_name)
except:
- raise TransportImportError("Unavailable")
+ raise TransportImportError("Error importing transport %s: %s" %
+ (transport_name, sys.exc_info()[1]))
try:
- cls = getattr(mod, transport_name)
- return cls(setup)
+ return getattr(mod, transport_name)(setup)
except:
- raise TransportImportError("Transport unavailable: %s" %
- traceback.format_exc().splitlines()[-1])
+ raise TransportImportError("Error instantiating transport %s: %s" %
+ (transport_name, sys.exc_info()[1]))
+
def load_transport_from_config(setup):
"""Load the transport in the config... eventually"""
@@ -32,4 +33,3 @@ def load_transport_from_config(setup):
return load_transport(setup['reporting_transport'], setup)
except KeyError:
raise TransportImportError('Transport missing in config')
-
diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py
index e63c180a8..2e75c1d1a 100644
--- a/src/lib/Bcfg2/Reporting/models.py
+++ b/src/lib/Bcfg2/Reporting/models.py
@@ -3,7 +3,7 @@ import sys
from django.core.exceptions import ImproperlyConfigured
try:
- from django.db import models
+ from django.db import models, backend, connection
except ImproperlyConfigured:
e = sys.exc_info()[1]
print("Reports: unable to import django models: %s" % e)
@@ -49,6 +49,20 @@ def hash_entry(entry_dict):
return hash(cPickle.dumps(dataset))
+_our_backend = None
+def _quote(value):
+ """
+ Quote a string to use as a table name or column
+
+ Newer versions and various drivers require an argument
+ https://code.djangoproject.com/ticket/13630
+ """
+ global _our_backend
+ if not _our_backend:
+ _our_backend = backend.DatabaseOperations(connection)
+ return _our_backend.quote_name(value)
+
+
class Client(models.Model):
"""Object representing every client we have seen stats for."""
creation = models.DateTimeField(auto_now_add=True)
@@ -77,16 +91,20 @@ class InteractionManager(models.Manager):
cursor = connection.cursor()
cfilter = "expiration is null"
- sql = 'select ri.id, x.client_id from (select client_id, MAX(timestamp) ' + \
- 'as timer from Reporting_interaction'
+ sql = 'select ri.id, x.client_id from ' + \
+ '(select client_id, MAX(timestamp) as timer from ' + \
+ _quote('Reporting_interaction')
if maxdate:
if not isinstance(maxdate, datetime):
raise ValueError('Expected a datetime object')
sql = sql + " where timestamp <= '%s' " % maxdate
cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate)
- sql = sql + ' GROUP BY client_id) x, Reporting_interaction ri where ' + \
- 'ri.client_id = x.client_id AND ri.timestamp = x.timer'
- sql = sql + " and x.client_id in (select id from Reporting_client where %s)" % cfilter
+ sql = sql + ' GROUP BY client_id) x, ' + \
+ _quote('Reporting_interaction') + \
+ ' ri where ri.client_id = x.client_id AND' + \
+ ' ri.timestamp = x.timer and x.client_id in' + \
+ ' (select id from %s where %s)' % \
+ (_quote('Reporting_client'), cfilter)
try:
cursor.execute(sql)
return [item[0] for item in cursor.fetchall()]
diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py
index d317cc03b..06a419354 100644
--- a/src/lib/Bcfg2/Server/Admin/__init__.py
+++ b/src/lib/Bcfg2/Server/Admin/__init__.py
@@ -128,6 +128,7 @@ class MetadataCore(Mode):
except Bcfg2.Server.Core.CoreInitError:
msg = sys.exc_info()[1]
self.errExit("Core load failed: %s" % msg)
+ self.bcore.load_plugins()
self.bcore.fam.handle_event_set()
self.metadata = self.bcore.metadata
diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py
index 48455819d..b05ad9d41 100644
--- a/src/lib/Bcfg2/Server/BuiltinCore.py
+++ b/src/lib/Bcfg2/Server/BuiltinCore.py
@@ -118,11 +118,11 @@ class Core(BaseCore):
self.logger.error("Server startup failed: %s" % err)
self.context.close()
return False
- self.server.register_instance(self)
return True
def _block(self):
""" Enter the blocking infinite loop. """
+ self.server.register_instance(self)
try:
self.server.serve_forever()
finally:
diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py
index 8bfb76461..7c3b2d9cc 100644
--- a/src/lib/Bcfg2/Server/Lint/Comments.py
+++ b/src/lib/Bcfg2/Server/Lint/Comments.py
@@ -1,8 +1,9 @@
-""" check files for various required comments """
+""" Check files for various required comments. """
import os
import lxml.etree
import Bcfg2.Server.Lint
+from Bcfg2.Server import XI_NAMESPACE
from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator \
import CfgPlaintextGenerator
from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator
@@ -11,7 +12,10 @@ from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML
class Comments(Bcfg2.Server.Lint.ServerPlugin):
- """ check files for various required headers """
+ """ The Comments lint plugin checks files for header comments that
+ give information about the files. For instance, you can require
+ SVN keywords in a comment, or require the name of the maintainer
+ of a Genshi template, and so on. """
def __init__(self, *args, **kwargs):
Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs)
self.config_cache = {}
@@ -27,21 +31,43 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
def Errors(cls):
return {"unexpanded-keywords": "warning",
"keywords-not-found": "warning",
- "comments-not-found": "warning"}
+ "comments-not-found": "warning",
+ "broken-xinclude-chain": "warning"}
def required_keywords(self, rtype):
- """ given a file type, fetch the list of required VCS keywords
- from the bcfg2-lint config """
+ """ Given a file type, fetch the list of required VCS keywords
+ from the bcfg2-lint config. Valid file types are documented
+ in :manpage:`bcfg2-lint.conf(5)`.
+
+ :param rtype: The file type
+ :type rtype: string
+ :returns: list - the required items
+ """
return self.required_items(rtype, "keyword")
def required_comments(self, rtype):
- """ given a file type, fetch the list of required comments
- from the bcfg2-lint config """
+ """ Given a file type, fetch the list of required comments
+ from the bcfg2-lint config. Valid file types are documented
+ in :manpage:`bcfg2-lint.conf(5)`.
+
+ :param rtype: The file type
+ :type rtype: string
+ :returns: list - the required items
+ """
return self.required_items(rtype, "comment")
def required_items(self, rtype, itype):
- """ given a file type and item type (comment or keyword),
- fetch the list of required items from the bcfg2-lint config """
+ """ Given a file type and item type (``comment`` or
+ ``keyword``), fetch the list of required items from the
+ bcfg2-lint config. Valid file types are documented in
+ :manpage:`bcfg2-lint.conf(5)`.
+
+ :param rtype: The file type
+ :type rtype: string
+ :param itype: The item type (``comment`` or ``keyword``)
+ :type itype: string
+ :returns: list - the required items
+ """
if itype not in self.config_cache:
self.config_cache[itype] = {}
@@ -62,7 +88,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
return self.config_cache[itype][rtype]
def check_bundles(self):
- """ check bundle files for required headers """
+ """ Check bundle files for required comments. """
if 'Bundler' in self.core.plugins:
for bundle in self.core.plugins['Bundler'].entries.values():
xdata = None
@@ -78,15 +104,41 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
self.check_xml(bundle.name, xdata, rtype)
def check_properties(self):
- """ check properties files for required headers """
+ """ Check Properties files for required comments. """
if 'Properties' in self.core.plugins:
props = self.core.plugins['Properties']
- for propfile, pdata in props.store.entries.items():
+ for propfile, pdata in props.entries.items():
if os.path.splitext(propfile)[1] == ".xml":
self.check_xml(pdata.name, pdata.xdata, 'properties')
+ def has_all_xincludes(self, mfile):
+ """ Return True if :attr:`Bcfg2.Server.Lint.Plugin.files`
+ includes all XIncludes listed in the specified metadata type,
+ false otherwise. In other words, this returns True if
+ bcfg2-lint is dealing with complete metadata.
+
+ :param mfile: The metadata file ("clients.xml" or
+ "groups.xml") to check for XIncludes
+ :type mfile: string
+ :returns: bool
+ """
+ if self.files is None:
+ return True
+ else:
+ path = os.path.join(self.metadata.data, mfile)
+ if path in self.files:
+ xdata = lxml.etree.parse(path)
+ for el in xdata.findall('./%sinclude' % XI_NAMESPACE):
+ if not self.has_all_xincludes(el.get('href')):
+ self.LintError("broken-xinclude-chain",
+ "Broken XInclude chain: could not "
+ "include %s" % path)
+ return False
+
+ return True
+
def check_metadata(self):
- """ check metadata files for required headers """
+ """ Check Metadata files for required comments. """
if self.has_all_xincludes("groups.xml"):
self.check_xml(os.path.join(self.metadata.data, "groups.xml"),
self.metadata.groups_xml.data,
@@ -97,7 +149,8 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
"metadata")
def check_cfg(self):
- """ check Cfg files and info.xml files for required headers """
+ """ Check Cfg files and ``info.xml`` files for required
+ comments. """
if 'Cfg' in self.core.plugins:
for entryset in self.core.plugins['Cfg'].entries.values():
for entry in entryset.entries.values():
@@ -117,29 +170,57 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin):
self.check_plaintext(entry.name, entry.data, rtype)
def check_probes(self):
- """ check probes for required headers """
+ """ Check Probes for required comments """
if 'Probes' in self.core.plugins:
for probe in self.core.plugins['Probes'].probes.entries.values():
self.check_plaintext(probe.name, probe.data, "probes")
def check_xml(self, filename, xdata, rtype):
- """ check generic XML files for required headers """
+ """ Generic check to check an XML file for required comments.
+
+ :param filename: The filename
+ :type filename: string
+ :param xdata: The file data
+ :type xdata: lxml.etree._Element
+ :param rtype: The type of file. Available types are
+ documented in :manpage:`bcfg2-lint.conf(5)`.
+ :type rtype: string
+ """
self.check_lines(filename,
[str(el)
for el in xdata.getiterator(lxml.etree.Comment)],
rtype)
def check_plaintext(self, filename, data, rtype):
- """ check generic plaintext files for required headers """
+ """ Generic check to check a plain text file for required
+ comments.
+
+ :param filename: The filename
+ :type filename: string
+ :param data: The file data
+ :type data: string
+ :param rtype: The type of file. Available types are
+ documented in :manpage:`bcfg2-lint.conf(5)`.
+ :type rtype: string
+ """
self.check_lines(filename, data.splitlines(), rtype)
def check_lines(self, filename, lines, rtype):
- """ generic header check for a set of lines """
+ """ Generic header check for a set of lines.
+
+ :param filename: The filename
+ :type filename: string
+ :param lines: The data to check
+ :type lines: list of strings
+ :param rtype: The type of file. Available types are
+ documented in :manpage:`bcfg2-lint.conf(5)`.
+ :type rtype: string
+ """
if self.HandlesFile(filename):
# found is trivalent:
- # False == not found
- # None == found but not expanded
- # True == found and expanded
+ # False == keyword not found
+ # None == keyword found but not expanded
+ # True == keyword found and expanded
found = dict((k, False) for k in self.required_keywords(rtype))
for line in lines:
diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py
index caee238bc..a1d0b7fa1 100755
--- a/src/lib/Bcfg2/Server/Lint/Genshi.py
+++ b/src/lib/Bcfg2/Server/Lint/Genshi.py
@@ -1,4 +1,4 @@
-""" Check Genshi templates for syntax errors """
+""" Check Genshi templates for syntax errors. """
import sys
import Bcfg2.Server.Lint
@@ -8,10 +8,9 @@ from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator
class Genshi(Bcfg2.Server.Lint.ServerPlugin):
- """ Check Genshi templates for syntax errors """
+ """ Check Genshi templates for syntax errors. """
def Run(self):
- """ run plugin """
if 'Cfg' in self.core.plugins:
self.check_cfg()
if 'Bundler' in self.core.plugins:
@@ -22,7 +21,7 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin):
return {"genshi-syntax-error": "error"}
def check_cfg(self):
- """ Check genshi templates in Cfg for syntax errors """
+ """ Check genshi templates in Cfg for syntax errors. """
for entryset in self.core.plugins['Cfg'].entries.values():
for entry in entryset.entries.values():
if (self.HandlesFile(entry.name) and
@@ -37,7 +36,7 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin):
"Genshi syntax error: %s" % err)
def check_bundler(self):
- """ Check templates in Bundler for syntax errors """
+ """ Check templates in Bundler for syntax errors. """
loader = TemplateLoader()
for entry in self.core.plugins['Bundler'].entries.values():
diff --git a/src/lib/Bcfg2/Server/Lint/GroupNames.py b/src/lib/Bcfg2/Server/Lint/GroupNames.py
index e41ed867e..730f32750 100644
--- a/src/lib/Bcfg2/Server/Lint/GroupNames.py
+++ b/src/lib/Bcfg2/Server/Lint/GroupNames.py
@@ -1,4 +1,4 @@
-""" ensure that all named groups are valid group names """
+""" Ensure that all named groups are valid group names. """
import os
import re
@@ -6,8 +6,15 @@ import Bcfg2.Server.Lint
class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
- """ ensure that all named groups are valid group names """
+ """ Ensure that all named groups are valid group names. """
+
+ #: A string regex that matches only valid group names. Currently,
+ #: a group name is considered valid if it contains only
+ #: non-whitespace characters.
pattern = r'\S+$'
+
+ #: A compiled regex for
+ #: :attr:`Bcfg2.Server.Lint.GroupNames.GroupNames.pattern`
valid = re.compile(r'^' + pattern)
def Run(self):
@@ -26,7 +33,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
return {"invalid-group-name": "error"}
def check_rules(self):
- """ Check groups used in the Rules plugin for validity """
+ """ Check groups used in the Rules plugin for validity. """
for rules in self.core.plugins['Rules'].entries.values():
if not self.HandlesFile(rules.name):
continue
@@ -35,7 +42,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
os.path.join(self.config['repo'], rules.name))
def check_bundles(self):
- """ Check groups used in the Bundler plugin for validity """
+ """ Check groups used in the Bundler plugin for validity. """
for bundle in self.core.plugins['Bundler'].entries.values():
if self.HandlesFile(bundle.name) and bundle.template is None:
self.check_entries(bundle.xdata.xpath("//Group"),
@@ -43,7 +50,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
def check_metadata(self):
""" Check groups used or declared in the Metadata plugin for
- validity """
+ validity. """
self.check_entries(self.metadata.groups_xml.xdata.xpath("//Group"),
os.path.join(self.config['repo'],
self.metadata.groups_xml.name))
@@ -61,7 +68,7 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
def check_cfg(self):
""" Check groups used in group-specific files in the Cfg
- plugin for validity """
+ plugin for validity. """
for root, _, files in os.walk(self.core.plugins['Cfg'].data):
for fname in files:
basename = os.path.basename(root)
@@ -74,7 +81,14 @@ class GroupNames(Bcfg2.Server.Lint.ServerPlugin):
def check_entries(self, entries, fname):
""" Check a generic list of XML entries for <Group> tags with
- invalid name attributes """
+ invalid name attributes.
+
+ :param entries: A list of XML <Group> tags whose ``name``
+ attributes will be validated.
+ :type entries: list of lxml.etree._Element
+ :param fname: The filename the entry list came from
+ :type fname: string
+ """
for grp in entries:
if not self.valid.search(grp.get("name")):
self.LintError("invalid-group-name",
diff --git a/src/lib/Bcfg2/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py
index f2349f861..184f657b7 100644
--- a/src/lib/Bcfg2/Server/Lint/InfoXML.py
+++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py
@@ -1,4 +1,4 @@
-""" ensure that all config files have an info.xml file"""
+""" Ensure that all config files have a valid info.xml file. """
import os
import Bcfg2.Options
@@ -7,7 +7,14 @@ from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML
class InfoXML(Bcfg2.Server.Lint.ServerPlugin):
- """ ensure that all config files have an info.xml file"""
+ """ Ensure that all config files have a valid info.xml file. This
+ plugin can check for:
+
+ * Missing ``info.xml`` files;
+ * Use of deprecated ``info``/``:info`` files;
+ * Paranoid mode disabled in an ``info.xml`` file;
+ * Required attributes missing from ``info.xml``
+ """
def Run(self):
if 'Cfg' not in self.core.plugins:
return
@@ -29,11 +36,10 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin):
def Errors(cls):
return {"no-infoxml": "warning",
"paranoid-false": "warning",
- "broken-xinclude-chain": "warning",
"required-infoxml-attrs-missing": "error"}
def check_infoxml(self, fname, xdata):
- """ verify that info.xml contains everything it should """
+ """ Verify that info.xml contains everything it should. """
for info in xdata.getroottree().findall("//Info"):
required = []
if "required_attrs" in self.config:
diff --git a/src/lib/Bcfg2/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py
index 44d02c2ff..2419c3d43 100644
--- a/src/lib/Bcfg2/Server/Lint/MergeFiles.py
+++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py
@@ -57,7 +57,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin):
else:
threshold = 0.75
rv = []
- elist = entries.items()
+ elist = list(entries.items())
while elist:
result = self._find_similar(elist.pop(0), copy.copy(elist),
threshold)
diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
index 7a2fd3fe9..f2464b585 100644
--- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
+++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
@@ -1,5 +1,5 @@
-""" verify attributes for configuration entries that cannot be
-verified with an XML schema alone"""
+""" Verify attributes for configuration entries that cannot be
+verified with an XML schema alone. """
import os
import re
@@ -14,7 +14,8 @@ except ImportError:
HAS_GENSHI = False
-# format verifying functions
+# format verifying functions. TODO: These should be moved into XML
+# schemas where possible.
def is_filename(val):
""" Return True if val is a string describing a valid full path
"""
@@ -52,8 +53,8 @@ def is_device_mode(val):
class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
- """ verify attributes for configuration entries that cannot be
- verified with an XML schema alone """
+ """ Verify attributes for configuration entries that cannot be
+ verified with an XML schema alone. """
def __init__(self, *args, **kwargs):
Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs)
self.required_attrs = dict(
@@ -114,8 +115,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
SEInterface={None: dict(name=None, selinuxtype=is_selinux_type)},
SEPermissive={None: dict(name=is_selinux_type)},
POSIXGroup={None: dict(name=is_username)},
- POSIXUser={None: dict(name=is_username)},
- MemberOf={None: dict(__text__=is_username)})
+ POSIXUser={None: dict(name=is_username)})
def Run(self):
self.check_packages()
@@ -135,7 +135,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
"extra-attrs": "warning"}
def check_packages(self):
- """ check package sources for Source entries with missing attrs """
+ """ Check Packages sources for Source entries with missing
+ attributes. """
if 'Packages' not in self.core.plugins:
return
@@ -175,7 +176,8 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
rules.name))
def check_bundles(self):
- """ check bundles for BoundPath entries with missing attrs """
+ """ Check bundles for BoundPath entries with missing
+ attrs. """
if 'Bundler' not in self.core.plugins:
return
@@ -186,7 +188,13 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
self.check_entry(path, bundle.name)
def check_entry(self, entry, filename):
- """ generic entry check """
+ """ Generic entry check.
+
+ :param entry: The XML entry to check for missing attributes.
+ :type entry: lxml.etree._Element
+ :param filename: The filename the entry came from
+ :type filename: string
+ """
if self.HandlesFile(filename):
name = entry.get('name')
tag = entry.tag
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index 946ef8270..ca9f138ef 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -1,4 +1,5 @@
-""" Ensure that the repo validates """
+""" Ensure that all XML files in the Bcfg2 repository validate
+according to their respective schemas. """
import os
import sys
@@ -10,10 +11,19 @@ from Bcfg2.Utils import Executor
class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
- """ Ensure that the repo validates """
+ """ Ensure that all XML files in the Bcfg2 repository validate
+ according to their respective schemas. """
def __init__(self, *args, **kwargs):
Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs)
+
+ #: A dict of <file glob>: <schema file> that maps files in the
+ #: Bcfg2 specification to their schemas. The globs are
+ #: extended :mod:`fnmatch` globs that also support ``**``,
+ #: which matches any number of any characters, including
+ #: forward slashes. The schema files are relative to the
+ #: schema directory, which can be controlled by the
+ #: ``bcfg2-lint --schema`` option.
self.filesets = \
{"Metadata/groups.xml": "metadata.xsd",
"Metadata/clients.xml": "clients.xsd",
@@ -76,7 +86,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
"input-output-error": "error"}
def check_properties(self):
- """ check Properties files against their schemas """
+ """ Check Properties files against their schemas. """
for filename in self.filelists['props']:
schemafile = "%s.xsd" % os.path.splitext(filename)[0]
if os.path.exists(schemafile):
@@ -90,7 +100,11 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
def parse(self, filename):
""" Parse an XML file, raising the appropriate LintErrors if
it can't be parsed or read. Return the
- lxml.etree._ElementTree parsed from the file. """
+ lxml.etree._ElementTree parsed from the file.
+
+ :param filename: The full path to the file to parse
+ :type filename: string
+ :returns: lxml.etree._ElementTree - the parsed data"""
try:
return lxml.etree.parse(filename)
except SyntaxError:
@@ -105,8 +119,20 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
return False
def validate(self, filename, schemafile, schema=None):
- """validate a file against the given lxml.etree.Schema.
- return True on success, False on failure """
+ """ Validate a file against the given schema.
+
+ :param filename: The full path to the file to validate
+ :type filename: string
+ :param schemafile: The full path to the schema file to
+ validate against
+ :type schemafile: string
+ :param schema: The loaded schema to validate against. This
+ can be used to avoid parsing a single schema
+ file for every file that needs to be validate
+ against it.
+ :type schema: lxml.etree.Schema
+ :returns: bool - True if the file validates, false otherwise
+ """
if schema is None:
# if no schema object was provided, instantiate one
schema = self._load_schema(schemafile)
@@ -127,7 +153,14 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
return True
def get_filelists(self):
- """ get lists of different kinds of files to validate """
+ """ Get lists of different kinds of files to validate. This
+ doesn't return anything, but it sets
+ :attr:`Bcfg2.Server.Lint.Validate.Validate.filelists` to a
+ dict whose keys are path globs given in
+ :attr:`Bcfg2.Server.Lint.Validate.Validate.filesets` and whose
+ values are lists of the full paths to all files in the Bcfg2
+ repository (or given with ``bcfg2-lint --stdin``) that match
+ the glob."""
if self.files is not None:
listfiles = lambda p: fnmatch.filter(self.files,
os.path.join('*', p))
@@ -154,7 +187,13 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
self.filelists['props'] = listfiles("Properties/*.xml")
def _load_schema(self, filename):
- """ load an XML schema document, returning the Schema object """
+ """ Load an XML schema document, returning the Schema object
+ and raising appropriate lint errors on failure.
+
+ :param filename: The full path to the schema file to load.
+ :type filename: string
+ :returns: lxml.etree.Schema - The loaded schema data
+ """
try:
return lxml.etree.XMLSchema(lxml.etree.parse(filename))
except IOError:
diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py
index 11afdd75d..28644263f 100644
--- a/src/lib/Bcfg2/Server/Lint/__init__.py
+++ b/src/lib/Bcfg2/Server/Lint/__init__.py
@@ -9,10 +9,9 @@ import lxml.etree
import fcntl
import termios
import struct
-from Bcfg2.Server import XI_NAMESPACE
from Bcfg2.Compat import walk_packages
-__all__ = [m[1] for m in walk_packages(path=__path__)]
+plugins = [m[1] for m in walk_packages(path=__path__)] # pylint: disable=C0103
def _ioctl_GWINSZ(fd): # pylint: disable=C0103
@@ -45,30 +44,56 @@ def get_termsize():
class Plugin(object):
- """ base class for ServerlessPlugin and ServerPlugin """
+ """ Base class for all bcfg2-lint plugins """
def __init__(self, config, errorhandler=None, files=None):
+ """
+ :param config: A :mod:`Bcfg2.Options` setup dict
+ :type config: dict
+ :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler`
+ that will be used to handle lint errors.
+ If one is not provided, a new one will be
+ instantiated.
+ :type errorhandler: Bcfg2.Server.Lint.ErrorHandler
+ :param files: A list of files to run bcfg2-lint against. (See
+ the bcfg2-lint ``--stdin`` option.)
+ :type files: list of strings
+ """
+
+ #: The list of files that bcfg2-lint should be run against
self.files = files
+
+ #: The Bcfg2.Options setup dict
self.config = config
+
self.logger = logging.getLogger('bcfg2-lint')
if errorhandler is None:
+ #: The error handler
self.errorhandler = ErrorHandler()
else:
self.errorhandler = errorhandler
self.errorhandler.RegisterErrors(self.Errors())
def Run(self):
- """ run the plugin. must be overloaded by child classes """
- pass
+ """ Run the plugin. Must be overloaded by child classes. """
+ raise NotImplementedError
@classmethod
def Errors(cls):
- """ returns a dict of errors the plugin supplies. must be
- overloaded by child classes """
+ """ Returns a dict of errors the plugin supplies, in a format
+ suitable for passing to
+ :func:`Bcfg2.Server.Lint.ErrorHandler.RegisterErrors`.
+
+ Must be overloaded by child classes.
+
+ :returns: dict
+ """
+ raise NotImplementedError
def HandlesFile(self, fname):
- """ returns true if the given file should be handled by the
- plugin according to the files list, false otherwise """
+ """ Returns True if the given file should be handled by the
+ plugin according to :attr:`Bcfg2.Server.Lint.Plugin.files`,
+ False otherwise. """
return (self.files is None or
fname in self.files or
os.path.join(self.config['repo'], fname) in self.files or
@@ -77,12 +102,27 @@ class Plugin(object):
fname)) in self.files)
def LintError(self, err, msg):
- """ record an error in the lint process """
+ """ Raise an error from the lint process.
+
+ :param err: The name of the error being raised. This name
+ must be a key in the dict returned by
+ :func:`Bcfg2.Server.Lint.Plugin.Errors`.
+ :type err: string
+ :param msg: The freeform message to display to the end user.
+ :type msg: string
+ """
self.errorhandler.dispatch(err, msg)
def RenderXML(self, element, keep_text=False):
- """render an XML element for error output -- line number
- prefixed, no children"""
+ """ Render an XML element for error output. This prefixes the
+ line number and removes children for nicer display.
+
+ :param element: The element to render
+ :type element: lxml.etree._Element
+ :param keep_text: Do not discard text content from the element
+ for display
+ :type keep_text: boolean
+ """
xml = None
if len(element) or element.text:
el = copy(element)
@@ -100,11 +140,18 @@ class Plugin(object):
return " line %s: %s" % (element.sourceline, xml)
-class ErrorHandler (object):
- """ a class to handle errors for bcfg2-lint plugins """
+class ErrorHandler(object):
+ """ A class to handle errors for bcfg2-lint plugins """
- def __init__(self, config=None):
+ def __init__(self, errors=None):
+ """
+ :param config: An initial dict of errors to register
+ :type config: dict
+ """
+ #: The number of errors passed to this error handler
self.errors = 0
+
+ #: The number of warnings passed to this error handler
self.warnings = 0
self.logger = logging.getLogger('bcfg2-lint')
@@ -114,17 +161,25 @@ class ErrorHandler (object):
twrap = textwrap.TextWrapper(initial_indent=" ",
subsequent_indent=" ",
width=termsize[0])
+ #: A function to wrap text to the width of the terminal
self._wrapper = twrap.wrap
else:
self._wrapper = lambda s: [s]
+ #: A dict of registered errors
self.errortypes = dict()
- if config is not None:
- self.RegisterErrors(dict(config.items()))
+ if errors is not None:
+ self.RegisterErrors(dict(errors.items()))
def RegisterErrors(self, errors):
- """ Register a dict of errors (name: default level) that a
- plugin may raise """
+ """ Register a dict of errors that a plugin may raise. The
+ keys of the dict are short strings that describe each error;
+ the values are the default error handling for that error
+ ("error", "warning", or "silent").
+
+ :param errors: The error dict
+ :type errors: dict
+ """
for err, action in errors.items():
if err not in self.errortypes:
if "warn" in action:
@@ -135,7 +190,16 @@ class ErrorHandler (object):
self.errortypes[err] = self.debug
def dispatch(self, err, msg):
- """ Dispatch an error to the correct handler """
+ """ Dispatch an error to the correct handler.
+
+ :param err: The name of the error being raised. This name
+ must be a key in
+ :attr:`Bcfg2.Server.Lint.ErrorHandler.errortypes`,
+ the dict of registered errors.
+ :type err: string
+ :param msg: The freeform message to display to the end user.
+ :type msg: string
+ """
if err in self.errortypes:
self.errortypes[err](msg)
self.logger.debug(" (%s)" % err)
@@ -145,22 +209,34 @@ class ErrorHandler (object):
self.logger.warning("Unknown error %s" % err)
def error(self, msg):
- """ log an error condition """
+ """ Log an error condition.
+
+ :param msg: The freeform message to display to the end user.
+ :type msg: string
+ """
self.errors += 1
self._log(msg, self.logger.error, prefix="ERROR: ")
def warn(self, msg):
- """ log a warning condition """
+ """ Log a warning condition.
+
+ :param msg: The freeform message to display to the end user.
+ :type msg: string
+ """
self.warnings += 1
self._log(msg, self.logger.warning, prefix="WARNING: ")
def debug(self, msg):
- """ log a silent/debug condition """
+ """ Log a silent/debug condition.
+
+ :param msg: The freeform message to display to the end user.
+ :type msg: string
+ """
self._log(msg, self.logger.debug)
def _log(self, msg, logfunc, prefix=""):
""" Generic log function that logs a message with the given
- function after wrapping it for the terminal width """
+ function after wrapping it for the terminal width. """
# a message may itself consist of multiple lines. wrap() will
# elide them all into a single paragraph, which we don't want.
# so we split the message into its paragraphs and wrap each
@@ -180,37 +256,37 @@ class ErrorHandler (object):
logfunc(line)
-class ServerlessPlugin (Plugin):
- """ base class for plugins that are run before the server starts
- up (i.e., plugins that check things that may prevent the server
- from starting up) """
+class ServerlessPlugin(Plugin): # pylint: disable=W0223
+ """ Base class for bcfg2-lint plugins that are run before the
+ server starts up (i.e., plugins that check things that may prevent
+ the server from starting up). """
pass
-class ServerPlugin (Plugin):
- """ base class for plugins that check things that require the
- running Bcfg2 server """
- def __init__(self, core, config, **kwargs):
- Plugin.__init__(self, config, **kwargs)
+class ServerPlugin(Plugin): # pylint: disable=W0223
+ """ Base class for bcfg2-lint plugins that check things that
+ require the running Bcfg2 server. """
+
+ def __init__(self, core, config, errorhandler=None, files=None):
+ """
+ :param core: The Bcfg2 server core
+ :type core: Bcfg2.Server.Core.BaseCore
+ :param config: A :mod:`Bcfg2.Options` setup dict
+ :type config: dict
+ :param errorhandler: A :class:`Bcfg2.Server.Lint.ErrorHandler`
+ that will be used to handle lint errors.
+ If one is not provided, a new one will be
+ instantiated.
+ :type errorhandler: Bcfg2.Server.Lint.ErrorHandler
+ :param files: A list of files to run bcfg2-lint against. (See
+ the bcfg2-lint ``--stdin`` option.)
+ :type files: list of strings
+ """
+ Plugin.__init__(self, config, errorhandler=errorhandler, files=files)
+
+ #: The server core
self.core = core
self.logger = self.core.logger
- self.metadata = self.core.metadata
- self.errorhandler.RegisterErrors({"broken-xinclude-chain": "warning"})
- def has_all_xincludes(self, mfile):
- """ return true if self.files includes all XIncludes listed in
- the specified metadata type, false otherwise"""
- if self.files is None:
- return True
- else:
- path = os.path.join(self.metadata.data, mfile)
- if path in self.files:
- xdata = lxml.etree.parse(path)
- for el in xdata.findall('./%sinclude' % XI_NAMESPACE):
- if not self.has_all_xincludes(el.get('href')):
- self.LintError("broken-xinclude-chain",
- "Broken XInclude chain: could not "
- "include %s" % path)
- return False
-
- return True
+ #: The metadata plugin
+ self.metadata = self.core.metadata
diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py
index d8290d844..d114b0873 100644
--- a/src/lib/Bcfg2/Server/Plugins/Bundler.py
+++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py
@@ -126,10 +126,10 @@ class Bundler(Bcfg2.Server.Plugin.Plugin,
class BundlerLint(Bcfg2.Server.Lint.ServerPlugin):
- """ Perform various bundle checks """
+ """ Perform various :ref:`Bundler
+ <server-plugins-structures-bundler-index>` checks. """
def Run(self):
- """ run plugin """
self.missing_bundles()
for bundle in self.core.plugins['Bundler'].entries.values():
if self.HandlesFile(bundle.name):
@@ -143,7 +143,8 @@ class BundlerLint(Bcfg2.Server.Lint.ServerPlugin):
"genshi-extension-bundle": "error"}
def missing_bundles(self):
- """ find bundles listed in Metadata but not implemented in Bundler """
+ """ Find bundles listed in Metadata but not implemented in
+ Bundler. """
if self.files is None:
# when given a list of files on stdin, this check is
# useless, so skip it
diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
index 9042a979e..3e5508160 100644
--- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
+++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
@@ -125,7 +125,12 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin,
class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin):
- """ bcfg2-lint plugin for GroupPatterns """
+ """ ``bcfg2-lint`` plugin to check all given :ref:`GroupPatterns
+ <server-plugins-grouping-grouppatterns>` patterns for validity.
+ This is simply done by trying to create a
+ :class:`Bcfg2.Server.Plugins.GroupPatterns.PatternMap` object for
+ each pattern, and catching exceptions and presenting them as
+ ``bcfg2-lint`` errors."""
def Run(self):
cfg = self.core.plugins['GroupPatterns'].config
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 49e36f72b..2bc82caa9 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -945,7 +945,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
self.debug_log("Client %s set as nonexistent group %s"
% (client, group))
- def set_profile(self, client, profile, addresspair):
+ def set_profile(self, client, profile, # pylint: disable=W0221
+ addresspair, require_public=True):
"""Set group parameter for provided client."""
self.logger.info("Asserting client %s profile to %s" % (client,
profile))
@@ -957,7 +958,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
self.logger.error(msg)
raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
group = self.groups[profile]
- if not group.is_public:
+ if require_public and not group.is_public:
msg = "Cannot set client %s to private group %s" % (client,
profile)
self.logger.error(msg)
@@ -1128,7 +1129,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
pgroup = self.default
if pgroup:
- self.set_profile(client, pgroup, (None, None))
+ self.set_profile(client, pgroup, (None, None),
+ require_public=False)
profile = _add_group(pgroup)
else:
msg = "Cannot add new client %s; no default group set" % client
@@ -1477,7 +1479,16 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
- """ bcfg2-lint plugin for Metadata """
+ """ ``bcfg2-lint`` plugin for :ref:`Metadata
+ <server-plugins-grouping-metadata>`. This checks for several things:
+
+ * ``<Client>`` tags nested inside other ``<Client>`` tags;
+ * Deprecated options (like ``location="floating"``);
+ * Profiles that don't exist, or that aren't profile groups;
+ * Groups or clients that are defined multiple times;
+ * Multiple default groups or a default group that isn't a profile
+ group.
+ """
def Run(self):
self.nested_clients()
@@ -1500,8 +1511,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
"default-is-not-profile": "error"}
def deprecated_options(self):
- """ check for the location='floating' option, which has been
- deprecated in favor of floating='true' """
+ """ Check for the ``location='floating'`` option, which has
+ been deprecated in favor of ``floating='true'``. """
if not hasattr(self.metadata, "clients_xml"):
# using metadata database
return
@@ -1519,8 +1530,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
(loc, floating, self.RenderXML(el)))
def nested_clients(self):
- """ check for a Client tag inside a Client tag, which doesn't
- make any sense """
+ """ Check for a ``<Client/>`` tag inside a ``<Client/>`` tag,
+ which is either redundant or will never match. """
groupdata = self.metadata.groups_xml.xdata
for el in groupdata.xpath("//Client//Client"):
self.LintError("nested-client-tags",
@@ -1528,8 +1539,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
(el.get("name"), self.RenderXML(el)))
def bogus_profiles(self):
- """ check for clients that have profiles that are either not
- flagged as public groups in groups.xml, or don't exist """
+ """ Check for clients that have profiles that are either not
+ flagged as profile groups in ``groups.xml``, or don't exist. """
if not hasattr(self.metadata, "clients_xml"):
# using metadata database
return
@@ -1547,20 +1558,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
(profile, client.get("name"), profile,
self.RenderXML(client)))
- def duplicate_groups(self):
- """ check for groups that are defined twice. We count a group
- tag as a definition if it a) has profile or public set; or b)
- has any children. """
- self.duplicate_entries(
- self.metadata.groups_xml.xdata.xpath("//Groups/Group") +
- self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"),
- "group",
- include=lambda g: (g.get("profile") or
- g.get("public") or
- g.getchildren()))
-
def duplicate_default_groups(self):
- """ check for multiple default groups """
+ """ Check for multiple default groups. """
defaults = []
for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \
self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"):
@@ -1572,7 +1571,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
"\n".join(defaults))
def duplicate_clients(self):
- """ check for clients that are defined twice. """
+ """ Check for clients that are defined more than once. """
if not hasattr(self.metadata, "clients_xml"):
# using metadata database
return
@@ -1580,17 +1579,34 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
self.metadata.clients_xml.xdata.xpath("//Client"),
"client")
- def duplicate_entries(self, allentries, etype, include=None):
- """ generic duplicate entry finder """
- if include is None:
- include = lambda e: True
+ def duplicate_groups(self):
+ """ Check for groups that are defined more than once. We
+ count a group tag as a definition if it a) has profile or
+ public set; or b) has any children."""
+ allgroups = [
+ g
+ for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") +
+ self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group")
+ if g.get("profile") or g.get("public") or g.getchildren()]
+ self.duplicate_entries(allgroups, "group")
+
+ def duplicate_entries(self, allentries, etype):
+ """ Generic duplicate entry finder.
+
+ :param allentries: A list of all entries to check for
+ duplicates.
+ :type allentries: list of lxml.etree._Element
+ :param etype: The entry type. This will be used to determine
+ the error name (``duplicate-<etype>``) and for
+ display to the end user.
+ :type etype: string
+ """
entries = dict()
for el in allentries:
- if include(el):
- if el.get("name") in entries:
- entries[el.get("name")].append(self.RenderXML(el))
- else:
- entries[el.get("name")] = [self.RenderXML(el)]
+ if el.get("name") in entries:
+ entries[el.get("name")].append(self.RenderXML(el))
+ else:
+ entries[el.get("name")] = [self.RenderXML(el)]
for ename, els in entries.items():
if len(els) > 1:
self.LintError("duplicate-%s" % etype,
@@ -1598,7 +1614,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
(etype.title(), ename, "\n".join(els)))
def default_is_profile(self):
- """ ensure that the default group is a profile group """
+ """ Ensure that the default group is a profile group. """
if (self.metadata.default and
not self.metadata.groups[self.metadata.default].is_profile):
xdata = \
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index 48c580be1..dba56eed2 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -39,6 +39,11 @@ class AptCollection(Collection):
else:
lines.append("deb %s %s %s" % (source.url, source.version,
" ".join(source.components)))
+ if source.debsrc:
+ lines.append("deb-src %s %s %s" %
+ (source.url,
+ source.version,
+ " ".join(source.components)))
lines.append("")
return "\n".join(lines)
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 3319dda78..767ac13ac 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -152,6 +152,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
#: this source
self.whitelist = [item.text for item in xsource.findall('Whitelist')]
+ #: Whether or not to include deb-src lines in the generated APT
+ #: configuration
+ self.debsrc = xsource.get('debsrc', 'false') == 'true'
+
#: A dict of repository options that will be included in the
#: configuration generated on the server side (if such is
#: applicable; most backends do not generate any sort of
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 052c362ab..567a16c40 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -9,7 +9,7 @@ import shutil
import lxml.etree
import Bcfg2.Logger
import Bcfg2.Server.Plugin
-from Bcfg2.Compat import ConfigParser, urlopen, HTTPError
+from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError
from Bcfg2.Server.Plugins.Packages.Collection import Collection, \
get_collection_class
from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources
@@ -445,7 +445,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
try:
open(localfile, 'w').write(urlopen(key).read())
keys.append(key)
- except HTTPError:
+ except (URLError, HTTPError):
err = sys.exc_info()[1]
self.logger.error("Packages: Error downloading %s: %s"
% (key, err))
diff --git a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py
index 7dac907e1..a1dcb575f 100644
--- a/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py
+++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py
@@ -177,7 +177,10 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir):
class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin):
- """ find duplicate Pkgmgr entries with the same priority """
+ """ Find duplicate :ref:`Pkgmgr
+ <server-plugins-generators-pkgmgr>` entries with the same
+ priority. """
+
def Run(self):
pset = set()
for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr',
@@ -202,12 +205,13 @@ class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin):
# check if package is already listed with same
# priority, type, grp
if ptuple in pset:
- self.LintError("duplicate-package",
- "Duplicate Package %s, priority:%s, type:%s" %
- (pkg.get('name'), priority, ptype))
+ self.LintError(
+ "duplicate-package",
+ "Duplicate Package %s, priority:%s, type:%s" %
+ (pkg.get('name'), priority, ptype))
else:
pset.add(ptuple)
-
+
@classmethod
def Errors(cls):
- return {"duplicate-packages":"error"}
+ return {"duplicate-packages": "error"}
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index 7e4935d74..e97607093 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -65,7 +65,7 @@ class ProbeData(str): # pylint: disable=E0012,R0924
.json, and .yaml properties to provide convenient ways to use
ProbeData objects as XML, JSON, or YAML data """
def __new__(cls, data):
- return str.__new__(cls, data.encode('utf-8'))
+ return str.__new__(cls, data)
def __init__(self, data): # pylint: disable=W0613
str.__init__(self)
@@ -224,15 +224,9 @@ class Probes(Bcfg2.Server.Plugin.Probing,
lxml.etree.SubElement(top, 'Client', name=client,
timestamp=str(int(probedata.timestamp)))
for probe in sorted(probedata):
- try:
- lxml.etree.SubElement(
- ctag, 'Probe', name=probe,
- value=str(
- self.probedata[client][probe]).decode('utf-8'))
- except AttributeError:
- lxml.etree.SubElement(
- ctag, 'Probe', name=probe,
- value=str(self.probedata[client][probe]))
+ lxml.etree.SubElement(
+ ctag, 'Probe', name=probe,
+ value=self.probedata[client][probe])
for group in sorted(self.cgroups[client]):
lxml.etree.SubElement(ctag, "Group", name=group)
try:
diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
index 1264fd1cf..84dcf2780 100644
--- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py
+++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
@@ -173,8 +173,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
for name in names[cmeta.hostname]:
newnames.add(name.split('.')[0])
try:
- newips.add(self.get_ipcache_entry(name)[0])
- except PluginExecutionError:
+ newips.update(self.get_ipcache_entry(name)[0])
+ except: # pylint: disable=W0702
continue
names[cmeta.hostname].update(newnames)
names[cmeta.hostname].update(cmeta.addresses)
@@ -290,7 +290,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
else:
# need to add entry
try:
- ipaddr = socket.gethostbyname(client)
+ ipaddr = set([info[4][0]
+ for info in socket.getaddrinfo(client, None)])
self.ipcache[client] = (ipaddr, client)
return (ipaddr, client)
except socket.gaierror:
diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
index e834759c2..050ba3b3e 100644
--- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
+++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
@@ -96,7 +96,18 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin,
class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin):
- """ find duplicate Pkgmgr entries with the same priority """
+ """ ``bcfg2-lint`` plugin to ensure that all :ref:`TemplateHelper
+ <server-plugins-connectors-templatehelper>` modules are valid.
+ This can check for:
+
+ * A TemplateHelper module that cannot be imported due to syntax or
+ other compile-time errors;
+ * A TemplateHelper module that does not have an ``__export__``
+ attribute, or whose ``__export__`` is not a list;
+ * Bogus symbols listed in ``__export__``, including symbols that
+ don't exist, that are reserved, or that start with underscores.
+ """
+
def __init__(self, *args, **kwargs):
Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs)
self.reserved_keywords = dir(HelperModule("foo.py"))
@@ -107,7 +118,11 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin):
self.check_helper(helper.name)
def check_helper(self, helper):
- """ check a helper module for export errors """
+ """ Check a single helper module.
+
+ :param helper: The filename of the helper module
+ :type helper: string
+ """
module_name = MODULE_RE.search(helper).group(1)
try:
diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py
index 13c756049..eea2183f7 100644
--- a/src/lib/Bcfg2/Server/SSLServer.py
+++ b/src/lib/Bcfg2/Server/SSLServer.py
@@ -346,7 +346,7 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer,
:param register: Presence should be reported to service-location
:type register: bool
:param allow_none: Allow None values in XML-RPC
- :type allow_non: bool
+ :type allow_none: bool
:param encoding: Encoding to use for XML-RPC
"""
diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py
index 68282331e..69c3264f9 100644
--- a/src/lib/Bcfg2/Utils.py
+++ b/src/lib/Bcfg2/Utils.py
@@ -2,6 +2,7 @@
used by both client and server. Stuff that doesn't fit anywhere
else. """
+import shlex
import fcntl
import logging
import threading
@@ -217,6 +218,7 @@ class Executor(object):
"""
if isinstance(command, str):
cmdstr = command
+ command = shlex.split(cmdstr)
else:
cmdstr = " ".join(command)
self.logger.debug("Running: %s" % cmdstr)
diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info
index 101530cac..4654359f7 100755
--- a/src/sbin/bcfg2-info
+++ b/src/sbin/bcfg2-info
@@ -626,30 +626,34 @@ Bcfg2 client itself.""")
self.fam.debug = True
def do_packageresolve(self, args):
- """ packageresolve <hostname> <package> [<package>...] -
- Resolve the specified set of packages """
+ """ packageresolve <hostname> [<package> [<package>...]] -
+ Resolve packages for the given host, optionally specifying a
+ set of packages """
arglist = args.split(" ")
- if len(arglist) < 2:
+ if len(arglist) < 1:
print(self._get_usage(self.do_packageresolve))
return
- if 'Packages' not in self.plugins:
+ try:
+ pkgs = self.plugins['Packages']
+ except KeyError:
print("Packages plugin not enabled")
return
- self.plugins['Packages'].toggle_debug()
-
- indep = lxml.etree.Element("Independent")
- structures = [lxml.etree.Element("Bundle", name="packages")]
- for arg in arglist[1:]:
- lxml.etree.SubElement(structures[0], "Package", name=arg)
+ pkgs.toggle_debug()
hostname = arglist[0]
metadata = self.build_metadata(hostname)
- # pylint: disable=W0212
- self.plugins['Packages']._build_packages(metadata, indep, structures)
- # pylint: enable=W0212
+ indep = lxml.etree.Element("Independent")
+ if len(arglist) > 1:
+ structures = [lxml.etree.Element("Bundle", name="packages")]
+ for arg in arglist[1:]:
+ lxml.etree.SubElement(structures[0], "Package", name=arg)
+ else:
+ structures = self.GetStructures(metadata)
+ pkgs._build_packages(metadata, indep, # pylint: disable=W0212
+ structures)
print("%d new packages added" % len(indep.getchildren()))
if len(indep.getchildren()):
print(" %s" % "\n ".join(lxml.etree.tostring(p)
diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint
index 4f81df89c..2ae5e02d5 100755
--- a/src/sbin/bcfg2-lint
+++ b/src/sbin/bcfg2-lint
@@ -58,16 +58,17 @@ def get_errorhandler():
""" get a Bcfg2.Server.Lint.ErrorHandler object """
setup = Bcfg2.Options.get_option_parser()
if setup.cfp.has_section("errors"):
- conf = dict(setup.cfp.items("errors"))
+ errors = dict(setup.cfp.items("errors"))
else:
- conf = None
- return Bcfg2.Server.Lint.ErrorHandler(config=conf)
+ errors = None
+ return Bcfg2.Server.Lint.ErrorHandler(errors=errors)
def load_server():
""" load server """
core = Bcfg2.Server.Core.BaseCore()
- core.fam.handle_events_in_interval(4)
+ core.load_plugins()
+ core.fam.handle_events_in_interval(0.1)
return core
@@ -93,7 +94,7 @@ def load_plugins():
elif setup['lint_plugins']:
plugin_list = setup['lint_plugins']
else:
- plugin_list = Bcfg2.Server.Lint.__all__
+ plugin_list = Bcfg2.Server.Lint.plugins
allplugins = dict()
for plugin in plugin_list:
diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test
index 510bb898b..f13240879 100755
--- a/src/sbin/bcfg2-test
+++ b/src/sbin/bcfg2-test
@@ -155,6 +155,7 @@ class ClientTest(TestCase):
def get_core(setup):
""" Get a server core, with events handled """
core = Bcfg2.Server.Core.BaseCore(setup)
+ core.load_plugins()
core.fam.handle_events_in_interval(0.1)
return core
diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py
index 211c39732..6d4644ea5 100644
--- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py
+++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py
@@ -226,8 +226,7 @@ class TestPOSIXUsers(TestTool):
users.user_supplementary_groups.assert_called_with(entry)
reset()
- m1 = lxml.etree.SubElement(entry, "MemberOf")
- m1.text = "wheel"
+ m1 = lxml.etree.SubElement(entry, "MemberOf", group="wheel")
m2 = lxml.etree.SubElement(entry, "MemberOf")
m2.text = "users"
self.assertTrue(users.VerifyPOSIXUser(entry, []))
@@ -236,8 +235,7 @@ class TestPOSIXUsers(TestTool):
users.user_supplementary_groups.assert_called_with(entry)
reset()
- m3 = lxml.etree.SubElement(entry, "MemberOf")
- m3.text = "extra"
+ m3 = lxml.etree.SubElement(entry, "MemberOf", group="extra")
self.assertFalse(users.VerifyPOSIXUser(entry, []))
users.populate_user_entry.assert_called_with(entry)
users._verify.assert_called_with(users.populate_user_entry.return_value)
@@ -371,8 +369,7 @@ class TestPOSIXUsers(TestTool):
entry = lxml.etree.Element("POSIXUser", name="test", group="test",
home="/home/test", shell="/bin/zsh",
gecos="Test McTest")
- m1 = lxml.etree.SubElement(entry, "MemberOf")
- m1.text = "wheel"
+ m1 = lxml.etree.SubElement(entry, "MemberOf", group="wheel")
m2 = lxml.etree.SubElement(entry, "MemberOf")
m2.text = "users"
diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py
index 3cd3786f9..93314f1e3 100755
--- a/tools/bcfg2-profile-templates.py
+++ b/tools/bcfg2-profile-templates.py
@@ -1,25 +1,35 @@
#!/usr/bin/python -Ott
+# -*- coding: utf-8 -*-
""" Benchmark template rendering times """
-import os
import sys
import time
+import math
import logging
import operator
import Bcfg2.Logger
+import Bcfg2.Options
import Bcfg2.Server.Core
-LOGGER = None
+
+def stdev(nums):
+ mean = float(sum(nums)) / len(nums)
+ return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums)))
def main():
- optinfo = \
- dict(client=Bcfg2.Options.Option("Benchmark templates for one client",
- cmd="--client",
- odesc="<client>",
- long_arg=True,
- default=None),
- )
+ optinfo = dict(
+ client=Bcfg2.Options.Option("Benchmark templates for one client",
+ cmd="--client",
+ odesc="<client>",
+ long_arg=True,
+ default=None),
+ runs=Bcfg2.Options.Option("Number of rendering passes per template",
+ cmd="--runs",
+ odesc="<runs>",
+ long_arg=True,
+ default=5,
+ cook=int))
optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS)
optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS)
setup = Bcfg2.Options.OptionParser(optinfo)
@@ -40,12 +50,11 @@ def main():
core = Bcfg2.Server.Core.BaseCore(setup)
logger.info("Bcfg2 server core loaded")
+ core.load_plugins()
+ logger.debug("Plugins loaded")
core.fam.handle_events_in_interval(0.1)
logger.debug("Repository events processed")
- # how many times to render each template for each client
- runs = 5
-
if setup['args']:
templates = setup['args']
else:
@@ -57,41 +66,57 @@ def main():
clients = [core.build_metadata(setup['client'])]
times = dict()
+ client_count = 0
for metadata in clients:
- for struct in core.GetStructures(metadata):
- logger.info("Rendering templates from structure %s:%s" %
- (struct.tag, struct.get("name")))
- for entry in struct.xpath("//Path"):
- path = entry.get("name")
- logger.info("Rendering %s..." % path)
- times[path] = dict()
- avg = 0.0
- for i in range(runs):
+ client_count += 1
+ logger.info("Rendering templates for client %s (%s/%s)" %
+ (metadata.hostname, client_count, len(clients)))
+ structs = core.GetStructures(metadata)
+ struct_count = 0
+ for struct in structs:
+ struct_count += 1
+ logger.info("Rendering templates from structure %s:%s (%s/%s)" %
+ (struct.tag, struct.get("name"), struct_count,
+ len(structs)))
+ entries = struct.xpath("//Path")
+ entry_count = 0
+ for entry in entries:
+ entry_count += 1
+ if templates and entry.get("name") not in templates:
+ continue
+ logger.info("Rendering Path:%s (%s/%s)..." %
+ (entry.get("name"), entry_count, len(entries)))
+ ptimes = times.setdefault(entry.get("name"), [])
+ for i in range(setup['runs']):
start = time.time()
try:
core.Bind(entry, metadata)
- avg += (time.time() - start) / runs
+ ptimes.append(time.time() - start)
except:
break
- if avg:
- logger.debug(" %s: %.02f sec" % (metadata.hostname, avg))
- times[path][metadata.hostname] = avg
+ if ptimes:
+ avg = sum(ptimes) / len(ptimes)
+ if avg:
+ logger.debug(" %s: %.02f sec" %
+ (metadata.hostname, avg))
# print out per-file results
tmpltimes = []
- for tmpl, clients in times.items():
+ for tmpl, ptimes in times.items():
try:
- avg = sum(clients.values()) / len(clients)
+ mean = float(sum(ptimes)) / len(ptimes)
except ZeroDivisionError:
continue
- if avg > 0.01 or templates:
- tmpltimes.append((tmpl, avg))
- print("%-50s %s" % ("Template", "Average Render Time"))
- for tmpl, avg in reversed(sorted(tmpltimes, key=operator.itemgetter(1))):
- print("%-50s %.02f" % (tmpl, avg))
-
- # TODO: complain about templates that on average were quick but
- # for which some clients were slow
+ ptimes.sort()
+ median = ptimes[len(ptimes) / 2]
+ std = stdev(ptimes)
+ if mean > 0.01 or median > 0.01 or std > 1 or templates:
+ tmpltimes.append((tmpl, mean, median, std))
+ print("%-50s %-9s %-11s %6s" %
+ ("Template", "Mean Time", "Median Time", "σ"))
+ for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))):
+ print("%-50s %9.02f %11.02f %6.02f" % info)
+ core.shutdown()
if __name__ == "__main__":
diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py
index edb5a7101..8c164e52e 100755
--- a/tools/bcfg2_local.py
+++ b/tools/bcfg2_local.py
@@ -19,7 +19,8 @@ class LocalCore(BaseCore):
setup['logging'] = None
Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup)
setup['syslog'], setup['logging'] = saved
- self.fam.handle_events_in_interval(4)
+ self.load_plugins()
+ self.fam.handle_events_in_interval(0.1)
def _daemonize(self):
return True
diff --git a/tools/export.py b/tools/export.py
index 716c831d9..0f4724e6b 100755
--- a/tools/export.py
+++ b/tools/export.py
@@ -227,9 +227,6 @@ E.G. 1.2.0pre1 is a valid version.
'VERSION="%s"\n' % version,
startswith=True,
dryrun=options.dryrun)
- # set new version in setup.py
- find_and_replace('setup.py', 'version=', ' version="%s",\n' % version,
- dryrun=options.dryrun)
# set new version in Bcfg2/version.py
find_and_replace('src/lib/Bcfg2/version.py',
'__version__ =',
diff --git a/tools/posixusers_baseline.py b/tools/posixusers_baseline.py
index a4abca42d..c45e54f1a 100755
--- a/tools/posixusers_baseline.py
+++ b/tools/posixusers_baseline.py
@@ -61,8 +61,8 @@ def main():
if entry.tag == 'POSIXUser':
entry.set("group", grp.getgrgid(data[3])[0])
for group in users.user_supplementary_groups(entry):
- memberof = lxml.etree.SubElement(entry, "MemberOf")
- memberof.text = group[0]
+ memberof = lxml.etree.SubElement(entry, "MemberOf",
+ group=group[0])
entry.tag = "Bound" + entry.tag
baseline.append(entry)
diff --git a/tools/selinux_baseline.py b/tools/selinux_baseline.py
index 06f6e6b98..507a16f43 100755
--- a/tools/selinux_baseline.py
+++ b/tools/selinux_baseline.py
@@ -42,7 +42,10 @@ def main():
baseline.append(lxml.etree.Comment("%s entries" % etype))
extra = handler.FindExtra()
for entry in extra:
- entry.tag = "Bound%s" % etype
+ if etype != "SEModule":
+ entry.tag = "Bound%s" % etype
+ else:
+ entry.tag = "%s" % etype
baseline.extend(extra)
print(lxml.etree.tostring(baseline, pretty_print=True))
diff --git a/tools/upgrade/1.3/migrate_dbstats.py b/tools/upgrade/1.3/migrate_dbstats.py
index cbd2a6099..07def2ac8 100755
--- a/tools/upgrade/1.3/migrate_dbstats.py
+++ b/tools/upgrade/1.3/migrate_dbstats.py
@@ -21,6 +21,7 @@ logger = logging.getLogger(__name__)
_our_backend = None
+
def _quote(value):
"""
Quote a string to use as a table name or column
@@ -44,12 +45,12 @@ def _migrate_perms():
fperms = {}
logger.info("Creating FilePerms objects")
- for data in ( ('owner', 'group', 'perms'),
+ for data in (('owner', 'group', 'perms'),
('current_owner', 'current_group', 'current_perms')):
for grp in legacy_models.Reason.objects.values_list(*data).distinct():
if grp in fperms:
continue
- fp = new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2])
+ fp = new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2])
fp.save()
fperms[grp] = fp
@@ -60,7 +61,7 @@ def _migrate_perms():
def _migrate_transaction(inter, entries, fperms):
"""helper"""
- logger.debug("Migrating interaction %s for %s" %
+ logger.debug("Migrating interaction %s for %s" %
(inter.id, inter.client.name))
newint = new_models.Interaction(id=inter.id,
@@ -107,7 +108,7 @@ def _migrate_transaction(inter, entries, fperms):
elif ent.kind == 'Package':
act_dict['target_version'] = ei.reason.version
act_dict['current_version'] = ei.reason.current_version
- logger.debug("Adding package %s %s" %
+ logger.debug("Adding package %s %s" %
(name, act_dict['target_version']))
updates['packages'].append(new_models.PackageEntry.entry_get_or_create(act_dict))
elif ent.kind == 'Path':
@@ -116,7 +117,7 @@ def _migrate_transaction(inter, entries, fperms):
act_dict['target_perms'] = fperms[(
ei.reason.owner,
- ei.reason.group,
+ ei.reason.group,
ei.reason.perms
)]
@@ -141,7 +142,6 @@ def _migrate_transaction(inter, entries, fperms):
act_dict['detail_type'] = new_models.PathEntry.DETAIL_PRUNED
act_dict['details'] = ei.reason.unpruned
-
if ei.reason.is_sensitive:
act_dict['detail_type'] = new_models.PathEntry.DETAIL_SENSITIVE
elif ei.reason.is_binary:
@@ -164,7 +164,7 @@ def _migrate_transaction(inter, entries, fperms):
for entry_type in updates.keys():
i = 0
while(i < len(updates[entry_type])):
- getattr(newint, entry_type).add(*updates[entry_type][i:i+100])
+ getattr(newint, entry_type).add(*updates[entry_type][i:i + 100])
i += 100
for perf in inter.performance_items.all():
@@ -220,8 +220,8 @@ def _restructure():
# run any migrations from the previous schema
try:
- from Bcfg2.Server.Reports.updatefix import update_database
- update_database()
+ from Bcfg2.Server.Reports.updatefix import update_database
+ update_database()
except:
logger.error("Failed to run legacy schema updates", exc_info=1)
return False
@@ -295,4 +295,3 @@ if __name__ == '__main__':
Reports(setup).__call__(['update'])
_restructure()
-
diff --git a/tools/yum-listpkgs-xml.py b/tools/yum-listpkgs-xml.py
index a052e75af..b4c5f6589 100755
--- a/tools/yum-listpkgs-xml.py
+++ b/tools/yum-listpkgs-xml.py
@@ -39,5 +39,5 @@ try:
sys.argv = [sys.argv[0], '-d', '0', 'list']
yummain.main(sys.argv[1:])
except KeyboardInterrupt:
- print("\n\nExiting on user cancel.", file=sys.stderr)
+ sys.stderr.write("\n\nExiting on user cancel.")
sys.exit(1)