summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChris St. Pierre <chris.a.st.pierre@gmail.com>2013-04-24 13:47:31 -0400
committerChris St. Pierre <chris.a.st.pierre@gmail.com>2013-04-24 13:47:31 -0400
commit0ff6b2788de683dd89203c7ae1393ea922a62c32 (patch)
tree54ce843377ab26c6336de7f1abf3ec906d49aa69
parent46a47b4120b3d892b8149a5e181e4d976ad87f99 (diff)
parent29399cbc599919fd9c88448bde692132c803e69b (diff)
downloadbcfg2-0ff6b2788de683dd89203c7ae1393ea922a62c32.tar.gz
bcfg2-0ff6b2788de683dd89203c7ae1393ea922a62c32.tar.bz2
bcfg2-0ff6b2788de683dd89203c7ae1393ea922a62c32.zip
Merge branch 'maint'
Conflicts: src/lib/Bcfg2/Client/Client.py src/lib/Bcfg2/Client/Frame.py src/lib/Bcfg2/Client/Tools/YUM.py src/lib/Bcfg2/Options.py src/lib/Bcfg2/Server/Admin/Perf.py src/lib/Bcfg2/Server/Admin/Xcmd.py src/lib/Bcfg2/Server/Admin/__init__.py src/lib/Bcfg2/Server/Core.py src/lib/Bcfg2/Server/FileMonitor/Fam.py src/lib/Bcfg2/Server/Lint/RequiredAttrs.py src/lib/Bcfg2/Server/Plugin/helpers.py src/lib/Bcfg2/Server/Plugins/Base.py src/lib/Bcfg2/Server/Plugins/Bundler.py src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py src/lib/Bcfg2/Server/Plugins/Cvs.py src/lib/Bcfg2/Server/Plugins/Darcs.py src/lib/Bcfg2/Server/Plugins/Decisions.py src/lib/Bcfg2/Server/Plugins/FileProbes.py src/lib/Bcfg2/Server/Plugins/Fossil.py src/lib/Bcfg2/Server/Plugins/Git.py src/lib/Bcfg2/Server/Plugins/Metadata.py src/lib/Bcfg2/Server/Plugins/NagiosGen.py src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py src/lib/Bcfg2/Server/Plugins/Packages/Source.py src/lib/Bcfg2/Server/Plugins/Packages/Yum.py src/lib/Bcfg2/Server/Plugins/Properties.py src/lib/Bcfg2/Server/Plugins/__init__.py src/lib/Bcfg2/Server/__init__.py src/sbin/bcfg2-build-reports src/sbin/bcfg2-crypt testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py
-rw-r--r--.travis.yml7
-rw-r--r--doc/development/compat.txt2
-rw-r--r--doc/reports/dynamic.txt64
-rw-r--r--doc/server/plugins/connectors/grouplogic.txt122
-rw-r--r--doc/server/plugins/generators/packages.txt40
-rw-r--r--doc/server/plugins/generators/rules.txt9
-rw-r--r--doc/server/plugins/generators/sslca.txt2
-rw-r--r--doc/server/plugins/grouping/metadata.txt17
-rw-r--r--doc/server/selinux.txt10
-rw-r--r--misc/bcfg2.spec3
-rw-r--r--schemas/grouplogic.xsd110
-rw-r--r--schemas/types.xsd6
-rw-r--r--src/lib/Bcfg2/Client/Client.py26
-rw-r--r--src/lib/Bcfg2/Client/Frame.py48
-rw-r--r--src/lib/Bcfg2/Client/Proxy.py2
-rw-r--r--src/lib/Bcfg2/Client/Tools/APK.py2
-rw-r--r--src/lib/Bcfg2/Client/Tools/Action.py4
-rw-r--r--src/lib/Bcfg2/Client/Tools/DebInit.py2
-rw-r--r--src/lib/Bcfg2/Client/Tools/Encap.py2
-rw-r--r--src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py4
-rw-r--r--src/lib/Bcfg2/Client/Tools/IPS.py6
-rw-r--r--src/lib/Bcfg2/Client/Tools/MacPorts.py2
-rw-r--r--src/lib/Bcfg2/Client/Tools/POSIX/Device.py4
-rw-r--r--src/lib/Bcfg2/Client/Tools/POSIX/File.py10
-rw-r--r--src/lib/Bcfg2/Client/Tools/POSIX/base.py20
-rw-r--r--src/lib/Bcfg2/Client/Tools/Pacman.py6
-rw-r--r--src/lib/Bcfg2/Client/Tools/Portage.py6
-rw-r--r--src/lib/Bcfg2/Client/Tools/RcUpdate.py24
-rw-r--r--src/lib/Bcfg2/Client/Tools/SELinux.py32
-rw-r--r--src/lib/Bcfg2/Client/Tools/SMF.py23
-rw-r--r--src/lib/Bcfg2/Client/Tools/SYSV.py4
-rw-r--r--src/lib/Bcfg2/Client/Tools/Upstart.py4
-rw-r--r--src/lib/Bcfg2/Client/Tools/YUM.py93
-rw-r--r--src/lib/Bcfg2/Client/__init__.py2
-rw-r--r--src/lib/Bcfg2/Compat.py10
-rw-r--r--src/lib/Bcfg2/Logger.py14
-rw-r--r--src/lib/Bcfg2/Options.py23
-rw-r--r--src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py4
-rw-r--r--src/lib/Bcfg2/Reporting/models.py8
-rw-r--r--src/lib/Bcfg2/Server/Admin/Init.py4
-rw-r--r--src/lib/Bcfg2/Server/Admin/Minestruct.py5
-rw-r--r--src/lib/Bcfg2/Server/Admin/Perf.py8
-rw-r--r--src/lib/Bcfg2/Server/Admin/Pull.py12
-rw-r--r--src/lib/Bcfg2/Server/Admin/Xcmd.py2
-rw-r--r--src/lib/Bcfg2/Server/Admin/__init__.py30
-rw-r--r--src/lib/Bcfg2/Server/BuiltinCore.py14
-rw-r--r--src/lib/Bcfg2/Server/Core.py220
-rwxr-xr-xsrc/lib/Bcfg2/Server/Encryption.py6
-rw-r--r--src/lib/Bcfg2/Server/FileMonitor/Inotify.py8
-rw-r--r--src/lib/Bcfg2/Server/FileMonitor/Pseudo.py2
-rw-r--r--src/lib/Bcfg2/Server/Lint/RequiredAttrs.py30
-rw-r--r--src/lib/Bcfg2/Server/Lint/Validate.py28
-rw-r--r--src/lib/Bcfg2/Server/Plugin/base.py16
-rw-r--r--src/lib/Bcfg2/Server/Plugin/helpers.py137
-rw-r--r--src/lib/Bcfg2/Server/Plugin/interfaces.py15
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py9
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py18
-rw-r--r--src/lib/Bcfg2/Server/Plugins/FileProbes.py10
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Git.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/GroupLogic.py47
-rw-r--r--src/lib/Bcfg2/Server/Plugins/GroupPatterns.py11
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Metadata.py145
-rw-r--r--src/lib/Bcfg2/Server/Plugins/NagiosGen.py21
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Ohai.py31
-rw-r--r--src/lib/Bcfg2/Server/Plugins/POSIXCompat.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Apt.py12
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py16
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Source.py10
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Yum.py14
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/__init__.py30
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Probes.py52
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Properties.py39
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Reporting.py7
-rw-r--r--src/lib/Bcfg2/Server/Plugins/SSHbase.py7
-rw-r--r--src/lib/Bcfg2/Server/Plugins/SSLCA.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/ServiceCompat.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/TemplateHelper.py2
-rw-r--r--src/lib/Bcfg2/Server/Plugins/__init__.py6
-rw-r--r--src/lib/Bcfg2/Server/SSLServer.py5
-rw-r--r--src/lib/Bcfg2/Server/Statistics.py11
-rw-r--r--src/lib/Bcfg2/Server/__init__.py5
-rw-r--r--src/lib/Bcfg2/Utils.py34
-rw-r--r--src/lib/Bcfg2/__init__.py2
-rw-r--r--src/lib/Bcfg2/settings.py4
-rw-r--r--src/lib/Bcfg2/version.py2
-rwxr-xr-xsrc/sbin/bcfg2-crypt2
-rwxr-xr-xsrc/sbin/bcfg2-info38
-rwxr-xr-xsrc/sbin/bcfg2-report-collector10
-rwxr-xr-xsrc/sbin/bcfg2-reports3
-rwxr-xr-xsrc/sbin/bcfg2-yum-helper36
-rw-r--r--testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py8
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py22
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py61
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py11
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py1
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py15
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py20
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py192
-rw-r--r--testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py27
-rw-r--r--testsuite/Testsrc/Testlib/TestStatistics.py44
-rwxr-xr-xtestsuite/before_install.sh9
-rwxr-xr-xtestsuite/install.sh12
-rw-r--r--testsuite/requirements.txt4
-rwxr-xr-xtools/bcfg2_local.py6
-rwxr-xr-xtools/selinux_baseline.py2
-rwxr-xr-xtools/upgrade/1.3/migrate_perms_to_mode.py5
107 files changed, 1562 insertions, 840 deletions
diff --git a/.travis.yml b/.travis.yml
index f5aade735..73b8a9594 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -3,16 +3,9 @@ python:
- "2.5"
- "2.6"
- "2.7"
- - "3.2"
env:
- WITH_OPTIONAL_DEPS=yes
- WITH_OPTIONAL_DEPS=no
-matrix:
- exclude:
- - python: "3.2"
- env: WITH_OPTIONAL_DEPS=yes
- - python: "3.3"
- env: WITH_OPTIONAL_DEPS=yes
before_install:
- testsuite/before_install.sh
install:
diff --git a/doc/development/compat.txt b/doc/development/compat.txt
index 90df45676..f90274ce5 100644
--- a/doc/development/compat.txt
+++ b/doc/development/compat.txt
@@ -100,6 +100,8 @@ behavior (e.g., :func:`input`) do not cause unexpected side-effects.
+---------------------------------+--------------------------------------------------+---------------------------------------------------------+
| long | :func:`long` | :func:`int` |
+---------------------------------+--------------------------------------------------+---------------------------------------------------------+
+| cmp | :func:`cmp` | Not implemented |
++---------------------------------+--------------------------------------------------+---------------------------------------------------------+
Python 2.4 compatibility
------------------------
diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt
index b3028e9e1..9de3f868f 100644
--- a/doc/reports/dynamic.txt
+++ b/doc/reports/dynamic.txt
@@ -53,40 +53,41 @@ Prerequisites
Install
-------
-Be sure to include the specified fields included in the example
-``bcfg2.conf`` file. These can be specified in either ``/etc/bcfg2.conf``,
-if it is readable by the webserver user, or ``/etc/bcfg2-web.conf``. Any
-database supported by `Django <http://www.djangoproject.com>`_ can be used.
-As of version 1.3, `South <http://south.aeracode.org>`_ is used to control
-schema changes. If your database is not supported by South, any updates
-will need to be applied manually. Sqlite is configured by default.
-Please see the :ref:`reporting-databases` section to configure alternative
-databases.
-.. warning::
+1. Be sure to include the specified fields included in the example
+ ``bcfg2.conf`` file. These can be specified in either
+ ``/etc/bcfg2.conf``, if it is readable by the webserver user,
+ or ``/etc/bcfg2-web.conf``. Any database supported by `Django
+ <http://www.djangoproject.com>`_ can be used. As of version 1.3,
+ `South <http://south.aeracode.org>`_ is used to control schema changes.
+ If your database is not supported by South, any updates will need to
+ be applied manually. Sqlite is configured by default. Please see the
+ :ref:`reporting-databases` section to configure alternative databases.
- If you are using an sqlite database, the directory containing the
- database file will need to be writable by the web server. The reason
- for this is that sqlite will create another file for its journal
- when it tries to update the database file.
+ .. warning::
-.. note::
+ If you are using an sqlite database, the directory containing the
+ database file will need to be writable by the web server. The reason
+ for this is that sqlite will create another file for its journal
+ when it tries to update the database file.
+
+ .. note::
- Distributed environments can share a single remote database for
- reporting.
+ Distributed environments can share a single remote database for
+ reporting.
-After configuring your database be sure to run `bcfg2-admin reports init`
-to create the schema.
+2. After configuring your database be sure to run ``bcfg2-admin reports
+ init`` to create the schema.
-To enable statistics collection in the bcfg2-server, add
-:ref:`server-plugins-statistics-reporting` to the **plugins**
-line in your ``bcfg2.conf`` and restart the bcfg2-server. A report collecting
-daemon should be run to import the collected statistics into the backend.
-Please see the section :ref:`Report Collector <report_collector>` for more
-information.
+3. To enable statistics collection in the bcfg2-server, add
+ :ref:`server-plugins-statistics-reporting` to the **plugins**
+ line in your ``bcfg2.conf`` and restart the bcfg2-server. A report
+ collecting daemon should be run to import the collected statistics
+ into the backend. Please see the section :ref:`Report Collector
+ <report_collector>` for more information.
-Detailed installation instructions can be found :ref:`here
-<appendix-guides-web-reports-install>`.
+ Detailed installation instructions can be found :ref:`here
+ <appendix-guides-web-reports-install>`.
.. _dynamic-http-install:
@@ -175,7 +176,7 @@ Upgrading
.. note::
After the database is upgraded all of the old tables are left
- intact. To remove them any table starting with reports_ can
+ intact. To remove them any table starting with **reports\_** can
be dropped.
4. `(Optional)` Run the :ref:`Report Collector <report_collector>`
@@ -199,11 +200,6 @@ An example using the defaults is listed below::
host =
port =
- [statistics]
- config = /etc/bcfg2-web.conf
- time_zone =
- web_debug = False
-
[reporting]
transport = DirectStore
web_prefix =
@@ -241,6 +237,8 @@ section:
statistics
^^^^^^^^^^
+.. deprecated: 1.3.0
+
* config: The config file to be read for additional reporting
data. This is used to restrict what can be read by the web
server.
diff --git a/doc/server/plugins/connectors/grouplogic.txt b/doc/server/plugins/connectors/grouplogic.txt
new file mode 100644
index 000000000..b9a5b00d6
--- /dev/null
+++ b/doc/server/plugins/connectors/grouplogic.txt
@@ -0,0 +1,122 @@
+.. -*- mode: rst -*-
+
+.. _server-plugins-connectors-grouplogic:
+
+==========
+GroupLogic
+==========
+
+.. versionadded:: 1.3.2
+
+GroupLogic is a connector plugin that lets you use an XML Genshi
+template to dynamically set additional groups for clients.
+
+Usage
+=====
+
+To use the GroupLogic plugin, first do ``mkdir
+/var/lib/bcfg2/GroupLogic``. Add ``GroupLogic`` to your ``plugins``
+line in ``/etc/bcfg2.conf``. Next, create
+``/var/lib/bcfg2/GroupLogic/groups.xml``:
+
+.. code-block:: xml
+
+ <GroupLogic xmlns:py="http://genshi.edgewall.org/">
+ </GroupLogic>
+
+``groups.xml`` is structured very similarly to the
+:ref:`server-plugins-grouping-metadata` ``groups.xml``. A Group tag
+that contains no children is a declaration of membership; a Group or
+Client tag that does contain children is a conditional.
+
+Unlike ``Metadata/groups.xml``, GroupLogic supports genshi templating,
+so you can dynamically create groups. ``GroupLogic/groups.xml`` is
+rendered for each client, and the groups set in it are added to the
+client metadata.
+
+.. note::
+
+ Also unlike ``Metadata/groups.xml``, GroupLogic can not be used to
+ associate bundles with clients directly, or to negate groups. But
+ you can use GroupLogic to assign a group that is associated with a
+ bundle in Metadata.
+
+Consider the case where you have four environments -- dev, test,
+staging, and production -- and four components to a web application --
+the frontend, the API, the database server, and the caching proxy. In
+order to make files specific to the component *and* to the
+environment, you need groups to describe each combination:
+webapp-frontend-dev, webapp-frontend-test, and so on. You *could* do
+this in ``Metadata/groups.xml``:
+
+.. code-block:: xml
+
+ <Groups>
+ <Group name="webapp-frontend">
+ <Group name="dev">
+ <Group name="webapp-frontend-dev"/>
+ </Group>
+ <Group name="test">
+ <Group name="webapp-frontend-test"/>
+ </Group>
+ ...
+ </Group>
+ <Group name="webapp-api">
+ ...
+ </Group>
+ ...
+ </Groups>
+
+Creating the sixteen groups this way is incredibly tedious, and this
+is a quite *small* site. GroupLogic can automate this process.
+
+Assume that we've declared the groups thusly in
+``Metadata/groups.xml``:
+
+.. code-block:: xml
+
+ <Groups>
+ <Group name="webapp-frontend" category="webapp-component"/>
+ <Group name="webapp-api" category="webapp-component"/>
+ <Group name="webapp-db" category="webapp-component"/>
+ <Group name="webapp-proxy" category="webapp-component"/>
+ <Group name="dev" category="environment"/>
+ <Group name="test" category="environment"/>
+ <Group name="staging" category="environment"/>
+ <Group name="prod" category="environment"/>
+ </Groups>
+
+One way to automate the creation of the groups would be to simply
+generate the tedious config:
+
+.. code-block:: xml
+
+ <GroupLogic xmlns:py="http://genshi.edgewall.org/">
+ <py:for each="component in metadata.query.all_groups_in_category("webapp-component")>
+ <Group name="${component}">
+ <py:for each="env in metadata.query.all_groups_in_category("environment")>
+ <Group name="${env}">
+ <Group name="${component}-${env}"/>
+ </Group>
+ </py:for>
+ </Group>
+ </py:for>
+ </GroupLogic>
+
+But, since ``GroupLogic/groups.xml`` is rendered for each client
+individually, there's a more elegant way to accomplish the same thing:
+
+.. code-block:: xml
+
+ <GroupLogic xmlns:py="http://genshi.edgewall.org/">
+ <?python
+component = metadata.group_in_category("webapp-component")
+env = metadata.group_in_category("environment")
+ ?>
+ <py:if test="component and env">
+ <Group name="${component}-${env}"/>
+ </py:if>
+ </GroupLogic>
+
+This gets only the component and environment for the current client,
+and, if both are set, sets the single appropriate group.
diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt
index 95bf4238a..092cff1ae 100644
--- a/doc/server/plugins/generators/packages.txt
+++ b/doc/server/plugins/generators/packages.txt
@@ -372,7 +372,7 @@ configs. Simply add entries like these to the appropriate bundles:
.. code-block:: xml
<Path name="/etc/yum.repos.d/bcfg2.repo"/>
- <Path name="/etc/apt/sources.d/bcfg2"/>
+ <Path name="/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list"/>
If you want to change the path to either of those files, you can set
``yum_config`` or ``apt_config`` in ``bcfg2.conf`` to the path to the
@@ -640,25 +640,25 @@ It understands the following directives:
[packages] section
------------------
-+-------------+------------------------------------------------------+----------+-----------------------------+
-| Name | Description | Values | Default |
-+=============+======================================================+==========+=============================+
-| resolver | Enable dependency resolution | Boolean | True |
-+-------------+------------------------------------------------------+----------+-----------------------------+
-| metadata | Enable metadata processing. Disabling ``metadata`` | Boolean | True |
-| | implies disabling ``resolver`` as well. | | |
-+-------------+------------------------------------------------------+----------+-----------------------------+
-| yum_config | The path at which to generate Yum configs. | String | /etc/yum.repos.d/bcfg2.repo |
-+-------------+------------------------------------------------------+----------+-----------------------------+
-| apt_config | The path at which to generate APT configs. | String | /etc/apt/sources.d/bcfg2 |
-+-------------+------------------------------------------------------+----------+-----------------------------+
-| gpg_keypath | The path on the client RPM GPG keys will be copied | String | /etc/pki/rpm-gpg |
-| | to before they are imported on the client. | | |
-+-------------+------------------------------------------------------+----------+-----------------------------+
-| version | Set the version attribute used when binding Packages | any|auto | auto |
-+-------------+------------------------------------------------------+----------+-----------------------------+
-| cache | Path where Packages will store its cache | String | <repo>/Packages/cache |
-+-------------+------------------------------------------------------+----------+-----------------------------+
++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+
+| Name | Description | Values | Default |
++=============+======================================================+==========+===================================================================+
+| resolver | Enable dependency resolution | Boolean | True |
++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+
+| metadata | Enable metadata processing. Disabling ``metadata`` | Boolean | True |
+| | implies disabling ``resolver`` as well. | | |
++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+
+| yum_config | The path at which to generate Yum configs. | String | /etc/yum.repos.d/bcfg2.repo |
++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+
+| apt_config | The path at which to generate APT configs. | String | /etc/apt/sources.list.d/bcfg2-packages-generated-sources.list |
++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+
+| gpg_keypath | The path on the client RPM GPG keys will be copied | String | /etc/pki/rpm-gpg |
+| | to before they are imported on the client. | | |
++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+
+| version | Set the version attribute used when binding Packages | any|auto | auto |
++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+
+| cache | Path where Packages will store its cache | String | <repo>/Packages/cache |
++-------------+------------------------------------------------------+----------+-------------------------------------------------------------------+
[packages:yum] section
diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt
index bcd7342c9..ccde65eb0 100644
--- a/doc/server/plugins/generators/rules.txt
+++ b/doc/server/plugins/generators/rules.txt
@@ -99,8 +99,13 @@ describe the attributes available for various Path types.
Note that ``secontext`` below expects a full context, not just the
type. For instance, "``system_u:object_r:etc_t:s0``", not just
``etc_t``. You can also specify "``__default__``", which will restore
-the context of the file to the default set by policy. See
-:ref:`server-selinux` for more information.
+the context of the file to the default set by policy. If a file has
+no default context rule, and you don't wish to set one, you can
+specify ``secontext=''`` (i.e., an empty ``secontext``), in which case
+the client will not try to manage the SELinux context of the file at
+all.
+
+See :ref:`server-selinux` for more information.
Attributes common to all Path tags:
diff --git a/doc/server/plugins/generators/sslca.txt b/doc/server/plugins/generators/sslca.txt
index 9c3a0806d..2a7e3ecad 100644
--- a/doc/server/plugins/generators/sslca.txt
+++ b/doc/server/plugins/generators/sslca.txt
@@ -156,7 +156,7 @@ Example
.. code-block:: xml
<CertInfo>
- <SubjectAltName>test.example.com</SubjectAltName>
+ <subjectAltName>test.example.com</subjectAltName>
<Group name="apache">
<Cert key="/etc/pki/tls/private/foo.key" days="730"/>
</Group>
diff --git a/doc/server/plugins/grouping/metadata.txt b/doc/server/plugins/grouping/metadata.txt
index 9fe87c878..832b1a13f 100644
--- a/doc/server/plugins/grouping/metadata.txt
+++ b/doc/server/plugins/grouping/metadata.txt
@@ -121,20 +121,19 @@ a simple ``groups.xml`` file:
<Group name='oracle-server'>
<Group name='selinux-enabled' negate='true'/>
</Group>
- <Client name='foo.eample.com'>
+ <Client name='foo.example.com'>
<Group name='oracle-server'/>
<Group name='apache-server'/>
</Client>
</Groups>
-A Group or Client tag that does not contain any child tags is a
-declaration of membership; a Group or Client tag that does contain
-children is a conditional. So the example above does not assign
-either the ``rhel5`` or ``rhel6`` groups to machines in the
-``mail-server`` group, but conditionally assigns the
-``sendmail-server`` or ``postfix-server`` groups depending on the OS
-of the client. (Presumably in this example the OS groups are set by a
-probe.)
+A Group tag that does not contain any child tags is a declaration of
+membership; a Group or Client tag that does contain children is a
+conditional. So the example above does not assign either the
+``rhel5`` or ``rhel6`` groups to machines in the ``mail-server``
+group, but conditionally assigns the ``sendmail-server`` or
+``postfix-server`` groups depending on the OS of the client.
+(Presumably in this example the OS groups are set by a probe.)
Consequently, a client that is RHEL 5 and a member of the
``mail-server`` profile group would also be a member of the
diff --git a/doc/server/selinux.txt b/doc/server/selinux.txt
index 9f54b0d68..79384970a 100644
--- a/doc/server/selinux.txt
+++ b/doc/server/selinux.txt
@@ -142,13 +142,13 @@ necessary.
Duplicate Entries
-----------------
-It may be necessary to use `BoundSELinux` tags if a single fcontext
+It may be necessary to use `BoundSEFcontext` tags if a single fcontext
needs two different SELinux types depending on whether it's a symlink
or a plain file. For instance:
.. code-block:: xml
- <BoundSELinux type="fcontext" filetype="symlink"
- name="/etc/localtime" selinuxtype="etc_t"/>
- <BoundSELinux type="fcontext" filetype="regular"
- name="/etc/localtime" selinuxtype="locale_t"/>
+ <BoundSEFcontext filetype="symlink"
+ name="/etc/localtime" selinuxtype="etc_t"/>
+ <BoundSEFcontext filetype="regular"
+ name="/etc/localtime" selinuxtype="locale_t"/>
diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec
index 13bc6474e..0b975cb37 100644
--- a/misc/bcfg2.spec
+++ b/misc/bcfg2.spec
@@ -153,7 +153,7 @@ Requires: bcfg2-server = %{version}
# cherrypy 3.2.3 actually doesn't exist yet, but 3.2.2 has bugs that
# prevent it from working:
# https://bitbucket.org/cherrypy/cherrypy/issue/1154/assertionerror-in-recv-when-ssl-is-enabled
-Requires: python-cherrypy > 3.2.2
+Requires: python-cherrypy > 3.3
%description server-cherrypy
Bcfg2 helps system administrators produce a consistent, reproducible,
@@ -522,4 +522,3 @@ fi
* Fri Sep 15 2006 Narayan Desai <desai@mcs.anl.gov> - 0.8.4-1
- Initial log
-
diff --git a/schemas/grouplogic.xsd b/schemas/grouplogic.xsd
new file mode 100644
index 000000000..bf43bceb3
--- /dev/null
+++ b/schemas/grouplogic.xsd
@@ -0,0 +1,110 @@
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+ xmlns:py="http://genshi.edgewall.org/" xml:lang="en">
+
+ <xsd:annotation>
+ <xsd:documentation>
+ GroupLogic schema for bcfg2
+ </xsd:documentation>
+ </xsd:annotation>
+
+ <xsd:import namespace="http://genshi.edgewall.org/"
+ schemaLocation="genshi.xsd"/>
+
+ <xsd:complexType name="GroupLogicDeclarationType">
+ <xsd:annotation>
+ <xsd:documentation>
+ A **GroupLogicDeclarationType** declares a Group to be added
+ to a client.
+ </xsd:documentation>
+ </xsd:annotation>
+ <xsd:attribute type='xsd:string' name='name' use='required'>
+ <xsd:annotation>
+ <xsd:documentation>
+ The group name
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
+ <xsd:attributeGroup ref="py:genshiAttrs"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="GroupLogicType">
+ <xsd:annotation>
+ <xsd:documentation>
+ The top-level tag of a GroupLogic configuration file.
+ </xsd:documentation>
+ </xsd:annotation>
+ <xsd:choice minOccurs="1" maxOccurs="unbounded">
+ <xsd:group ref="py:genshiElements"/>
+ <xsd:element name='Group' type='GroupLogicDeclarationType'/>
+ <xsd:element name='Group' type='GroupLogicContainerType'>
+ <xsd:annotation>
+ <xsd:documentation>
+ Elements within Group tags only apply to clients that are
+ members of that group (or vice-versa; see #element_negate
+ below)
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:element>
+ <xsd:element name='Client' type='GroupLogicContainerType'>
+ <xsd:annotation>
+ <xsd:documentation>
+ Elements within Client tags only apply to the named client
+ (or vice-versa; see #element_negate below)
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:element>
+ <xsd:element name='GroupLogic' type='GroupLogicType'>
+ <xsd:annotation>
+ <xsd:documentation>
+ Nesting GroupLogic tags is allowed in order to support
+ XInclude.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:element>
+ </xsd:choice>
+ <xsd:attributeGroup ref="py:genshiAttrs"/>
+ </xsd:complexType>
+
+ <xsd:complexType name="GroupLogicContainerType">
+ <xsd:annotation>
+ <xsd:documentation>
+ A **GroupLogicContainerType** is a tag used to provide logic.
+ Child entries of a GroupLogicContainerType tag only apply to
+ machines that match the condition specified -- either
+ membership in a group, or a matching client name.
+ :xml:attribute:`GroupLogicContainerType:negate` can be set to
+ negate the sense of the match.
+ </xsd:documentation>
+ </xsd:annotation>
+ <xsd:complexContent>
+ <xsd:extension base="GroupLogicType">
+ <xsd:attribute type='xsd:string' name='name' use='required'>
+ <xsd:annotation>
+ <xsd:documentation>
+ The group name
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
+ <xsd:attribute type='xsd:string' name='negate'>
+ <xsd:annotation>
+ <xsd:documentation>
+ Negate the sense of this group or client; i.e., entries
+ within this tag are only used on clients that are not
+ members of the group, or that have hostnames that do not
+ match.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
+ </xsd:extension>
+ </xsd:complexContent>
+ </xsd:complexType>
+
+ <xsd:element name='GroupLogic' type='GroupLogicType'>
+ <xsd:annotation>
+ <xsd:documentation>
+ A GroupLogic file is a genshi file that can be used to
+ dynamically add additional groups to a client.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:element>
+</xsd:schema>
diff --git a/schemas/types.xsd b/schemas/types.xsd
index 144ef7337..9c4a0a48e 100644
--- a/schemas/types.xsd
+++ b/schemas/types.xsd
@@ -430,8 +430,10 @@
<xsd:attribute type="xsd:string" name="gecos">
<xsd:annotation>
<xsd:documentation>
- Human-readable user name or comment. If this is not set,
- the GECOS will be the same as the username.
+ This field is typically used to record general information
+ about the account or its user(s) such as their real name
+ and phone number. If this is not set, the GECOS will be
+ the same as the username.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py
index 66c1ce430..10a15f160 100644
--- a/src/lib/Bcfg2/Client/Client.py
+++ b/src/lib/Bcfg2/Client/Client.py
@@ -91,7 +91,10 @@ class Client(object):
try:
script.write("#!%s\n" %
(probe.attrib.get('interpreter', '/bin/sh')))
- script.write(probe.text)
+ if sys.hexversion >= 0x03000000:
+ script.write(probe.text)
+ else:
+ script.write(probe.text.encode('utf-8'))
script.close()
os.chmod(scriptname,
stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH |
@@ -105,7 +108,10 @@ class Client(object):
self._probe_failure(name, "Return value %s" % rv)
self.logger.info("Probe %s has result:" % name)
self.logger.info(rv.stdout)
- ret.text = rv.stdout
+ if sys.hexversion >= 0x03000000:
+ ret.text = rv.stdout
+ else:
+ ret.text = rv.stdout.decode('utf-8')
finally:
os.unlink(scriptname)
except SystemExit:
@@ -164,10 +170,11 @@ class Client(object):
if len(probes.findall(".//probe")) > 0:
try:
# upload probe responses
- self.proxy.RecvProbeData(Bcfg2.Client.XML.tostring(
+ self.proxy.RecvProbeData(
+ Bcfg2.Client.XML.tostring(
probedata,
- xml_declaration=False).decode('UTF-8'))
- except Bcfg2.Client.Proxy.ProxyError:
+ xml_declaration=False).decode('utf-8'))
+ except Bcfg2.Proxy.ProxyError:
err = sys.exc_info()[1]
self.fatal_error("Failed to upload probe data: %s" % err)
@@ -228,7 +235,7 @@ class Client(object):
self.fatal_error("Failed to get decision list: %s" % err)
try:
- rawconfig = self.proxy.GetConfig().encode('UTF-8')
+ rawconfig = self.proxy.GetConfig().encode('utf-8')
except Bcfg2.Client.Proxy.ProxyError:
err = sys.exc_info()[1]
self.fatal_error("Failed to download configuration from "
@@ -246,7 +253,7 @@ class Client(object):
self.logger.info("Starting Bcfg2 client run at %s" % times['start'])
- rawconfig = self.get_config(times=times)
+ rawconfig = self.get_config(times=times).decode('utf-8')
if self.setup['cache']:
try:
@@ -317,9 +324,10 @@ class Client(object):
feedback = self.tools.GenerateStats()
try:
- self.proxy.RecvStats(Bcfg2.Client.XML.tostring(
+ self.proxy.RecvStats(
+ Bcfg2.Client.XML.tostring(
feedback,
- xml_declaration=False).decode('UTF-8'))
+ xml_declaration=False).decode('utf-8'))
except Bcfg2.Client.Proxy.ProxyError:
err = sys.exc_info()[1]
self.logger.error("Failed to upload configuration statistics: "
diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py
index 6ef686c10..11a82fcc0 100644
--- a/src/lib/Bcfg2/Client/Frame.py
+++ b/src/lib/Bcfg2/Client/Frame.py
@@ -6,8 +6,8 @@ import fnmatch
import logging
import Bcfg2.Client.Tools
from Bcfg2.Client import prompt
-from Bcfg2.Compat import any, all # pylint: disable=W0622
from Bcfg2.Options import get_option_parser
+from Bcfg2.Compat import any, all, cmp # pylint: disable=W0622
def cmpent(ent1, ent2):
@@ -67,8 +67,8 @@ class Frame(object):
self.logger = logging.getLogger(__name__)
drivers = self.setup['drivers']
for driver in drivers[:]:
- if driver not in Bcfg2.Client.Tools.__all__ and \
- isinstance(driver, str):
+ if (driver not in Bcfg2.Client.Tools.__all__ and
+ isinstance(driver, str)):
self.logger.error("Tool driver %s is not available" % driver)
drivers.remove(driver)
@@ -134,7 +134,7 @@ class Frame(object):
if entry.tag == 'Package']
if pkgs:
self.logger.debug("The following packages are specified in bcfg2:")
- self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == None])
+ self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] is None])
self.logger.debug("The following packages are prereqs added by "
"Packages:")
self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages'])
@@ -193,19 +193,19 @@ class Frame(object):
"""
# Need to process decision stuff early so that dryrun mode
# works with it
- self.whitelist = [entry for entry in self.states \
+ self.whitelist = [entry for entry in self.states
if not self.states[entry]]
if not self.setup['file']:
if self.setup['decision'] == 'whitelist':
dwl = self.setup['decision_list']
- w_to_rem = [e for e in self.whitelist \
+ w_to_rem = [e for e in self.whitelist
if not matches_white_list(e, dwl)]
if w_to_rem:
self.logger.info("In whitelist mode: "
"suppressing installation of:")
self.logger.info(["%s:%s" % (e.tag, e.get('name'))
for e in w_to_rem])
- self.whitelist = [x for x in self.whitelist \
+ self.whitelist = [x for x in self.whitelist
if x not in w_to_rem]
elif self.setup['decision'] == 'blacklist':
b_to_rem = \
@@ -236,7 +236,7 @@ class Frame(object):
cfile not in self.whitelist):
continue
tools = [t for t in self.tools
- if t.handlesEntry(cfile) and t.canVerify(cfile)]
+ if t.handlesEntry(cfile) and t.canVerify(cfile)]
if not tools:
continue
if (self.setup['interactive'] and not
@@ -395,8 +395,8 @@ class Frame(object):
"""Install all entries."""
self.DispatchInstallCalls(self.whitelist)
mods = self.modified
- mbundles = [struct for struct in self.config.findall('Bundle') if \
- [mod for mod in mods if mod in struct]]
+ mbundles = [struct for struct in self.config.findall('Bundle')
+ if any(True for mod in mods if mod in struct)]
if self.modified:
# Handle Bundle interdeps
@@ -411,33 +411,34 @@ class Frame(object):
self.logger.error("%s.Inventory() call failed:" %
tool.name,
exc_info=1)
- clobbered = [entry for bundle in mbundles for entry in bundle \
+ clobbered = [entry for bundle in mbundles for entry in bundle
if (not self.states[entry] and
entry not in self.blacklist)]
if clobbered:
self.logger.debug("Found clobbered entries:")
- self.logger.debug(["%s:%s" % (entry.tag, entry.get('name')) \
+ self.logger.debug(["%s:%s" % (entry.tag, entry.get('name'))
for entry in clobbered])
if not self.setup['interactive']:
self.DispatchInstallCalls(clobbered)
for bundle in self.config.findall('.//Bundle'):
- if self.setup['bundle'] and \
- bundle.get('name') not in self.setup['bundle']:
+ if (self.setup['bundle'] and
+ bundle.get('name') not in self.setup['bundle']):
# prune out unspecified bundles when running with -b
continue
+ if bundle in mbundles:
+ self.logger.debug("Bundle %s was modified" % bundle)
+ func = "BundleUpdated"
+ else:
+ self.logger.debug("Bundle %s was not modified" % bundle)
+ func = "BundleNotUpdated"
for tool in self.tools:
- if bundle in mbundles:
- func = tool.BundleUpdated
- else:
- func = tool.BundleNotUpdated
try:
- self.states.update(func(bundle))
+ self.states.update(getattr(tool, func)(bundle))
except:
self.logger.error("%s.%s(%s:%s) call failed:" %
- (tool.name, func.im_func.func_name,
- bundle.tag, bundle.get("name")),
- exc_info=1)
+ (tool.name, func, bundle.tag,
+ bundle.get("name")), exc_info=1)
for indep in self.config.findall('.//Independent'):
for tool in self.tools:
@@ -468,7 +469,8 @@ class Frame(object):
self.logger.info('Incorrect entries: %d' %
list(self.states.values()).count(False))
if phase == 'final' and list(self.states.values()).count(False):
- for entry in self.states.keys():
+ for entry in sorted(self.states.keys(), key=lambda e: e.tag + ":" +
+ e.get('name')):
if not self.states[entry]:
etype = entry.get('type')
if etype:
diff --git a/src/lib/Bcfg2/Client/Proxy.py b/src/lib/Bcfg2/Client/Proxy.py
index f8817bb27..57be34369 100644
--- a/src/lib/Bcfg2/Client/Proxy.py
+++ b/src/lib/Bcfg2/Client/Proxy.py
@@ -327,6 +327,7 @@ class XMLRPCTransport(xmlrpclib.Transport):
return self.parse_response(response)
if sys.hexversion < 0x03000000:
+ # pylint: disable=E1101
def send_request(self, host, handler, request_body, debug):
""" send_request() changed significantly in py3k."""
conn = self.make_connection(host)
@@ -335,6 +336,7 @@ class XMLRPCTransport(xmlrpclib.Transport):
self.send_user_agent(conn)
self.send_content(conn, request_body)
return conn
+ # pylint: enable=E1101
def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None,
diff --git a/src/lib/Bcfg2/Client/Tools/APK.py b/src/lib/Bcfg2/Client/Tools/APK.py
index 553339fc9..46f46bb1c 100644
--- a/src/lib/Bcfg2/Client/Tools/APK.py
+++ b/src/lib/Bcfg2/Client/Tools/APK.py
@@ -27,7 +27,7 @@ class APK(Bcfg2.Client.Tools.PkgTool):
"""Verify Package status for entry."""
if not 'version' in entry.attrib:
self.logger.info("Cannot verify unversioned package %s" %
- (entry.attrib['name']))
+ entry.attrib['name'])
return False
if entry.attrib['name'] in self.installed:
diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py
index 7b62f61c7..fd2c467d7 100644
--- a/src/lib/Bcfg2/Client/Tools/Action.py
+++ b/src/lib/Bcfg2/Client/Tools/Action.py
@@ -19,12 +19,12 @@ class Action(Bcfg2.Client.Tools.Tool):
the whitelist or blacklist """
if self.setup['decision'] == 'whitelist' and \
not matches_white_list(action, self.setup['decision_list']):
- self.logger.info("In whitelist mode: suppressing Action:" + \
+ self.logger.info("In whitelist mode: suppressing Action: %s" %
action.get('name'))
return False
if self.setup['decision'] == 'blacklist' and \
not passes_black_list(action, self.setup['decision_list']):
- self.logger.info("In blacklist mode: suppressing Action:" + \
+ self.logger.info("In blacklist mode: suppressing Action: %s" %
action.get('name'))
return False
return True
diff --git a/src/lib/Bcfg2/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py
index ca556e98b..d916b1662 100644
--- a/src/lib/Bcfg2/Client/Tools/DebInit.py
+++ b/src/lib/Bcfg2/Client/Tools/DebInit.py
@@ -16,7 +16,7 @@ class DebInit(Bcfg2.Client.Tools.SvcTool):
__handles__ = [('Service', 'deb')]
__req__ = {'Service': ['name', 'status']}
svcre = \
- re.compile("/etc/.*/(?P<action>[SK])(?P<sequence>\d+)(?P<name>\S+)")
+ re.compile(r'/etc/.*/(?P<action>[SK])(?P<sequence>\d+)(?P<name>\S+)')
# implement entry (Verify|Install) ops
def VerifyService(self, entry, _):
diff --git a/src/lib/Bcfg2/Client/Tools/Encap.py b/src/lib/Bcfg2/Client/Tools/Encap.py
index 678e0f00c..270f0a5f2 100644
--- a/src/lib/Bcfg2/Client/Tools/Encap.py
+++ b/src/lib/Bcfg2/Client/Tools/Encap.py
@@ -13,7 +13,7 @@ class Encap(Bcfg2.Client.Tools.PkgTool):
__req__ = {'Package': ['version', 'url']}
pkgtype = 'encap'
pkgtool = ("/usr/local/bin/epkg -l -f -q %s", ("%s", ["url"]))
- splitter = re.compile('.*/(?P<name>[\w-]+)\-(?P<version>[\w\.+-]+)')
+ splitter = re.compile(r'.*/(?P<name>[\w-]+)\-(?P<version>[\w\.+-]+)')
def RefreshPackages(self):
"""Try to find encap packages."""
diff --git a/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py
index 635318805..31925fa3c 100644
--- a/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py
+++ b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py
@@ -21,7 +21,7 @@ class FreeBSDPackage(Bcfg2.Client.Tools.PkgTool):
def RefreshPackages(self):
self.installed = {}
packages = self.cmd.run("/usr/sbin/pkg_info -a -E").stdout.splitlines()
- pattern = re.compile('(.*)-(\d.*)')
+ pattern = re.compile(r'(.*)-(\d.*)')
for pkg in packages:
if pattern.match(pkg):
name = pattern.match(pkg).group(1)
@@ -31,7 +31,7 @@ class FreeBSDPackage(Bcfg2.Client.Tools.PkgTool):
def VerifyPackage(self, entry, _):
if not 'version' in entry.attrib:
self.logger.info("Cannot verify unversioned package %s" %
- (entry.attrib['name']))
+ entry.attrib['name'])
return False
if entry.attrib['name'] in self.installed:
if self.installed[entry.attrib['name']] == entry.attrib['version']:
diff --git a/src/lib/Bcfg2/Client/Tools/IPS.py b/src/lib/Bcfg2/Client/Tools/IPS.py
index e4c9d605d..c998ff083 100644
--- a/src/lib/Bcfg2/Client/Tools/IPS.py
+++ b/src/lib/Bcfg2/Client/Tools/IPS.py
@@ -50,9 +50,9 @@ class IPS(Bcfg2.Client.Tools.PkgTool):
pass
else:
if entry.get('version') != self.installed[pname]:
- self.logger.debug("IPS: Package %s: have %s want %s" \
- % (pname, self.installed[pname],
- entry.get('version')))
+ self.logger.debug("IPS: Package %s: have %s want %s" %
+ (pname, self.installed[pname],
+ entry.get('version')))
return False
# need to implement pkg chksum validation
diff --git a/src/lib/Bcfg2/Client/Tools/MacPorts.py b/src/lib/Bcfg2/Client/Tools/MacPorts.py
index 5b395a170..dcf58cfec 100644
--- a/src/lib/Bcfg2/Client/Tools/MacPorts.py
+++ b/src/lib/Bcfg2/Client/Tools/MacPorts.py
@@ -33,7 +33,7 @@ class MacPorts(Bcfg2.Client.Tools.PkgTool):
"""Verify Package status for entry."""
if not 'version' in entry.attrib:
self.logger.info("Cannot verify unversioned package %s" %
- (entry.attrib['name']))
+ entry.attrib['name'])
return False
if entry.attrib['name'] in self.installed:
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py
index d5aaf069d..9b84adad0 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIX/Device.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIX/Device.py
@@ -12,8 +12,8 @@ class POSIXDevice(POSIXTool):
def fully_specified(self, entry):
if entry.get('dev_type') in ['block', 'char']:
# check if major/minor are properly specified
- if (entry.get('major') == None or
- entry.get('minor') == None):
+ if (entry.get('major') is None or
+ entry.get('minor') is None):
return False
return True
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py
index 9b95d2234..168c35c98 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py
@@ -34,13 +34,11 @@ class POSIXFile(POSIXTool):
def _get_data(self, entry):
""" Get a tuple of (<file data>, <is binary>) for the given entry """
- is_binary = False
- if entry.get('encoding', 'ascii') == 'base64':
- tempdata = b64decode(entry.text)
- is_binary = True
-
- elif entry.get('empty', 'false') == 'true':
+ is_binary = entry.get('encoding', 'ascii') == 'base64'
+ if entry.get('empty', 'false') == 'true' or not entry.text:
tempdata = ''
+ elif is_binary:
+ tempdata = b64decode(entry.text)
else:
tempdata = entry.text
if isinstance(tempdata, unicode) and unicode != str:
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py
index f46875743..16fe0acb5 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py
@@ -275,7 +275,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
if path is None:
path = entry.get("name")
context = entry.get("secontext")
- if context is None:
+ if not context:
# no context listed
return True
@@ -520,13 +520,19 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
"Current mtime is %s but should be %s" %
(path, mtime, entry.get('mtime')))
- if HAS_SELINUX and entry.get("secontext"):
+ if HAS_SELINUX:
+ wanted_secontext = None
if entry.get("secontext") == "__default__":
- wanted_secontext = \
- selinux.matchpathcon(path, 0)[1].split(":")[2]
+ try:
+ wanted_secontext = \
+ selinux.matchpathcon(path, 0)[1].split(":")[2]
+ except OSError:
+ errors.append("%s has no default SELinux context" %
+ entry.get("name"))
else:
wanted_secontext = entry.get("secontext")
- if attrib['current_secontext'] != wanted_secontext:
+ if (wanted_secontext and
+ attrib['current_secontext'] != wanted_secontext):
errors.append("SELinux context for path %s is incorrect. "
"Current context is %s but should be %s" %
(path, attrib['current_secontext'],
@@ -712,8 +718,8 @@ class POSIXTool(Bcfg2.Client.Tools.Tool):
tmpentry.set('mode', oct_mode(newmode))
for acl in tmpentry.findall('ACL'):
acl.set('perms',
- oct_mode(self._norm_acl_perms(acl.get('perms')) | \
- ACL_MAP['x']))
+ oct_mode(self._norm_acl_perms(acl.get('perms')) |
+ ACL_MAP['x']))
for cpath in created:
rv &= self._set_perms(tmpentry, path=cpath)
return rv
diff --git a/src/lib/Bcfg2/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py
index 15fab53bd..d7d60a66d 100644
--- a/src/lib/Bcfg2/Client/Tools/Pacman.py
+++ b/src/lib/Bcfg2/Client/Tools/Pacman.py
@@ -25,12 +25,12 @@ class Pacman(Bcfg2.Client.Tools.PkgTool):
def VerifyPackage(self, entry, _):
'''Verify Package status for entry'''
- self.logger.info("VerifyPackage : %s : %s" % entry.get('name'),
- entry.get('version'))
+ self.logger.info("VerifyPackage: %s : %s" % (entry.get('name'),
+ entry.get('version')))
if not 'version' in entry.attrib:
self.logger.info("Cannot verify unversioned package %s" %
- (entry.attrib['name']))
+ entry.attrib['name'])
return False
if entry.attrib['name'] in self.installed:
diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py
index 32afa8cbf..5e319a414 100644
--- a/src/lib/Bcfg2/Client/Tools/Portage.py
+++ b/src/lib/Bcfg2/Client/Tools/Portage.py
@@ -13,8 +13,8 @@ class Portage(Bcfg2.Client.Tools.PkgTool):
__req__ = {'Package': ['name', 'version']}
pkgtype = 'ebuild'
# requires a working PORTAGE_BINHOST in make.conf
- _binpkgtool = ('emerge --getbinpkgonly %s', ('=%s-%s', \
- ['name', 'version']))
+ _binpkgtool = ('emerge --getbinpkgonly %s', ('=%s-%s', ['name',
+ 'version']))
pkgtool = ('emerge %s', ('=%s-%s', ['name', 'version']))
def __init__(self, config):
@@ -22,7 +22,7 @@ class Portage(Bcfg2.Client.Tools.PkgTool):
Bcfg2.Client.Tools.PkgTool.__init__(self, config)
self._initialised = True
self.__important__ = self.__important__ + ['/etc/make.conf']
- self._pkg_pattern = re.compile('(.*)-(\d.*)')
+ self._pkg_pattern = re.compile(r'(.*)-(\d.*)')
self._ebuild_pattern = re.compile('(ebuild|binary)')
self.installed = {}
self._binpkgonly = self.setup.get('portage_binpkgonly', False)
diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py
index 552b27842..4b78581f7 100644
--- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py
+++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py
@@ -12,6 +12,15 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool):
__handles__ = [('Service', 'rc-update')]
__req__ = {'Service': ['name', 'status']}
+ def get_enabled_svcs(self):
+ """
+ Return a list of all enabled services.
+ """
+ return [line.split()[0]
+ for line in self.cmd.run(['/bin/rc-status',
+ '-s']).stdout.splitlines()
+ if 'started' in line]
+
def VerifyService(self, entry, _):
"""
Verify Service status for entry.
@@ -21,9 +30,12 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool):
if entry.get('status') == 'ignore':
return True
+ # get a list of all started services
+ allsrv = self.get_enabled_svcs()
+
# check if service is enabled
- result = self.cmd.run(["/sbin/rc-update", "show", "default"])
- is_enabled = entry.get("name") in result.stdout
+ result = self.cmd.run(["/sbin/rc-update", "show", "default"]).stdout
+ is_enabled = entry.get("name") in result
# check if init script exists
try:
@@ -34,8 +46,7 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool):
return False
# check if service is enabled
- result = self.cmd.run(self.get_svc_command(entry, "status"))
- is_running = "started" in result.stdout
+ is_running = entry.get('name') in allsrv
if entry.get('status') == 'on' and not (is_enabled and is_running):
entry.set('current_status', 'off')
@@ -70,10 +81,7 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool):
def FindExtra(self):
"""Locate extra rc-update services."""
- allsrv = [line.split()[0]
- for line in self.cmd.run(['/bin/rc-status',
- '-s']).stdout.splitlines()
- if 'started' in line]
+ allsrv = self.get_enabled_svcs()
self.logger.debug('Found active services:')
self.logger.debug(allsrv)
specified = [srv.get('name') for srv in self.getSupportedEntries()]
diff --git a/src/lib/Bcfg2/Client/Tools/SELinux.py b/src/lib/Bcfg2/Client/Tools/SELinux.py
index baf930610..f38615062 100644
--- a/src/lib/Bcfg2/Client/Tools/SELinux.py
+++ b/src/lib/Bcfg2/Client/Tools/SELinux.py
@@ -13,6 +13,7 @@ import seobject
import Bcfg2.Client.XML
import Bcfg2.Client.Tools
from Bcfg2.Client.Tools.POSIX.File import POSIXFile
+from Bcfg2.Compat import long # pylint: disable=W0622
def pack128(int_val):
@@ -48,7 +49,7 @@ def netmask_itoa(netmask, proto="ipv4"):
if netmask > size:
raise ValueError("Netmask too large: %s" % netmask)
- res = 0L
+ res = long(0)
for i in range(netmask):
res |= 1 << (size - i - 1)
netmask = socket.inet_ntop(family, pack128(res))
@@ -167,7 +168,7 @@ class SELinuxEntryHandler(object):
key_format = ("name",)
value_format = ()
str_format = '%(name)s'
- custom_re = re.compile(' (?P<name>\S+)$')
+ custom_re = re.compile(r' (?P<name>\S+)$')
custom_format = None
def __init__(self, tool, config):
@@ -200,7 +201,16 @@ class SELinuxEntryHandler(object):
type, if the records object supports the customized() method
"""
if hasattr(self.records, "customized") and self.custom_re:
- return dict([(k, self.all_records[k]) for k in self.custom_keys])
+ rv = dict()
+ for key in self.custom_keys:
+ if key in self.all_records:
+ rv[key] = self.all_records[key]
+ else:
+ self.logger.warning("SELinux %s %s customized, but no "
+ "record found. This may indicate an "
+ "error in your SELinux policy." %
+ (self.etype, key))
+ return rv
else:
# ValueError is really a pretty dumb exception to raise,
# but that's what the seobject customized() method raises
@@ -504,14 +514,14 @@ class SELinuxSefcontextHandler(SELinuxEntryHandler):
char="-c",
door="-D")
filetypenames = dict(all="all files",
- regular="regular file",
- directory="directory",
- symlink="symbolic link",
- pipe="named pipe",
- socket="socket",
- block="block device",
- char="character device",
- door="door")
+ regular="regular file",
+ directory="directory",
+ symlink="symbolic link",
+ pipe="named pipe",
+ socket="socket",
+ block="block device",
+ char="character device",
+ door="door")
filetypeattrs = dict([v, k] for k, v in filetypenames.iteritems())
custom_re = re.compile(r'-f \'(?P<filetype>[a-z ]+)\'.*? \'(?P<name>.*)\'')
diff --git a/src/lib/Bcfg2/Client/Tools/SMF.py b/src/lib/Bcfg2/Client/Tools/SMF.py
index 68d8b2965..8b23a4a37 100644
--- a/src/lib/Bcfg2/Client/Tools/SMF.py
+++ b/src/lib/Bcfg2/Client/Tools/SMF.py
@@ -48,12 +48,12 @@ class SMF(Bcfg2.Client.Tools.SvcTool):
gname = "/etc/rc*.d/%s" % filename
files = glob.glob(gname.replace('_', '.'))
if files:
- self.logger.debug("Matched %s with %s" % \
+ self.logger.debug("Matched %s with %s" %
(entry.get("FMRI"), ":".join(files)))
return entry.get('status') == 'on'
else:
- self.logger.debug("No service matching %s" % \
- (entry.get("FMRI")))
+ self.logger.debug("No service matching %s" %
+ entry.get("FMRI"))
return entry.get('status') == 'off'
try:
srvdata = \
@@ -76,13 +76,12 @@ class SMF(Bcfg2.Client.Tools.SvcTool):
if entry.get("FMRI").startswith('lrc'):
try:
loc = entry.get("FMRI")[4:].replace('_', '.')
- self.logger.debug("Renaming file %s to %s" % \
+ self.logger.debug("Renaming file %s to %s" %
(loc, loc.replace('/S', '/DISABLED.S')))
os.rename(loc, loc.replace('/S', '/DISABLED.S'))
return True
except OSError:
- self.logger.error("Failed to rename init script %s" % \
- (loc))
+ self.logger.error("Failed to rename init script %s" % loc)
return False
else:
return self.cmd.run("/usr/sbin/svcadm disable %s" %
@@ -118,12 +117,12 @@ class SMF(Bcfg2.Client.Tools.SvcTool):
def FindExtra(self):
"""Find Extra SMF Services."""
- allsrv = [name for name, version in \
- [srvc.split()
- for srvc in self.cmd.run([
- "/usr/bin/svcs", "-a", "-H",
- "-o", "FMRI,STATE"]).stdout.splitlines()]
- if version != 'disabled']
+ allsrv = []
+ for srvc in self.cmd.run(["/usr/bin/svcs", "-a", "-H",
+ "-o", "FMRI,STATE"]).stdout.splitlines():
+ name, version = srvc.split()
+ if version != 'disabled':
+ allsrv.append(name)
for svc in self.getSupportedEntries():
if svc.get("FMRI") in allsrv:
diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py
index b5afa629d..7be7b6fa3 100644
--- a/src/lib/Bcfg2/Client/Tools/SYSV.py
+++ b/src/lib/Bcfg2/Client/Tools/SYSV.py
@@ -41,7 +41,7 @@ class SYSV(Bcfg2.Client.Tools.PkgTool):
self.noaskfile.write(noask)
# flush admin file contents to disk
self.noaskfile.flush()
- self.pkgtool = (self.pkgtool[0] % ("-a %s" % (self.noaskname)), \
+ self.pkgtool = (self.pkgtool[0] % ("-a %s" % (self.noaskname)),
self.pkgtool[1])
except: # pylint: disable=W0702
self.pkgtool = (self.pkgtool[0] % "", self.pkgtool[1])
@@ -66,7 +66,7 @@ class SYSV(Bcfg2.Client.Tools.PkgTool):
desired_version = entry.get('version')
if desired_version == 'any':
desired_version = self.installed.get(entry.get('name'),
- desired_version)
+ desired_version)
if not self.cmd.run(["/usr/bin/pkginfo", "-q", "-v",
desired_version, entry.get('name')]):
diff --git a/src/lib/Bcfg2/Client/Tools/Upstart.py b/src/lib/Bcfg2/Client/Tools/Upstart.py
index cd1c4a2bc..c96eab69d 100644
--- a/src/lib/Bcfg2/Client/Tools/Upstart.py
+++ b/src/lib/Bcfg2/Client/Tools/Upstart.py
@@ -46,9 +46,9 @@ class Upstart(Bcfg2.Client.Tools.SvcTool):
entry.get('name'))
return False
- match = re.compile("%s( \(.*\))? (start|stop)/(running|waiting)" %
+ match = re.compile(r'%s( \(.*\))? (start|stop)/(running|waiting)' %
entry.get('name')).match(output)
- if match == None:
+ if match is None:
# service does not exist
entry.set('current_status', 'off')
status = False
diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py
index 57ca06e77..147615f47 100644
--- a/src/lib/Bcfg2/Client/Tools/YUM.py
+++ b/src/lib/Bcfg2/Client/Tools/YUM.py
@@ -133,10 +133,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
def __init__(self, config):
self.yumbase = self._loadYumBase()
Bcfg2.Client.Tools.PkgTool.__init__(self, config)
- self.ignores = [entry.get('name') for struct in config \
- for entry in struct \
- if entry.tag == 'Path' and \
- entry.get('type') == 'ignore']
+ self.ignores = []
+ for struct in config:
+ self.ignores.extend([entry.get('name')
+ for entry in struct
+ if (entry.tag == 'Path' and
+ entry.get('type') == 'ignore')])
self.instance_status = {}
self.extra_instances = []
self.modlists = {}
@@ -296,8 +298,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
group. """
missing = Bcfg2.Client.Tools.PkgTool.missing_attrs(self, entry)
- if entry.get('name', None) == None and \
- entry.get('group', None) == None:
+ if (entry.get('name', None) is None and
+ entry.get('group', None) is None):
missing += ['name', 'group']
return missing
@@ -425,10 +427,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
if entry.get('group'):
self.logger.debug("Verifying packages for group %s" %
- entry.get('group'))
+ entry.get('group'))
else:
self.logger.debug("Verifying package instances for %s" %
- entry.get('name'))
+ entry.get('name'))
self.verify_cache = dict() # Used for checking multilib packages
self.modlists[entry] = modlist
@@ -437,10 +439,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
package_fail = False
qtext_versions = []
virt_pkg = False
- pkg_checks = self.pkg_checks and \
- entry.get('pkg_checks', 'true').lower() == 'true'
- pkg_verify = self.pkg_verify and \
- entry.get('pkg_verify', 'true').lower() == 'true'
+ pkg_checks = (self.pkg_checks and
+ entry.get('pkg_checks', 'true').lower() == 'true')
+ pkg_verify = (self.pkg_verify and
+ entry.get('pkg_verify', 'true').lower() == 'true')
yum_group = False
if entry.get('name') == 'gpg-pubkey':
@@ -458,15 +460,13 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
if d]
group_type = entry.get('choose', 'default')
if group_type in ['default', 'optional', 'all']:
- group_packages += [p
- for p, d in
- group.default_packages.items()
- if d]
+ group_packages += [
+ p for p, d in group.default_packages.items()
+ if d]
if group_type in ['optional', 'all']:
- group_packages += [p
- for p, d in
- group.optional_packages.items()
- if d]
+ group_packages += [
+ p for p, d in group.optional_packages.items()
+ if d]
if len(group_packages) == 0:
self.logger.error("No packages found for group %s" %
entry.get("group"))
@@ -492,7 +492,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
else:
all_pkg_objs = \
self.yumbase.rpmdb.searchNevra(name=entry.get('name'))
- if len(all_pkg_objs) == 0 and yum_group != True:
+ if len(all_pkg_objs) == 0 and yum_group is not True:
# Some sort of virtual capability? Try to resolve it
all_pkg_objs = self.yumbase.rpmdb.searchProvides(entry.get('name'))
if len(all_pkg_objs) > 0:
@@ -570,9 +570,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
pkg_objs = [po for po in all_pkg_objs]
else:
pkg_objs = [po for po in all_pkg_objs
- if po.checkPrco('provides',
- (nevra["name"], 'EQ',
- tuple(vlist)))]
+ if po.checkPrco('provides',
+ (nevra["name"], 'EQ',
+ tuple(vlist)))]
elif entry.get('name') == 'gpg-pubkey':
if 'version' not in nevra:
self.logger.warning("Skipping verify: gpg-pubkey without "
@@ -625,7 +625,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
if self.setup.get('quick', False):
# Passed -q on the command line
continue
- if not (pkg_verify and \
+ if not (pkg_verify and
inst.get('pkg_verify', 'true').lower() == 'true'):
continue
@@ -651,8 +651,8 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
# Now take out the Yum specific objects / modlists / unproblems
ignores = [ig.get('name') for ig in entry.findall('Ignore')] + \
- [ig.get('name') for ig in inst.findall('Ignore')] + \
- self.ignores
+ [ig.get('name') for ig in inst.findall('Ignore')] + \
+ self.ignores
for fname, probs in list(vrfy_result.items()):
if fname in modlist:
self.logger.debug(" %s in modlist, skipping" % fname)
@@ -740,8 +740,9 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
for pkg in pkg_objs:
self.logger.debug(" Extra Instance Found: %s" % str(pkg))
Bcfg2.Client.XML.SubElement(extra_entry, 'Instance',
- epoch=pkg.epoch, name=pkg.name, version=pkg.version,
- release=pkg.release, arch=pkg.arch)
+ epoch=pkg.epoch, name=pkg.name,
+ version=pkg.version,
+ release=pkg.release, arch=pkg.arch)
if pkg_objs == []:
return None
@@ -785,7 +786,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
ver = yum.misc.keyIdToRPMVer(gpg['keyid'])
rel = yum.misc.keyIdToRPMVer(gpg['timestamp'])
if not (ver == inst.get('version') and rel == inst.get('release')):
- self.logger.info("GPG key file %s does not match gpg-pubkey-%s-%s"\
+ self.logger.info("GPG key file %s does not match gpg-pubkey-%s-%s"
% (key_file, inst.get('version'),
inst.get('release')))
return False
@@ -794,20 +795,21 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
gpg['timestamp']) == 0:
result = tset.pgpImportPubkey(yum.misc.procgpgkey(rawkey))
else:
- self.logger.debug("gpg-pubkey-%s-%s already installed"\
- % (inst.get('version'),
- inst.get('release')))
+ self.logger.debug("gpg-pubkey-%s-%s already installed" %
+ (inst.get('version'), inst.get('release')))
return True
if result != 0:
- self.logger.debug("Unable to install %s-%s" % \
- (self.instance_status[inst].get('pkg').get('name'),
- nevra2string(inst)))
+ self.logger.debug(
+ "Unable to install %s-%s" %
+ (self.instance_status[inst].get('pkg').get('name'),
+ nevra2string(inst)))
return False
else:
- self.logger.debug("Installed %s-%s-%s" % \
- (self.instance_status[inst].get('pkg').get('name'),
- inst.get('version'), inst.get('release')))
+ self.logger.debug(
+ "Installed %s-%s-%s" %
+ (self.instance_status[inst].get('pkg').get('name'),
+ inst.get('version'), inst.get('release')))
return True
def _runYumTransaction(self):
@@ -902,7 +904,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
# Remove extra instances.
# Can not reverify because we don't have a package entry.
if self.extra_instances is not None and len(self.extra_instances) > 0:
- if (self.setup.get('remove') == 'all' or \
+ if (self.setup.get('remove') == 'all' or
self.setup.get('remove') == 'packages'):
self.Remove(self.extra_instances)
else:
@@ -917,7 +919,7 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
# Figure out which instances of the packages actually need something
# doing to them and place in the appropriate work 'queue'.
for pkg in packages:
- insts = [pinst for pinst in pkg \
+ insts = [pinst for pinst in pkg
if pinst.tag in ['Instance', 'Package']]
if insts:
for inst in insts:
@@ -1010,10 +1012,11 @@ class YUM(Bcfg2.Client.Tools.PkgTool):
if not self.setup['kevlar']:
for pkg_entry in [p for p in packages if self.canVerify(p)]:
- self.logger.debug("Reverifying Failed Package %s" \
- % (pkg_entry.get('name')))
- states[pkg_entry] = self.VerifyPackage(pkg_entry,
- self.modlists.get(pkg_entry, []))
+ self.logger.debug("Reverifying Failed Package %s" %
+ pkg_entry.get('name'))
+ states[pkg_entry] = \
+ self.VerifyPackage(pkg_entry,
+ self.modlists.get(pkg_entry, []))
self.modified.extend(ent for ent in packages if states[ent])
return states
diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py
index dd5ae1e83..e40ef750b 100644
--- a/src/lib/Bcfg2/Client/__init__.py
+++ b/src/lib/Bcfg2/Client/__init__.py
@@ -1,7 +1,5 @@
"""This contains all Bcfg2 Client modules"""
-__all__ = ["Frame", "Tools", "XML", "Client"]
-
import os
import sys
import select
diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py
index 44c76303c..4e9239e26 100644
--- a/src/lib/Bcfg2/Compat.py
+++ b/src/lib/Bcfg2/Compat.py
@@ -10,7 +10,7 @@ Python 2.4 and such-like """
import sys
-# pylint: disable=E0611,W0611,W0622,C0103
+# pylint: disable=E0601,E0602,E0611,W0611,W0622,C0103
try:
from email.Utils import formatdate
@@ -270,3 +270,11 @@ try:
except NameError:
# longs are just ints in py3k
long = int
+
+
+try:
+ cmp = cmp
+except NameError:
+ def cmp(a, b):
+ """ Py3k implementation of cmp() """
+ return (a > b) - (a < b)
diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py
index 5bbc9ff96..e537b6148 100644
--- a/src/lib/Bcfg2/Logger.py
+++ b/src/lib/Bcfg2/Logger.py
@@ -144,9 +144,9 @@ def add_console_handler(level=logging.DEBUG):
# tell the handler to use this format
console.setFormatter(TermiosFormatter())
try:
- console.set_name("console")
+ console.set_name("console") # pylint: disable=E1101
except AttributeError:
- console.name = "console"
+ console.name = "console" # pylint: disable=W0201
logging.root.addHandler(console)
@@ -162,9 +162,9 @@ def add_syslog_handler(procname, syslog_facility, level=logging.DEBUG):
('localhost', 514),
syslog_facility)
try:
- syslog.set_name("syslog")
+ syslog.set_name("syslog") # pylint: disable=E1101
except AttributeError:
- syslog.name = "syslog"
+ syslog.name = "syslog" # pylint: disable=W0201
syslog.setLevel(level)
syslog.setFormatter(
logging.Formatter('%(name)s[%(process)d]: %(message)s'))
@@ -179,9 +179,9 @@ def add_file_handler(to_file, level=logging.DEBUG):
"""Add a logging handler that logs to to_file."""
filelog = logging.FileHandler(to_file)
try:
- filelog.set_name("file")
+ filelog.set_name("file") # pylint: disable=E1101
except AttributeError:
- filelog.name = "file"
+ filelog.name = "file" # pylint: disable=W0201
filelog.setLevel(level)
filelog.setFormatter(
logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(message)s'))
@@ -197,7 +197,7 @@ def setup_logging(procname, to_console=True, to_syslog=True,
params = []
if to_console:
- if to_console == True:
+ if to_console is True:
to_console = logging.WARNING
if level == 0:
clvl = to_console
diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py
index e5aeccf4d..67dcf901e 100644
--- a/src/lib/Bcfg2/Options.py
+++ b/src/lib/Bcfg2/Options.py
@@ -308,14 +308,14 @@ def list_split(c_string):
""" split an option string on commas, optionally surrounded by
whitespace, returning a list """
if c_string:
- return re.split("\s*,\s*", c_string)
+ return re.split(r'\s*,\s*', c_string)
return []
def colon_split(c_string):
""" split an option string on colons, returning a list """
if c_string:
- return c_string.split(':')
+ return c_string.split(r':')
return []
@@ -349,7 +349,7 @@ def get_size(value):
'512m', '2g'), get the absolute number of bytes as an integer """
if value == -1:
return value
- mat = re.match("(\d+)([KkMmGg])?", value)
+ mat = re.match(r'(\d+)([KkMmGg])?', value)
if not mat:
raise ValueError("Not a valid size", value)
rvalue = int(mat.group(1))
@@ -395,7 +395,8 @@ CFILE = \
Option('Specify configuration file',
default=DEFAULT_CONFIG_LOCATION,
cmd='-C',
- odesc='<conffile>')
+ odesc='<conffile>',
+ env="BCFG2_CONFIG")
LOCKFILE = \
Option('Specify lockfile',
default='/var/lock/bcfg2.run',
@@ -1014,6 +1015,15 @@ VERBOSE = \
cmd='-v',
cook=get_bool,
cf=('logging', 'verbose'))
+LOG_PERFORMANCE = \
+ Option("Periodically log performance statistics",
+ default=False,
+ cf=('logging', 'performance'))
+PERFLOG_INTERVAL = \
+ Option("Performance statistics logging interval in seconds",
+ default=300.0,
+ cook=get_timeout,
+ cf=('logging', 'performance_interval'))
# Plugin-specific options
CFG_VALIDATION = \
@@ -1097,7 +1107,9 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY,
web_configfile=WEB_CFILE,
backend=SERVER_BACKEND,
vcs_root=SERVER_VCS_ROOT,
- authentication=SERVER_AUTHENTICATION)
+ authentication=SERVER_AUTHENTICATION,
+ perflog=LOG_PERFORMANCE,
+ perflog_interval=PERFLOG_INTERVAL)
CRYPT_OPTIONS = dict(encrypt=ENCRYPT,
decrypt=DECRYPT,
@@ -1206,6 +1218,7 @@ INFO_COMMON_OPTIONS = dict(ppath=PARANOID_PATH,
INFO_COMMON_OPTIONS.update(CLI_COMMON_OPTIONS)
INFO_COMMON_OPTIONS.update(SERVER_COMMON_OPTIONS)
+
class OptionParser(OptionSet):
""" OptionParser bootstraps option parsing, getting the value of
the config file. This should only be instantiated by
diff --git a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py
index 0a0f032e5..c7d5c512a 100644
--- a/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py
+++ b/src/lib/Bcfg2/Reporting/Transport/LocalFilesystem.py
@@ -87,7 +87,7 @@ class LocalFilesystem(TransportBase):
# using a tmpfile to hopefully avoid the file monitor from grabbing too
# soon
- saved = open(tmp_file, 'w')
+ saved = open(tmp_file, 'wb')
try:
saved.write(payload)
except IOError:
@@ -123,7 +123,7 @@ class LocalFilesystem(TransportBase):
self.debug_log("Handling event %s" % event.filename)
payload = os.path.join(self.work_path, event.filename)
try:
- payloadfd = open(payload, "r")
+ payloadfd = open(payload, "rb")
interaction = cPickle.load(payloadfd)
payloadfd.close()
os.unlink(payload)
diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py
index 4be509f53..e63c180a8 100644
--- a/src/lib/Bcfg2/Reporting/models.py
+++ b/src/lib/Bcfg2/Reporting/models.py
@@ -139,9 +139,11 @@ class Interaction(models.Model):
posixgroups = models.ManyToManyField("POSIXGroupEntry")
failures = models.ManyToManyField("FailureEntry")
- entry_types = ('actions', 'packages', 'paths', 'services', 'sebooleans',
- 'seports', 'sefcontexts', 'senodes', 'selogins', 'seusers',
- 'seinterfaces', 'sepermissives', 'semodules', 'posixusers',
+ entry_types = ('actions', 'failures', 'packages',
+ 'paths', 'services', 'sebooleans',
+ 'seports', 'sefcontexts', 'senodes',
+ 'selogins', 'seusers', 'seinterfaces',
+ 'sepermissives', 'semodules', 'posixusers',
'posixgroups')
# Formerly InteractionMetadata
diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py
index 884405786..870a31480 100644
--- a/src/lib/Bcfg2/Server/Admin/Init.py
+++ b/src/lib/Bcfg2/Server/Admin/Init.py
@@ -231,8 +231,8 @@ class Init(Bcfg2.Server.Admin.Mode):
def _prompt_password(self):
"""Ask for a password or generate one if none is provided."""
newpassword = getpass.getpass(
- "Input password used for communication verification "
- "(without echoing; leave blank for a random): ").strip()
+ "Input password used for communication verification "
+ "(without echoing; leave blank for a random): ").strip()
if len(newpassword) != 0:
self.data['password'] = newpassword
diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py
index 6d0dab106..93e42305c 100644
--- a/src/lib/Bcfg2/Server/Admin/Minestruct.py
+++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py
@@ -3,6 +3,7 @@ import getopt
import lxml.etree
import sys
import Bcfg2.Server.Admin
+from Bcfg2.Server.Plugin import PullSource
class Minestruct(Bcfg2.Server.Admin.StructureMode):
@@ -39,12 +40,12 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode):
try:
extra = set()
- for source in self.bcore.pull_sources:
+ for source in self.bcore.plugins_by_type(PullSource):
for item in source.GetExtra(client):
extra.add(item)
except:
self.log.error("Failed to find extra entry info for client %s" %
- client)
+ client)
raise SystemExit(1)
root = lxml.etree.Element("Base")
self.log.info("Found %d extra entries" % (len(extra)))
diff --git a/src/lib/Bcfg2/Server/Admin/Perf.py b/src/lib/Bcfg2/Server/Admin/Perf.py
index a7e67c956..1a772e6fc 100644
--- a/src/lib/Bcfg2/Server/Admin/Perf.py
+++ b/src/lib/Bcfg2/Server/Admin/Perf.py
@@ -31,8 +31,8 @@ class Perf(Bcfg2.Server.Admin.Mode):
timeout=setup['timeout'])
data = proxy.get_statistics()
for key in sorted(data.keys()):
- output.append((key, ) +
- tuple(["%.06f" % item
- for item in data[key][:-1]] + \
- [data[key][-1]]))
+ output.append(
+ (key, ) +
+ tuple(["%.06f" % item
+ for item in data[key][:-1]] + [data[key][-1]]))
self.print_table(output)
diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py
index 1905fac3c..e883c432f 100644
--- a/src/lib/Bcfg2/Server/Admin/Pull.py
+++ b/src/lib/Bcfg2/Server/Admin/Pull.py
@@ -6,6 +6,7 @@ import sys
import getopt
import select
import Bcfg2.Server.Admin
+from Bcfg2.Server.Plugin import PullSource, Generator
from Bcfg2.Compat import input # pylint: disable=W0622
@@ -62,13 +63,14 @@ class Pull(Bcfg2.Server.Admin.MetadataCore):
given client/entry from statistics.
"""
new_entry = {'type': etype, 'name': ename}
- for plugin in self.bcore.pull_sources:
+ pull_sources = self.bcore.plugins_by_type(PullSource)
+ for plugin in pull_sources:
try:
(owner, group, mode, contents) = \
- plugin.GetCurrentEntry(client, etype, ename)
+ plugin.GetCurrentEntry(client, etype, ename)
break
except Bcfg2.Server.Plugin.PluginExecutionError:
- if plugin == self.bcore.pull_sources[-1]:
+ if plugin == pull_sources[-1]:
print("Pull Source failure; could not fetch current state")
raise SystemExit(1)
@@ -121,8 +123,8 @@ class Pull(Bcfg2.Server.Admin.MetadataCore):
meta = self.bcore.build_metadata(client)
# Find appropriate plugin in bcore
- glist = [gen for gen in self.bcore.generators if
- ename in gen.Entries.get(etype, {})]
+ glist = [gen for gen in self.bcore.plugins_by_type(Generator)
+ if ename in gen.Entries.get(etype, {})]
if len(glist) != 1:
self.errExit("Got wrong numbers of matching generators for entry:"
"%s" % ([g.name for g in glist]))
diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py
index 6f411c2e4..ba4777c93 100644
--- a/src/lib/Bcfg2/Server/Admin/Xcmd.py
+++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py
@@ -51,5 +51,5 @@ class Xcmd(Bcfg2.Server.Admin.Mode):
print("Proxy Error: %s" % err)
return
- if data != None:
+ if data is not None:
print(data)
diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py
index 0c4764642..3fbdf8fa8 100644
--- a/src/lib/Bcfg2/Server/Admin/__init__.py
+++ b/src/lib/Bcfg2/Server/Admin/__init__.py
@@ -1,30 +1,14 @@
""" Base classes for admin modes """
-__all__ = [
- 'Backup',
- 'Bundle',
- 'Client',
- 'Compare',
- 'Group',
- 'Init',
- 'Minestruct',
- 'Perf',
- 'Pull',
- 'Query',
- 'Reports',
- 'Syncdb',
- 'Tidy',
- 'Viz',
- 'Xcmd'
- ]
-
import re
import sys
import logging
import lxml.etree
import Bcfg2.Server.Core
import Bcfg2.Options
-from Bcfg2.Compat import ConfigParser
+from Bcfg2.Compat import ConfigParser, walk_packages
+
+__all__ = [m[1] for m in walk_packages(path=__path__)]
class Mode(object):
@@ -104,15 +88,15 @@ class Mode(object):
# Calculate column widths (longest item in each column
# plus padding on both sides)
cols = list(zip(*rows))
- col_widths = [max([len(str(item)) + 2 * padding for \
- item in col]) for col in cols]
+ col_widths = [max([len(str(item)) + 2 * padding
+ for item in col]) for col in cols]
borderline = vdelim.join([w * hdelim for w in col_widths])
# Print out the table
print(borderline)
for row in rows:
- print(vdelim.join([justify(str(item), width) for \
- (item, width) in zip(row, col_widths)]))
+ print(vdelim.join([justify(str(item), width)
+ for (item, width) in zip(row, col_widths)]))
if hdr:
print(borderline)
hdr = False
diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py
index 663ee6f92..48455819d 100644
--- a/src/lib/Bcfg2/Server/BuiltinCore.py
+++ b/src/lib/Bcfg2/Server/BuiltinCore.py
@@ -9,12 +9,12 @@ from Bcfg2.Server.Core import BaseCore, NoExposedMethod
from Bcfg2.Compat import xmlrpclib, urlparse
from Bcfg2.Server.SSLServer import XMLRPCServer
-from lockfile import LockFailed
+from lockfile import LockFailed, LockTimeout
# pylint: disable=E0611
try:
- from daemon.pidfile import PIDLockFile
+ from daemon.pidfile import TimeoutPIDLockFile
except ImportError:
- from daemon.pidlockfile import PIDLockFile
+ from daemon.pidlockfile import TimeoutPIDLockFile
# pylint: enable=E0611
@@ -33,7 +33,8 @@ class Core(BaseCore):
gid=self.setup['daemon_gid'],
umask=int(self.setup['umask'], 8))
if self.setup['daemon']:
- daemon_args['pidfile'] = PIDLockFile(self.setup['daemon'])
+ daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'],
+ acquire_timeout=5)
#: The :class:`daemon.DaemonContext` used to drop
#: privileges, write the PID file (with :class:`PidFile`),
#: and daemonize this core.
@@ -90,6 +91,11 @@ class Core(BaseCore):
err = sys.exc_info()[1]
self.logger.error("Failed to daemonize %s: %s" % (self.name, err))
return False
+ except LockTimeout:
+ err = sys.exc_info()[1]
+ self.logger.error("Failed to daemonize %s: Failed to acquire lock "
+ "on %s" % (self.name, self.setup['daemon']))
+ return False
def _run(self):
""" Create :attr:`server` to start the server listening. """
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index c69e8b055..b0cb4cc87 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -20,7 +20,9 @@ from itertools import chain
from Bcfg2.Server.Cache import Cache
from Bcfg2.Options import get_option_parser, SERVER_FAM_IGNORE
from Bcfg2.Compat import xmlrpclib # pylint: disable=W0622
-from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError
+from Bcfg2.Server.Plugin.exceptions import * # pylint: disable=W0401,W0614
+from Bcfg2.Server.Plugin.interfaces import * # pylint: disable=W0401,W0614
+from Bcfg2.Server.Plugin import track_statistics
try:
import psyco
@@ -93,6 +95,7 @@ class BaseCore(object):
.. automethod:: _block
.. -----
.. automethod:: _file_monitor_thread
+ .. automethod:: _perflog_thread
"""
#: The Bcfg2 options dict
self.setup = get_option_parser()
@@ -179,6 +182,9 @@ class BaseCore(object):
#: the first one loaded wins.
self.plugin_blacklist = {}
+ #: The Metadata plugin
+ self.metadata = None
+
#: Revision of the Bcfg2 specification. This will be sent to
#: the client in the configuration, and can be set by a
#: :class:`Bcfg2.Server.Plugin.interfaces.Version` plugin.
@@ -237,71 +243,6 @@ class BaseCore(object):
self.logger.error("Failed to set ownership of database "
"at %s: %s" % (db_settings['NAME'], err))
- if '' in self.setup['plugins']:
- self.setup['plugins'].remove('')
-
- for plugin in self.setup['plugins']:
- if not plugin in self.plugins:
- self.init_plugin(plugin)
- # Remove blacklisted plugins
- for plugin, blacklist in list(self.plugin_blacklist.items()):
- if len(blacklist) > 0:
- self.logger.error("The following plugins conflict with %s;"
- "Unloading %s" % (plugin, blacklist))
- for plug in blacklist:
- del self.plugins[plug]
-
- # Log experimental plugins
- expl = [plug for plug in list(self.plugins.values())
- if plug.experimental]
- if expl:
- self.logger.info("Loading experimental plugin(s): %s" %
- (" ".join([x.name for x in expl])))
- self.logger.info("NOTE: Interfaces subject to change")
-
- # Log deprecated plugins
- depr = [plug for plug in list(self.plugins.values())
- if plug.deprecated]
- if depr:
- self.logger.info("Loading deprecated plugin(s): %s" %
- (" ".join([x.name for x in depr])))
-
- # Find the metadata plugin and set self.metadata
- mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata)
- if len(mlist) >= 1:
- #: The Metadata plugin
- self.metadata = mlist[0]
- if len(mlist) > 1:
- self.logger.error("Multiple Metadata plugins loaded; "
- "using %s" % self.metadata)
- else:
- self.logger.error("No Metadata plugin loaded; "
- "failed to instantiate Core")
- raise CoreInitError("No Metadata Plugin")
-
- #: The list of plugins that handle
- #: :class:`Bcfg2.Server.Plugin.interfaces.Statistics`
- self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics)
-
- #: The list of plugins that implement the
- #: :class:`Bcfg2.Server.Plugin.interfaces.PullSource`
- #: interface
- self.pull_sources = \
- self.plugins_by_type(Bcfg2.Server.Plugin.PullSource)
-
- #: The list of
- #: :class:`Bcfg2.Server.Plugin.interfaces.Generator` plugins
- self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator)
-
- #: The list of plugins that handle
- #: :class:`Bcfg2.Server.Plugin.interfaces.Structure`
- #: generation
- self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure)
-
- #: The list of plugins that implement the
- #: :class:`Bcfg2.Server.Plugin.interfaces.Connector` interface
- self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector)
-
#: The CA that signed the server cert
self.ca = self.setup['ca']
@@ -319,6 +260,12 @@ class BaseCore(object):
threading.Thread(name="%sFAMThread" % self.setup['filemonitor'],
target=self._file_monitor_thread)
+ self.perflog_thread = None
+ if self.setup['perflog']:
+ self.perflog_thread = \
+ threading.Thread(name="PerformanceLoggingThread",
+ target=self._perflog_thread)
+
#: A :func:`threading.Lock` for use by
#: :func:`Bcfg2.Server.FileMonitor.FileMonitor.handle_event_set`
self.lock = threading.Lock()
@@ -327,10 +274,6 @@ class BaseCore(object):
#: metadata
self.metadata_cache = Cache()
- if self.debug_flag:
- # enable debugging on everything else.
- self.plugins[plugin].set_debug(self.debug_flag)
-
def plugins_by_type(self, base_cls):
""" Return a list of loaded plugins that match the passed type.
@@ -351,11 +294,23 @@ class BaseCore(object):
if isinstance(plugin, base_cls)],
key=lambda p: (p.sort_order, p.name))
+ def _perflog_thread(self):
+ """ The thread that periodically logs performance statistics
+ to syslog. """
+ self.logger.debug("Performance logging thread starting")
+ while not self.terminate.isSet():
+ self.terminate.wait(self.setup['perflog_interval'])
+ for name, stats in self.get_statistics(None).items():
+ self.logger.info("Performance statistics: "
+ "%s min=%.06f, max=%.06f, average=%.06f, "
+ "count=%d" % ((name, ) + stats))
+
def _file_monitor_thread(self):
""" The thread that runs the
:class:`Bcfg2.Server.FileMonitor.FileMonitor`. This also
queries :class:`Bcfg2.Server.Plugin.interfaces.Version`
plugins for the current revision of the Bcfg2 repo. """
+ self.logger.debug("File monitor thread starting")
famfd = self.fam.fileno()
terminate = self.terminate
while not terminate.isSet():
@@ -374,7 +329,7 @@ class BaseCore(object):
def _update_vcs_revision(self):
""" Update the revision of the current configuration on-disk
from the VCS plugin """
- for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version):
+ for plugin in self.plugins_by_type(Version):
try:
newrev = plugin.get_revision()
if newrev != self.revision:
@@ -386,6 +341,59 @@ class BaseCore(object):
(plugin.name, sys.exc_info()[1]))
self.revision = '-1'
+ def load_plugins(self):
+ """ Load all plugins, setting
+ :attr:`Bcfg2.Server.Core.BaseCore.plugins` and
+ :attr:`Bcfg2.Server.Core.BaseCore.metadata` as side effects.
+ This does not start plugin threads; that is done later, in
+ :func:`Bcfg2.Server.Core.BaseCore.run` """
+ while '' in self.setup['plugins']:
+ self.setup['plugins'].remove('')
+
+ for plugin in self.setup['plugins']:
+ if not plugin in self.plugins:
+ self.init_plugin(plugin)
+
+ # Remove blacklisted plugins
+ for plugin, blacklist in list(self.plugin_blacklist.items()):
+ if len(blacklist) > 0:
+ self.logger.error("The following plugins conflict with %s;"
+ "Unloading %s" % (plugin, blacklist))
+ for plug in blacklist:
+ del self.plugins[plug]
+
+ # Log deprecated and experimental plugins
+ expl = []
+ depr = []
+ for plug in list(self.plugins.values()):
+ if plug.experimental:
+ expl.append(plug)
+ if plug.deprecated:
+ depr.append(plug)
+ if expl:
+ self.logger.info("Loading experimental plugin(s): %s" %
+ (" ".join([x.name for x in expl])))
+ self.logger.info("NOTE: Interfaces subject to change")
+ if depr:
+ self.logger.info("Loading deprecated plugin(s): %s" %
+ (" ".join([x.name for x in depr])))
+
+ # Find the metadata plugin and set self.metadata
+ mlist = self.plugins_by_type(Metadata)
+ if len(mlist) >= 1:
+ self.metadata = mlist[0]
+ if len(mlist) > 1:
+ self.logger.error("Multiple Metadata plugins loaded; using %s"
+ % self.metadata)
+ else:
+ self.logger.error("No Metadata plugin loaded; "
+ "failed to instantiate Core")
+ raise CoreInitError("No Metadata Plugin")
+
+ if self.debug_flag:
+ # enable debugging on plugins
+ self.plugins[plugin].set_debug(self.debug_flag)
+
def init_plugin(self, plugin):
""" Import and instantiate a single plugin. The plugin is
stored to :attr:`plugins`.
@@ -399,7 +407,7 @@ class BaseCore(object):
self.logger.debug("Loading plugin %s" % plugin)
try:
mod = getattr(__import__("Bcfg2.Server.Plugins.%s" %
- (plugin)).Server.Plugins, plugin)
+ (plugin)).Server.Plugins, plugin)
except ImportError:
try:
mod = __import__(plugin, globals(), locals(),
@@ -422,6 +430,10 @@ class BaseCore(object):
except PluginInitError:
self.logger.error("Failed to instantiate plugin %s" % plugin,
exc_info=1)
+ except OSError:
+ err = sys.exc_info()[1]
+ self.logger.error("Failed to add a file monitor while "
+ "instantiating plugin %s: %s" % (plugin, err))
except:
self.logger.error("Unexpected instantiation failure for plugin %s"
% plugin, exc_info=1)
@@ -468,8 +480,7 @@ class BaseCore(object):
metadata.hostname))
start = time.time()
try:
- for plugin in \
- self.plugins_by_type(Bcfg2.Server.Plugin.ClientRunHooks):
+ for plugin in self.plugins_by_type(ClientRunHooks):
try:
getattr(plugin, hook)(metadata)
except AttributeError:
@@ -500,11 +511,10 @@ class BaseCore(object):
:type data: list of lxml.etree._Element objects
"""
self.logger.debug("Validating structures for %s" % metadata.hostname)
- for plugin in \
- self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator):
+ for plugin in self.plugins_by_type(StructureValidator):
try:
plugin.validate_structures(metadata, data)
- except Bcfg2.Server.Plugin.ValidationError:
+ except ValidationError:
err = sys.exc_info()[1]
self.logger.error("Plugin %s structure validation failed: %s" %
(plugin.name, err))
@@ -527,10 +537,10 @@ class BaseCore(object):
:type data: list of lxml.etree._Element objects
"""
self.logger.debug("Validating goals for %s" % metadata.hostname)
- for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator):
+ for plugin in self.plugins_by_type(GoalValidator):
try:
plugin.validate_goals(metadata, data)
- except Bcfg2.Server.Plugin.ValidationError:
+ except ValidationError:
err = sys.exc_info()[1]
self.logger.error("Plugin %s goal validation failed: %s" %
(plugin.name, err.message))
@@ -548,8 +558,9 @@ class BaseCore(object):
:returns: list of :class:`lxml.etree._Element` objects
"""
self.logger.debug("Getting structures for %s" % metadata.hostname)
- structures = list(chain(*[struct.BuildStructures(metadata)
- for struct in self.structures]))
+ structures = list(
+ chain(*[struct.BuildStructures(metadata)
+ for struct in self.plugins_by_type(Structure)]))
sbundles = [b.get('name') for b in structures if b.tag == 'Bundle']
missing = [b for b in metadata.bundles if b not in sbundles]
if missing:
@@ -634,8 +645,9 @@ class BaseCore(object):
self.logger.error("Falling back to %s:%s" %
(entry.tag, entry.get('name')))
- glist = [gen for gen in self.generators if
- entry.get('name') in gen.Entries.get(entry.tag, {})]
+ generators = self.plugins_by_type(Generator)
+ glist = [gen for gen in generators
+ if entry.get('name') in gen.Entries.get(entry.tag, {})]
if len(glist) == 1:
return glist[0].Entries[entry.tag][entry.get('name')](entry,
metadata)
@@ -643,8 +655,8 @@ class BaseCore(object):
generators = ", ".join([gen.name for gen in glist])
self.logger.error("%s %s served by multiple generators: %s" %
(entry.tag, entry.get('name'), generators))
- g2list = [gen for gen in self.generators if
- gen.HandlesEntry(entry, metadata)]
+ g2list = [gen for gen in generators
+ if gen.HandlesEntry(entry, metadata)]
try:
if len(g2list) == 1:
return g2list[0].HandleEntry(entry, metadata)
@@ -671,7 +683,7 @@ class BaseCore(object):
revision=self.revision)
try:
meta = self.build_metadata(client)
- except Bcfg2.Server.Plugin.MetadataConsistencyError:
+ except MetadataConsistencyError:
self.logger.error("Metadata consistency error for client %s" %
client)
return lxml.etree.Element("error", type='metadata error')
@@ -718,7 +730,8 @@ class BaseCore(object):
:type event: Bcfg2.Server.FileMonitor.Event
"""
if event.filename != self.cfile:
- print("Got event for unknown file: %s" % event.filename)
+ self.logger.error("Got event for unknown file: %s" %
+ event.filename)
return
if event.code2str() == 'deleted':
return
@@ -755,11 +768,15 @@ class BaseCore(object):
return False
try:
+ self.load_plugins()
+
self.fam.start()
self.fam_thread.start()
self.fam.AddMonitor(self.cfile, self)
+ if self.perflog_thread is not None:
+ self.perflog_thread.start()
- for plug in self.plugins_by_type(Bcfg2.Server.Plugin.Threaded):
+ for plug in self.plugins_by_type(Threaded):
plug.start_threads()
except:
self.shutdown()
@@ -801,7 +818,7 @@ class BaseCore(object):
"""
self.logger.debug("Getting decision list for %s" % metadata.hostname)
result = []
- for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision):
+ for plugin in self.plugins_by_type(Decision):
try:
result.extend(plugin.GetDecisions(metadata, mode))
except:
@@ -862,7 +879,7 @@ class BaseCore(object):
"""
if not hasattr(self, 'metadata'):
# some threads start before metadata is even loaded
- raise Bcfg2.Server.Plugin.MetadataRuntimeError
+ raise MetadataRuntimeError("Metadata not loaded yet")
if self.metadata_cache_mode == 'initial':
# the Metadata plugin handles loading the cached data if
# we're only caching the initial metadata object
@@ -872,10 +889,11 @@ class BaseCore(object):
if not imd:
self.logger.debug("Building metadata for %s" % client_name)
imd = self.metadata.get_initial_metadata(client_name)
- for conn in self.connectors:
+ connectors = self.plugins_by_type(Connector)
+ for conn in connectors:
grps = conn.get_additional_groups(imd)
self.metadata.merge_additional_groups(imd, grps)
- for conn in self.connectors:
+ for conn in connectors:
data = conn.get_additional_data(imd)
self.metadata.merge_additional_data(imd, conn.name, data)
imd.query.by_name = self.build_metadata
@@ -896,7 +914,7 @@ class BaseCore(object):
meta = self.build_metadata(client_name)
state = statistics.find(".//Statistics")
if state.get('version') >= '2.0':
- for plugin in self.statistics:
+ for plugin in self.plugins_by_type(Statistics):
try:
plugin.process_statistics(meta, statistics)
except:
@@ -938,11 +956,11 @@ class BaseCore(object):
meta = self.build_metadata(client)
else:
meta = None
- except Bcfg2.Server.Plugin.MetadataConsistencyError:
+ except MetadataConsistencyError:
err = sys.exc_info()[1]
self.critical_error("Client metadata resolution error for %s: %s" %
(address[0], err))
- except Bcfg2.Server.Plugin.MetadataRuntimeError:
+ except MetadataRuntimeError:
err = sys.exc_info()[1]
self.critical_error('Metadata system runtime failure for %s: %s' %
(address[0], err))
@@ -1038,8 +1056,7 @@ class BaseCore(object):
version))
try:
self.metadata.set_version(client, version)
- except (Bcfg2.Server.Plugin.MetadataConsistencyError,
- Bcfg2.Server.Plugin.MetadataRuntimeError):
+ except (MetadataConsistencyError, MetadataRuntimeError):
err = sys.exc_info()[1]
self.critical_error("Unable to set version for %s: %s" %
(client, err))
@@ -1059,7 +1076,7 @@ class BaseCore(object):
client, metadata = self.resolve_client(address, cleanup_cache=True)
self.logger.debug("Getting probes for %s" % client)
try:
- for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing):
+ for plugin in self.plugins_by_type(Probing):
for probe in plugin.GetProbes(metadata):
resp.append(probe)
return lxml.etree.tostring(resp,
@@ -1129,11 +1146,10 @@ class BaseCore(object):
self.logger.debug("%s sets its profile to %s" % (client, profile))
try:
self.metadata.set_profile(client, profile, address)
- except (Bcfg2.Server.Plugin.MetadataConsistencyError,
- Bcfg2.Server.Plugin.MetadataRuntimeError):
+ except (MetadataConsistencyError, MetadataRuntimeError):
err = sys.exc_info()[1]
self.critical_error("Unable to assert profile for %s: %s" %
- (client, err))
+ (client, err))
return True
@exposed
@@ -1152,7 +1168,7 @@ class BaseCore(object):
config = self.BuildConfiguration(client)
return lxml.etree.tostring(config,
xml_declaration=False).decode('UTF-8')
- except Bcfg2.Server.Plugin.MetadataConsistencyError:
+ except MetadataConsistencyError:
self.critical_error("Metadata consistency failure for %s" % client)
@exposed
diff --git a/src/lib/Bcfg2/Server/Encryption.py b/src/lib/Bcfg2/Server/Encryption.py
index b46337eb0..ee1c1cab9 100755
--- a/src/lib/Bcfg2/Server/Encryption.py
+++ b/src/lib/Bcfg2/Server/Encryption.py
@@ -21,7 +21,7 @@ DECRYPT = 0
#: Default initialization vector. For best security, you should use a
#: unique IV for each message. :func:`ssl_encrypt` does this in an
#: automated fashion.
-IV = '\0' * 16
+IV = r'\0' * 16
#: The config file section encryption options and passphrases are
#: stored in
@@ -121,9 +121,11 @@ def ssl_decrypt(data, passwd, algorithm=ALGORITHM):
# base64-decode the data
data = b64decode(data)
salt = data[8:16]
+ # pylint: disable=E1101
hashes = [md5(passwd + salt).digest()]
for i in range(1, 3):
hashes.append(md5(hashes[i - 1] + passwd + salt).digest())
+ # pylint: enable=E1101
key = hashes[0] + hashes[1]
iv = hashes[2]
@@ -149,9 +151,11 @@ def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None):
if salt is None:
salt = Rand.rand_bytes(8)
+ # pylint: disable=E1101
hashes = [md5(passwd + salt).digest()]
for i in range(1, 3):
hashes.append(md5(hashes[i - 1] + passwd + salt).digest())
+ # pylint: enable=E1101
key = hashes[0] + hashes[1]
iv = hashes[2]
diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
index cdd52dbb9..2cdf27ed8 100644
--- a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
+++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py
@@ -3,14 +3,11 @@ support. """
import os
import errno
-import logging
import pyinotify
from Bcfg2.Compat import reduce # pylint: disable=W0622
from Bcfg2.Server.FileMonitor import Event
from Bcfg2.Server.FileMonitor.Pseudo import Pseudo
-LOGGER = logging.getLogger(__name__)
-
class Inotify(Pseudo, pyinotify.ProcessEvent):
""" File monitor backend with `inotify
@@ -123,8 +120,9 @@ class Inotify(Pseudo, pyinotify.ProcessEvent):
try:
watch = self.watchmgr.watches[ievent.wd]
except KeyError:
- LOGGER.error("Error handling event %s for %s: Watch %s not found" %
- (action, ievent.pathname, ievent.wd))
+ self.logger.error("Error handling event %s for %s: "
+ "Watch %s not found" %
+ (action, ievent.pathname, ievent.wd))
return
# FAM-style file monitors return the full path to the parent
# directory that is being watched, relative paths to anything
diff --git a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py
index 24cd099d0..b1e1adab7 100644
--- a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py
+++ b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py
@@ -24,6 +24,6 @@ class Pseudo(FileMonitor):
self.events.append(Event(handleID, fname, 'exists'))
self.events.append(Event(handleID, path, 'endExist'))
- if obj != None:
+ if obj is not None:
self.handles[handleID] = obj
return handleID
diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
index 497e8fac6..7a2fd3fe9 100644
--- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
+++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py
@@ -4,9 +4,14 @@ verified with an XML schema alone"""
import os
import re
import Bcfg2.Server.Lint
-import Bcfg2.Client.Tools.POSIX
import Bcfg2.Client.Tools.VCS
from Bcfg2.Server.Plugins.Packages import Apt, Yum
+from Bcfg2.Client.Tools.POSIX.base import device_map
+try:
+ from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile
+ HAS_GENSHI = True
+except ImportError:
+ HAS_GENSHI = False
# format verifying functions
@@ -53,10 +58,10 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs)
self.required_attrs = dict(
Path=dict(
- device=dict(name=is_filename, owner=is_username,
+ device=dict(name=is_filename,
+ owner=is_username,
group=is_username,
- dev_type=lambda v: \
- v in Bcfg2.Client.Tools.POSIX.base.device_map),
+ dev_type=lambda v: v in device_map),
directory=dict(name=is_filename, owner=is_username,
group=is_username, mode=is_octal_mode),
file=dict(name=is_filename, owner=is_username,
@@ -81,21 +86,21 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
command=None)},
ACL=dict(
default=dict(scope=lambda v: v in ['user', 'group'],
- perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}',
+ perms=lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}',
v)),
access=dict(scope=lambda v: v in ['user', 'group'],
- perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}',
+ perms=lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}',
v)),
- mask=dict(perms=lambda v: re.match('^([0-7]|[rwx\-]{0,3}',
+ mask=dict(perms=lambda v: re.match(r'^([0-7]|[rwx\-]{0,3}',
v))),
Package={"__any__": dict(name=None)},
SEBoolean={None: dict(name=None,
value=lambda v: v in ['on', 'off'])},
SEModule={None: dict(name=None, __text__=None)},
- SEPort={None:
- dict(name=lambda v: re.match(r'^\d+(-\d+)?/(tcp|udp)',
- v),
- selinuxtype=is_selinux_type)},
+ SEPort={
+ None: dict(name=lambda v: re.match(r'^\d+(-\d+)?/(tcp|udp)',
+ v),
+ selinuxtype=is_selinux_type)},
SEFcontext={None: dict(name=None, selinuxtype=is_selinux_type)},
SENode={None: dict(name=lambda v: "/" in v,
selinuxtype=is_selinux_type,
@@ -110,8 +115,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin):
SEPermissive={None: dict(name=is_selinux_type)},
POSIXGroup={None: dict(name=is_username)},
POSIXUser={None: dict(name=is_username)},
- MemberOf={None: dict(__text__=is_username)},
- )
+ MemberOf={None: dict(__text__=is_username)})
def Run(self):
self.check_packages()
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index dd45ac62e..946ef8270 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -39,7 +39,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
"NagiosGen/config.xml": "nagiosgen.xsd",
"FileProbes/config.xml": "fileprobes.xsd",
"SSLCA/**/cert.xml": "sslca-cert.xsd",
- "SSLCA/**/key.xml": "sslca-key.xsd"
+ "SSLCA/**/key.xml": "sslca-key.xsd",
+ "GroupLogic/groups.xml": "grouplogic.xsd"
}
self.filelists = {}
@@ -83,17 +84,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
else:
self.LintError("properties-schema-not-found",
"No schema found for %s" % filename)
+ # ensure that it at least parses
+ self.parse(filename)
- def validate(self, filename, schemafile, schema=None):
- """validate a file against the given lxml.etree.Schema.
- return True on success, False on failure """
- if schema is None:
- # if no schema object was provided, instantiate one
- schema = self._load_schema(schemafile)
- if not schema:
- return False
+ def parse(self, filename):
+ """ Parse an XML file, raising the appropriate LintErrors if
+ it can't be parsed or read. Return the
+ lxml.etree._ElementTree parsed from the file. """
try:
- datafile = lxml.etree.parse(filename)
+ return lxml.etree.parse(filename)
except SyntaxError:
result = self.cmd.run(["xmllint", filename])
self.LintError("xml-failed-to-parse",
@@ -105,6 +104,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
"Failed to open file %s" % filename)
return False
+ def validate(self, filename, schemafile, schema=None):
+ """validate a file against the given lxml.etree.Schema.
+ return True on success, False on failure """
+ if schema is None:
+ # if no schema object was provided, instantiate one
+ schema = self._load_schema(schemafile)
+ if not schema:
+ return False
+ datafile = self.parse(filename)
if not schema.validate(datafile):
cmd = ["xmllint"]
if self.files is None:
diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py
index f7bc08717..ecd970b54 100644
--- a/src/lib/Bcfg2/Server/Plugin/base.py
+++ b/src/lib/Bcfg2/Server/Plugin/base.py
@@ -97,15 +97,21 @@ class Plugin(Debuggable):
:param datastore: The path to the Bcfg2 repository on the
filesystem
:type datastore: string
- :raises: :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError`
+ :raises: :exc:`OSError` if adding a file monitor failed;
+ :class:`Bcfg2.Server.Plugin.exceptions.PluginInitError`
+ on other errors
.. autoattribute:: Bcfg2.Server.Plugin.base.Debuggable.__rmi__
"""
+ Debuggable.__init__(self, name=self.name)
self.Entries = {}
self.core = core
self.data = os.path.join(datastore, self.name)
+ if not os.path.exists(self.data):
+ self.logger.warning("%s: %s does not exist, creating" %
+ (self.name, self.data))
+ os.makedirs(self.data)
self.running = True
- Debuggable.__init__(self, name=self.name)
@classmethod
def init_repo(cls, repo):
@@ -125,5 +131,11 @@ class Plugin(Debuggable):
self.debug_log("Shutting down %s plugin" % self.name)
self.running = False
+ def set_debug(self, debug):
+ for entry in self.Entries.values():
+ if isinstance(entry, Debuggable):
+ entry.set_debug(debug)
+ return Debuggable.set_debug(self, debug)
+
def __str__(self):
return "%s Plugin" % self.__class__.__name__
diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py
index ded7dd8dc..e5ea5188a 100644
--- a/src/lib/Bcfg2/Server/Plugin/helpers.py
+++ b/src/lib/Bcfg2/Server/Plugin/helpers.py
@@ -30,7 +30,38 @@ try:
except ImportError:
HAS_DJANGO = False
-LOGGER = logging.getLogger(__name__)
+class track_statistics(object): # pylint: disable=C0103
+ """ Decorator that tracks execution time for the given
+ :class:`Plugin` method with :mod:`Bcfg2.Statistics` for reporting
+ via ``bcfg2-admin perf`` """
+
+ def __init__(self, name=None):
+ """
+ :param name: The name under which statistics for this function
+ will be tracked. By default, the name will be
+ the name of the function concatenated with the
+ name of the class the function is a member of.
+ :type name: string
+ """
+ # if this is None, it will be set later during __call_
+ self.name = name
+
+ def __call__(self, func):
+ if self.name is None:
+ self.name = func.__name__
+
+ @wraps(func)
+ def inner(obj, *args, **kwargs):
+ """ The decorated function """
+ name = "%s:%s" % (obj.__class__.__name__, self.name)
+
+ start = time.time()
+ try:
+ return func(obj, *args, **kwargs)
+ finally:
+ Bcfg2.Statistics.stats.add_value(name, time.time() - start)
+
+ return inner
def removecomment(stream):
@@ -188,7 +219,7 @@ class FileBacked(Debuggable):
return "%s: %s" % (self.__class__.__name__, self.name)
-class DirectoryBacked(object):
+class DirectoryBacked(Debuggable):
""" DirectoryBacked objects represent a directory that contains
files, represented by objects of the type listed in
:attr:`__child__`, and other directories recursively. It monitors
@@ -219,7 +250,7 @@ class DirectoryBacked(object):
.. -----
.. autoattribute:: __child__
"""
- object.__init__(self)
+ Debuggable.__init__(self)
self.data = os.path.normpath(data)
self.fam = Bcfg2.Server.FileMonitor.get_fam()
@@ -238,11 +269,29 @@ class DirectoryBacked(object):
self.handles = {}
# Monitor everything in the plugin's directory
+ if not os.path.exists(self.data):
+ self.logger.warning("%s does not exist, creating" % self.data)
+ os.makedirs(self.data)
self.add_directory_monitor('')
+ def set_debug(self, debug):
+ for entry in self.entries.values():
+ if isinstance(entry, Debuggable):
+ entry.set_debug(debug)
+ return Debuggable.set_debug(self, debug)
+
def __getitem__(self, key):
return self.entries[key]
+ def __len__(self):
+ return len(self.entries)
+
+ def __delitem__(self, key):
+ del self.entries[key]
+
+ def __setitem__(self, key, val):
+ self.entries[key] = val
+
def __iter__(self):
return iter(list(self.entries.items()))
@@ -259,7 +308,7 @@ class DirectoryBacked(object):
dirpathname = os.path.join(self.data, relative)
if relative not in self.handles.values():
if not os.path.isdir(dirpathname):
- LOGGER.error("%s is not a directory" % dirpathname)
+ self.logger.error("%s is not a directory" % dirpathname)
return
reqid = self.fam.AddMonitor(dirpathname, self)
self.handles[reqid] = relative
@@ -303,8 +352,8 @@ class DirectoryBacked(object):
return
if event.requestID not in self.handles:
- LOGGER.warn("Got %s event with unknown handle (%s) for %s" %
- (action, event.requestID, event.filename))
+ self.logger.warn("Got %s event with unknown handle (%s) for %s" %
+ (action, event.requestID, event.filename))
return
# Clean up path names
@@ -314,7 +363,7 @@ class DirectoryBacked(object):
event.filename = event.filename[len(self.data) + 1:]
if self.ignore and self.ignore.search(event.filename):
- LOGGER.debug("Ignoring event %s" % event.filename)
+ self.logger.debug("Ignoring event %s" % event.filename)
return
# Calculate the absolute and relative paths this event refers to
@@ -349,19 +398,20 @@ class DirectoryBacked(object):
# class doesn't support canceling, so at least let
# the user know that a restart might be a good
# idea.
- LOGGER.warn("Directory properties for %s changed, please "
- " consider restarting the server" % abspath)
+ self.logger.warn("Directory properties for %s changed, "
+ "please consider restarting the server" %
+ abspath)
else:
# Got a "changed" event for a directory that we
# didn't know about. Go ahead and treat it like a
# "created" event, but log a warning, because this
# is unexpected.
- LOGGER.warn("Got %s event for unexpected dir %s" %
- (action, abspath))
+ self.logger.warn("Got %s event for unexpected dir %s" %
+ (action, abspath))
self.add_directory_monitor(relpath)
else:
- LOGGER.warn("Got unknown dir event %s %s %s" %
- (event.requestID, event.code2str(), abspath))
+ self.logger.warn("Got unknown dir event %s %s %s" %
+ (event.requestID, event.code2str(), abspath))
elif self.patterns.search(event.filename):
if action in ['exists', 'created']:
self.add_entry(relpath, event)
@@ -373,16 +423,15 @@ class DirectoryBacked(object):
# know about. Go ahead and treat it like a
# "created" event, but log a warning, because this
# is unexpected.
- LOGGER.warn("Got %s event for unexpected file %s" %
- (action,
- abspath))
+ self.logger.warn("Got %s event for unexpected file %s" %
+ (action, abspath))
self.add_entry(relpath, event)
else:
- LOGGER.warn("Got unknown file event %s %s %s" %
- (event.requestID, event.code2str(), abspath))
+ self.logger.warn("Got unknown file event %s %s %s" %
+ (event.requestID, event.code2str(), abspath))
else:
- LOGGER.warn("Could not process filename %s; ignoring" %
- event.filename)
+ self.logger.warn("Could not process filename %s; ignoring" %
+ event.filename)
class XMLFileBacked(FileBacked):
@@ -397,7 +446,11 @@ class XMLFileBacked(FileBacked):
#: behavior, set ``__identifier__`` to ``None``.
__identifier__ = 'name'
- def __init__(self, filename, should_monitor=False):
+ #: If ``create`` is set, then it overrides the ``create`` argument
+ #: to the constructor.
+ create = None
+
+ def __init__(self, filename, should_monitor=False, create=None):
"""
:param filename: The full path to the file to cache and monitor
:type filename: string
@@ -409,6 +462,13 @@ class XMLFileBacked(FileBacked):
:class:`Bcfg2.Server.Plugin.helpers.XMLDirectoryBacked`
object).
:type should_monitor: bool
+ :param create: Create the file if it doesn't exist.
+ ``create`` can be either an
+ :class:`lxml.etree._Element` object, which will
+ be used as initial content, or a string, which
+ will be used as the name of the (empty) tag
+ that will be the initial content of the file.
+ :type create: lxml.etree._Element or string
.. -----
.. autoattribute:: __identifier__
@@ -432,6 +492,21 @@ class XMLFileBacked(FileBacked):
#: "Extra" files included in this file by XInclude.
self.extras = []
+ #: Extra FAM monitors set by this object for files included by
+ #: XInclude.
+ self.extra_monitors = []
+
+ if ((create is not None or self.create not in [None, False]) and
+ not os.path.exists(self.name)):
+ toptag = create or self.create
+ self.logger.warning("%s does not exist, creating" % self.name)
+ if hasattr(toptag, "getroottree"):
+ el = toptag
+ else:
+ el = lxml.etree.Element(toptag)
+ el.getroottree().write(self.name, xml_declaration=False,
+ pretty_print=True)
+
#: Whether or not to monitor this file for changes.
self.should_monitor = should_monitor
if should_monitor:
@@ -471,9 +546,11 @@ class XMLFileBacked(FileBacked):
parent.remove(el)
for extra in extras:
if extra != self.name and extra not in self.extras:
- self.add_monitor(extra)
+ self.extras.append(extra)
lxml.etree.SubElement(parent, xinclude, href=extra)
self._follow_xincludes(fname=extra)
+ if extra not in self.extra_monitors:
+ self.add_monitor(extra)
def Index(self):
self.xdata = lxml.etree.XML(self.data, base_url=self.name,
@@ -502,7 +579,7 @@ class XMLFileBacked(FileBacked):
:type fpath: string
:returns: None
"""
- self.extras.append(fpath)
+ self.extra_monitors.append(fpath)
if self.should_monitor:
self.fam.AddMonitor(fpath, self)
@@ -837,8 +914,8 @@ class XMLSrc(XMLFileBacked):
__cacheobj__ = dict
__priority_required__ = True
- def __init__(self, filename, should_monitor=False):
- XMLFileBacked.__init__(self, filename, should_monitor)
+ def __init__(self, filename, should_monitor=False, create=None):
+ XMLFileBacked.__init__(self, filename, should_monitor, create)
self.items = {}
self.cache = None
self.pnode = None
@@ -938,7 +1015,7 @@ class XMLDirectoryBacked(DirectoryBacked):
#: Only track and include files whose names (not paths) match this
#: compiled regex.
- patterns = re.compile('^.*\.xml$')
+ patterns = re.compile(r'^.*\.xml$')
#: The type of child objects to create for files contained within
#: the directory that is tracked. Default is
@@ -1195,7 +1272,7 @@ class EntrySet(Debuggable):
#: file is encountered that does not match the ``basename``
#: argument passed to the constructor or ``ignore``, then a
#: warning will be produced.
- ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\\.genshi_include)$")
+ ignore = re.compile(r'^(\.#.*|.*~|\..*\.(sw[px])|.*\.genshi_include)$')
# The ``basename`` argument passed to the constructor will be
#: processed as a string that contains a regular expression (i.e.,
@@ -1258,8 +1335,8 @@ class EntrySet(Debuggable):
base_pat = basename
else:
base_pat = re.escape(basename)
- pattern = '(.*/)?%s(\.((H_(?P<hostname>\S+))|' % base_pat
- pattern += '(G(?P<prio>\d+)_(?P<group>\S+))))?$'
+ pattern = r'(.*/)?' + base_pat + \
+ r'(\.((H_(?P<hostname>\S+))|(G(?P<prio>\d+)_(?P<group>\S+))))?$'
#: ``specific`` is a regular expression that is used to
#: determine the specificity of a file in this entry set. It
@@ -1520,8 +1597,6 @@ class GroupSpool(Plugin, Generator):
def __init__(self, core, datastore):
Plugin.__init__(self, core, datastore)
Generator.__init__(self)
- if self.data[-1] == '/':
- self.data = self.data[:-1]
self.fam = Bcfg2.Server.FileMonitor.get_fam()
diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py
index 11a61ff9c..d460cc45d 100644
--- a/src/lib/Bcfg2/Server/Plugin/interfaces.py
+++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py
@@ -337,12 +337,11 @@ class ThreadedStatistics(Statistics, Threaded, threading.Thread):
pending_data = []
try:
while not self.work_queue.empty():
- (metadata, data) = self.work_queue.get_nowait()
- pending_data.append(
- (metadata.hostname,
- lxml.etree.tostring(
- data,
- xml_declaration=False).decode("UTF-8")))
+ (metadata, xdata) = self.work_queue.get_nowait()
+ data = \
+ lxml.etree.tostring(xdata,
+ xml_declaration=False).decode("UTF-8")
+ pending_data.append((metadata.hostname, data))
except Empty:
pass
@@ -409,7 +408,7 @@ class ThreadedStatistics(Statistics, Threaded, threading.Thread):
def run(self):
if not self._load():
return
- while not self.terminate.isSet() and self.work_queue != None:
+ while not self.terminate.isSet() and self.work_queue is not None:
try:
(client, xdata) = self.work_queue.get(block=True, timeout=2)
except Empty:
@@ -419,7 +418,7 @@ class ThreadedStatistics(Statistics, Threaded, threading.Thread):
self.logger.error("ThreadedStatistics: %s" % err)
continue
self.handle_statistic(client, xdata)
- if self.work_queue != None and not self.work_queue.empty():
+ if self.work_queue is not None and not self.work_queue.empty():
self._save()
def process_statistics(self, metadata, data):
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
index b3781e299..e056c871a 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py
@@ -68,7 +68,7 @@ class CfgGenshiGenerator(CfgGenerator):
#: exception in a Genshi template so we can provide a decent error
#: message that actually tells the end user where an error
#: occurred.
- pyerror_re = re.compile('<\w+ u?[\'"](.*?)\s*\.\.\.[\'"]>')
+ pyerror_re = re.compile(r'<\w+ u?[\'"](.*?)\s*\.\.\.[\'"]>')
def __init__(self, fname, spec, encoding):
CfgGenerator.__init__(self, fname, spec, encoding)
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py
index 7ebce192c..735f23a1c 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py
@@ -9,7 +9,7 @@ from Bcfg2.Server.Plugin import StructFile
from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError
from Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator import CfgPublicKeyCreator
try:
- from Bcfg2.Server.Encryption import get_passphrases, ssl_encrypt
+ import Bcfg2.Server.Encryption
HAS_CRYPTO = True
except ImportError:
HAS_CRYPTO = False
@@ -51,8 +51,8 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile):
if (HAS_CRYPTO and
self.setup.cfp.has_section("sshkeys") and
self.setup.cfp.has_option("sshkeys", "passphrase")):
- return get_passphrases()[self.setup.cfp.get("sshkeys",
- "passphrase")]
+ return Bcfg2.Encrypption.get_passphrases()[
+ self.setup.cfp.get("sshkeys", "passphrase")]
return None
def handle_event(self, event):
@@ -72,7 +72,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile):
the given client metadata, and may be obtained by
doing ``self.XMLMatch(metadata)``
:type spec: lxml.etree._Element
- :returns: None
+ :returns: string - The filename of the private key
"""
if spec is None:
spec = self.XMLMatch(metadata)
@@ -142,7 +142,6 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile):
if spec is None:
spec = self.XMLMatch(metadata)
category = spec.get("category", self.category)
- print("category=%s" % category)
if category is None:
per_host_default = "true"
else:
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
index 2301de725..3e464af49 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
@@ -77,7 +77,7 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData,
self.encoding = encoding
self.setup = Bcfg2.Options.get_option_parser()
__init__.__doc__ = Bcfg2.Server.Plugin.SpecificData.__init__.__doc__ + \
-"""
+ """
.. -----
.. autoattribute:: CfgBaseFileMatcher.__basenames__
.. autoattribute:: CfgBaseFileMatcher.__extensions__
@@ -101,12 +101,12 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData,
components = ['^(?P<basename>%s)' % '|'.join(re.escape(b)
for b in basenames)]
if cls.__specific__:
- components.append('(|\\.H_(?P<hostname>\S+?)|' +
- '\.G(?P<prio>\d+)_(?P<group>\S+?))')
+ components.append(r'(|\.H_(?P<hostname>\S+?)|' +
+ r'\.G(?P<prio>\d+)_(?P<group>\S+?))')
if cls.__extensions__:
- components.append('\\.(?P<extension>%s)' %
- '|'.join(cls.__extensions__))
- components.append('$')
+ components.append(r'\.(?P<extension>%s)' %
+ r'|'.join(cls.__extensions__))
+ components.append(r'$')
return re.compile("".join(components))
@classmethod
@@ -563,6 +563,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet,
else:
try:
if not isinstance(data, unicode):
+ if not isinstance(data, str):
+ data = data.decode('utf-8')
data = u_str(data, self.encoding)
except UnicodeDecodeError:
msg = "Failed to decode %s: %s" % (entry.get('name'),
@@ -715,8 +717,8 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet,
pass
if not rv or not rv[0].hostname:
- rv.append(Bcfg2.Server.Plugin.Specificity(
- hostname=metadata.hostname))
+ rv.append(
+ Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname))
return rv
def build_filename(self, specific):
diff --git a/src/lib/Bcfg2/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py
index 33914bd45..316e4bc53 100644
--- a/src/lib/Bcfg2/Server/Plugins/FileProbes.py
+++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py
@@ -67,7 +67,8 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin,
self.config = \
Bcfg2.Server.Plugin.StructFile(os.path.join(self.data,
'config.xml'),
- should_monitor=True)
+ should_monitor=True,
+ create=self.name)
self.entries = dict()
self.probes = dict()
@@ -225,11 +226,8 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin,
root = lxml.etree.Element("FileInfo")
root.append(info)
try:
- open(infoxml,
- "w").write(
- lxml.etree.tostring(root,
- xml_declaration=False,
- pretty_print=True).decode('UTF-8'))
+ root.getroottree().write(infoxml, xml_declaration=False,
+ pretty_print=True)
except IOError:
err = sys.exc_info()[1]
self.logger.error("Could not write %s: %s" % (infoxml, err))
diff --git a/src/lib/Bcfg2/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py
index 781413e1a..58a5c58f0 100644
--- a/src/lib/Bcfg2/Server/Plugins/Git.py
+++ b/src/lib/Bcfg2/Server/Plugins/Git.py
@@ -46,7 +46,7 @@ class Git(Version):
else:
cmd = ["git", "--git-dir", self.vcs_path,
"--work-tree", self.vcs_root, "rev-parse", "HEAD"]
- self.debug_log("Git: Running cmd")
+ self.debug_log("Git: Running %s" % cmd)
result = self.cmd.run(cmd)
if not result.success:
raise Exception(result.stderr)
diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py
new file mode 100644
index 000000000..810b273af
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py
@@ -0,0 +1,47 @@
+""" GroupLogic is a connector plugin that lets you use an XML Genshi
+template to dynamically set additional groups for clients. """
+
+import os
+import lxml.etree
+import Bcfg2.Server.Plugin
+try:
+ from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile
+except ImportError:
+ # BundleTemplateFile missing means that genshi is missing. we
+ # import genshi to get the _real_ error
+ import genshi # pylint: disable=W0611
+
+
+class GroupLogicConfig(BundleTemplateFile):
+ """ Representation of the GroupLogic groups.xml file """
+ create = lxml.etree.Element("GroupLogic",
+ nsmap=dict(py="http://genshi.edgewall.org/"))
+
+ def __init__(self, name, fam):
+ BundleTemplateFile.__init__(self, name,
+ Bcfg2.Server.Plugin.Specificity(), None)
+ self.fam = fam
+ self.should_monitor = True
+ self.fam.AddMonitor(self.name, self)
+
+ def _match(self, item, metadata):
+ if item.tag == 'Group' and not len(item.getchildren()):
+ return [item]
+ return BundleTemplateFile._match(self, item, metadata)
+
+
+class GroupLogic(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Connector):
+ """ GroupLogic is a connector plugin that lets you use an XML
+ Genshi template to dynamically set additional groups for
+ clients. """
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"),
+ core.fam)
+
+ def get_additional_groups(self, metadata):
+ return [el.get("name")
+ for el in self.config.get_xml_value(metadata).findall("Group")]
diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
index 99f01201b..9042a979e 100644
--- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
+++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py
@@ -3,7 +3,6 @@
import os
import re
import sys
-import logging
import Bcfg2.Server.Lint
import Bcfg2.Server.Plugin
from Bcfg2.Utils import PackedDigitRange
@@ -16,16 +15,16 @@ class PatternMap(object):
self.pattern = pattern
self.rangestr = rangestr
self.groups = groups
- if pattern != None:
+ if pattern is not None:
self.re = re.compile(pattern)
self.process = self.process_re
- elif rangestr != None:
+ elif rangestr is not None:
if '\\' in rangestr:
raise Exception("Backslashes are not allowed in NameRanges")
range_finder = r'\[\[[\d\-,]+\]\]'
self.process = self.process_range
- self.re = re.compile('^' + re.sub(range_finder, '(\d+)',
- rangestr))
+ self.re = re.compile(r'^' + re.sub(range_finder, r'(\d+)',
+ rangestr))
dmatcher = re.compile(re.sub(range_finder,
r'\[\[([\d\-,]+)\]\]',
rangestr))
@@ -67,13 +66,13 @@ class PatternMap(object):
class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked):
""" representation of GroupPatterns config.xml """
__identifier__ = None
+ create = 'GroupPatterns'
def __init__(self, filename, core=None):
Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename,
should_monitor=True)
self.core = core
self.patterns = []
- self.logger = logging.getLogger(self.__class__.__name__)
def Index(self):
Bcfg2.Server.Plugin.XMLFileBacked.Index(self)
diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py
index 7f8db7b6d..49e36f72b 100644
--- a/src/lib/Bcfg2/Server/Plugins/Metadata.py
+++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py
@@ -75,6 +75,7 @@ if HAS_DJANGO:
yield client.hostname
def keys(self):
+ """ Get keys for the mapping """
return [c.hostname for c in MetadataClientModel.objects.all()]
def __contains__(self, key):
@@ -94,8 +95,10 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked):
# then we immediately set should_monitor to the proper value,
# so that XInclude'd files get properly watched
fpath = os.path.join(metadata.data, basefile)
+ toptag = os.path.splitext(basefile)[0].title()
Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, fpath,
- should_monitor=False)
+ should_monitor=False,
+ create=toptag)
self.should_monitor = watch_clients
self.metadata = metadata
self.basefile = basefile
@@ -325,6 +328,11 @@ class ClientMetadata(object):
return grp
return ''
+ def __repr__(self):
+ return "%s(%s, profile=%s, groups=%s)" % (self.__class__.__name__,
+ self.hostname,
+ self.profile, self.groups)
+
class MetadataQuery(object):
""" This class provides query methods for the metadata of all
@@ -438,7 +446,7 @@ class MetadataQuery(object):
return [self.by_name(name) for name in self.all_clients()]
-class MetadataGroup(tuple):
+class MetadataGroup(tuple): # pylint: disable=E0012,R0924
""" representation of a metadata group. basically just a named tuple """
# pylint: disable=R0913,W0613
@@ -595,7 +603,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
def _add_xdata(self, config, tag, name, attribs=None, alias=False):
""" Generic method to add XML data (group, client, etc.) """
node = self._search_xdata(tag, name, config.xdata, alias=alias)
- if node != None:
+ if node is not None:
raise Bcfg2.Server.Plugin.MetadataConsistencyError("%s \"%s\" "
"already exists"
% (tag, name))
@@ -655,7 +663,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
def _update_xdata(self, config, tag, name, attribs, alias=False):
""" Generic method to modify XML data (group, client, etc.) """
node = self._search_xdata(tag, name, config.xdata, alias=alias)
- if node == None:
+ if node is None:
self.logger.error("%s \"%s\" does not exist" % (tag, name))
raise Bcfg2.Server.Plugin.MetadataConsistencyError
xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' %
@@ -672,7 +680,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
"""Update a groups attributes."""
if self._use_db:
msg = "Metadata does not support updating groups with " + \
- "use_database enabled"
+ "use_database enabled"
self.logger.error(msg)
raise Bcfg2.Server.Plugin.PluginExecutionError(msg)
else:
@@ -700,7 +708,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
def _remove_xdata(self, config, tag, name):
""" Generic method to remove XML data (group, client, etc.) """
node = self._search_xdata(tag, name, config.xdata)
- if node == None:
+ if node is None:
self.logger.error("%s \"%s\" does not exist" % (tag, name))
raise Bcfg2.Server.Plugin.MetadataConsistencyError
xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' %
@@ -936,16 +944,11 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
if group not in self.groups:
self.debug_log("Client %s set as nonexistent group %s"
% (client, group))
- for gname, ginfo in list(self.groups.items()):
- for group in ginfo.groups:
- if group not in self.groups:
- self.debug_log("Group %s set as nonexistent group %s" %
- (gname, group))
def set_profile(self, client, profile, addresspair):
"""Set group parameter for provided client."""
- self.logger.info("Asserting client %s profile to %s" %
- (client, profile))
+ self.logger.info("Asserting client %s profile to %s" % (client,
+ profile))
if False in list(self.states.values()):
raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not "
"been read yet")
@@ -996,19 +999,18 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
self.clients_xml.write()
def set_version(self, client, version):
- """Set group parameter for provided client."""
- if client in self.clients:
- if client not in self.versions or version != self.versions[client]:
- self.logger.info("Setting client %s version to %s" %
- (client, version))
- if not self._use_db:
- self.update_client(client, dict(version=version))
- self.clients_xml.write()
- self.versions[client] = version
- else:
- msg = "Cannot set version on non-existent client %s" % client
- self.logger.error(msg)
- raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
+ """Set version for provided client."""
+ if client not in self.clients:
+ # this creates the client as a side effect
+ self.get_initial_metadata(client)
+
+ if client not in self.versions or version != self.versions[client]:
+ self.logger.info("Setting client %s version to %s" % (client,
+ version))
+ if not self._use_db:
+ self.update_client(client, dict(version=version))
+ self.clients_xml.write()
+ self.versions[client] = version
def resolve_client(self, addresspair, cleanup_cache=False):
"""Lookup address locally or in DNS to get a hostname."""
@@ -1085,7 +1087,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
raise Bcfg2.Server.Plugin.MetadataRuntimeError("Metadata has not "
"been read yet")
client = client.lower()
-
if client in self.core.metadata_cache:
return self.core.metadata_cache[client]
@@ -1096,6 +1097,29 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
categories = dict()
profile = None
+ def _add_group(grpname):
+ """ Add a group to the set of groups for this client.
+ Handles setting categories and category suppression.
+ Returns the new profile for the client (which might be
+ unchanged). """
+ groups.add(grpname)
+ if grpname in self.groups:
+ group = self.groups[grpname]
+ category = group.category
+ if category:
+ if category in categories:
+ self.logger.warning("%s: Group %s suppressed by "
+ "category %s; %s already a member "
+ "of %s" %
+ (self.name, grpname, category,
+ client, categories[category]))
+ return
+ categories[category] = grpname
+ if not profile and group.is_profile:
+ return grpname
+ else:
+ return profile
+
if client not in self.clients:
pgroup = None
if client in self.clientgroups:
@@ -1105,41 +1129,28 @@ class Metadata(Bcfg2.Server.Plugin.Metadata,
if pgroup:
self.set_profile(client, pgroup, (None, None))
- groups.add(pgroup)
- category = self.groups[pgroup].category
- if category:
- categories[category] = pgroup
- if (pgroup in self.groups and self.groups[pgroup].is_profile):
- profile = pgroup
+ profile = _add_group(pgroup)
else:
msg = "Cannot add new client %s; no default group set" % client
self.logger.error(msg)
raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg)
- if client in self.clientgroups:
- for cgroup in self.clientgroups[client]:
- if cgroup in groups:
- continue
- if cgroup not in self.groups:
- self.groups[cgroup] = MetadataGroup(cgroup)
- category = self.groups[cgroup].category
- if category and category in categories:
- self.logger.warning("%s: Group %s suppressed by "
- "category %s; %s already a member "
- "of %s" %
- (self.name, cgroup, category,
- client, categories[category]))
- continue
- if category:
- categories[category] = cgroup
- groups.add(cgroup)
- # favor client groups for setting profile
- if not profile and self.groups[cgroup].is_profile:
- profile = cgroup
+ for cgroup in self.clientgroups.get(client, []):
+ if cgroup in groups:
+ continue
+ if cgroup not in self.groups:
+ self.groups[cgroup] = MetadataGroup(cgroup)
+ profile = _add_group(cgroup)
groups, categories = self._merge_groups(client, groups,
categories=categories)
+ if len(groups) == 0 and self.default:
+ # no initial groups; add the default profile
+ profile = _add_group(self.default)
+ groups, categories = self._merge_groups(client, groups,
+ categories=categories)
+
bundles = set()
for group in groups:
try:
@@ -1475,6 +1486,7 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
self.duplicate_groups()
self.duplicate_default_groups()
self.duplicate_clients()
+ self.default_is_profile()
@classmethod
def Errors(cls):
@@ -1484,11 +1496,15 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
"non-profile-set-as-profile": "error",
"duplicate-group": "error",
"duplicate-client": "error",
- "multiple-default-groups": "error"}
+ "multiple-default-groups": "error",
+ "default-is-not-profile": "error"}
def deprecated_options(self):
""" check for the location='floating' option, which has been
deprecated in favor of floating='true' """
+ if not hasattr(self.metadata, "clients_xml"):
+ # using metadata database
+ return
clientdata = self.metadata.clients_xml.xdata
for el in clientdata.xpath("//Client"):
loc = el.get("location")
@@ -1514,6 +1530,9 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
def bogus_profiles(self):
""" check for clients that have profiles that are either not
flagged as public groups in groups.xml, or don't exist """
+ if not hasattr(self.metadata, "clients_xml"):
+ # using metadata database
+ return
for client in self.metadata.clients_xml.xdata.findall('.//Client'):
profile = client.get("profile")
if profile not in self.metadata.groups:
@@ -1533,8 +1552,8 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
tag as a definition if it a) has profile or public set; or b)
has any children. """
self.duplicate_entries(
- self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \
- self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"),
+ self.metadata.groups_xml.xdata.xpath("//Groups/Group") +
+ self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"),
"group",
include=lambda g: (g.get("profile") or
g.get("public") or
@@ -1554,6 +1573,9 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
def duplicate_clients(self):
""" check for clients that are defined twice. """
+ if not hasattr(self.metadata, "clients_xml"):
+ # using metadata database
+ return
self.duplicate_entries(
self.metadata.clients_xml.xdata.xpath("//Client"),
"client")
@@ -1574,3 +1596,14 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin):
self.LintError("duplicate-%s" % etype,
"%s %s is defined multiple times:\n%s" %
(etype.title(), ename, "\n".join(els)))
+
+ def default_is_profile(self):
+ """ ensure that the default group is a profile group """
+ if (self.metadata.default and
+ not self.metadata.groups[self.metadata.default].is_profile):
+ xdata = \
+ self.metadata.groups_xml.xdata.xpath("//Group[@name='%s']" %
+ self.metadata.default)[0]
+ self.LintError("default-is-not-profile",
+ "Default group is not a profile group:\n%s" %
+ self.RenderXML(xdata))
diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py
index d5ea0cb24..9603cd518 100644
--- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py
+++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py
@@ -5,23 +5,22 @@ import re
import sys
import glob
import socket
-import Bcfg2.Server.Plugin
+from Bcfg2.Server.Plugin import Plugin, Generator, StructFile, \
+ PluginExecutionError
-class NagiosGen(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Generator):
+class NagiosGen(Plugin, Generator):
""" NagiosGen is a Bcfg2 plugin that dynamically generates Nagios
configuration file based on Bcfg2 data. """
__author__ = 'bcfg-dev@mcs.anl.gov'
line_fmt = '\t%-32s %s'
def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Generator.__init__(self)
+ Plugin.__init__(self, core, datastore)
+ Generator.__init__(self)
self.config = \
- Bcfg2.Server.Plugin.StructFile(os.path.join(self.data,
- 'config.xml'),
- should_monitor=True)
+ StructFile(os.path.join(self.data, 'config.xml'),
+ should_monitor=True, create=self.name)
self.Entries = {'Path':
{'/etc/nagiosgen.status': self.createhostconfig,
'/etc/nagios/nagiosgen.cfg': self.createserverconfig}}
@@ -42,9 +41,9 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin,
try:
host_address = socket.gethostbyname(metadata.hostname)
except socket.gaierror:
- self.logger.error("Failed to find IP address for %s" %
- metadata.hostname)
- raise Bcfg2.Server.Plugin.PluginExecutionError
+ self.logger.error()
+ raise PluginExecutionError("Failed to find IP address for %s" %
+ metadata.hostname)
host_groups = [grp for grp in metadata.groups
if os.path.isfile('%s/%s-group.cfg' % (self.data, grp))]
host_config = ['define host {',
diff --git a/src/lib/Bcfg2/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py
index 8b73f0ba7..18261be10 100644
--- a/src/lib/Bcfg2/Server/Plugins/Ohai.py
+++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py
@@ -2,8 +2,10 @@
operating system using ohai
(http://wiki.opscode.com/display/chef/Ohai) """
-import lxml.etree
import os
+import sys
+import glob
+import lxml.etree
import Bcfg2.Server.Plugin
try:
@@ -31,22 +33,39 @@ class OhaiCache(object):
self.dirname = dirname
self.cache = dict()
+ def hostpath(self, host):
+ """ Get the path to the file that contains Ohai data for the
+ given host """
+ return os.path.join(self.dirname, "%s.json" % host)
+
def __setitem__(self, item, value):
- if value == None:
+ if value is None:
# simply return if the client returned nothing
return
self.cache[item] = json.loads(value)
- open("%s/%s.json" % (self.dirname, item), 'w').write(value)
+ open(self.hostpath(item), 'w').write(value)
def __getitem__(self, item):
if item not in self.cache:
try:
- data = open("%s/%s.json" % (self.dirname, item)).read()
+ data = open(self.hostpath(item)).read()
except:
raise KeyError(item)
self.cache[item] = json.loads(data)
return self.cache[item]
+ def __delitem__(self, item):
+ if item in self.cache:
+ del self.cache[item]
+ try:
+ os.unlink(self.hostpath(item))
+ except:
+ raise IndexError("Could not unlink %s: %s" % (self.hostpath(item),
+ sys.exc_info()[1]))
+
+ def __len__(self):
+ return len(glob.glob(self.hostpath('*')))
+
def __iter__(self):
data = list(self.cache.keys())
data.extend([x[:-5] for x in os.listdir(self.dirname)])
@@ -67,10 +86,6 @@ class Ohai(Bcfg2.Server.Plugin.Plugin,
self.probe = lxml.etree.Element('probe', name='Ohai', source='Ohai',
interpreter='/bin/sh')
self.probe.text = PROBECODE
- try:
- os.stat(self.data)
- except OSError:
- os.makedirs(self.data)
self.cache = OhaiCache(self.data)
def GetProbes(self, _):
diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py
index 490ee6f20..1736becc7 100644
--- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py
+++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py
@@ -15,7 +15,7 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin,
def validate_goals(self, metadata, goals):
"""Verify that we are generating correct old POSIX entries."""
- if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0):
+ if metadata.version_info and metadata.version_info >= (1, 3, 0, '', 0):
# do not care about a client that is _any_ 1.3.0 release
# (including prereleases and RCs)
return
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index 57f802bb5..48c580be1 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -29,8 +29,8 @@ class AptCollection(Collection):
""" Get an APT configuration file (i.e., ``sources.list``).
:returns: string """
- lines = ["# This config was generated automatically by the Bcfg2 " \
- "Packages plugin", '']
+ lines = ["# This config was generated automatically by the Bcfg2 "
+ "Packages plugin", '']
for source in self:
if source.rawurl:
@@ -88,6 +88,8 @@ class AptSource(Source):
self.logger.error("Packages: Failed to read file %s" % fname)
raise
for line in reader.readlines():
+ if not isinstance(line, str):
+ line = line.decode('utf-8')
words = str(line.strip()).split(':', 1)
if words[0] == 'Package':
pkgname = words[1].strip().rstrip()
@@ -99,8 +101,8 @@ class AptSource(Source):
vindex = 0
for dep in words[1].split(','):
if '|' in dep:
- cdeps = [re.sub('\s+', '',
- re.sub('\(.*\)', '', cdep))
+ cdeps = [re.sub(r'\s+', '',
+ re.sub(r'\(.*\)', '', cdep))
for cdep in dep.split('|')]
dyn_dname = "choice-%s-%s-%s" % (pkgname,
barch,
@@ -109,7 +111,7 @@ class AptSource(Source):
bdeps[barch][pkgname].append(dyn_dname)
bprov[barch][dyn_dname] = set(cdeps)
else:
- raw_dep = re.sub('\(.*\)', '', dep)
+ raw_dep = re.sub(r'\(.*\)', '', dep)
raw_dep = raw_dep.rstrip().strip()
bdeps[barch][pkgname].append(raw_dep)
elif words[0] == 'Provides':
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
index e9744c777..aa6127f57 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
@@ -9,6 +9,7 @@ from Bcfg2.Server.Statistics import track_statistics
from Bcfg2.Server.Plugins.Packages.Source import SourceInitError
+# pylint: disable=E0012,R0924
class PackagesSources(Bcfg2.Server.Plugin.StructFile,
Bcfg2.Server.Plugin.Debuggable):
""" PackagesSources handles parsing of the
@@ -17,6 +18,9 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile,
:class:`Bcfg2.Server.Plugins.Packages.Source.Source` object for
each ``Source`` tag. """
+ __identifier__ = None
+ create = "Sources"
+
def __init__(self, filename, cachepath, packages):
"""
:param filename: The full path to ``sources.xml``
@@ -34,14 +38,8 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile,
If ``sources.xml`` cannot be read
"""
Bcfg2.Server.Plugin.Debuggable.__init__(self)
- try:
- Bcfg2.Server.Plugin.StructFile.__init__(self, filename,
- should_monitor=True)
- except OSError:
- err = sys.exc_info()[1]
- msg = "Packages: Failed to read configuration file: %s" % err
- self.logger.error(msg)
- raise Bcfg2.Server.Plugin.PluginInitError(msg)
+ Bcfg2.Server.Plugin.StructFile.__init__(self, filename,
+ should_monitor=True)
#: The full path to the directory where
#: :class:`Bcfg2.Server.Plugins.Packages.Source.Source` data
@@ -124,7 +122,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile,
""" Create a
:class:`Bcfg2.Server.Plugins.Packages.Source.Source` subclass
object from XML representation of a source in ``sources.xml``.
- ``source_from-xml`` determines the appropriate subclass of
+ ``source_from_xml`` determines the appropriate subclass of
``Source`` to instantiate according to the ``type`` attribute
of the ``Source`` tag.
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 5cf90e188..3319dda78 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -52,8 +52,8 @@ import sys
import Bcfg2.Server.Plugin
from Bcfg2.Options import get_option_parser
from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \
- HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \
- urlopen, cPickle, md5
+ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \
+ cPickle, md5
from Bcfg2.Server.Statistics import track_statistics
@@ -66,7 +66,7 @@ def fetch_url(url):
:raises: URLError - Failure fetching URL
:returns: string - the content of the page at the given URL """
if '@' in url:
- mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url)
+ mobj = re.match(r'(\w+://)([^:]+):([^@]+)@(.*)$', url)
if not mobj:
raise ValueError("Invalid URL")
user = mobj.group(2)
@@ -308,7 +308,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
:raises: OSError - If the saved data cannot be read
:raises: cPickle.UnpicklingError - If the saved data is corrupt """
- data = open(self.cachefile)
+ data = open(self.cachefile, 'rb')
(self.pkgnames, self.deps, self.provides,
self.essentialpkgs) = cPickle.load(data)
@@ -608,7 +608,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
self.logger.info("Packages: Updating %s" % url)
fname = self.escape_url(url)
try:
- open(fname, 'w').write(fetch_url(url))
+ open(fname, 'wb').write(fetch_url(url))
except ValueError:
self.logger.error("Packages: Bad url string %s" % url)
raise
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index 4535fb76d..ab96d3f59 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -68,7 +68,7 @@ from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \
# pylint: enable=W0622
from Bcfg2.Server.Plugins.Packages.Collection import Collection
from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \
- fetch_url
+ fetch_url
from Bcfg2.Server.Statistics import track_statistics
LOGGER = logging.getLogger(__name__)
@@ -279,7 +279,7 @@ class YumCollection(Collection):
#: Define a unique cache file for this collection to use
#: for cached yum metadata
self.cachefile = os.path.join(self.cachepath,
- "cache-%s" % self.cachekey)
+ "cache-%s" % self.cachekey)
if not os.path.exists(self.cachefile):
os.mkdir(self.cachefile)
@@ -420,7 +420,7 @@ class YumCollection(Collection):
config.add_section(reponame)
added = True
except ConfigParser.DuplicateSectionError:
- match = re.search("-(\d+)", reponame)
+ match = re.search(r'-(\d+)', reponame)
if match:
rid = int(match.group(1)) + 1
else:
@@ -1153,7 +1153,7 @@ class YumSource(Source):
if entry.get('name').startswith('/'):
self.needed_paths.add(entry.get('name'))
pro = pdata.find(RP + 'provides')
- if pro != None:
+ if pro is not None:
for entry in pro.getchildren():
prov = entry.get('name')
if prov not in self.provides[arch]:
@@ -1169,9 +1169,9 @@ class YumSource(Source):
try:
groupid = group.xpath('id')[0].text
self.yumgroups[groupid] = {'mandatory': list(),
- 'default': list(),
- 'optional': list(),
- 'conditional': list()}
+ 'default': list(),
+ 'optional': list(),
+ 'conditional': list()}
except IndexError:
continue
try:
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index 2175cf0aa..052c362ab 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -5,7 +5,6 @@ determine the completeness of the client configuration. """
import os
import sys
import glob
-import copy
import shutil
import lxml.etree
import Bcfg2.Logger
@@ -20,7 +19,8 @@ from Bcfg2.Server.Statistics import track_statistics
YUM_CONFIG_DEFAULT = "/etc/yum.repos.d/bcfg2.repo"
#: The default path for generated apt configs
-APT_CONFIG_DEFAULT = "/etc/apt/sources.d/bcfg2"
+APT_CONFIG_DEFAULT = \
+ "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list"
class Packages(Bcfg2.Server.Plugin.Plugin,
@@ -177,6 +177,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
for (key, value) in list(attrib.items()):
entry.attrib.__setitem__(key, value)
+ def get_config(self, metadata):
+ """ Get yum/apt config, as a string, for the specified client.
+
+ :param metadata: The client to create the config for.
+ :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata
+ """
+ return self.get_collection(metadata).get_config()
+
def HandleEntry(self, entry, metadata):
""" Bind configuration entries. ``HandleEntry`` handles
entries two different ways:
@@ -226,14 +234,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
return True
elif entry.tag == 'Path':
# managed entries for yum/apt configs
- if (entry.get("name") == \
- self.core.setup.cfp.get("packages",
- "yum_config",
- default=YUM_CONFIG_DEFAULT) or
- entry.get("name") == \
- self.core.setup.cfp.get("packages",
- "apt_config",
- default=APT_CONFIG_DEFAULT)):
+ if (entry.get("name") ==
+ self.core.setup.cfp.get("packages",
+ "yum_config",
+ default=YUM_CONFIG_DEFAULT) or
+ entry.get("name") ==
+ self.core.setup.cfp.get("packages",
+ "apt_config",
+ default=APT_CONFIG_DEFAULT)):
return True
return False
@@ -523,7 +531,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
"""
collection = self.get_collection(metadata)
return dict(sources=collection.get_additional_data(),
- allsources=copy.deepcopy(self.sources))
+ get_config=self.get_config)
def end_client_run(self, metadata):
""" Hook to clear the cache for this client in
diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py
index 2ea5088de..7e4935d74 100644
--- a/src/lib/Bcfg2/Server/Plugins/Probes.py
+++ b/src/lib/Bcfg2/Server/Plugins/Probes.py
@@ -60,12 +60,12 @@ class ClientProbeDataSet(dict):
dict.__init__(self, *args, **kwargs)
-class ProbeData(str):
+class ProbeData(str): # pylint: disable=E0012,R0924
""" a ProbeData object emulates a str object, but also has .xdata,
.json, and .yaml properties to provide convenient ways to use
ProbeData objects as XML, JSON, or YAML data """
def __new__(cls, data):
- return str.__new__(cls, data)
+ return str.__new__(cls, data.encode('utf-8'))
def __init__(self, data): # pylint: disable=W0613
str.__init__(self)
@@ -113,15 +113,15 @@ class ProbeData(str):
class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
""" Handle universal and group- and host-specific probe files """
- ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$")
+ ignore = re.compile(r'^(\.#.*|.*~|\..*\.(tmp|sw[px])|probed\.xml)$')
probename = \
- re.compile("(.*/)?(?P<basename>\S+?)(\.(?P<mode>(?:G\d\d)|H)_\S+)?$")
- bangline = re.compile('^#!\s*(?P<interpreter>.*)$')
+ re.compile(r'(.*/)?(?P<basename>\S+?)(\.(?P<mode>(?:G\d\d)|H)_\S+)?$')
+ bangline = re.compile(r'^#!\s*(?P<interpreter>.*)$')
basename_is_regex = True
def __init__(self, path, encoding, plugin_name):
self.plugin_name = plugin_name
- Bcfg2.Server.Plugin.EntrySet.__init__(self, '[0-9A-Za-z_\-]+', path,
+ Bcfg2.Server.Plugin.EntrySet.__init__(self, r'[0-9A-Za-z_\-]+', path,
Bcfg2.Server.Plugin.SpecificData,
encoding)
Bcfg2.Server.FileMonitor.get_fam().AddMonitor(path, self)
@@ -155,7 +155,20 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet):
probe = lxml.etree.Element('probe')
probe.set('name', os.path.basename(name))
probe.set('source', self.plugin_name)
- probe.text = entry.data
+ if (metadata.version_info and
+ metadata.version_info > (1, 3, 1, '', 0)):
+ try:
+ probe.text = entry.data.decode('utf-8')
+ except AttributeError:
+ probe.text = entry.data
+ else:
+ try:
+ probe.text = entry.data
+ except: # pylint: disable=W0702
+ self.logger.error("Client unable to handle unicode "
+ "probes. Skipping %s" %
+ probe.get('name'))
+ continue
match = self.bangline.match(entry.data.split('\n')[0])
if match:
probe.set('interpreter', match.group('interpreter'))
@@ -211,15 +224,21 @@ class Probes(Bcfg2.Server.Plugin.Probing,
lxml.etree.SubElement(top, 'Client', name=client,
timestamp=str(int(probedata.timestamp)))
for probe in sorted(probedata):
- lxml.etree.SubElement(ctag, 'Probe', name=probe,
- value=str(self.probedata[client][probe]))
+ try:
+ lxml.etree.SubElement(
+ ctag, 'Probe', name=probe,
+ value=str(
+ self.probedata[client][probe]).decode('utf-8'))
+ except AttributeError:
+ lxml.etree.SubElement(
+ ctag, 'Probe', name=probe,
+ value=str(self.probedata[client][probe]))
for group in sorted(self.cgroups[client]):
lxml.etree.SubElement(ctag, "Group", name=group)
try:
- datafile = open(os.path.join(self.data, 'probed.xml'), 'w')
- datafile.write(lxml.etree.tostring(
- top, xml_declaration=False,
- pretty_print='true').decode('UTF-8'))
+ top.getroottree().write(os.path.join(self.data, 'probed.xml'),
+ xml_declaration=False,
+ pretty_print='true')
except IOError:
err = sys.exc_info()[1]
self.logger.error("Failed to write probed.xml: %s" % err)
@@ -234,9 +253,10 @@ class Probes(Bcfg2.Server.Plugin.Probing,
if pdata.data != data:
pdata.data = data
pdata.save()
+
ProbesDataModel.objects.filter(
hostname=client.hostname).exclude(
- probe__in=self.probedata[client.hostname]).delete()
+ probe__in=self.probedata[client.hostname]).delete()
for group in self.cgroups[client.hostname]:
try:
@@ -248,7 +268,7 @@ class Probes(Bcfg2.Server.Plugin.Probing,
grp.save()
ProbesGroupsModel.objects.filter(
hostname=client.hostname).exclude(
- group__in=self.cgroups[client.hostname]).delete()
+ group__in=self.cgroups[client.hostname]).delete()
def load_data(self):
""" Load probe data from the appropriate backend (probed.xml
@@ -322,7 +342,7 @@ class Probes(Bcfg2.Server.Plugin.Probing,
def ReceiveDataItem(self, client, data, cgroups, cprobedata):
"""Receive probe results pertaining to client."""
- if data.text == None:
+ if data.text is None:
self.logger.info("Got null response to probe %s from %s" %
(data.get('name'), client.hostname))
cprobedata[data.get('name')] = ProbeData('')
diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py
index 762f9f8f0..f091acf01 100644
--- a/src/lib/Bcfg2/Server/Plugins/Properties.py
+++ b/src/lib/Bcfg2/Server/Plugins/Properties.py
@@ -216,8 +216,13 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile):
return repr(self.xdata)
-class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
- """ A collection of properties files. """
+class Properties(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Connector,
+ Bcfg2.Server.Plugin.DirectoryBacked):
+ """ The properties plugin maps property files into client metadata
+ instances. """
+
+ #: Extensions that are understood by Properties.
extensions = ["xml"]
if HAS_JSON:
extensions.append("json")
@@ -234,14 +239,17 @@ class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
#: Ignore XML schema (``.xsd``) files
ignore = re.compile(r'.*\.xsd$')
- def __init__(self, data):
- Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, data)
+ def __init__(self, core, datastore):
+ global SETUP # pylint: disable=W0603
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data)
#: Instead of creating children of this object with a static
#: object, we use :func:`property_dispatcher` to create a
#: child of the appropriate subclass of :class:`PropertyFile`
self.__child__ = self.property_dispatcher
- __init__.__doc__ = Bcfg2.Server.Plugin.DirectoryBacked.__init__.__doc__
+ __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__
def property_dispatcher(self, fname):
""" Dispatch an event on a Properties file to the
@@ -262,28 +270,9 @@ class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
raise Bcfg2.Server.Plugin.PluginExecutionError(
"Properties: Unknown extension %s" % fname)
-
-class Properties(Bcfg2.Server.Plugin.Plugin,
- Bcfg2.Server.Plugin.Connector):
- """ The properties plugin maps property files into client metadata
- instances. """
-
- def __init__(self, core, datastore):
- Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
- Bcfg2.Server.Plugin.Connector.__init__(self)
- try:
- self.store = PropDirectoryBacked(self.data)
- except OSError:
- err = sys.exc_info()[1]
- self.logger.error("Error while creating Properties store: %s" %
- err)
- raise Bcfg2.Server.Plugin.PluginInitError
-
- __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__
-
def get_additional_data(self, metadata):
rv = dict()
- for fname, pfile in self.store.entries.items():
+ for fname, pfile in self.entries.items():
rv[fname] = pfile.get_additional_data(metadata)
return rv
get_additional_data.__doc__ = \
diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py
index a6dc2c1ef..3354763d4 100644
--- a/src/lib/Bcfg2/Server/Plugins/Reporting.py
+++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py
@@ -92,10 +92,11 @@ class Reporting(Statistics, Threaded, PullSource, Debuggable):
# try 3 times to store the data
for i in [1, 2, 3]:
try:
- self.transport.store(client.hostname, cdata,
- lxml.etree.tostring(
+ self.transport.store(
+ client.hostname, cdata,
+ lxml.etree.tostring(
stats,
- xml_declaration=False).decode('UTF-8'))
+ xml_declaration=False))
self.debug_log("%s: Queued statistics data for %s" %
(self.__class__.__name__, client.hostname))
return
diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
index fb5bd50bf..1264fd1cf 100644
--- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py
+++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py
@@ -202,10 +202,11 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
if specific.hostname and specific.hostname in names:
hostnames = names[specific.hostname]
elif specific.group:
- hostnames = list(chain(
+ hostnames = list(
+ chain(
*[names[cmeta.hostname]
- for cmeta in \
- mquery.by_groups([specific.group])]))
+ for cmeta in
+ mquery.by_groups([specific.group])]))
elif specific.all:
# a generic key for all hosts? really?
hostnames = list(chain(*list(names.values())))
diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py
index d52d9325c..b21732666 100644
--- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py
+++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py
@@ -69,7 +69,7 @@ class SSLCACertSpec(SSLCAXMLSpec):
def get_spec(self, metadata):
rv = SSLCAXMLSpec.get_spec(self, metadata)
rv['subjectaltname'] = [e.text for e in self.Match(metadata)
- if e.tag == "SubjectAltName"]
+ if e.tag == "subjectAltName"]
return rv
diff --git a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py
index 0aea439f9..c3a2221f6 100644
--- a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py
+++ b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py
@@ -14,7 +14,7 @@ class ServiceCompat(Bcfg2.Server.Plugin.Plugin,
def validate_goals(self, metadata, config):
""" Apply defaults """
- if metadata.version_info and metadata.version_info > (1, 3, 0, '', 0):
+ if metadata.version_info and metadata.version_info >= (1, 3, 0, '', 0):
# do not care about a client that is _any_ 1.3.0 release
# (including prereleases and RCs)
return
diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
index ad3eb65bc..e834759c2 100644
--- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
+++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py
@@ -81,7 +81,7 @@ class TemplateHelper(Bcfg2.Server.Plugin.Plugin,
Bcfg2.Server.Plugin.DirectoryBacked):
""" A plugin to provide helper classes and functions to templates """
__author__ = 'chris.a.st.pierre@gmail.com'
- ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$")
+ ignore = re.compile(r'^(\.#.*|.*~|\..*\.(sw[px])|.*\.py[co])$')
patterns = MODULE_RE
__child__ = HelperModule
diff --git a/src/lib/Bcfg2/Server/Plugins/__init__.py b/src/lib/Bcfg2/Server/Plugins/__init__.py
index 1f85702f0..ad51cf368 100644
--- a/src/lib/Bcfg2/Server/Plugins/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/__init__.py
@@ -1 +1,5 @@
-""" Bcfg2 Plugins """
+"""Imports for Bcfg2.Server.Plugins."""
+
+from Bcfg2.Compat import walk_packages
+
+__all__ = [m[1] for m in walk_packages(path=__path__)]
diff --git a/src/lib/Bcfg2/Server/SSLServer.py b/src/lib/Bcfg2/Server/SSLServer.py
index 28450aa1a..13c756049 100644
--- a/src/lib/Bcfg2/Server/SSLServer.py
+++ b/src/lib/Bcfg2/Server/SSLServer.py
@@ -51,10 +51,11 @@ class XMLRPCDispatcher(SimpleXMLRPCServer.SimpleXMLRPCDispatcher):
allow_none=self.allow_none,
encoding=self.encoding)
except:
+ err = sys.exc_info()
self.logger.error("Unexpected handler error", exc_info=1)
# report exception back to server
raw_response = xmlrpclib.dumps(
- xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value)),
+ xmlrpclib.Fault(1, "%s:%s" % (err[0].__name__, err[1])),
allow_none=self.allow_none, encoding=self.encoding)
return raw_response
@@ -199,8 +200,10 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
try:
username, password = auth_content.split(":")
except TypeError:
+ # pylint: disable=E0602
username, pw = auth_content.split(bytes(":", encoding='utf-8'))
password = pw.decode('utf-8')
+ # pylint: enable=E0602
except ValueError:
username = auth_content
password = ""
diff --git a/src/lib/Bcfg2/Server/Statistics.py b/src/lib/Bcfg2/Server/Statistics.py
index 8a6ff54c4..dfb698b61 100644
--- a/src/lib/Bcfg2/Server/Statistics.py
+++ b/src/lib/Bcfg2/Server/Statistics.py
@@ -30,10 +30,10 @@ class Statistic(object):
:param value: The value to add to this statistic
:type value: int or float
"""
- self.min = min(self.min, value)
- self.max = max(self.max, value)
- self.ave = (((self.ave * (self.count - 1)) + value) / self.count)
+ self.min = min(self.min, float(value))
+ self.max = max(self.max, float(value))
self.count += 1
+ self.ave = (((self.ave * (self.count - 1)) + value) / self.count)
def get_value(self):
""" Get a tuple of all the stats tracked on this named item.
@@ -48,6 +48,11 @@ class Statistic(object):
"""
return (self.name, (self.min, self.max, self.ave, self.count))
+ def __repr__(self):
+ return "%s(%s, (min=%s, avg=%s, max=%s, count=%s))" % (
+ self.__class__.__name__,
+ self.name, self.min, self.ave, self.max, self.count)
+
class Statistics(object):
""" A collection of named :class:`Statistic` objects. """
diff --git a/src/lib/Bcfg2/Server/__init__.py b/src/lib/Bcfg2/Server/__init__.py
index bcf3b4dea..6bd0ff9eb 100644
--- a/src/lib/Bcfg2/Server/__init__.py
+++ b/src/lib/Bcfg2/Server/__init__.py
@@ -1,9 +1,14 @@
"""This is the set of modules for Bcfg2.Server."""
import lxml.etree
+from Bcfg2.Compat import walk_packages
+<<<<<<< HEAD
__all__ = ["Admin", "Core", "FileMonitor", "Plugin", "Plugins",
"Reports", "XMLParser", "XI", "XI_NAMESPACE"]
+=======
+__all__ = [m[1] for m in walk_packages(path=__path__)]
+>>>>>>> maint
XI = 'http://www.w3.org/2001/XInclude'
XI_NAMESPACE = '{%s}' % XI
diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py
index dd76f04d3..ef88a1a36 100644
--- a/src/lib/Bcfg2/Utils.py
+++ b/src/lib/Bcfg2/Utils.py
@@ -22,7 +22,7 @@ class ClassName(object):
return owner.__name__
-class PackedDigitRange(object):
+class PackedDigitRange(object): # pylint: disable=E0012,R0924
""" Representation of a set of integer ranges. A range is
described by a comma-delimited string of integers and ranges,
e.g.::
@@ -108,10 +108,16 @@ class ExecutorResult(object):
def __init__(self, stdout, stderr, retval):
#: The output of the command
- self.stdout = stdout
+ if isinstance(stdout, str):
+ self.stdout = stdout
+ else:
+ self.stdout = stdout.decode('utf-8')
#: The error produced by the command
- self.stderr = stderr
+ if isinstance(stdout, str):
+ self.stderr = stderr
+ else:
+ self.stderr = stderr.decode('utf-8')
#: The return value of the command.
self.retval = retval
@@ -145,6 +151,19 @@ class ExecutorResult(object):
returned a tuple of (return value, stdout split by lines). """
return (self.retval, self.stdout.splitlines())[idx]
+ def __len__(self):
+ """ This provides compatibility with the old Executor, which
+ returned a tuple of (return value, stdout split by lines). """
+ return 2
+
+ def __delitem__(self, _):
+ raise TypeError("'%s' object doesn't support item deletion" %
+ self.__class__.__name__)
+
+ def __setitem__(self, idx, val):
+ raise TypeError("'%s' object does not support item assignment" %
+ self.__class__.__name__)
+
def __nonzero__(self):
return self.__bool__()
@@ -172,7 +191,7 @@ class Executor(object):
:param proc: The process to kill upon timeout.
:type proc: subprocess.Popen
:returns: None """
- if proc.poll() == None:
+ if proc.poll() is None:
try:
proc.kill()
self.logger.warning("Process exceeeded timeout, killing")
@@ -216,6 +235,13 @@ class Executor(object):
for line in inputdata.splitlines():
self.logger.debug('> %s' % line)
(stdout, stderr) = proc.communicate(input=inputdata)
+
+ # py3k fixes
+ if not isinstance(stdout, str):
+ stdout = stdout.decode('utf-8')
+ if not isinstance(stderr, str):
+ stderr = stderr.decode('utf-8')
+
for line in stdout.splitlines(): # pylint: disable=E1103
self.logger.debug('< %s' % line)
for line in stderr.splitlines(): # pylint: disable=E1103
diff --git a/src/lib/Bcfg2/__init__.py b/src/lib/Bcfg2/__init__.py
index 3fe2a0d75..74a871f2a 100644
--- a/src/lib/Bcfg2/__init__.py
+++ b/src/lib/Bcfg2/__init__.py
@@ -1,3 +1 @@
"""Base modules definition."""
-
-__all__ = ['Server', 'Client', 'Logger', 'Options', 'Proxy', 'Statistics']
diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py
index 87f2a0df0..f3697d66f 100644
--- a/src/lib/Bcfg2/settings.py
+++ b/src/lib/Bcfg2/settings.py
@@ -54,11 +54,11 @@ DEFAULT_CONFIG = _default_config()
def read_config(cfile=DEFAULT_CONFIG, repo=None):
""" read the config file and set django settings based on it """
- # pylint: disable=W0603
+ # pylint: disable=W0602,W0603
global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \
DATABASE_HOST, DATABASE_PORT, DEBUG, TEMPLATE_DEBUG, TIME_ZONE, \
MEDIA_URL
- # pylint: enable=W0603
+ # pylint: enable=W0602,W0603
if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG):
print("%s does not exist, using %s for database configuration" %
diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py
index 6f3ba3e49..12fc584fe 100644
--- a/src/lib/Bcfg2/version.py
+++ b/src/lib/Bcfg2/version.py
@@ -5,7 +5,7 @@ import re
__version__ = "1.3.1"
-class Bcfg2VersionInfo(tuple):
+class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924
""" object to make granular version operations (particularly
comparisons) easier """
diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt
index f7deba90c..c4d5f0747 100755
--- a/src/sbin/bcfg2-crypt
+++ b/src/sbin/bcfg2-crypt
@@ -337,7 +337,7 @@ class PropertiesEncryptor(Encryptor):
# actually need to unchunk anything
xdata = data[0]
# find root element
- while xdata.getparent() != None:
+ while xdata.getparent() is not None:
xdata = xdata.getparent()
return lxml.etree.tostring(xdata,
xml_declaration=False,
diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info
index ad35bbeeb..101530cac 100755
--- a/src/sbin/bcfg2-info
+++ b/src/sbin/bcfg2-info
@@ -120,7 +120,6 @@ class InfoCore(cmd.Cmd, Bcfg2.Server.Core.BaseCore):
Bcfg2.Server.Core.BaseCore.__init__(self)
self.prompt = '> '
self.cont = True
- self.fam.handle_events_in_interval(4)
def _get_client_list(self, hostglobs):
""" given a host glob, get a list of clients that match it """
@@ -379,7 +378,7 @@ Bcfg2 client itself.""")
xml_declaration=False).decode('UTF-8')
except Exception:
print("Failed to build entry %s for host %s: %s" %
- (fname, client, traceback.format_exc().splitlines()[-1]))
+ (fname, client, traceback.format_exc().splitlines()[-1]))
raise
try:
if outfile:
@@ -457,9 +456,7 @@ Bcfg2 client itself.""")
def do_clients(self, _):
""" clients - Print out client/profile info """
data = [('Client', 'Profile')]
- clist = self.metadata.clients
- clist.sort()
- for client in clist:
+ for client in sorted(self.metadata.list_clients()):
imd = self.metadata.get_initial_metadata(client)
data.append((client, imd.profile))
print_tabular(data)
@@ -467,19 +464,18 @@ Bcfg2 client itself.""")
def do_config(self, _):
""" config - Print out the current configuration of Bcfg2"""
output = [
- ('Description', 'Value'),
- ('Path Bcfg2 repository', self.setup['repo']),
- ('Plugins', self.setup['plugins']),
- ('Password', self.setup['password']),
- ('Server Metadata Connector', self.setup['mconnect']),
- ('Filemonitor', self.setup['filemonitor']),
- ('Server address', self.setup['location']),
- ('Path to key', self.setup['key']),
- ('Path to SSL certificate', self.setup['cert']),
- ('Path to SSL CA certificate', self.setup['ca']),
- ('Protocol', self.setup['protocol']),
- ('Logging', self.setup['logging'])
- ]
+ ('Description', 'Value'),
+ ('Path Bcfg2 repository', self.setup['repo']),
+ ('Plugins', self.setup['plugins']),
+ ('Password', self.setup['password']),
+ ('Server Metadata Connector', self.setup['mconnect']),
+ ('Filemonitor', self.setup['filemonitor']),
+ ('Server address', self.setup['location']),
+ ('Path to key', self.setup['key']),
+ ('Path to SSL certificate', self.setup['cert']),
+ ('Path to SSL CA certificate', self.setup['ca']),
+ ('Protocol', self.setup['protocol']),
+ ('Logging', self.setup['logging'])]
print_tabular(output)
def do_probes(self, args):
@@ -606,7 +602,7 @@ Bcfg2 client itself.""")
# Dump all mappings unless type specified
data = [('Plugin', 'Type', 'Name')]
arglen = len(args.split())
- for generator in self.generators:
+ for generator in self.plugins_by_type(Bcfg2.Server.Plugin.Generator):
if arglen == 0:
etypes = list(generator.Entries.keys())
else:
@@ -712,6 +708,8 @@ Bcfg2 client itself.""")
def run(self, args): # pylint: disable=W0221
try:
+ self.load_plugins()
+ self.fam.handle_events_in_interval(1)
if args:
self.onecmd(" ".join(args))
else:
@@ -738,7 +736,7 @@ def build_usage():
# shim for python 2.4, __func__ is im_func
funcattr = getattr(attr, "__func__", getattr(attr, "im_func", None))
- if (funcattr != None and
+ if (funcattr is not None and
funcattr.func_name not in cmd_blacklist and
funcattr.func_name.startswith("do_") and
funcattr.func_doc):
diff --git a/src/sbin/bcfg2-report-collector b/src/sbin/bcfg2-report-collector
index 403775251..ae6d3b167 100755
--- a/src/sbin/bcfg2-report-collector
+++ b/src/sbin/bcfg2-report-collector
@@ -12,12 +12,10 @@ from Bcfg2.Reporting.Collector import ReportingCollector, ReportingError
def main():
logger = logging.getLogger('bcfg2-report-collector')
- optinfo = dict(
- daemon=Bcfg2.Options.DAEMON,
- repo=Bcfg2.Options.SERVER_REPOSITORY,
- filemonitor=Bcfg2.Options.SERVER_FILEMONITOR,
- web_configfile=Bcfg2.Options.WEB_CFILE,
- )
+ optinfo = dict(daemon=Bcfg2.Options.DAEMON,
+ repo=Bcfg2.Options.SERVER_REPOSITORY,
+ filemonitor=Bcfg2.Options.SERVER_FILEMONITOR,
+ web_configfile=Bcfg2.Options.WEB_CFILE)
optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS)
optinfo.update(Bcfg2.Options.REPORTING_COMMON_OPTIONS)
setup = Bcfg2.Options.load_option_parser(optinfo)
diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports
index 2c4a918be..bb45e0009 100755
--- a/src/sbin/bcfg2-reports
+++ b/src/sbin/bcfg2-reports
@@ -233,7 +233,8 @@ def main():
try:
entries = [l.strip().split(":")
for l in open(options.file)]
- except IOError, err:
+ except IOError:
+ err = sys.exc_info()[1]
print("Cannot read entries from %s: %s" % (options.file,
err))
return 2
diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper
index 7e5c03fd5..7dbdad16b 100755
--- a/src/sbin/bcfg2-yum-helper
+++ b/src/sbin/bcfg2-yum-helper
@@ -9,33 +9,13 @@ import os
import sys
import yum
import logging
+import Bcfg2.Logger
from optparse import OptionParser
try:
import json
except ImportError:
import simplejson as json
-LOGGER = None
-
-
-def get_logger(verbose=0):
- """ set up logging according to the verbose level given on the
- command line """
- global LOGGER
- if LOGGER is None:
- LOGGER = logging.getLogger(sys.argv[0])
- stderr = logging.StreamHandler()
- if verbose:
- level = logging.DEBUG
- else:
- level = logging.WARNING
- LOGGER.setLevel(level)
- LOGGER.addHandler(stderr)
- syslog = logging.handlers.SysLogHandler("/dev/log")
- syslog.setFormatter(logging.Formatter("%(name)s: %(message)s"))
- LOGGER.addHandler(syslog)
- return LOGGER
-
def pkg_to_tuple(package):
""" json doesn't distinguish between tuples and lists, but yum
@@ -76,7 +56,7 @@ class DepSolver(object):
except AttributeError:
self.yumbase._getConfig(cfgfile, debuglevel=verbose)
# pylint: enable=E1121,W0212
- self.logger = get_logger(verbose)
+ self.logger = logging.getLogger(self.__class__.__name__)
self._groups = None
def get_groups(self):
@@ -220,7 +200,17 @@ def main():
parser.add_option("-v", "--verbose", help="Verbosity level",
action="count")
(options, args) = parser.parse_args()
- logger = get_logger(options.verbose)
+
+ if options.verbose:
+ level = logging.DEBUG
+ clevel = logging.DEBUG
+ else:
+ level = logging.WARNING
+ clevel = logging.INFO
+ Bcfg2.Logger.setup_logging('bcfg2-yum-helper', to_syslog=True,
+ to_console=clevel, level=level)
+ logger = logging.getLogger('bcfg2-yum-helper')
+
try:
cmd = args[0]
except IndexError:
diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py
index 662e0e1b6..8f933e08f 100644
--- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py
+++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/TestFile.py
@@ -64,10 +64,18 @@ class TestPOSIXFile(TestPOSIXTool):
self.assertEqual(ptool._get_data(entry), ("test", True))
entry = copy.deepcopy(orig_entry)
+ entry.set("encoding", "base64")
+ entry.set("empty", "true")
+ self.assertEqual(ptool._get_data(entry), ("", True))
+
+ entry = copy.deepcopy(orig_entry)
entry.set("empty", "true")
self.assertEqual(ptool._get_data(entry), ("", False))
entry = copy.deepcopy(orig_entry)
+ self.assertEqual(ptool._get_data(entry), ("", False))
+
+ entry = copy.deepcopy(orig_entry)
entry.text = "test"
self.assertEqual(ptool._get_data(entry), ("test", False))
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py
index a1e624824..318f5ceaa 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py
@@ -72,14 +72,32 @@ class TestPlugin(TestDebuggable):
if core is None:
core = Mock()
core.setup = MagicMock()
- return self.test_obj(core, datastore)
+ @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock())
+ def inner():
+ return self.test_obj(core, datastore)
+ return inner()
- def test__init(self):
+ @patch("os.makedirs")
+ @patch("os.path.exists")
+ def test__init(self, mock_exists, mock_makedirs):
core = Mock()
core.setup = MagicMock()
+
+ mock_exists.return_value = True
+ p = self.get_obj(core=core)
+ self.assertEqual(p.data, os.path.join(datastore, p.name))
+ self.assertEqual(p.core, core)
+ mock_exists.assert_any_call(p.data)
+ self.assertFalse(mock_makedirs.called)
+
+ mock_exists.reset_mock()
+ mock_makedirs.reset_mock()
+ mock_exists.return_value = False
p = self.get_obj(core=core)
self.assertEqual(p.data, os.path.join(datastore, p.name))
self.assertEqual(p.core, core)
+ mock_exists.assert_any_call(p.data)
+ mock_makedirs.assert_any_call(p.data)
@patch("os.makedirs")
def test_init_repo(self, mock_makedirs):
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py
index 6187880b7..929f665b1 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py
@@ -133,7 +133,11 @@ class TestDirectoryBacked(Bcfg2TestCase):
""" ensure that the child object has the correct interface """
self.assertTrue(hasattr(self.test_obj.__child__, "HandleEvent"))
- def get_obj(self):
+ @patch("os.makedirs", Mock())
+ def get_obj(self, fam=None):
+ if fam is None:
+ fam = Mock()
+
@patch("%s.%s.add_directory_monitor" % (self.test_obj.__module__,
self.test_obj.__name__),
Mock())
@@ -142,12 +146,26 @@ class TestDirectoryBacked(Bcfg2TestCase):
self.test_obj.__name__))
return inner()
- def test__init(self):
+ @patch("os.makedirs")
+ @patch("os.path.exists")
+ def test__init(self, mock_exists, mock_makedirs):
@patch("%s.%s.add_directory_monitor" % (self.test_obj.__module__,
self.test_obj.__name__))
def inner(mock_add_monitor):
db = self.test_obj(datastore)
+ mock_exists.return_value = True
+ mock_add_monitor.assert_called_with('')
+ mock_exists.assert_called_with(db.data)
+ self.assertFalse(mock_makedirs.called)
+
+ mock_add_monitor.reset_mock()
+ mock_exists.reset_mock()
+ mock_makedirs.reset_mock()
+ mock_exists.return_value = False
+ db = self.test_obj(datastore)
mock_add_monitor.assert_called_with('')
+ mock_exists.assert_called_with(db.data)
+ mock_makedirs.assert_called_with(db.data)
inner()
@@ -367,20 +385,24 @@ class TestXMLFileBacked(TestFileBacked):
def get_obj(self, path=None, should_monitor=False):
if path is None:
path = self.path
- return self.test_obj(path, should_monitor=should_monitor)
+
+ @patchIf(not isinstance(os.path.exists, Mock),
+ "os.path.exists", Mock())
+ def inner():
+ return self.test_obj(path, should_monitor=should_monitor)
+ return inner()
@patch("Bcfg2.Server.FileMonitor.get_fam")
def test__init(self, mock_get_fam):
xfb = self.get_obj()
self.assertEqual(xfb.fam, mock_get_fam.return_value)
- if self.should_monitor is not True:
- xfb = self.get_obj()
- self.assertFalse(xfb.fam.AddMonitor.called)
-
- if self.should_monitor is not False:
+ if self.should_monitor:
xfb = self.get_obj(should_monitor=True)
xfb.fam.AddMonitor.assert_called_with(self.path, xfb)
+ else:
+ xfb = self.get_obj()
+ self.assertFalse(xfb.fam.AddMonitor.called)
@patch("glob.glob")
@patch("lxml.etree.parse")
@@ -571,21 +593,21 @@ class TestXMLFileBacked(TestFileBacked):
def test_add_monitor(self):
xfb = self.get_obj()
xfb.add_monitor("/test/test2.xml")
- self.assertIn("/test/test2.xml", xfb.extras)
+ self.assertIn("/test/test2.xml", xfb.extra_monitors)
if self.should_monitor is not True:
xfb = self.get_obj()
xfb.fam = Mock()
xfb.add_monitor("/test/test3.xml")
self.assertFalse(xfb.fam.AddMonitor.called)
- self.assertIn("/test/test3.xml", xfb.extras)
+ self.assertIn("/test/test3.xml", xfb.extra_monitors)
if self.should_monitor is not False:
xfb = self.get_obj(should_monitor=True)
xfb.fam = Mock()
xfb.add_monitor("/test/test4.xml")
xfb.fam.AddMonitor.assert_called_with("/test/test4.xml", xfb)
- self.assertIn("/test/test4.xml", xfb.extras)
+ self.assertIn("/test/test4.xml", xfb.extra_monitors)
class TestStructFile(TestXMLFileBacked):
@@ -1370,13 +1392,18 @@ class TestXMLDirectoryBacked(TestDirectoryBacked):
class TestPrioDir(TestPlugin, TestGenerator, TestXMLDirectoryBacked):
test_obj = PrioDir
- @patch("Bcfg2.Server.Plugin.helpers.%s.add_directory_monitor" %
- test_obj.__name__,
- Mock())
def get_obj(self, core=None):
if core is None:
core = Mock()
- return self.test_obj(core, datastore)
+
+ @patch("%s.%s.add_directory_monitor" %
+ (self.test_obj.__module__, self.test_obj.__name__),
+ Mock())
+ @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock())
+ def inner():
+ return self.test_obj(core, datastore)
+
+ return inner()
def test_HandleEvent(self):
TestXMLDirectoryBacked.test_HandleEvent(self)
@@ -1622,7 +1649,8 @@ class TestEntrySet(TestDebuggable):
bogus)))
for ignore in self.ignore:
- self.assertTrue(eset.ignore.match(ignore))
+ self.assertTrue(eset.ignore.match(ignore),
+ "%s should be ignored but wasn't" % ignore)
self.assertFalse(eset.ignore.match(basename))
self.assertFalse(eset.ignore.match(basename + ".G20_foo"))
@@ -1961,6 +1989,7 @@ class TestGroupSpool(TestPlugin, TestGenerator):
return inner()
def test__init(self):
+ @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock())
@patch("%s.%s.AddDirectoryMonitor" % (self.test_obj.__module__,
self.test_obj.__name__))
def inner(mock_Add):
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py
index 6effe05de..ac0454f84 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testinterfaces.py
@@ -97,11 +97,6 @@ class TestProbing(Bcfg2TestCase):
class TestStatistics(TestPlugin):
test_obj = Statistics
- def get_obj(self, core=None):
- if core is None:
- core = Mock()
- return self.test_obj(core, datastore)
-
def test_process_statistics(self):
s = self.get_obj()
self.assertRaises(NotImplementedError,
@@ -354,12 +349,6 @@ class TestGoalValidator(Bcfg2TestCase):
class TestVersion(TestPlugin):
test_obj = Version
- def get_obj(self, core=None):
- if core is None:
- core = Mock()
- core.setup = MagicMock()
- return self.test_obj(core, datastore)
-
def test_get_revision(self):
d = self.get_obj()
self.assertRaises(NotImplementedError, d.get_revision)
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py
index 154d6a8db..b73670fb7 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgGenshiGenerator.py
@@ -2,6 +2,7 @@ import os
import sys
import lxml.etree
from mock import Mock, MagicMock, patch
+import Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator
from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import *
from Bcfg2.Server.Plugin import PluginExecutionError
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py
index a9346156c..c6e6f5ef7 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestGroupPatterns.py
@@ -92,7 +92,12 @@ class TestPatternFile(TestXMLFileBacked):
core.fam = fam
elif not core:
core = Mock()
- return self.test_obj(path, core=core)
+
+ @patchIf(not isinstance(lxml.etree.Element, Mock),
+ "lxml.etree.Element", Mock())
+ def inner():
+ return self.test_obj(path, core=core)
+ return inner()
@patch("Bcfg2.Server.Plugins.GroupPatterns.PatternMap")
def test_Index(self, mock_PatternMap):
@@ -135,6 +140,14 @@ class TestPatternFile(TestXMLFileBacked):
class TestGroupPatterns(TestPlugin, TestConnector):
test_obj = GroupPatterns
+ def get_obj(self, core=None):
+ @patchIf(not isinstance(lxml.etree.Element, Mock),
+ "lxml.etree.Element", Mock())
+ def inner():
+ return TestPlugin.get_obj(self, core=core)
+ return inner()
+
+
def test_get_additional_groups(self):
gp = self.get_obj()
gp.config = Mock()
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py
index 221eb8a3c..a9e9d9701 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py
@@ -94,7 +94,13 @@ def get_metadata_object(core=None, watch_clients=False, use_db=False):
core.setup = MagicMock()
core.metadata_cache = MagicMock()
core.setup.cfp.getboolean = Mock(return_value=use_db)
- return Metadata(core, datastore, watch_clients=watch_clients)
+
+ @patchIf(not isinstance(os.makedirs, Mock), "os.makedirs", Mock())
+ @patchIf(not isinstance(lxml.etree.Element, Mock),
+ "lxml.etree.Element", Mock())
+ def inner():
+ return Metadata(core, datastore, watch_clients=watch_clients)
+ return inner()
class TestMetadataDB(DBModelTestCase):
@@ -203,7 +209,11 @@ class TestXMLMetadataConfig(TestXMLFileBacked):
def get_obj(self, basefile="clients.xml", core=None, watch_clients=False):
self.metadata = get_metadata_object(core=core,
watch_clients=watch_clients)
- return XMLMetadataConfig(self.metadata, watch_clients, basefile)
+ @patchIf(not isinstance(lxml.etree.Element, Mock),
+ "lxml.etree.Element", Mock())
+ def inner():
+ return XMLMetadataConfig(self.metadata, watch_clients, basefile)
+ return inner()
@patch("Bcfg2.Server.FileMonitor.get_fam", Mock())
def test__init(self):
@@ -1531,7 +1541,11 @@ class TestMetadata_ClientsXML(TestMetadataBase):
metadata = self.get_obj()
fam = Bcfg2.Server.FileMonitor._FAM
Bcfg2.Server.FileMonitor._FAM = MagicMock()
- metadata.clients_xml = metadata._handle_file("clients.xml")
+ @patchIf(not isinstance(lxml.etree.Element, Mock),
+ "lxml.etree.Element", Mock())
+ def inner():
+ metadata.clients_xml = metadata._handle_file("clients.xml")
+ inner()
metadata = TestMetadata.load_clients_data(self, metadata=metadata,
xdata=xdata)
rv = TestMetadataBase.load_clients_data(self, metadata=metadata,
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py
index 958dba4ff..30b08ef2f 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py
@@ -1,7 +1,9 @@
import os
import sys
+import copy
import time
import lxml.etree
+import Bcfg2.version
import Bcfg2.Server
import Bcfg2.Server.Plugin
from mock import Mock, MagicMock, patch
@@ -25,6 +27,47 @@ test_data = dict(a=1, b=[1, 2, 3], c="test",
d=dict(a=1, b=dict(a=1), c=(1, "2", 3)))
+class FakeElement(lxml.etree._Element):
+ getroottree = Mock()
+
+ def __init__(self, el):
+ self._element = el
+
+ def __getattribute__(self, attr):
+ el = lxml.etree._Element.__getattribute__(self,
+ '__dict__')['_element']
+ if attr == "getroottree":
+ return FakeElement.getroottree
+ elif attr == "_element":
+ return el
+ else:
+ return getattr(el, attr)
+
+
+class StoringElement(object):
+ OriginalElement = copy.copy(lxml.etree.Element)
+
+ def __init__(self):
+ self.element = None
+ self.return_value = None
+
+ def __call__(self, *args, **kwargs):
+ self.element = self.OriginalElement(*args, **kwargs)
+ self.return_value = FakeElement(self.element)
+ return self.return_value
+
+
+class StoringSubElement(object):
+ OriginalSubElement = copy.copy(lxml.etree.SubElement)
+
+ def __call__(self, parent, tag, **kwargs):
+ try:
+ return self.OriginalSubElement(parent._element, tag,
+ **kwargs)
+ except AttributeError:
+ return self.OriginalSubElement(parent, tag, **kwargs)
+
+
class FakeList(list):
pass
@@ -173,6 +216,8 @@ group-specific"""
ps.get_matching.return_value = matching
metadata = Mock()
+ metadata.version_info = \
+ Bcfg2.version.Bcfg2VersionInfo(Bcfg2.version.__version__)
pdata = ps.get_probe_data(metadata)
ps.get_matching.assert_called_with(metadata)
# we can't create a matching operator.attrgetter object, and I
@@ -199,9 +244,7 @@ class TestProbes(TestProbing, TestConnector, TestDatabaseBacked):
test_obj = Probes
def get_obj(self, core=None):
- if core is None:
- core = MagicMock()
- return self.test_obj(core, datastore)
+ return TestDatabaseBacked.get_obj(self, core=core)
def get_test_probedata(self):
test_xdata = lxml.etree.Element("test")
@@ -245,9 +288,10 @@ text
# test__init(), which relies on being able to check the calls
# of load_data(), and thus on load_data() being consistently
# mocked)
- @patch("Bcfg2.Server.Plugins.Probes.Probes.load_data", new=load_data)
+ @patch("%s.%s.load_data" % (self.test_obj.__module__,
+ self.test_obj.__name__), new=load_data)
def inner():
- return Probes(core, datastore)
+ return self.get_obj(core)
return inner()
@@ -284,61 +328,71 @@ text
probes._write_data_db.assert_called_with("test")
self.assertFalse(probes._write_data_xml.called)
- @patch("%s.open" % builtins)
- def test__write_data_xml(self, mock_open):
+ def test__write_data_xml(self):
probes = self.get_probes_object(use_db=False)
probes.probedata = self.get_test_probedata()
probes.cgroups = self.get_test_cgroups()
- probes._write_data_xml(None)
-
- mock_open.assert_called_with(os.path.join(datastore, probes.name,
- "probed.xml"), "w")
- data = lxml.etree.XML(mock_open.return_value.write.call_args[0][0])
- self.assertEqual(len(data.xpath("//Client")), 2)
-
- foodata = data.find("Client[@name='foo.example.com']")
- self.assertIsNotNone(foodata)
- self.assertIsNotNone(foodata.get("timestamp"))
- self.assertEqual(len(foodata.findall("Probe")),
- len(probes.probedata['foo.example.com']))
- self.assertEqual(len(foodata.findall("Group")),
- len(probes.cgroups['foo.example.com']))
- xml = foodata.find("Probe[@name='xml']")
- self.assertIsNotNone(xml)
- self.assertIsNotNone(xml.get("value"))
- xdata = lxml.etree.XML(xml.get("value"))
- self.assertIsNotNone(xdata)
- self.assertIsNotNone(xdata.find("test"))
- self.assertEqual(xdata.find("test").get("foo"), "foo")
- text = foodata.find("Probe[@name='text']")
- self.assertIsNotNone(text)
- self.assertIsNotNone(text.get("value"))
- multiline = foodata.find("Probe[@name='multiline']")
- self.assertIsNotNone(multiline)
- self.assertIsNotNone(multiline.get("value"))
- self.assertGreater(len(multiline.get("value").splitlines()), 1)
-
- bardata = data.find("Client[@name='bar.example.com']")
- self.assertIsNotNone(bardata)
- self.assertIsNotNone(bardata.get("timestamp"))
- self.assertEqual(len(bardata.findall("Probe")),
- len(probes.probedata['bar.example.com']))
- self.assertEqual(len(bardata.findall("Group")),
- len(probes.cgroups['bar.example.com']))
- empty = bardata.find("Probe[@name='empty']")
- self.assertIsNotNone(empty)
- self.assertIsNotNone(empty.get("value"))
- self.assertEqual(empty.get("value"), "")
- if HAS_JSON:
- jdata = bardata.find("Probe[@name='json']")
- self.assertIsNotNone(jdata)
- self.assertIsNotNone(jdata.get("value"))
- self.assertItemsEqual(test_data, json.loads(jdata.get("value")))
- if HAS_YAML:
- ydata = bardata.find("Probe[@name='yaml']")
- self.assertIsNotNone(ydata)
- self.assertIsNotNone(ydata.get("value"))
- self.assertItemsEqual(test_data, yaml.load(ydata.get("value")))
+
+ @patch("lxml.etree.Element")
+ @patch("lxml.etree.SubElement", StoringSubElement())
+ def inner(mock_Element):
+ mock_Element.side_effect = StoringElement()
+ probes._write_data_xml(None)
+
+ top = mock_Element.side_effect.return_value
+ write = top.getroottree.return_value.write
+ self.assertEqual(write.call_args[0][0],
+ os.path.join(datastore, probes.name,
+ "probed.xml"))
+
+ data = top._element
+ foodata = data.find("Client[@name='foo.example.com']")
+ self.assertIsNotNone(foodata)
+ self.assertIsNotNone(foodata.get("timestamp"))
+ self.assertEqual(len(foodata.findall("Probe")),
+ len(probes.probedata['foo.example.com']))
+ self.assertEqual(len(foodata.findall("Group")),
+ len(probes.cgroups['foo.example.com']))
+ xml = foodata.find("Probe[@name='xml']")
+ self.assertIsNotNone(xml)
+ self.assertIsNotNone(xml.get("value"))
+ xdata = lxml.etree.XML(xml.get("value"))
+ self.assertIsNotNone(xdata)
+ self.assertIsNotNone(xdata.find("test"))
+ self.assertEqual(xdata.find("test").get("foo"), "foo")
+ text = foodata.find("Probe[@name='text']")
+ self.assertIsNotNone(text)
+ self.assertIsNotNone(text.get("value"))
+ multiline = foodata.find("Probe[@name='multiline']")
+ self.assertIsNotNone(multiline)
+ self.assertIsNotNone(multiline.get("value"))
+ self.assertGreater(len(multiline.get("value").splitlines()), 1)
+
+ bardata = data.find("Client[@name='bar.example.com']")
+ self.assertIsNotNone(bardata)
+ self.assertIsNotNone(bardata.get("timestamp"))
+ self.assertEqual(len(bardata.findall("Probe")),
+ len(probes.probedata['bar.example.com']))
+ self.assertEqual(len(bardata.findall("Group")),
+ len(probes.cgroups['bar.example.com']))
+ empty = bardata.find("Probe[@name='empty']")
+ self.assertIsNotNone(empty)
+ self.assertIsNotNone(empty.get("value"))
+ self.assertEqual(empty.get("value"), "")
+ if HAS_JSON:
+ jdata = bardata.find("Probe[@name='json']")
+ self.assertIsNotNone(jdata)
+ self.assertIsNotNone(jdata.get("value"))
+ self.assertItemsEqual(test_data,
+ json.loads(jdata.get("value")))
+ if HAS_YAML:
+ ydata = bardata.find("Probe[@name='yaml']")
+ self.assertIsNotNone(ydata)
+ self.assertIsNotNone(ydata.get("value"))
+ self.assertItemsEqual(test_data,
+ yaml.load(ydata.get("value")))
+
+ inner()
@skipUnless(HAS_DJANGO, "Django not found, skipping")
def test__write_data_db(self):
@@ -410,18 +464,24 @@ text
probes._load_data_db.assert_any_call()
self.assertFalse(probes._load_data_xml.called)
- @patch("%s.open" % builtins)
@patch("lxml.etree.parse")
- def test__load_data_xml(self, mock_parse, mock_open):
+ def test__load_data_xml(self, mock_parse):
probes = self.get_probes_object(use_db=False)
- # to get the value for lxml.etree.parse to parse, we call
- # _write_data_xml, mock the open() call, and grab the data
- # that gets "written" to probed.xml
probes.probedata = self.get_test_probedata()
probes.cgroups = self.get_test_cgroups()
- probes._write_data_xml(None)
- xdata = \
- lxml.etree.XML(str(mock_open.return_value.write.call_args[0][0]))
+
+ # to get the value for lxml.etree.parse to parse, we call
+ # _write_data_xml, mock the lxml.etree._ElementTree.write()
+ # call, and grab the data that gets "written" to probed.xml
+ @patch("lxml.etree.Element")
+ @patch("lxml.etree.SubElement", StoringSubElement())
+ def inner(mock_Element):
+ mock_Element.side_effect = StoringElement()
+ probes._write_data_xml(None)
+ top = mock_Element.side_effect.return_value
+ return top._element
+
+ xdata = inner()
mock_parse.return_value = xdata.getroottree()
probes.probedata = dict()
probes.cgroups = dict()
@@ -559,5 +619,3 @@ text
metadata.hostname = "nonexistent"
self.assertEqual(probes.get_additional_data(metadata),
ClientProbeDataSet())
-
-
diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py
index b63d08524..92dc85fb1 100644
--- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py
+++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProperties.py
@@ -295,8 +295,8 @@ class TestXMLPropertyFile(TestPropertyFile, TestStructFile):
self.assertFalse(mock_copy.called)
-class TestPropDirectoryBacked(TestDirectoryBacked):
- test_obj = PropDirectoryBacked
+class TestProperties(TestPlugin, TestConnector, TestDirectoryBacked):
+ test_obj = Properties
testfiles = ['foo.xml', 'bar.baz.xml']
if HAS_JSON:
testfiles.extend(["foo.json", "foo.xml.json"])
@@ -305,16 +305,13 @@ class TestPropDirectoryBacked(TestDirectoryBacked):
ignore = ['foo.xsd', 'bar.baz.xsd', 'quux.xml.xsd']
badevents = ['bogus.txt']
-
-class TestProperties(TestPlugin, TestConnector):
- test_obj = Properties
-
- def test__init(self):
- TestPlugin.test__init(self)
-
- core = Mock()
- p = self.get_obj(core=core)
- self.assertIsInstance(p.store, PropDirectoryBacked)
+ def get_obj(self, core=None):
+ @patch("%s.%s.add_directory_monitor" % (self.test_obj.__module__,
+ self.test_obj.__name__),
+ Mock())
+ def inner():
+ return TestPlugin.get_obj(self, core=core)
+ return inner()
@patch("copy.copy")
def test_get_additional_data(self, mock_copy):
@@ -322,11 +319,11 @@ class TestProperties(TestPlugin, TestConnector):
p = self.get_obj()
metadata = Mock()
- p.store.entries = {"foo.xml": Mock(),
- "foo.yml": Mock()}
+ p.entries = {"foo.xml": Mock(),
+ "foo.yml": Mock()}
rv = p.get_additional_data(metadata)
expected = dict()
- for name, entry in p.store.entries.items():
+ for name, entry in p.entries.items():
entry.get_additional_data.assert_called_with(metadata)
expected[name] = entry.get_additional_data.return_value
self.assertItemsEqual(rv, expected)
diff --git a/testsuite/Testsrc/Testlib/TestStatistics.py b/testsuite/Testsrc/Testlib/TestStatistics.py
new file mode 100644
index 000000000..496cbac28
--- /dev/null
+++ b/testsuite/Testsrc/Testlib/TestStatistics.py
@@ -0,0 +1,44 @@
+import os
+import sys
+from mock import Mock, MagicMock, patch
+
+# add all parent testsuite directories to sys.path to allow (most)
+# relative imports in python 2.4
+path = os.path.dirname(__file__)
+while path != "/":
+ if os.path.basename(path).lower().startswith("test"):
+ sys.path.append(path)
+ if os.path.basename(path) == "testsuite":
+ break
+ path = os.path.dirname(path)
+from common import *
+
+from Bcfg2.Statistics import *
+
+
+class TestStatistic(Bcfg2TestCase):
+ def test_stat(self):
+ stat = Statistic("test", 1)
+ self.assertEqual(stat.get_value(), ("test", (1.0, 1.0, 1.0, 1)))
+ stat.add_value(10)
+ self.assertEqual(stat.get_value(), ("test", (1.0, 10.0, 5.5, 2)))
+ stat.add_value(100)
+ self.assertEqual(stat.get_value(), ("test", (1.0, 100.0, 37.0, 3)))
+ stat.add_value(12.345)
+ self.assertEqual(stat.get_value(), ("test", (1.0, 100.0, 30.83625, 4)))
+ stat.add_value(0.655)
+ self.assertEqual(stat.get_value(), ("test", (0.655, 100.0, 24.8, 5)))
+
+
+class TestStatistics(Bcfg2TestCase):
+ def test_stats(self):
+ stats = Statistics()
+ self.assertEqual(stats.display(), dict())
+ stats.add_value("test1", 1)
+ self.assertEqual(stats.display(), dict(test1=(1.0, 1.0, 1.0, 1)))
+ stats.add_value("test2", 1.23)
+ self.assertEqual(stats.display(), dict(test1=(1.0, 1.0, 1.0, 1),
+ test2=(1.23, 1.23, 1.23, 1)))
+ stats.add_value("test1", 10)
+ self.assertEqual(stats.display(), dict(test1=(1.0, 10.0, 5.5, 2),
+ test2=(1.23, 1.23, 1.23, 1)))
diff --git a/testsuite/before_install.sh b/testsuite/before_install.sh
index 884971e45..5f1a59aaf 100755
--- a/testsuite/before_install.sh
+++ b/testsuite/before_install.sh
@@ -2,9 +2,12 @@
# before_install script for Travis-CI
+PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))')
+
sudo apt-get update -qq
-sudo apt-get install -qq swig pylint libxml2-utils
+sudo apt-get install -qq swig libxml2-utils
if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then
- sudo apt-get install -qq python-selinux python-pylibacl python-pyinotify \
- python-yaml yum
+ if [[ ${PYVER:0:1} == "2" ]]; then
+ sudo apt-get install -qq python-selinux python-pylibacl yum
+ fi
fi
diff --git a/testsuite/install.sh b/testsuite/install.sh
index c1685f831..817ed5911 100755
--- a/testsuite/install.sh
+++ b/testsuite/install.sh
@@ -7,12 +7,16 @@ pip install -r testsuite/requirements.txt --use-mirrors
PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))')
if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then
+ pip install --use-mirrors genshi PyYAML pyinotify
if [[ $PYVER == "2.5" ]]; then
- # markdown 2.2.0 is broken on py2.5, so until 2.2.1 is released use 2.1
- pip install --use-mirrors 'markdown<2.2'
- pip install --use-mirrors simplejson
+ # markdown 2.2+ doesn't work on py2.5
+ pip install --use-mirrors simplejson 'markdown<2.2'
+ fi
+ if [[ ${PYVER:0:1} == "2" ]]; then
+ # django supports py3k, but South doesn't, and the django bits
+ # in bcfg2 require South
+ pip install cheetah 'django<1.5' South M2Crypto
fi
- pip install --use-mirrors genshi cheetah 'django<1.4' South M2Crypto
else
# python < 2.6 requires M2Crypto for SSL communication, not just
# for encryption support
diff --git a/testsuite/requirements.txt b/testsuite/requirements.txt
index 8529b247f..2d6dbc557 100644
--- a/testsuite/requirements.txt
+++ b/testsuite/requirements.txt
@@ -2,4 +2,6 @@ lxml
nose
mock
sphinx
-daemon
+pylint
+pep8
+python-daemon
diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py
index 2b9d39342..edb5a7101 100755
--- a/tools/bcfg2_local.py
+++ b/tools/bcfg2_local.py
@@ -64,6 +64,12 @@ class LocalClient(Client):
def main():
optinfo = Bcfg2.Options.CLIENT_COMMON_OPTIONS
optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS)
+ if 'bundle_quick' in optinfo:
+ # CLIENT_BUNDLEQUICK option uses -Q, just like the server repo
+ # option. the server repo is more important for this
+ # application.
+ optinfo['bundle_quick'] = Bcfg2.Options.Option('bundlequick',
+ default=False)
setup = Bcfg2.Options.OptionParser(optinfo)
setup.parse(sys.argv[1:])
diff --git a/tools/selinux_baseline.py b/tools/selinux_baseline.py
index b6997bb29..06f6e6b98 100755
--- a/tools/selinux_baseline.py
+++ b/tools/selinux_baseline.py
@@ -42,7 +42,7 @@ def main():
baseline.append(lxml.etree.Comment("%s entries" % etype))
extra = handler.FindExtra()
for entry in extra:
- entry.tag = "BoundSELinux"
+ entry.tag = "Bound%s" % etype
baseline.extend(extra)
print(lxml.etree.tostring(baseline, pretty_print=True))
diff --git a/tools/upgrade/1.3/migrate_perms_to_mode.py b/tools/upgrade/1.3/migrate_perms_to_mode.py
index e061558d3..18abffec2 100755
--- a/tools/upgrade/1.3/migrate_perms_to_mode.py
+++ b/tools/upgrade/1.3/migrate_perms_to_mode.py
@@ -13,6 +13,7 @@ def setmodeattr(elem):
elem.set('mode', elem.get('perms'))
del elem.attrib['perms']
return True
+ return False
def writefile(f, xdata):
@@ -32,7 +33,7 @@ def convertinfo(ifile):
return
found = False
for i in xdata.findall('//Info'):
- found = setmodeattr(i)
+ found |= setmodeattr(i)
if found:
writefile(ifile, xdata)
@@ -47,7 +48,7 @@ def convertstructure(structfile):
return
found = False
for path in xdata.xpath('//BoundPath|//Path'):
- found = setmodeattr(path)
+ found |= setmodeattr(path)
if found:
writefile(structfile, xdata)