From b573521f7c2d3262171ea2890e3e9b4c4e759661 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 25 Sep 2013 16:27:44 -0400 Subject: docs: clarify when JSON/YAML Properties files were added --- doc/server/plugins/connectors/properties.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/server/plugins/connectors/properties.txt b/doc/server/plugins/connectors/properties.txt index 836524def..47e82fdbf 100644 --- a/doc/server/plugins/connectors/properties.txt +++ b/doc/server/plugins/connectors/properties.txt @@ -181,6 +181,8 @@ XML tag should be ````. JSON Property Files ------------------- +.. versionadded:: 1.3.0 + The data in a JSON property file can be accessed with the ``json`` attribute, which is the loaded JSON data. The JSON properties interface does not provide any additional functionality beyond the @@ -189,6 +191,8 @@ interface does not provide any additional functionality beyond the YAML Property Files ------------------- +.. versionadded:: 1.3.0 + The data in a YAML property file can be accessed with the ``yaml`` attribute, which is the loaded YAML data. Only a single YAML document may be included in a file. -- cgit v1.2.3-1-g7c22 From 35b53c77c4b7edad7cf84146abf5722ea5323eba Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 26 Sep 2013 13:48:43 -0400 Subject: New plugin: AWSTags AWSTags allows querying tags from EC2, and setting groups based on the tag names or values. --- doc/development/lint.txt | 5 + doc/server/plugins/connectors/awstags.txt | 124 ++++++++++++ misc/bcfg2.spec | 1 + schemas/awstags.xsd | 73 +++++++ src/lib/Bcfg2/Server/Lint/Validate.py | 1 + src/lib/Bcfg2/Server/Plugins/AWSTags.py | 217 +++++++++++++++++++++ .../Testlib/TestServer/TestPlugins/TestAWSTags.py | 140 +++++++++++++ 7 files changed, 561 insertions(+) create mode 100644 doc/server/plugins/connectors/awstags.txt create mode 100644 schemas/awstags.xsd create mode 100644 src/lib/Bcfg2/Server/Plugins/AWSTags.py create mode 100644 testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py diff --git a/doc/development/lint.txt b/doc/development/lint.txt index 6a4651f92..6c0be960d 100644 --- a/doc/development/lint.txt +++ b/doc/development/lint.txt @@ -106,6 +106,11 @@ Basics Existing ``bcfg2-lint`` Plugins =============================== +AWSTagsLint +----------- + +.. autoclass:: Bcfg2.Server.Plugins.AWSTags.AWSTagsLint + BundlerLint ----------- diff --git a/doc/server/plugins/connectors/awstags.txt b/doc/server/plugins/connectors/awstags.txt new file mode 100644 index 000000000..b884ca065 --- /dev/null +++ b/doc/server/plugins/connectors/awstags.txt @@ -0,0 +1,124 @@ +.. -*- mode: rst -*- + +.. _server-plugins-connectors-awstags: + +========= + AWSTags +========= + +The AWSTags plugin is a connector that retrieves tags from instances +in EC2, and can assign optionally assign +group membership pased on patterns in the tags. See `Using Tags +`_ +for details on using tags in EC2. + +AWSTags queries EC2 for instances whose ``private-dns-name`` property +matches the hostname of the client. + +Setup +===== + +#. Add ``AWSTags`` to the ``plugins`` option in ``/etc/bcfg2.conf`` +#. Configure AWS credentials in ``/etc/bcfg2.conf`` (See + `Configuration`_ below for details.) +#. Optionally, create ``AWSTags/config.xml`` (See `Assigning Groups`_ + below for details.) +#. Restart the Bcfg2 server. + +Using Tag Data +============== + +AWSTags exposes the data in templates as a dict available as +``metadata.AWSTags``. E.g., in a :ref:`Genshi template +`, you could do: + +.. code-block:: genshitext + + Known tags on ${metadata.hostname}: + {% for key, val in metadata.AWSTags.items() %}\ + ${key} ${val} + {% end %}\ + +This would produce something like:: + + Known tags on foo.example.com: + Name foo.example.com + some random tag the value + +Assigning Groups +================ + +AWSTags can assign groups based on the tag data. This functionality +is configured in ``AWSTags/config.xml``. + +Example +------- + +.. code-block:: xml + + + + foo + + + bar + + + $1 + + + +In this example, any machine with a tag named ``foo`` would be added +to the ``foo`` group. Any machine with a tag named ``bar`` whose +value was also ``bar`` would be added to the ``bar`` group. Finally, +any machine with a tag named ``bcfg2 group`` would be added to the +group named in the value of that tag. + +Note that both the ``name`` and ``value`` attributes are *always* +regular expressions. + +If a ```` element has only a ``name`` attribute, then it only +checks for existence of a matching tag. If it has both ``name`` and +``value``, then it checks for a matching tag with a matching value. + +You can use backreferences (``$1``, ``$2``, etc.) in the group names. +If only ``name`` is specified, then the backreferences will refer to +groups in the ``name`` regex. If ``name`` and ``value`` are both +specified, then backreferences will refer to groups in the ``value`` +regex. If you specify both ``name`` and ``value``, it is not possible +to refer to groups in the ``name`` regex. + +Schema Reference +---------------- + +.. xml:schema:: awstags.xsd + +Configuration +============= + +AWSTags recognizes several options in ``/etc/bcfg2.conf``; at a +minimum, you must configure an AWS access key ID and secret key. All +of the following options are in the ``[awstags]`` section: + ++-----------------------+-----------------------------------------------------+ +| Option | Description | ++=======================+=====================================================+ +| ``access_key_id`` | The AWS access key ID | ++-----------------------+-----------------------------------------------------+ +| ``secret_access_key`` | The AWS secret access key | ++-----------------------+-----------------------------------------------------+ +| ``cache`` | Whether or not to cache tag lookups. See `Caching`_ | +| | for details. Default is to cache. | ++-----------------------+-----------------------------------------------------+ + +Caching +======= + +Since the AWS API isn't always very quick to respond, AWSTags caches +its results by default. The cache is fairly short-lived: the cache +for each host is expired when it starts a client run, so it will start +the run with fresh data. + +If you frequently update tags on your instances, you may wish to +disable caching. That's probably a bad idea, and would tend to +suggest that updating tags frequently is perhaps the Wrong Thing. diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 47a1a9548..66c0c8de1 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -188,6 +188,7 @@ Group: Documentation/HTML %else Group: Documentation %endif +BuildRequires: python-boto %if 0%{?suse_version} BuildRequires: python-M2Crypto BuildRequires: python-Genshi diff --git a/schemas/awstags.xsd b/schemas/awstags.xsd new file mode 100644 index 000000000..72be0366f --- /dev/null +++ b/schemas/awstags.xsd @@ -0,0 +1,73 @@ + + + + + + :ref:`AWSTags <server-plugins-connectors-awstags>` config + schema for bcfg2 + + + + + + + + + + + The group to assign to machines with tags that match the + enclosing Tag expression. More than one group can be + specified. + + + + + + + + The name pattern to match against. This is a regular + expression. It is not anchored. + + + + + + + The value pattern to match against. This is a regular + expression. It is not anchored. + + + + + + + + + Top-level tag for ``AWSTags/config.xml``. + + + + + + + Representation of a pattern that matches AWS tags. Tags can be + matched in one of two ways: + + * If only :xml:attribute:`TagType:name` is specified, then + AWSTags will only look for a tag with a matching name, and + the value of tags is ignored. + * If both :xml:attribute:`TagType:name` and + :xml:attribute:`TagType:value` are specified, a tag must + have a matching name *and* a matching value. + + + + + + + + + + diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 09f3f3d25..c537877a0 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -47,6 +47,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Decisions/*.xml": "decisions.xsd", "Packages/sources.xml": "packages.xsd", "GroupPatterns/config.xml": "grouppatterns.xsd", + "AWSTags/config.xml": "awstags.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", "SSLCA/**/cert.xml": "sslca-cert.xsd", diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py new file mode 100644 index 000000000..962b5d6a1 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -0,0 +1,217 @@ +""" Query tags from AWS via boto, optionally setting group membership """ + +import os +import re +import sys +import Bcfg2.Server.Lint +import Bcfg2.Server.Plugin +from boto import connect_ec2 +from Bcfg2.Cache import Cache +from Bcfg2.Compat import ConfigParser + + +class NoInstanceFound(Exception): + """ Raised when there's no AWS instance for a given hostname """ + + +class AWSTagPattern(object): + """ Handler for a single Tag entry """ + + def __init__(self, name, value, groups): + self.name = re.compile(name) + if value is not None: + self.value = re.compile(value) + else: + self.value = value + self.groups = groups + + def get_groups(self, tags): + """ Get groups that apply to the given tag set """ + for key, value in tags.items(): + name_match = self.name.search(key) + if name_match: + if self.value is not None: + value_match = self.value.search(value) + if value_match: + return self._munge_groups(value_match) + else: + return self._munge_groups(name_match) + break + return [] + + def _munge_groups(self, match): + """ Replace backreferences (``$1``, ``$2``) in Group tags with + their values in the regex. """ + rv = [] + sub = match.groups() + for group in self.groups: + newg = group + for idx in range(len(sub)): + newg = newg.replace('$%s' % (idx + 1), sub[idx]) + rv.append(newg) + return rv + + def __str__(self): + if self.value: + return "%s: %s=%s: %s" % (self.__class__.__name__, self.name, + self.value, self.groups) + else: + return "%s: %s: %s" % (self.__class__.__name__, self.name, + self.groups) + + +class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): + """ representation of AWSTags config.xml """ + __identifier__ = None + create = 'AWSTags' + + def __init__(self, filename, core=None): + try: + fam = core.fam + except AttributeError: + fam = None + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, + should_monitor=True) + self.core = core + self.tags = [] + + def Index(self): + Bcfg2.Server.Plugin.XMLFileBacked.Index(self) + if (self.core and + self.core.metadata_cache_mode in ['cautious', 'aggressive']): + self.core.metadata_cache.expire() + self.tags = [] + for entry in self.xdata.xpath('//Tag'): + try: + groups = [g.text for g in entry.findall('Group')] + self.tags.append(AWSTagPattern(entry.get("name"), + entry.get("value"), + groups)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to initialize pattern %s: " + "%s" % (entry.get("name"), + sys.exc_info()[1])) + + def get_groups(self, hostname, tags): + """ return a list of groups that should be added to the given + client based on patterns that match the hostname """ + ret = [] + for pattern in self.tags: + try: + ret.extend(pattern.get_groups(tags)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to process pattern %s for " + "%s" % (pattern, hostname), + exc_info=1) + return ret + + +class AWSTags(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, + Bcfg2.Server.Plugin.ClientRunHooks, + Bcfg2.Server.Plugin.Connector): + """ Query tags from AWS via boto, optionally setting group membership """ + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache'] + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) + try: + key_id = self.core.setup.cfp.get("awstags", "access_key_id") + secret_key = self.core.setup.cfp.get("awstags", + "secret_access_key") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + err = sys.exc_info()[1] + raise Bcfg2.Server.Plugin.PluginInitError( + "AWSTags is not configured in bcfg2.conf: %s" % err) + self.debug_log("%s: Connecting to EC2" % self.name) + self._ec2 = connect_ec2(aws_access_key_id=key_id, + aws_secret_access_key=secret_key) + self._tagcache = Cache() + try: + self._keep_cache = self.core.setup.cfp.getboolean("awstags", + "cache") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self._keep_cache = True + + self.config = PatternFile(os.path.join(self.data, 'config.xml'), + core=core) + + def _load_instance(self, hostname): + """ Load an instance from EC2 whose private DNS name matches + the given hostname """ + self.debug_log("AWSTags: Loading instance with private-dns-name=%s" % + hostname) + filters = {'private-dns-name': hostname} + reservations = self._ec2.get_all_instances(filters=filters) + if reservations: + res = reservations[0] + if res.instances: + return res.instances[0] + raise NoInstanceFound( + "AWSTags: No instance found with private-dns-name=%s" % + hostname) + + def _get_tags_from_ec2(self, hostname): + """ Get tags for the given host from EC2. This does not use + the local caching layer. """ + self.debug_log("AWSTags: Getting tags for %s from AWS" % + hostname) + try: + return self._load_instance(hostname).tags + except NoInstanceFound: + self.debug_log(sys.exc_info()[1]) + return dict() + + def get_tags(self, metadata): + """ Get tags for the given host. This caches the tags locally + if 'cache' in the ``[awstags]`` section of ``bcfg2.conf`` is + true. """ + if not self._keep_cache: + return self._get_tags_from_ec2(metadata) + + if metadata.hostname not in self._tagcache: + self._tagcache[metadata.hostname] = \ + self._get_tags_from_ec2(metadata.hostname) + return self._tagcache[metadata.hostname] + + def expire_cache(self, key=None): + self._tagcache.expire(key=key) + + def start_client_run(self, metadata): + self.expire_cache(self, key=metadata.hostname) + + def get_additional_data(self, metadata): + return self.get_tags(metadata) + + def get_additional_groups(self, metadata): + return self.config.get_groups(metadata.hostname, + self.get_tags(metadata)) + + +class AWSTagsLint(Bcfg2.Server.Lint.ServerPlugin): + """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags + ` patterns for validity. """ + + def Run(self): + cfg = self.core.plugins['AWSTags'].config + for entry in cfg.xdata.xpath('//Tag'): + self.check(entry, "name") + if entry.get("value"): + self.check(entry, "value") + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize": "error"} + + def check(self, entry, attr): + """ Check a single attribute (``name`` or ``value``) of a + single entry for validity. """ + try: + re.compile(entry.get(attr)) + except re.error: + self.LintError("pattern-fails-to-initialize", + "'%s' regex could not be compiled: %s\n %s" % + (attr, sys.exc_info()[1], entry.get("name"))) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py new file mode 100644 index 000000000..05e0bb9a1 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py @@ -0,0 +1,140 @@ +import os +import sys +import lxml.etree +import Bcfg2.Server.Plugin +from mock import Mock, MagicMock, patch +try: + from Bcfg2.Server.Plugins.AWSTags import * + HAS_BOTO = True +except ImportError: + HAS_BOTO = False + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * +from TestPlugin import TestPlugin, TestConnector, TestClientRunHooks + +config = ''' + + + group1 + group2 + + + group3 + + + group-$1 + + + group-$1 + + +''' + +tags = { + "empty.example.com": {}, + "no-matches.example.com": {"nameonly": "foo", + "Name": "no-matches", + "foo": "bar"}, + "foo.example.com": {"name-only": "name-only", + "name-and-value": "wrong", + "regex-name": "foo"}, + "bar.example.com": {"name-and-value": "value", + "regex-value": "bar"}} + +groups = { + "empty.example.com": [], + "no-matches.example.com": [], + "foo.example.com": ["group1", "group2", "group-name"], + "bar.example.com": ["group3", "group-value", "group-bar"]} + + +def make_instance(name): + rv = Mock() + rv.private_dns_name = name + rv.tags = tags[name] + return rv + + +instances = [make_instance(n) for n in tags.keys()] + + +def get_all_instances(filters=None): + insts = [i for i in instances + if i.private_dns_name == filters['private-dns-name']] + res = Mock() + res.instances = insts + return [res] + + +if HAS_BOTO: + class TestAWSTags(TestPlugin, TestClientRunHooks, TestConnector): + test_obj = AWSTags + + def get_obj(self, core=None): + @patchIf(not isinstance(Bcfg2.Server.Plugins.AWSTags.connect_ec2, + Mock), + "Bcfg2.Server.Plugins.AWSTags.connect_ec2", Mock()) + @patch("lxml.etree.Element", Mock()) + def inner(): + obj = TestPlugin.get_obj(self, core=core) + obj.config.data = config + obj.config.Index() + return obj + return inner() + + @patch("Bcfg2.Server.Plugins.AWSTags.connect_ec2") + def test_connect(self, mock_connect_ec2): + """ Test connection to EC2 """ + key_id = "a09sdbipasdf" + access_key = "oiilb234ipwe9" + + def cfp_get(section, option): + if option == "access_key_id": + return key_id + elif option == "secret_access_key": + return access_key + else: + return Mock() + + core = Mock() + core.setup.cfp.get = Mock(side_effect=cfp_get) + awstags = self.get_obj(core=core) + mock_connect_ec2.assert_called_with( + aws_access_key_id=key_id, + aws_secret_access_key=access_key) + + def test_get_additional_data(self): + """ Test AWSTags.get_additional_data() """ + awstags = self.get_obj() + awstags._ec2.get_all_instances = \ + Mock(side_effect=get_all_instances) + + for hostname, expected in tags.items(): + metadata = Mock() + metadata.hostname = hostname + self.assertItemsEqual(awstags.get_additional_data(metadata), + expected) + + def test_get_additional_groups_caching(self): + """ Test AWSTags.get_additional_groups() with caching enabled """ + awstags = self.get_obj() + awstags._ec2.get_all_instances = \ + Mock(side_effect=get_all_instances) + + for hostname, expected in groups.items(): + metadata = Mock() + metadata.hostname = hostname + actual = awstags.get_additional_groups(metadata) + msg = """%s has incorrect groups: +actual: %s +expected: %s""" % (hostname, actual, expected) + self.assertItemsEqual(actual, expected, msg) -- cgit v1.2.3-1-g7c22 From 0bf89e137796811478575455c6bcbb008bec71f8 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 26 Sep 2013 13:49:27 -0400 Subject: Metadata: better logging when updating XML data fails --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 04ba79c55..d47fd644b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -677,14 +677,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ Generic method to modify XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) if node is None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = "%s \"%s\" does not exist" % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = 'Unexpected error finding %s "%s"' % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) for key, val in list(attribs.items()): xdict['xquery'][0].set(key, val) config.write_xml(xdict['filename'], xdict['xmltree']) -- cgit v1.2.3-1-g7c22 From aaba827108daede76bcb0968f0e094cc041d95d3 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 26 Sep 2013 14:40:06 -0400 Subject: testsuite: install boto with optional dependencies --- testsuite/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/install.sh b/testsuite/install.sh index 1ca89f40f..c4a1bb192 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -7,7 +7,7 @@ pip install -r testsuite/requirements.txt --use-mirrors PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then - pip install --use-mirrors genshi PyYAML pyinotify + pip install --use-mirrors genshi PyYAML pyinotify boto if [[ $PYVER == "2.5" ]]; then # markdown 2.2+ doesn't work on py2.5 pip install --use-mirrors simplejson 'markdown<2.2' -- cgit v1.2.3-1-g7c22 From 21b5c747dbac9c500246f6d528da9dcf68b50e5e Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 26 Sep 2013 14:40:45 -0400 Subject: models: handle failure to import plugin gracefully --- src/lib/Bcfg2/Server/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 1f64111e7..7e2f5b09d 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -57,7 +57,7 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): # the second attempt. LOGGER.error("Failed to load plugin %s: %s" % (plugin, err)) - continue + continue for sym in dir(mod): obj = getattr(mod, sym) if hasattr(obj, "__bases__") and models.Model in obj.__bases__: -- cgit v1.2.3-1-g7c22 From 5eb0b282d4e5142e37018d12b4a11cae71c82e40 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 26 Sep 2013 15:03:40 -0400 Subject: AWSTags: fixed cache clearing at start of client run --- src/lib/Bcfg2/Server/Plugins/AWSTags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py index 962b5d6a1..147f37fbf 100644 --- a/src/lib/Bcfg2/Server/Plugins/AWSTags.py +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -181,7 +181,7 @@ class AWSTags(Bcfg2.Server.Plugin.Plugin, self._tagcache.expire(key=key) def start_client_run(self, metadata): - self.expire_cache(self, key=metadata.hostname) + self.expire_cache(key=metadata.hostname) def get_additional_data(self, metadata): return self.get_tags(metadata) -- cgit v1.2.3-1-g7c22 From 36641e89d28aeb411cc1886a7ecd90b8bcbf0f8f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 27 Sep 2013 06:40:04 -0400 Subject: GroupLogic: fixed thread-local variable initialization --- src/lib/Bcfg2/Server/Plugins/GroupLogic.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index aa71d2cfe..d74c16e8b 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -47,19 +47,21 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), core.fam) self._local = local() - # building is a thread-local set that tracks which machines - # GroupLogic is getting additional groups for. If a - # get_additional_groups() is called twice for a machine before - # the first call has completed, the second call returns an - # empty list. This is for infinite recursion protection; - # without this check, it'd be impossible to use things like - # metadata.query.in_group() in GroupLogic, since that requires - # building all metadata, which requires running - # GroupLogic.get_additional_groups() for all hosts, which - # requires building all metadata... - self._local.building = set() def get_additional_groups(self, metadata): + if not hasattr(self._local, "building"): + # building is a thread-local set that tracks which + # machines GroupLogic is getting additional groups for. + # If a get_additional_groups() is called twice for a + # machine before the first call has completed, the second + # call returns an empty list. This is for infinite + # recursion protection; without this check, it'd be + # impossible to use things like metadata.query.in_group() + # in GroupLogic, since that requires building all + # metadata, which requires running + # GroupLogic.get_additional_groups() for all hosts, which + # requires building all metadata... + self._local.building = set() if metadata.hostname in self._local.building: return [] self._local.building.add(metadata.hostname) -- cgit v1.2.3-1-g7c22 From aba6d94560658eb8d0abcc787560916038c9d7ac Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 30 Sep 2013 11:29:24 -0400 Subject: Core: Fixed error message on failed altsrc bind --- src/lib/Bcfg2/Server/Core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 271fb8237..5ec1b5bce 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -648,10 +648,10 @@ class BaseCore(object): del entry.attrib['realname'] return ret except: - entry.set('name', oldname) self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, entry.get('name'), - entry.get('altsrc'))) + (entry.tag, entry.get('realname'), + entry.get('name'))) + entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) -- cgit v1.2.3-1-g7c22 From 6aa55eda59107d89fe1272f0127fa01f11a61e98 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 1 Oct 2013 10:31:04 -0500 Subject: debian: Add build-depends for 35b53c7 Signed-off-by: Sol Jerome --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index 37b72a9f4..aee6f1c24 100644 --- a/debian/control +++ b/debian/control @@ -9,6 +9,7 @@ Build-Depends: debhelper (>= 7.0.50~), python-sphinx (>= 1.0.7+dfsg) | python3-sphinx, python-lxml, python-daemon, + python-boto, python-cherrypy3, python-gamin, python-genshi, -- cgit v1.2.3-1-g7c22 From 7f6c10db41c22b3924539aae19164a9ab9a80468 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 3 Oct 2013 16:24:05 -0400 Subject: Metadata: import any() from Compat --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index d47fd644b..4a0413a55 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -16,7 +16,9 @@ import Bcfg2.Server.Lint import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked -from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 +# pylint: disable=W0622 +from Bcfg2.Compat import MutableMapping, all, any, wraps +# pylint: enable=W0622 from Bcfg2.version import Bcfg2VersionInfo try: -- cgit v1.2.3-1-g7c22 From 9a6a231ccb4f509c0f6fa932c97bad647d29af50 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 4 Oct 2013 15:10:36 -0400 Subject: Metadata: read in clients.xml on every write This ensures consistency between the in-memory representation of clients.xml and the representation on disk. If we don't read our writes immediately, there's a race condition when creating a new client: If it asserts its profile or version before the FAM event from the clients.xml edit is processed, then the clients doesn't appear to exist yet, and Bcfg2 complains. --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 4a0413a55..047dd4f4e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -221,6 +221,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): sys.exc_info()[1]) self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg) + self.load_xml() def find_xml_for_xpath(self, xpath): """Find and load xml file containing the xpath query""" -- cgit v1.2.3-1-g7c22 From af6fb8f8a16d6141f5a34b66fa81d74d4693ec67 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Fri, 4 Oct 2013 16:56:17 -0400 Subject: Tests: Fix tests after 9a6a231 The addition of the call to load_xml in 9a6a231 causes the test to fail because load_xml() expects to read a clients.xml file. The actual actual open calls in write_xml are dummied out with Mock, so no file is written, and thus cannot be read back. This commit dummies out the load_xml and adds some more asserts for good measure. --- testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py index e839914d7..a07fffba1 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py @@ -339,6 +339,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): @patch('Bcfg2.Utils.locked', Mock(return_value=False)) @patch('fcntl.lockf', Mock()) + @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml") @patch('os.open') @patch('os.fdopen') @patch('os.unlink') @@ -346,7 +347,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): @patch('os.path.islink') @patch('os.readlink') def test_write_xml(self, mock_readlink, mock_islink, mock_rename, - mock_unlink, mock_fdopen, mock_open): + mock_unlink, mock_fdopen, mock_open, mock_load_xml): fname = "clients.xml" config = self.get_obj(fname) fpath = os.path.join(self.metadata.data, fname) @@ -360,6 +361,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): mock_unlink.reset_mock() mock_fdopen.reset_mock() mock_open.reset_mock() + mock_load_xml.reset_mock() mock_islink.return_value = False @@ -371,6 +373,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): self.assertTrue(mock_fdopen.return_value.write.called) mock_islink.assert_called_with(fpath) mock_rename.assert_called_with(tmpfile, fpath) + mock_load_xml.assert_called_with() # test: clients.xml.new is locked the first time we write it def rv(fname, mode): @@ -389,6 +392,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): self.assertTrue(mock_fdopen.return_value.write.called) mock_islink.assert_called_with(fpath) mock_rename.assert_called_with(tmpfile, fpath) + mock_load_xml.assert_called_with() # test writing a symlinked clients.xml reset() @@ -397,6 +401,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): mock_readlink.return_value = linkdest config.write_xml(fpath, get_clients_test_tree()) mock_rename.assert_called_with(tmpfile, linkdest) + mock_load_xml.assert_called_with() # test failure of os.rename() reset() -- cgit v1.2.3-1-g7c22 From 3a5eec174af0b9907b29fdfd3eb1e4fd7677beeb Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 9 Oct 2013 11:24:07 -0400 Subject: Packages: fixed metadata.Packages["sources"] --- src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 3cdcdc162..479138ef1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -651,7 +651,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, """ getter for the 'sources' key of the OnDemandDict returned by this function. This delays calling get_collection() until it's absolutely necessary. """ - return self.get_collection(metadata).get_additional_data + return self.get_collection(metadata).get_additional_data() return OnDemandDict( sources=get_sources, -- cgit v1.2.3-1-g7c22 From cde00779a8d4d9c2624a3b82c6911ccf1cb2aaef Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Wed, 9 Oct 2013 15:48:51 -0400 Subject: Reporting: start a new thread for each import When dealing with a high-latency database connection (eg. across a WAN), the bcfg2-report-collector process can fall behind in its import queue. The imports are very much bound by the response latency of the database server and not processing throughput. This patch fires off a new thread for each database interaction. The thread itself simply falls out of scope when the interaction is finished processing. The interaction object is still read from the disk serially in order avoid having to create a locking mechanism for that part of the process. This change does potentially create greater load on the database server, but ultimately the load is limited by rate at which the bcfg2 server can generate work. --- src/lib/Bcfg2/Reporting/Collector.py | 39 +++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 4556cda82..68e1d6a6d 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -20,10 +20,36 @@ from Bcfg2.Reporting.Transport.DirectStore import DirectStore from Bcfg2.Reporting.Storage import load_storage_from_config, \ StorageError, StorageImportError + class ReportingError(Exception): """Generic reporting exception""" pass + +class ReportingStoreThread(threading.Thread): + """Thread for calling the storage backend""" + def __init__(self, interaction, storage, group=None, target=None, + name=None, *args, **kwargs): + """Initialize the thread with a reference to the interaction + as well as the storage engine to use""" + threading.Thread.__init__(self, group, target, name, args, kwargs) + self.interaction = interaction + self.storage = storage + self.logger = logging.getLogger('bcfg2-report-collector') + + def run(self): + try: + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', ''), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + traceback.format_exc().splitlines()[-1]) + + class ReportingCollector(object): """The collecting process for reports""" @@ -77,7 +103,6 @@ class ReportingCollector(object): (self.storage.__class__.__name__, traceback.format_exc().splitlines()[-1])) - def run(self): """Startup the processing and go!""" self.terminate = threading.Event() @@ -103,15 +128,9 @@ class ReportingCollector(object): interaction = self.transport.fetch() if not interaction: continue - try: - start = time.time() - self.storage.import_interaction(interaction) - self.logger.info("Imported interaction for %s in %ss" % - (interaction.get('hostname', ''), - time.time() - start)) - except: - #TODO requeue? - raise + + t = ReportingStoreThread(interaction, self.storage) + t.start() except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") self.shutdown() -- cgit v1.2.3-1-g7c22 From f813f86f8ac2bc7b55f4eb6a0d936f1ce4f68ba7 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Wed, 9 Oct 2013 15:59:42 -0400 Subject: Reporting: Simple sanity check to avoid creating too many threads --- src/lib/Bcfg2/Reporting/Collector.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 68e1d6a6d..f348a60dc 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -130,6 +130,9 @@ class ReportingCollector(object): continue t = ReportingStoreThread(interaction, self.storage) + while len(threading.enumerate()) > 100: + self.logger.info("more than 100 threads running, sleeping") + time.sleep(1) t.start() except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") -- cgit v1.2.3-1-g7c22 From 9b8a2b6bfa0a0586f1c25358519c8d8db1260009 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Thu, 10 Oct 2013 14:21:07 -0400 Subject: Reporting: misc improvements to collector threading 1. Use a better convention for calling the threading.Thread constructor 2. Add docstring to ReportingStoreThread.run 3. Give the storage thread variable a better name --- src/lib/Bcfg2/Reporting/Collector.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index f348a60dc..febdfed13 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -29,15 +29,17 @@ class ReportingError(Exception): class ReportingStoreThread(threading.Thread): """Thread for calling the storage backend""" def __init__(self, interaction, storage, group=None, target=None, - name=None, *args, **kwargs): + name=None, args=(), kwargs=None): """Initialize the thread with a reference to the interaction as well as the storage engine to use""" - threading.Thread.__init__(self, group, target, name, args, kwargs) + threading.Thread.__init__(self, group, target, name, args, + kwargs or dict()) self.interaction = interaction self.storage = storage self.logger = logging.getLogger('bcfg2-report-collector') def run(self): + """Call the database storage procedure (aka import)""" try: start = time.time() self.storage.import_interaction(self.interaction) @@ -129,11 +131,11 @@ class ReportingCollector(object): if not interaction: continue - t = ReportingStoreThread(interaction, self.storage) + store_thread = ReportingStoreThread(interaction, self.storage) while len(threading.enumerate()) > 100: self.logger.info("more than 100 threads running, sleeping") time.sleep(1) - t.start() + store_thread.start() except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") self.shutdown() -- cgit v1.2.3-1-g7c22 From 171179777d0b6104440a87bae7f4aff0c82ba297 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Mon, 14 Oct 2013 14:33:44 -0400 Subject: Revert "Reporting: Simple sanity check to avoid creating too many threads" This reverts commit f813f86f8ac2bc7b55f4eb6a0d936f1ce4f68ba7. Premature optimization is the root of all evil, etc. Conflicts: src/lib/Bcfg2/Reporting/Collector.py --- src/lib/Bcfg2/Reporting/Collector.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index febdfed13..b42364d8d 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -132,9 +132,6 @@ class ReportingCollector(object): continue store_thread = ReportingStoreThread(interaction, self.storage) - while len(threading.enumerate()) > 100: - self.logger.info("more than 100 threads running, sleeping") - time.sleep(1) store_thread.start() except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") -- cgit v1.2.3-1-g7c22 From 1d27fdadac37eb95d56f98cf7bdd1721a339df81 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 28 Oct 2013 09:11:36 -0400 Subject: testsuite: install django 1.4.8 when testing py 2.5 --- testsuite/install.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testsuite/install.sh b/testsuite/install.sh index c4a1bb192..8eecd71d7 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -10,7 +10,9 @@ if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then pip install --use-mirrors genshi PyYAML pyinotify boto if [[ $PYVER == "2.5" ]]; then # markdown 2.2+ doesn't work on py2.5 - pip install --use-mirrors simplejson 'markdown<2.2' + pip install --use-mirrors simplejson 'markdown<2.2' 'django<1.4.9' + else + pip install 'django<1.5' fi if [[ ${PYVER:0:1} == "2" ]]; then # django supports py3k, but South doesn't, and the django bits -- cgit v1.2.3-1-g7c22