From e496fb95eaf9200f78248106f9fd7ec6b7d9e530 Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Fri, 22 Jul 2011 14:26:35 -0500 Subject: Fixed to accommodate changes made to Plugin.py in changeset 3291a875339a7e5569d4. The changes to the INode.Match() function in changeset 3291a875339a7e5569d4 caused breakage in the Deps plugin, as it inherits from INode. This commit adjusts the definition of the predicate function in Deps.py to mirror the changes made to INode in Plugin.py, eliminating the error about the wrong number of parameters being passed to the lambda function. --- src/lib/Server/Plugins/Deps.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py index b186258cb..389645232 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Server/Plugins/Deps.py @@ -8,18 +8,19 @@ import Bcfg2.Server.Plugin class DNode(Bcfg2.Server.Plugin.INode): """DNode provides supports for single predicate types for dependencies.""" - raw = {'Group': "lambda x:'%s' in x.groups and predicate(x)"} + raw = {'Group': "lambda m, e:'%(name)s' in m.groups and predicate(m, e)"} containers = ['Group'] def __init__(self, data, idict, parent=None): self.data = data self.contents = {} if parent == None: - self.predicate = lambda x: True + self.predicate = lambda x, d: True else: predicate = parent.predicate if data.tag in list(self.raw.keys()): - self.predicate = eval(self.raw[data.tag] % (data.get('name')), + self.predicate = eval(self.raw[data.tag] % + {'name': data.get('name')}, {'predicate': predicate}) else: raise Exception -- cgit v1.2.3-1-g7c22 From 42199a9835a548a0a78f48fd6c5f11c3153371dd Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Fri, 22 Jul 2011 23:17:14 -0500 Subject: Created a new method to return a sorted list of plugins by type. Replaced many list comprehensions in Core.py with a new method named plugins_by_type(), which does the same thing with an added twist: this new method also sorts the list of plugins by a new field named sort_order. It also uses the name of the plugin where the sort_order values are the same. This lets us control the processing sequence of plugins that need to build on the results of a plugin that runs prior to them. The immediate example is Deps, which should run after Packages has generated the full list of packages to be installed. Prior to this commit, it was impossible to control the order in which they ran. A future commit will (hopefully) take advantage of this capability. This commit also splits the Core.validate_data() method into two: one for validate_structures() and one for validate_goals(), instead of passing in a base class and using if logic. This approach seemed a little clearer to me. --- src/lib/Server/Plugins/Metadata.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index 7fc34f178..bfe1ac053 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -222,6 +222,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' name = "Metadata" + sort_order = 500 def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) -- cgit v1.2.3-1-g7c22 From 71b2daf28752af0f4f91e10139bfac8bce10b041 Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Sat, 23 Jul 2011 00:36:42 -0500 Subject: Refactor the validate_structure() method into two. New method named calculate_prereqs() handles pre-req calculation. Existing validate_structure() method now only handles memoization of the prereqs and use of the prereqs to update the configuration. Also added a sort_order value to cause this plugin to run after Packages, so we can use Deps to add dependencies to implicitly added Packages. NOTE: This doesn't work yet, as Packages adds BoundPackage entries, and Deps doesn't yet realize they are equivalent to the Package entries it knows about. This will be fixed in a future commit. --- src/lib/Server/Plugins/Deps.py | 70 +++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 25 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py index 389645232..cedee4d8b 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Server/Plugins/Deps.py @@ -49,6 +49,12 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = DepXMLSrc + # Override the default sort_order (of 500) so that this plugin + # gets handled after others running at the default. In particular, + # we want to run after Packages, so we can see the final set of + # packages that will be installed on the client. + sort_order = 750 + def __init__(self, core, datastore): Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) Bcfg2.Server.Plugin.StructureValidator.__init__(self) @@ -59,8 +65,10 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, Bcfg2.Server.Plugin.PrioDir.HandleEvent(self, event) def validate_structures(self, metadata, structures): + """Examine the passed structures and append any additional + prerequisite entries as defined by the files in Deps. + """ entries = [] - prereqs = [] for structure in structures: for entry in structure.getchildren(): if (entry.tag, entry.get('name')) not in entries \ @@ -71,33 +79,12 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, gdata = list(metadata.groups) gdata.sort() gdata = tuple(gdata) + + # Check to see if we have cached the prereqs already if (entries, gdata) in self.cache: prereqs = self.cache[(entries, gdata)] else: - [src.Cache(metadata) for src in list(self.entries.values())] - - toexamine = list(entries[:]) - while toexamine: - entry = toexamine.pop() - matching = [src for src in list(self.entries.values()) - if src.cache and entry[0] in src.cache[1] - and entry[1] in src.cache[1][entry[0]]] - if len(matching) > 1: - prio = [int(src.priority) for src in matching] - if prio.count(max(prio)) > 1: - self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" % - (entry[0].lower(), metadata.hostname, entry[1])) - raise Bcfg2.Server.Plugin.PluginExecutionError - index = prio.index(max(prio)) - matching = [matching[index]] - - if not matching: - continue - elif len(matching) == 1: - for prq in matching[0].cache[1][entry[0]][entry[1]]: - if prq not in prereqs and prq not in entries: - toexamine.append(prq) - prereqs.append(prq) + prereqs = self.calculate_prereqs(metadata, entries) self.cache[(entries, gdata)] = prereqs newstruct = lxml.etree.Element("Independent") @@ -107,3 +94,36 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, except: self.logger.error("Failed to add dep entry for %s:%s" % (tag, name)) structures.append(newstruct) + + + def calculate_prereqs(self, metadata, entries): + """Calculate the prerequisites defined in Deps for the passed + set of entries. + """ + prereqs = [] + [src.Cache(metadata) for src in list(self.entries.values())] + + import ipdb; ipdb.set_trace() + toexamine = list(entries[:]) + while toexamine: + entry = toexamine.pop() + matching = [src for src in list(self.entries.values()) + if src.cache and entry[0] in src.cache[1] + and entry[1] in src.cache[1][entry[0]]] + if len(matching) > 1: + prio = [int(src.priority) for src in matching] + if prio.count(max(prio)) > 1: + self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" % + (entry[0].lower(), metadata.hostname, entry[1])) + raise Bcfg2.Server.Plugin.PluginExecutionError + index = prio.index(max(prio)) + matching = [matching[index]] + elif len(matching) == 1: + for prq in matching[0].cache[1][entry[0]][entry[1]]: + if prq not in prereqs and prq not in entries: + toexamine.append(prq) + prereqs.append(prq) + else: + continue + + return prereqs -- cgit v1.2.3-1-g7c22 From 0b69a35bb21341d006a1463f8683d249eae3229a Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Sat, 23 Jul 2011 10:25:26 -0500 Subject: Treat Bound entries like unbound entries for calculating prerequisites in Deps. --- src/lib/Server/Plugins/Deps.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py index cedee4d8b..3a538292e 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Server/Plugins/Deps.py @@ -71,9 +71,12 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, entries = [] for structure in structures: for entry in structure.getchildren(): - if (entry.tag, entry.get('name')) not in entries \ - and not isinstance(entry, lxml.etree._Comment): - entries.append((entry.tag, entry.get('name'))) + tag = entry.tag + if tag.startswith('Bound'): + tag = tag[5:] + if (tag, entry.get('name')) not in entries \ + and not isinstance(entry, lxml.etree._Comment): + entries.append((tag, entry.get('name'))) entries.sort() entries = tuple(entries) gdata = list(metadata.groups) @@ -101,9 +104,8 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, set of entries. """ prereqs = [] - [src.Cache(metadata) for src in list(self.entries.values())] + [src.Cache(metadata) for src in self.entries.values()] - import ipdb; ipdb.set_trace() toexamine = list(entries[:]) while toexamine: entry = toexamine.pop() -- cgit v1.2.3-1-g7c22 From 17d801958e683c5ff4a9122f9fd172f0be67f5a5 Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Mon, 25 Jul 2011 17:23:46 -0500 Subject: Update to Deps to eliminate "Failed to add dep entry for :None" warning. Comments in the Deps file (inside a Package element) will cause the above error. In generating the prerequisites, we can eliminate this by looking for callable() items and excluding them. --- src/lib/Server/Plugins/Deps.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py index 3a538292e..482d457af 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Server/Plugins/Deps.py @@ -122,7 +122,12 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, matching = [matching[index]] elif len(matching) == 1: for prq in matching[0].cache[1][entry[0]][entry[1]]: - if prq not in prereqs and prq not in entries: + # XML comments seem to show up in the cache as a + # tuple with item 0 being callable. The logic + # below filters them out. Would be better to + # exclude them when we load the cache in the first + # place. + if prq not in prereqs and prq not in entries and not callable(prq[0]): toexamine.append(prq) prereqs.append(prq) else: -- cgit v1.2.3-1-g7c22 From bda0f01adda437a6bac1c5a890063bccbd1f5b78 Mon Sep 17 00:00:00 2001 From: Mike McCallister Date: Tue, 26 Jul 2011 09:34:39 -0500 Subject: Rewrote DirectoryBacked so it handles files contained in an arbitrary directory layout. Previously, DirectoryBacked (and as a result Bundler, Deps, Rules, Base, Pkgmgr, and others) only recognized XML files contained in the top-level plugin directory, for example: Deps/foo.xml Deps/subdir/foo.xml # <--- Ignored Bundler/bar.xml Bundler/subdir/baz.xml # <--- Ignored Now it can support the following as well: Deps/debian-lenny/foo.xml Deps/debian-squeeze/foo.xml Bundler/group-a/bar.xml Bundler/group-b/baz.xml Note that the directories and filenames do not factor into the semantic meaning of the configuration specification. The contents of foo.xml must stand alone, as if they were in the same single-level directory as before. In the case of Deps, Rules, Pkgmgr, and Svcmgr, you must use Groups and priorities within the XML files as needed to ensure that Bcfg2 can correctly resolve the configuration for your hosts. For example, prior to this change you would use a single file like the following: Deps/foo.xml: Now you can use a pair of files in separate directories like the following. Note how the groups within each file prevent there from being two sources for a single package: Deps/debian-lenny/foo.xml: Deps/debian-squeeze/foo.xml: In the case of Bundler, individual filenames must remain unique throughout the directory hierarchy, and they must match the bundle name. --- src/lib/Server/Plugins/Bundler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/lib/Server/Plugins') diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 01ad3c78b..bf0c42416 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -3,6 +3,7 @@ __revision__ = '$Revision$' import copy import lxml.etree +import os import re import sys @@ -74,8 +75,8 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, """Build all structures for client (metadata).""" bundleset = [] for bundlename in metadata.bundles: - entries = [item for (key, item) in list(self.entries.items()) if \ - self.patterns.match(key).group('name') == bundlename] + entries = [item for (key, item) in self.entries.items() if \ + self.patterns.match(os.path.basename(key)).group('name') == bundlename] if len(entries) == 0: continue elif len(entries) == 1: -- cgit v1.2.3-1-g7c22