diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/lib/Server/Core.py | 95 | ||||
-rw-r--r-- | src/lib/Server/Plugin.py | 154 | ||||
-rw-r--r-- | src/lib/Server/Plugins/Bundler.py | 5 | ||||
-rw-r--r-- | src/lib/Server/Plugins/Deps.py | 83 | ||||
-rw-r--r-- | src/lib/Server/Plugins/Metadata.py | 1 |
5 files changed, 230 insertions, 108 deletions
diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index 5adfa5381..91b6a3555 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -90,42 +90,49 @@ class Core(Component): "Unloading %s" % (p, bl)) for plug in bl: del self.plugins[plug] - # This section loads the experimental plugins + # This section logs the experimental plugins expl = [plug for (name, plug) in list(self.plugins.items()) if plug.experimental] if expl: logger.info("Loading experimental plugin(s): %s" % \ (" ".join([x.name for x in expl]))) logger.info("NOTE: Interfaces subject to change") + # This section logs the deprecated plugins depr = [plug for (name, plug) in list(self.plugins.items()) if plug.deprecated] - # This section loads the deprecated plugins if depr: logger.info("Loading deprecated plugin(s): %s" % \ (" ".join([x.name for x in depr]))) - mlist = [p for p in list(self.plugins.values()) if \ - isinstance(p, Bcfg2.Server.Plugin.Metadata)] + mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata) if len(mlist) == 1: self.metadata = mlist[0] else: logger.error("No Metadata Plugin loaded; failed to instantiate Core") raise CoreInitError("No Metadata Plugin") - self.statistics = [plugin for plugin in list(self.plugins.values()) - if isinstance(plugin, Bcfg2.Server.Plugin.Statistics)] - self.pull_sources = [plugin for plugin in self.statistics - if isinstance(plugin, Bcfg2.Server.Plugin.PullSource)] - self.generators = [plugin for plugin in list(self.plugins.values()) - if isinstance(plugin, Bcfg2.Server.Plugin.Generator)] - self.structures = [plugin for plugin in list(self.plugins.values()) - if isinstance(plugin, Bcfg2.Server.Plugin.Structure)] - self.connectors = [plugin for plugin in list(self.plugins.values()) - if isinstance(plugin, Bcfg2.Server.Plugin.Connector)] + self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics) + self.pull_sources = self.plugins_by_type(Bcfg2.Server.Plugin.PullSource) + self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator) + self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure) + self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector) self.ca = ca self.fam_thread = threading.Thread(target=self._file_monitor_thread) if start_fam_thread: self.fam_thread.start() + def plugins_by_type(self, base_cls): + """Return a list of loaded plugins that match the passed type. + + The returned list is sorted in ascending order by the Plugins' + sort_order value. The sort_order defaults to 500 in Plugin.py, + but can be overridden by individual plugins. Plugins with the + same numerical sort_order value are sorted in alphabetical + order by their name. + """ + return sorted([plugin for plugin in self.plugins.values() + if isinstance(plugin, base_cls)], + key=lambda p: (p.sort_order, p.name)) + def _file_monitor_thread(self): """The thread for monitor the files.""" famfd = self.fam.fileno() @@ -141,9 +148,8 @@ class Core(Component): except: continue # VCS plugin periodic updates - for plugin in list(self.plugins.values()): - if isinstance(plugin, Bcfg2.Server.Plugin.Version): - self.revision = plugin.get_revision() + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version): + self.revision = plugin.get_revision() def init_plugins(self, plugin): """Handling for the plugins.""" @@ -176,23 +182,33 @@ class Core(Component): for plugin in list(self.plugins.values()): plugin.shutdown() - def validate_data(self, metadata, data, base_cls): + def validate_structures(self, metadata, data): """Checks the data structure.""" - for plugin in list(self.plugins.values()): - if isinstance(plugin, base_cls): - try: - if base_cls == Bcfg2.Server.Plugin.StructureValidator: - plugin.validate_structures(metadata, data) - elif base_cls == Bcfg2.Server.Plugin.GoalValidator: - plugin.validate_goals(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: - err = sys.exc_info()[1] - logger.error("Plugin %s structure validation failed: %s" \ - % (plugin.name, err.message)) - raise - except: - logger.error("Plugin %s: unexpected structure validation failure" \ - % (plugin.name), exc_info=1) + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): + try: + plugin.validate_structures(metadata, data) + except Bcfg2.Server.Plugin.ValidationError: + err = sys.exc_info()[1] + logger.error("Plugin %s structure validation failed: %s" \ + % (plugin.name, err.message)) + raise + except: + logger.error("Plugin %s: unexpected structure validation failure" \ + % (plugin.name), exc_info=1) + + def validate_goals(self, metadata, data): + """Checks that the config matches the goals enforced by the plugins.""" + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator): + try: + plugin.validate_goals(metadata, data) + except Bcfg2.Server.Plugin.ValidationError: + err = sys.exc_info()[1] + logger.error("Plugin %s goal validation failed: %s" \ + % (plugin.name, err.message)) + raise + except: + logger.error("Plugin %s: unexpected goal validation failure" \ + % (plugin.name), exc_info=1) def GetStructures(self, metadata): """Get all structures for client specified by metadata.""" @@ -276,8 +292,7 @@ class Core(Component): logger.error("error in GetStructures", exc_info=1) return lxml.etree.Element("error", type='structure error') - self.validate_data(meta, structures, - Bcfg2.Server.Plugin.StructureValidator) + self.validate_structures(meta, structures) # Perform altsrc consistency checking esrcs = {} @@ -297,7 +312,7 @@ class Core(Component): config.append(astruct) except: logger.error("error in BindStructure", exc_info=1) - self.validate_data(meta, config, Bcfg2.Server.Plugin.GoalValidator) + self.validate_goals(meta, config) logger.info("Generated config for %s in %.03fs" % \ (client, time.time() - start)) return config @@ -305,10 +320,9 @@ class Core(Component): def GetDecisions(self, metadata, mode): """Get data for the decision list.""" result = [] - for plugin in list(self.plugins.values()): + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision): try: - if isinstance(plugin, Bcfg2.Server.Plugin.Decision): - result += plugin.GetDecisions(metadata, mode) + result += plugin.GetDecisions(metadata, mode) except: logger.error("Plugin: %s failed to generate decision list" \ % plugin.name, exc_info=1) @@ -354,8 +368,7 @@ class Core(Component): name = self.metadata.resolve_client(address) meta = self.build_metadata(name) - for plugin in [p for p in list(self.plugins.values()) \ - if isinstance(p, Bcfg2.Server.Plugin.Probing)]: + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): for probe in plugin.GetProbes(meta): resp.append(probe) return lxml.etree.tostring(resp, encoding='UTF-8', diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 17547be13..a7ab9feab 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -36,7 +36,7 @@ mdata_setup = Bcfg2.Options.OptionParser(opts) mdata_setup.parse([]) del mdata_setup['args'] -logger = logging.getLogger('Bcfg2.Plugin') +logger = logging.getLogger('Bcfg2.Server.Plugin') default_file_metadata = mdata_setup @@ -81,7 +81,17 @@ class Plugin(object): deprecated = False conflicts = [] + # Default sort_order to 500. Plugins of the same type are + # processed in order of ascending sort_order value. Plugins with + # the same sort_order are sorted alphabetically by their name. + sort_order = 500 + def __init__(self, core, datastore): + """Initialize the plugin. + + :param core: the Bcfg2.Server.Core initializing the plugin + :param datastore: the filesystem path of Bcfg2's repository + """ object.__init__(self) self.Entries = {} self.core = core @@ -370,9 +380,22 @@ class DirectoryBacked(object): object.__init__(self) self.name = name self.fam = fam + + # self.entries contains information about the files monitored + # by this object.... The keys of the dict are the relative + # paths to the files. The values are the objects (of type + # __child__) that handle their contents. self.entries = {} - self.inventory = False - fam.AddMonitor(name, self) + + # self.handles contains information about the directories + # monitored by this object. The keys of the dict are the + # values returned by the initial fam.AddMonitor() call (which + # appear to be integers). The values are the relative paths of + # the directories. + self.handles = {} + + # Monitor everything in the plugin's directory + self.add_directory_monitor('') def __getitem__(self, key): return self.entries[key] @@ -380,46 +403,103 @@ class DirectoryBacked(object): def __iter__(self): return iter(list(self.entries.items())) - def AddEntry(self, name): - """Add new entry to data structures upon file creation.""" - if name == '': - logger.info("got add for empty name") - elif name in self.entries: - self.entries[name].HandleEvent() - else: - if ((name[-1] == '~') or - (name[:2] == '.#') or - (name[-4:] == '.swp') or - (name in ['SCCS', '.svn'])): - return - if not self.patterns.match(name): + def add_directory_monitor(self, relative): + """Add a new directory to FAM structures for monitoring. + + :param relative: Path name to monitor. This must be relative + to the plugin's directory. An empty string value ("") will + cause the plugin directory itself to be monitored. + """ + dirpathname = os.path.join(self.data, relative) + if relative not in self.handles.values(): + if not posixpath.isdir(dirpathname): + logger.error("Failed to open directory %s" % (dirpathname)) return - self.entries[name] = self.__child__('%s/%s' % (self.name, name)) - self.entries[name].HandleEvent() + reqid = self.fam.AddMonitor(dirpathname, self) + self.handles[reqid] = relative def HandleEvent(self, event): - """Propagate fam events to underlying objects.""" + """Handle FAM/Gamin events. + + This method is invoked by FAM/Gamin when it detects a change + to a filesystem object we have requsted to be monitored. + + This method manages the lifecycle of events related to the + monitored objects, adding them to our indiciess and creating + objects of type __child__ that actually do the domain-specific + processing. When appropriate, it propogates events those + objects by invoking their HandleEvent in turn. + """ action = event.code2str() - if event.filename == '': - logger.info("Got event for blank filename") + + # Exclude events for actions and filesystem paths we don't + # care about + if action == 'endExist': return - if action == 'exists': - if event.filename != self.name: - self.AddEntry(event.filename) - elif action == 'created': - self.AddEntry(event.filename) - elif action == 'changed': - if event.filename in self.entries: - self.entries[event.filename].HandleEvent(event) - elif action == 'deleted': - if event.filename in self.entries: - del self.entries[event.filename] - elif action in ['endExist']: - pass + elif os.path.isabs(event.filename[0]): + # After AddDirectoryMonitor calls, we receive an 'exists' + # event with the just-added directory and its absolute + # path name. Ignore these. + return + elif event.filename == '': + logger.warning("Got event for blank filename") + return + + # Calculate the absolute and relative paths this event refers to + abspath = os.path.join(self.data, self.handles[event.requestID], + event.filename) + relpath = os.path.join(self.handles[event.requestID], event.filename) + + if action == 'deleted': + for key in self.entries.keys(): + if key.startswith(relpath): + del self.entries[key] + for handle in self.handles.keys(): + if self.handles[handle].startswith(relpath): + del self.handles[handle] + elif posixpath.isdir(abspath): + # Deal with events for directories + if action in ['exists', 'created']: + self.add_directory_monitor(relpath) + elif action == 'changed' and relpath in self.entries: + # Ownerships, permissions or timestamps changed on the + # directory. None of these should affect the contents + # of the files, though it could change our ability to + # access them. + # + # It seems like the right thing to do is to cancel + # monitoring the directory and then begin monitoring + # it again. But the current FileMonitor class doesn't + # support canceling, so at least let the user know + # that a restart might be a good idea. + logger.warn("Directory properties for %s changed, please " + + " consider restarting the server" % (abspath)) + else: + logger.warn("Got unknown dir event %s %s %s" % (event.requestID, + event.code2str(), + abspath)) else: - print("Got unknown event %s %s %s" % (event.requestID, - event.code2str(), - event.filename)) + # Deal with events for non-directories + if action in ['exists', 'created']: + if ((event.filename[-1] == '~') or + (event.filename[:2] == '.#') or + (event.filename[-4:] == '.swp') or + (event.filename in ['SCCS', '.svn'])): + return + if not self.patterns.match(event.filename): + return + self.entries[relpath] = self.__child__('%s/%s' % (self.name, + relpath)) + self.entries[relpath].HandleEvent(event) + elif action == 'changed': + if relpath in self.entries: + self.entries[relpath].HandleEvent(event) + else: + logger.warn("Got %s event for unexpected path %s" % (action, abspath)) + else: + logger.warn("Got unknown file event %s %s %s" % (event.requestID, + event.code2str(), + abspath)) class XMLFileBacked(FileBacked): diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 01ad3c78b..bf0c42416 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -3,6 +3,7 @@ __revision__ = '$Revision$' import copy import lxml.etree +import os import re import sys @@ -74,8 +75,8 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, """Build all structures for client (metadata).""" bundleset = [] for bundlename in metadata.bundles: - entries = [item for (key, item) in list(self.entries.items()) if \ - self.patterns.match(key).group('name') == bundlename] + entries = [item for (key, item) in self.entries.items() if \ + self.patterns.match(os.path.basename(key)).group('name') == bundlename] if len(entries) == 0: continue elif len(entries) == 1: diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py index 389645232..482d457af 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Server/Plugins/Deps.py @@ -49,6 +49,12 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = DepXMLSrc + # Override the default sort_order (of 500) so that this plugin + # gets handled after others running at the default. In particular, + # we want to run after Packages, so we can see the final set of + # packages that will be installed on the client. + sort_order = 750 + def __init__(self, core, datastore): Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) Bcfg2.Server.Plugin.StructureValidator.__init__(self) @@ -59,45 +65,29 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, Bcfg2.Server.Plugin.PrioDir.HandleEvent(self, event) def validate_structures(self, metadata, structures): + """Examine the passed structures and append any additional + prerequisite entries as defined by the files in Deps. + """ entries = [] - prereqs = [] for structure in structures: for entry in structure.getchildren(): - if (entry.tag, entry.get('name')) not in entries \ - and not isinstance(entry, lxml.etree._Comment): - entries.append((entry.tag, entry.get('name'))) + tag = entry.tag + if tag.startswith('Bound'): + tag = tag[5:] + if (tag, entry.get('name')) not in entries \ + and not isinstance(entry, lxml.etree._Comment): + entries.append((tag, entry.get('name'))) entries.sort() entries = tuple(entries) gdata = list(metadata.groups) gdata.sort() gdata = tuple(gdata) + + # Check to see if we have cached the prereqs already if (entries, gdata) in self.cache: prereqs = self.cache[(entries, gdata)] else: - [src.Cache(metadata) for src in list(self.entries.values())] - - toexamine = list(entries[:]) - while toexamine: - entry = toexamine.pop() - matching = [src for src in list(self.entries.values()) - if src.cache and entry[0] in src.cache[1] - and entry[1] in src.cache[1][entry[0]]] - if len(matching) > 1: - prio = [int(src.priority) for src in matching] - if prio.count(max(prio)) > 1: - self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" % - (entry[0].lower(), metadata.hostname, entry[1])) - raise Bcfg2.Server.Plugin.PluginExecutionError - index = prio.index(max(prio)) - matching = [matching[index]] - - if not matching: - continue - elif len(matching) == 1: - for prq in matching[0].cache[1][entry[0]][entry[1]]: - if prq not in prereqs and prq not in entries: - toexamine.append(prq) - prereqs.append(prq) + prereqs = self.calculate_prereqs(metadata, entries) self.cache[(entries, gdata)] = prereqs newstruct = lxml.etree.Element("Independent") @@ -107,3 +97,40 @@ class Deps(Bcfg2.Server.Plugin.PrioDir, except: self.logger.error("Failed to add dep entry for %s:%s" % (tag, name)) structures.append(newstruct) + + + def calculate_prereqs(self, metadata, entries): + """Calculate the prerequisites defined in Deps for the passed + set of entries. + """ + prereqs = [] + [src.Cache(metadata) for src in self.entries.values()] + + toexamine = list(entries[:]) + while toexamine: + entry = toexamine.pop() + matching = [src for src in list(self.entries.values()) + if src.cache and entry[0] in src.cache[1] + and entry[1] in src.cache[1][entry[0]]] + if len(matching) > 1: + prio = [int(src.priority) for src in matching] + if prio.count(max(prio)) > 1: + self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" % + (entry[0].lower(), metadata.hostname, entry[1])) + raise Bcfg2.Server.Plugin.PluginExecutionError + index = prio.index(max(prio)) + matching = [matching[index]] + elif len(matching) == 1: + for prq in matching[0].cache[1][entry[0]][entry[1]]: + # XML comments seem to show up in the cache as a + # tuple with item 0 being callable. The logic + # below filters them out. Would be better to + # exclude them when we load the cache in the first + # place. + if prq not in prereqs and prq not in entries and not callable(prq[0]): + toexamine.append(prq) + prereqs.append(prq) + else: + continue + + return prereqs diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index 7fc34f178..bfe1ac053 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -222,6 +222,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' name = "Metadata" + sort_order = 500 def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) |