diff options
Diffstat (limited to 'src/lib/Bcfg2/Client/Frame.py')
| -rw-r--r-- | src/lib/Bcfg2/Client/Frame.py | 453 |
1 files changed, 453 insertions, 0 deletions
diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py new file mode 100644 index 000000000..9ad669ad6 --- /dev/null +++ b/src/lib/Bcfg2/Client/Frame.py @@ -0,0 +1,453 @@ +""" +Frame is the Client Framework that verifies and +installs entries, and generates statistics. +""" + +import logging +import sys +import time +import Bcfg2.Client.Tools + + +def cmpent(ent1, ent2): + """Sort entries.""" + if ent1.tag != ent2.tag: + return cmp(ent1.tag, ent2.tag) + else: + return cmp(ent1.get('name'), ent2.get('name')) + + +def promptFilter(prompt, entries): + """Filter a supplied list based on user input.""" + ret = [] + entries.sort(cmpent) + for entry in entries[:]: + if 'qtext' in entry.attrib: + iprompt = entry.get('qtext') + else: + iprompt = prompt % (entry.tag, entry.get('name')) + try: + # py3k compatibility + try: + ans = raw_input(iprompt.encode(sys.stdout.encoding, 'replace')) + except NameError: + ans = input(iprompt) + if ans in ['y', 'Y']: + ret.append(entry) + except EOFError: + # python 2.4.3 on CentOS doesn't like ^C for some reason + break + except: + print("Error while reading input") + continue + return ret + + +def matches_entry(entryspec, entry): + # both are (tag, name) + if entryspec == entry: + return True + else: + for i in [0, 1]: + if entryspec[i] == entry[i]: + continue + elif entryspec[i] == '*': + continue + elif '*' in entryspec[i]: + starpt = entryspec[i].index('*') + if entry[i].startswith(entryspec[i][:starpt]): + continue + return False + return True + + +def matches_white_list(entry, whitelist): + return True in [matches_entry(we, (entry.tag, entry.get('name'))) + for we in whitelist] + + +def passes_black_list(entry, blacklist): + return True not in [matches_entry(be, (entry.tag, entry.get('name'))) + for be in blacklist] + + +class Frame: + """Frame is the container for all Tool objects and state information.""" + def __init__(self, config, setup, times, drivers, dryrun): + self.config = config + self.times = times + self.dryrun = dryrun + self.times['initialization'] = time.time() + self.setup = setup + self.tools = [] + self.states = {} + self.whitelist = [] + self.blacklist = [] + self.removal = [] + self.logger = logging.getLogger("Bcfg2.Client.Frame") + for driver in drivers[:]: + if driver not in Bcfg2.Client.Tools.drivers and \ + isinstance(driver, str): + self.logger.error("Tool driver %s is not available" % driver) + drivers.remove(driver) + + tclass = {} + for tool in drivers: + if not isinstance(tool, str): + tclass[time.time()] = tool + tool_class = "Bcfg2.Client.Tools.%s" % tool + try: + tclass[tool] = getattr(__import__(tool_class, globals(), + locals(), ['*']), + tool) + except ImportError: + continue + except: + self.logger.error("Tool %s unexpectedly failed to load" % tool, + exc_info=1) + + for tool in list(tclass.values()): + try: + self.tools.append(tool(self.logger, setup, config)) + except Bcfg2.Client.Tools.toolInstantiationError: + continue + except: + self.logger.error("Failed to instantiate tool %s" % \ + (tool), exc_info=1) + + for tool in self.tools[:]: + for conflict in getattr(tool, 'conflicts', []): + [self.tools.remove(item) for item in self.tools \ + if item.name == conflict] + + self.logger.info("Loaded tool drivers:") + self.logger.info([tool.name for tool in self.tools]) + + # find entries not handled by any tools + problems = [entry for struct in config for \ + entry in struct if entry not in self.handled] + + if problems: + self.logger.error("The following entries are not handled by any tool:") + self.logger.error(["%s:%s:%s" % (entry.tag, entry.get('type'), \ + entry.get('name')) for entry in problems]) + self.logger.error("") + entries = [(entry.tag, entry.get('name')) + for struct in config for entry in struct] + pkgs = [(entry.get('name'), entry.get('origin')) + for struct in config for entry in struct if entry.tag == 'Package'] + multi = [] + for entry in entries[:]: + if entries.count(entry) > 1: + multi.append(entry) + entries.remove(entry) + if multi: + self.logger.debug("The following entries are included multiple times:") + self.logger.debug(["%s:%s" % entry for entry in multi]) + self.logger.debug("") + if pkgs: + self.logger.debug("The following packages are specified in bcfg2:") + self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == None]) + self.logger.debug("The following packages are prereqs added by Packages:") + self.logger.debug([pkg[0] for pkg in pkgs if pkg[1] == 'Packages']) + + def __getattr__(self, name): + if name in ['extra', 'handled', 'modified', '__important__']: + ret = [] + for tool in self.tools: + ret += getattr(tool, name) + return ret + elif name in self.__dict__: + return self.__dict__[name] + raise AttributeError(name) + + def InstallImportant(self): + """Install important entries + + We also process the decision mode stuff here because we want to prevent + non-whitelisted/blacklisted 'important' entries from being installed + prior to determining the decision mode on the client. + """ + # Need to process decision stuff early so that dryrun mode works with it + self.whitelist = [entry for entry in self.states \ + if not self.states[entry]] + if not self.setup['file']: + if self.setup['decision'] == 'whitelist': + dwl = self.setup['decision_list'] + w_to_rem = [e for e in self.whitelist \ + if not matches_white_list(e, dwl)] + if w_to_rem: + self.logger.info("In whitelist mode: suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in w_to_rem]) + self.whitelist = [x for x in self.whitelist \ + if x not in w_to_rem] + elif self.setup['decision'] == 'blacklist': + b_to_rem = [e for e in self.whitelist \ + if not passes_black_list(e, self.setup['decision_list'])] + if b_to_rem: + self.logger.info("In blacklist mode: suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_rem]) + self.whitelist = [x for x in self.whitelist if x not in b_to_rem] + + # take care of important entries first + if not self.dryrun and not self.setup['bundle']: + for cfile in [cfl for cfl in self.config.findall(".//Path") \ + if cfl.get('name') in self.__important__ and \ + cfl.get('type') == 'file']: + if cfile not in self.whitelist: + continue + tl = [t for t in self.tools if t.handlesEntry(cfile) \ + and t.canVerify(cfile)] + if tl: + if self.setup['interactive'] and not \ + promptFilter("Install %s: %s? (y/N):", [cfile]): + self.whitelist.remove(cfile) + continue + try: + self.states[cfile] = tl[0].InstallPath(cfile) + if self.states[cfile]: + tl[0].modified.append(cfile) + except: + self.logger.error("Unexpected tool failure", + exc_info=1) + cfile.set('qtext', '') + if tl[0].VerifyPath(cfile, []): + self.whitelist.remove(cfile) + + def Inventory(self): + """ + Verify all entries, + find extra entries, + and build up workqueues + + """ + # initialize all states + for struct in self.config.getchildren(): + for entry in struct.getchildren(): + self.states[entry] = False + for tool in self.tools: + try: + tool.Inventory(self.states) + except: + self.logger.error("%s.Inventory() call failed:" % tool.name, exc_info=1) + + def Decide(self): + """Set self.whitelist based on user interaction.""" + prompt = "Install %s: %s? (y/N): " + rprompt = "Remove %s: %s? (y/N): " + if self.setup['remove']: + if self.setup['remove'] == 'all': + self.removal = self.extra + elif self.setup['remove'] in ['services', 'Services']: + self.removal = [entry for entry in self.extra + if entry.tag == 'Service'] + elif self.setup['remove'] in ['packages', 'Packages']: + self.removal = [entry for entry in self.extra + if entry.tag == 'Package'] + + candidates = [entry for entry in self.states + if not self.states[entry]] + + if self.dryrun: + if self.whitelist: + self.logger.info("In dryrun mode: suppressing entry installation for:") + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) + for entry in self.whitelist]) + self.whitelist = [] + if self.removal: + self.logger.info("In dryrun mode: suppressing entry removal for:") + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) + for entry in self.removal]) + self.removal = [] + return + # Here is where most of the work goes + # first perform bundle filtering + if self.setup['bundle']: + all_bundle_names = [b.get('name') for b in + self.config.findall('./Bundle')] + # warn if non-existent bundle given + for bundle in self.setup['bundle']: + if bundle not in all_bundle_names: + self.logger.info("Warning: Bundle %s not found" % bundle) + bundles = [b for b in self.config.findall('./Bundle') + if b.get('name') in self.setup['bundle']] + self.whitelist = [e for e in self.whitelist + if True in [e in b for b in bundles]] + elif self.setup['indep']: + bundles = [nb for nb in self.config.getchildren() + if nb.tag != 'Bundle'] + else: + bundles = self.config.getchildren() + + # first process prereq actions + for bundle in bundles[:]: + if bundle.tag != 'Bundle': + continue + bmodified = len([item for item in bundle if item in self.whitelist]) + actions = [a for a in bundle.findall('./Action') + if (a.get('timing') != 'post' and + (bmodified or a.get('when') == 'always'))] + # now we process all "always actions" + if self.setup['interactive']: + promptFilter(prompt, actions) + self.DispatchInstallCalls(actions) + + # need to test to fail entries in whitelist + if False in [self.states[a] for a in actions]: + # then display bundles forced off with entries + self.logger.info("Bundle %s failed prerequisite action" % + (bundle.get('name'))) + bundles.remove(bundle) + b_to_remv = [ent for ent in self.whitelist if ent in bundle] + if b_to_remv: + self.logger.info("Not installing entries from Bundle %s" % + (bundle.get('name'))) + self.logger.info(["%s:%s" % (e.tag, e.get('name')) + for e in b_to_remv]) + [self.whitelist.remove(ent) for ent in b_to_remv] + + if self.setup['interactive']: + self.whitelist = promptFilter(prompt, self.whitelist) + self.removal = promptFilter(rprompt, self.removal) + + for entry in candidates: + if entry not in self.whitelist: + self.blacklist.append(entry) + + def DispatchInstallCalls(self, entries): + """Dispatch install calls to underlying tools.""" + for tool in self.tools: + handled = [entry for entry in entries if tool.canInstall(entry)] + if not handled: + continue + try: + tool.Install(handled, self.states) + except: + self.logger.error("%s.Install() call failed:" % tool.name, exc_info=1) + + def Install(self): + """Install all entries.""" + self.DispatchInstallCalls(self.whitelist) + mods = self.modified + mbundles = [struct for struct in self.config.findall('Bundle') if \ + [mod for mod in mods if mod in struct]] + + if self.modified: + # Handle Bundle interdeps + if mbundles: + self.logger.info("The Following Bundles have been modified:") + self.logger.info([mbun.get('name') for mbun in mbundles]) + self.logger.info("") + tbm = [(t, b) for t in self.tools for b in mbundles] + for tool, bundle in tbm: + try: + tool.Inventory(self.states, [bundle]) + except: + self.logger.error("%s.Inventory() call failed:" % tool.name, exc_info=1) + clobbered = [entry for bundle in mbundles for entry in bundle \ + if not self.states[entry] and entry not in self.blacklist] + if clobbered: + self.logger.debug("Found clobbered entries:") + self.logger.debug(["%s:%s" % (entry.tag, entry.get('name')) \ + for entry in clobbered]) + if not self.setup['interactive']: + self.DispatchInstallCalls(clobbered) + + for bundle in self.config.findall('.//Bundle'): + if self.setup['bundle'] and \ + bundle.get('name') not in self.setup['bundle']: + # prune out unspecified bundles when running with -b + continue + for tool in self.tools: + try: + if bundle in mbundles: + tool.BundleUpdated(bundle, self.states) + else: + tool.BundleNotUpdated(bundle, self.states) + except: + self.logger.error("%s.BundleNotUpdated() call failed:" % \ + (tool.name), exc_info=1) + + def Remove(self): + """Remove extra entries.""" + for tool in self.tools: + extras = [entry for entry in self.removal if tool.handlesEntry(entry)] + if extras: + try: + tool.Remove(extras) + except: + self.logger.error("%s.Remove() failed" % tool.name, exc_info=1) + + def CondDisplayState(self, phase): + """Conditionally print tracing information.""" + self.logger.info('\nPhase: %s' % phase) + self.logger.info('Correct entries:\t%d' % list(self.states.values()).count(True)) + self.logger.info('Incorrect entries:\t%d' % list(self.states.values()).count(False)) + if phase == 'final' and list(self.states.values()).count(False): + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) for \ + entry in self.states if not self.states[entry]]) + self.logger.info('Total managed entries:\t%d' % len(list(self.states.values()))) + self.logger.info('Unmanaged entries:\t%d' % len(self.extra)) + if phase == 'final' and self.setup['extra']: + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) \ + for entry in self.extra]) + + self.logger.info("") + + if ((list(self.states.values()).count(False) == 0) and not self.extra): + self.logger.info('All entries correct.') + + def ReInventory(self): + """Recheck everything.""" + if not self.dryrun and self.setup['kevlar']: + self.logger.info("Rechecking system inventory") + self.Inventory() + + def Execute(self): + """Run all methods.""" + self.Inventory() + self.times['inventory'] = time.time() + self.CondDisplayState('initial') + self.InstallImportant() + self.Decide() + self.Install() + self.times['install'] = time.time() + self.Remove() + self.times['remove'] = time.time() + if self.modified: + self.ReInventory() + self.times['reinventory'] = time.time() + self.times['finished'] = time.time() + self.CondDisplayState('final') + + def GenerateStats(self): + """Generate XML summary of execution statistics.""" + feedback = Bcfg2.Client.XML.Element("upload-statistics") + stats = Bcfg2.Client.XML.SubElement(feedback, + 'Statistics', + total=str(len(self.states)), + version='2.0', + revision=self.config.get('revision', '-1')) + good = len([key for key, val in list(self.states.items()) if val]) + stats.set('good', str(good)) + if len([key for key, val in list(self.states.items()) if not val]) == 0: + stats.set('state', 'clean') + else: + stats.set('state', 'dirty') + + # List bad elements of the configuration + for (data, ename) in [(self.modified, 'Modified'), (self.extra, "Extra"), \ + ([entry for entry in self.states if not \ + self.states[entry]], "Bad")]: + container = Bcfg2.Client.XML.SubElement(stats, ename) + for item in data: + item.set('qtext', '') + container.append(item) + item.text = None + + timeinfo = Bcfg2.Client.XML.Element("OpStamps") + feedback.append(stats) + for (event, timestamp) in list(self.times.items()): + timeinfo.set(event, str(timestamp)) + stats.append(timeinfo) + return feedback |