diff options
| -rw-r--r-- | examples/bcfg2-lint.conf | 5 | ||||
| -rw-r--r-- | man/bcfg2-lint.8 | 5 | ||||
| -rw-r--r-- | man/bcfg2-lint.conf.5 | 8 | ||||
| -rw-r--r-- | src/lib/Server/Lint/MergeFiles.py | 71 | ||||
| -rw-r--r-- | src/lib/Server/Lint/__init__.py | 43 |
5 files changed, 121 insertions, 11 deletions
diff --git a/examples/bcfg2-lint.conf b/examples/bcfg2-lint.conf index abf969496..9c0d2c72a 100644 --- a/examples/bcfg2-lint.conf +++ b/examples/bcfg2-lint.conf @@ -1,5 +1,5 @@ [lint] -plugins=Duplicates,InfoXML,Bundles,Comments,RequiredAttrs,Validate +plugins=Duplicates,InfoXML,Bundles,Comments,RequiredAttrs,Validate,MergeFiles [errors] no-infoxml=error @@ -23,3 +23,6 @@ probe_comments = Maintainer,Purpose,Groups,Other Output [Validate] schema=/usr/share/bcfg2/schema + +[MergeFiles] +threshold=85 diff --git a/man/bcfg2-lint.8 b/man/bcfg2-lint.8 index b1fa9244b..f2d4f9e88 100644 --- a/man/bcfg2-lint.8 +++ b/man/bcfg2-lint.8 @@ -120,6 +120,11 @@ exists for all Cfg files, and that paranoid mode be enabled for all files. .TP +.BR MergeFiles +Suggest that similar probes and config files be merged into single +probes or TGenshi templates. + +.TP .BR Pkgmgr Check for duplicate packages specified in Pkgmgr. diff --git a/man/bcfg2-lint.conf.5 b/man/bcfg2-lint.conf.5 index 49a32bb22..10a812874 100644 --- a/man/bcfg2-lint.conf.5 +++ b/man/bcfg2-lint.conf.5 @@ -152,6 +152,14 @@ A comma-delimited list of attributes to require on tags. Default is "owner,group,perms". .TP +.BR MergeFiles + +\(bu +.B threshold +The threshold at which MergeFiles will suggest merging config files +and probes. Default is 75% similar. + +.TP .BR Validate \(bu diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py new file mode 100644 index 000000000..1e177acff --- /dev/null +++ b/src/lib/Server/Lint/MergeFiles.py @@ -0,0 +1,71 @@ +import os +from copy import deepcopy +from difflib import SequenceMatcher +import Bcfg2.Options +import Bcfg2.Server.Lint + +class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): + """ find Probes or Cfg files with multiple similar files that + might be merged into one """ + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + if 'Cfg' in self.core.plugins: + self.check_cfg() + if 'Probes' in self.core.plugins: + self.check_probes() + + def check_cfg(self): + for filename, entryset in self.core.plugins['Cfg'].entries.items(): + for mset in self.get_similar(entryset.entries): + self.LintError("merge-cfg", + "The following files are similar: %s. " + "Consider merging them into a single Genshi " + "template." % + ", ".join([os.path.join(filename, p) + for p in mset])) + + def check_probes(self): + probes = self.core.plugins['Probes'].probes.entries + for mset in self.get_similar(probes): + self.LintError("merge-cfg", + "The following probes are similar: %s. " + "Consider merging them into a single probe." % + ", ".join([p for p in mset])) + + def get_similar(self, entries): + if "threshold" in self.config: + # accept threshold either as a percent (e.g., "threshold=75") or + # as a ratio (e.g., "threshold=.75") + threshold = float(self.config['threshold']) + if threshold > 1: + threshold /= 100 + else: + threshold = 0.75 + rv = [] + elist = entries.items() + while elist: + result = self._find_similar(elist.pop(0), deepcopy(elist), + threshold) + if len(result) > 1: + elist = [(fname, fdata) + for fname, fdata in elist + if fname not in result] + rv.append(result) + return rv + + def _find_similar(self, ftuple, others, threshold): + fname, fdata = ftuple + rv = [fname] + while others: + cname, cdata = others.pop(0) + sm = SequenceMatcher(None, fdata.data, cdata.data) + # perform progressively more expensive comparisons + if (sm.real_quick_ratio() > threshold and + sm.quick_ratio() > threshold and + sm.ratio() > threshold): + rv.extend(self._find_similar((cname, cdata), deepcopy(others), + threshold)) + return rv + + diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index 3b89d1f9e..013cbf2ba 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -4,6 +4,7 @@ __all__ = ['Bundles', 'Comments', 'Duplicates', 'InfoXML', + 'MergeFiles', 'Pkgmgr', 'RequiredAttrs', 'Validate'] @@ -11,6 +12,7 @@ __all__ = ['Bundles', import logging import os.path from copy import copy +import textwrap import lxml.etree import Bcfg2.Logger @@ -84,7 +86,9 @@ class ErrorHandler (object): "properties-schema-not-found":"warning", "xml-failed-to-parse":"error", "xml-failed-to-read":"error", - "xml-failed-to-verify":"error",} + "xml-failed-to-verify":"error", + "merge-cfg":"warning", + "merge-probes":"warning",} def __init__(self, config=None): self.errors = 0 @@ -92,6 +96,9 @@ class ErrorHandler (object): self.logger = logging.getLogger('bcfg2-lint') + self._wrapper = textwrap.TextWrapper(initial_indent = " ", + subsequent_indent = " ") + self._handlers = {} if config is not None: for err, action in config.items(): @@ -116,26 +123,42 @@ class ErrorHandler (object): self._handlers[err](msg) self.logger.debug(" (%s)" % err) else: - self.logger.info("Unknown error %s" % err) + # assume that it's an error, but complain + self.error(msg) + self.logger.warning("Unknown error %s" % err) def error(self, msg): """ log an error condition """ self.errors += 1 - lines = msg.splitlines() - self.logger.error("ERROR: %s" % lines.pop()) - [self.logger.error(" %s" % l) for l in lines] + self._log(msg, self.logger.error, prefix="ERROR: ") def warn(self, msg): """ log a warning condition """ self.warnings += 1 - lines = msg.splitlines() - self.logger.warning("WARNING: %s" % lines.pop()) - [self.logger.warning(" %s" % l) for l in lines] + self._log(msg, self.logger.warning, prefix="WARNING: ") def debug(self, msg): """ log a silent/debug condition """ - lines = msg.splitlines() - [self.logger.debug("%s" % l) for l in lines] + self._log(msg, self.logger.debug) + + def _log(self, msg, logfunc, prefix=""): + # a message may itself consist of multiple lines. wrap() will + # elide them all into a single paragraph, which we don't want. + # so we split the message into its paragraphs and wrap each + # paragraph individually. this means, unfortunately, that we + # lose textwrap's built-in initial indent functionality, + # because we want to only treat the very first line of the + # first paragraph specially. so we do some silliness. + rawlines = msg.splitlines() + firstline = True + for rawline in rawlines: + lines = self._wrapper.wrap(rawline) + for line in lines: + if firstline: + logfunc("%s%s" % (prefix, line.lstrip())) + firstline = False + else: + logfunc(line) class ServerlessPlugin (Plugin): |