diff options
Diffstat (limited to 'src/lib/Server/Lint')
-rw-r--r-- | src/lib/Server/Lint/Comments.py | 2 | ||||
-rw-r--r-- | src/lib/Server/Lint/MergeFiles.py | 71 | ||||
-rw-r--r-- | src/lib/Server/Lint/Validate.py | 23 | ||||
-rw-r--r-- | src/lib/Server/Lint/__init__.py | 44 |
4 files changed, 121 insertions, 19 deletions
diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 8e86cc564..09443d4c0 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -70,7 +70,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): props = self.core.plugins['Properties'] for propfile, pdata in props.store.entries.items(): if os.path.splitext(propfile)[1] == ".xml": - self.check_xml(pdata.name, pdata.data, 'properties') + self.check_xml(pdata.name, pdata.xdata, 'properties') def check_metadata(self): """ check metadata files for required headers """ diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py new file mode 100644 index 000000000..1e177acff --- /dev/null +++ b/src/lib/Server/Lint/MergeFiles.py @@ -0,0 +1,71 @@ +import os +from copy import deepcopy +from difflib import SequenceMatcher +import Bcfg2.Options +import Bcfg2.Server.Lint + +class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): + """ find Probes or Cfg files with multiple similar files that + might be merged into one """ + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + if 'Cfg' in self.core.plugins: + self.check_cfg() + if 'Probes' in self.core.plugins: + self.check_probes() + + def check_cfg(self): + for filename, entryset in self.core.plugins['Cfg'].entries.items(): + for mset in self.get_similar(entryset.entries): + self.LintError("merge-cfg", + "The following files are similar: %s. " + "Consider merging them into a single Genshi " + "template." % + ", ".join([os.path.join(filename, p) + for p in mset])) + + def check_probes(self): + probes = self.core.plugins['Probes'].probes.entries + for mset in self.get_similar(probes): + self.LintError("merge-cfg", + "The following probes are similar: %s. " + "Consider merging them into a single probe." % + ", ".join([p for p in mset])) + + def get_similar(self, entries): + if "threshold" in self.config: + # accept threshold either as a percent (e.g., "threshold=75") or + # as a ratio (e.g., "threshold=.75") + threshold = float(self.config['threshold']) + if threshold > 1: + threshold /= 100 + else: + threshold = 0.75 + rv = [] + elist = entries.items() + while elist: + result = self._find_similar(elist.pop(0), deepcopy(elist), + threshold) + if len(result) > 1: + elist = [(fname, fdata) + for fname, fdata in elist + if fname not in result] + rv.append(result) + return rv + + def _find_similar(self, ftuple, others, threshold): + fname, fdata = ftuple + rv = [fname] + while others: + cname, cdata = others.pop(0) + sm = SequenceMatcher(None, fdata.data, cdata.data) + # perform progressively more expensive comparisons + if (sm.real_quick_ratio() > threshold and + sm.quick_ratio() > threshold and + sm.ratio() > threshold): + rv.extend(self._find_similar((cname, cdata), deepcopy(others), + threshold)) + return rv + + diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index 834608378..8a8406e73 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -1,10 +1,12 @@ +import fnmatch import glob import lxml.etree import os -import fnmatch +from subprocess import Popen, PIPE, STDOUT +import sys + import Bcfg2.Options import Bcfg2.Server.Lint -from subprocess import Popen, PIPE, STDOUT class Validate(Bcfg2.Server.Lint.ServerlessPlugin): """ Ensure that the repo validates """ @@ -14,7 +16,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.filesets = {"metadata:groups":"%s/metadata.xsd", "metadata:clients":"%s/clients.xsd", "info":"%s/info.xsd", - "%s/Bundler/*.{xml,genshi}":"%s/bundle.xsd", + "%s/Bundler/*.xml":"%s/bundle.xsd", + "%s/Bundler/*.genshi":"%s/bundle.xsd", "%s/Pkgmgr/*.xml":"%s/pkglist.xsd", "%s/Base/*.xml":"%s/base.xsd", "%s/Rules/*.xml":"%s/rules.xsd", @@ -33,23 +36,27 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): @Bcfg2.Server.Lint.returnErrors def Run(self): - self.schemadir = self.config['schema'] + schemadir = self.config['schema'] - for schemaname, path in self.filesets.items(): + for path, schemaname in self.filesets.items(): try: filelist = self.filelists[path] except KeyError: filelist = [] - + if filelist: # avoid loading schemas for empty file lists try: schema = lxml.etree.XMLSchema(lxml.etree.parse(schemaname % schemadir)) + except IOError: + e = sys.exc_info()[1] + self.LintError("input-output-error", e.message) + continue except: self.LintError("schema-failed-to-parse", - "Failed to process schema %s", - schemaname % schemadir) + "Failed to process schema %s" % + (schemaname % schemadir)) continue for filename in filelist: self.validate(filename, schemaname % schemadir, diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index 3b89d1f9e..63cb2463b 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -4,6 +4,7 @@ __all__ = ['Bundles', 'Comments', 'Duplicates', 'InfoXML', + 'MergeFiles', 'Pkgmgr', 'RequiredAttrs', 'Validate'] @@ -11,6 +12,7 @@ __all__ = ['Bundles', import logging import os.path from copy import copy +import textwrap import lxml.etree import Bcfg2.Logger @@ -84,7 +86,10 @@ class ErrorHandler (object): "properties-schema-not-found":"warning", "xml-failed-to-parse":"error", "xml-failed-to-read":"error", - "xml-failed-to-verify":"error",} + "xml-failed-to-verify":"error", + "merge-cfg":"warning", + "merge-probes":"warning", + "input-output-error": "error"} def __init__(self, config=None): self.errors = 0 @@ -92,6 +97,9 @@ class ErrorHandler (object): self.logger = logging.getLogger('bcfg2-lint') + self._wrapper = textwrap.TextWrapper(initial_indent = " ", + subsequent_indent = " ") + self._handlers = {} if config is not None: for err, action in config.items(): @@ -116,26 +124,42 @@ class ErrorHandler (object): self._handlers[err](msg) self.logger.debug(" (%s)" % err) else: - self.logger.info("Unknown error %s" % err) + # assume that it's an error, but complain + self.error(msg) + self.logger.warning("Unknown error %s" % err) def error(self, msg): """ log an error condition """ self.errors += 1 - lines = msg.splitlines() - self.logger.error("ERROR: %s" % lines.pop()) - [self.logger.error(" %s" % l) for l in lines] + self._log(msg, self.logger.error, prefix="ERROR: ") def warn(self, msg): """ log a warning condition """ self.warnings += 1 - lines = msg.splitlines() - self.logger.warning("WARNING: %s" % lines.pop()) - [self.logger.warning(" %s" % l) for l in lines] + self._log(msg, self.logger.warning, prefix="WARNING: ") def debug(self, msg): """ log a silent/debug condition """ - lines = msg.splitlines() - [self.logger.debug("%s" % l) for l in lines] + self._log(msg, self.logger.debug) + + def _log(self, msg, logfunc, prefix=""): + # a message may itself consist of multiple lines. wrap() will + # elide them all into a single paragraph, which we don't want. + # so we split the message into its paragraphs and wrap each + # paragraph individually. this means, unfortunately, that we + # lose textwrap's built-in initial indent functionality, + # because we want to only treat the very first line of the + # first paragraph specially. so we do some silliness. + rawlines = msg.splitlines() + firstline = True + for rawline in rawlines: + lines = self._wrapper.wrap(rawline) + for line in lines: + if firstline: + logfunc("%s%s" % (prefix, line.lstrip())) + firstline = False + else: + logfunc(line) class ServerlessPlugin (Plugin): |