diff options
author | Narayan Desai <desai@mcs.anl.gov> | 2011-06-22 17:01:40 -0500 |
---|---|---|
committer | Narayan Desai <desai@mcs.anl.gov> | 2011-06-22 17:01:40 -0500 |
commit | 4849378a62ab0dbf72f8ce4e6b2073dc73f2337a (patch) | |
tree | 14945dd6045c438f5dbc8e96f023258d31e1adf9 /src/lib | |
parent | 8d08d5195fa0b6f503752273183948e5d24f33d6 (diff) | |
parent | 2db056574f5f55c3c802da63abe7e3de10db5d76 (diff) | |
download | bcfg2-4849378a62ab0dbf72f8ce4e6b2073dc73f2337a.tar.gz bcfg2-4849378a62ab0dbf72f8ce4e6b2073dc73f2337a.tar.bz2 bcfg2-4849378a62ab0dbf72f8ce4e6b2073dc73f2337a.zip |
Merge branch 'master' of git.mcs.anl.gov:bcfg2
Diffstat (limited to 'src/lib')
38 files changed, 692 insertions, 268 deletions
diff --git a/src/lib/Client/Frame.py b/src/lib/Client/Frame.py index 60d158eb1..d8e308ee0 100644 --- a/src/lib/Client/Frame.py +++ b/src/lib/Client/Frame.py @@ -5,6 +5,7 @@ installs entries, and generates statistics. __revision__ = '$Revision$' import logging +import sys import time import Bcfg2.Client.Tools @@ -29,7 +30,7 @@ def promptFilter(prompt, entries): try: # py3k compatibility try: - ans = raw_input(iprompt) + ans = raw_input(iprompt.encode(sys.stdout.encoding, 'replace')) except NameError: ans = input(iprompt) if ans in ['y', 'Y']: @@ -122,22 +123,7 @@ class Frame: self.logger.info("Loaded tool drivers:") self.logger.info([tool.name for tool in self.tools]) - if not self.dryrun and not self.setup['bundle']: - for cfile in [cfl for cfl in config.findall(".//Path") \ - if cfl.get('name') in self.__important__ and \ - cfl.get('type') == 'file']: - tl = [t for t in self.tools if t.handlesEntry(cfile) \ - and t.canVerify(cfile)] - if tl: - if not tl[0].VerifyPath(cfile, []): - if self.setup['interactive'] and not \ - promptFilter("Install %s: %s? (y/N):", [cfile]): - continue - try: - self.states[cfile] = tl[0].InstallPath(cfile) - except: - self.logger.error("Unexpected tool failure", - exc_info=1) + # find entries not handled by any tools problems = [entry for struct in config for \ entry in struct if entry not in self.handled] @@ -176,6 +162,57 @@ class Frame: return self.__dict__[name] raise AttributeError(name) + def InstallImportant(self): + """Install important entries + + We also process the decision mode stuff here because we want to prevent + non-whitelisted/blacklisted 'important' entries from being installed + prior to determining the decision mode on the client. + """ + # Need to process decision stuff early so that dryrun mode works with it + self.whitelist = [entry for entry in self.states \ + if not self.states[entry]] + if self.setup['decision'] == 'whitelist': + dwl = self.setup['decision_list'] + w_to_rem = [e for e in self.whitelist \ + if not matches_white_list(e, dwl)] + if w_to_rem: + self.logger.info("In whitelist mode: suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in w_to_rem]) + self.whitelist = [x for x in self.whitelist \ + if x not in w_to_rem] + + elif self.setup['decision'] == 'blacklist': + b_to_rem = [e for e in self.whitelist \ + if not passes_black_list(e, self.setup['decision_list'])] + if b_to_rem: + self.logger.info("In blacklist mode: suppressing installation of:") + self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_rem]) + self.whitelist = [x for x in self.whitelist if x not in b_to_rem] + + # take care of important entries first + if not self.dryrun and not self.setup['bundle']: + for cfile in [cfl for cfl in self.config.findall(".//Path") \ + if cfl.get('name') in self.__important__ and \ + cfl.get('type') == 'file']: + if cfile not in self.whitelist: + continue + tl = [t for t in self.tools if t.handlesEntry(cfile) \ + and t.canVerify(cfile)] + if tl: + if self.setup['interactive'] and not \ + promptFilter("Install %s: %s? (y/N):", [cfile]): + self.whitelist.remove(cfile) + continue + try: + self.states[cfile] = tl[0].InstallPath(cfile) + except: + self.logger.error("Unexpected tool failure", + exc_info=1) + cfile.set('qtext', '') + if tl[0].VerifyPath(cfile, []): + self.whitelist.remove(cfile) + def Inventory(self): """ Verify all entries, @@ -209,26 +246,6 @@ class Frame: candidates = [entry for entry in self.states \ if not self.states[entry]] - self.whitelist = [entry for entry in self.states \ - if not self.states[entry]] - # Need to process decision stuff early so that dryrun mode works with it - if self.setup['decision'] == 'whitelist': - dwl = self.setup['decision_list'] - w_to_rem = [e for e in self.whitelist \ - if not matches_white_list(e, dwl)] - if w_to_rem: - self.logger.info("In whitelist mode: suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in w_to_rem]) - self.whitelist = [x for x in self.whitelist \ - if x not in w_to_rem] - - elif self.setup['decision'] == 'blacklist': - b_to_rem = [e for e in self.whitelist \ - if not passes_black_list(e, self.setup['decision_list'])] - if b_to_rem: - self.logger.info("In blacklist mode: suppressing installation of:") - self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_rem]) - self.whitelist = [x for x in self.whitelist if x not in b_to_rem] if self.dryrun: if self.whitelist: @@ -388,6 +405,7 @@ class Frame: self.Inventory() self.times['inventory'] = time.time() self.CondDisplayState('initial') + self.InstallImportant() self.Decide() self.Install() self.times['install'] = time.time() diff --git a/src/lib/Client/Tools/APT.py b/src/lib/Client/Tools/APT.py index a838f5e27..2b8cc3304 100644 --- a/src/lib/Client/Tools/APT.py +++ b/src/lib/Client/Tools/APT.py @@ -9,6 +9,8 @@ warnings.filterwarnings("ignore", "Accessed deprecated property Package.installe warnings.filterwarnings("ignore", "Accessed deprecated property Package.candidateVersion, please see the Version class for alternatives.", DeprecationWarning) warnings.filterwarnings("ignore", "Deprecated, please use 'is_installed' instead", DeprecationWarning) warnings.filterwarnings("ignore", "Attribute 'IsUpgradable' of the 'apt_pkg.DepCache' object is deprecated, use 'is_upgradable' instead.", DeprecationWarning) +warnings.filterwarnings("ignore", "Attribute 'VersionList' of the 'apt_pkg.Package' object is deprecated, use 'version_list' instead.", DeprecationWarning) +warnings.filterwarnings("ignore", "Attribute 'VerStr' of the 'apt_pkg.Version' object is deprecated, use 'ver_str' instead.", DeprecationWarning) import apt.cache import os @@ -71,7 +73,8 @@ class APT(Bcfg2.Client.Tools.Tool): self.cmd.run("%s clean" % APTGET) try: self.pkg_cache = apt.cache.Cache() - except SystemError, e: + except SystemError: + e = sys.exc_info()[1] self.logger.info("Failed to initialize APT cache: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError self.pkg_cache.update() diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Client/Tools/POSIX.py index af3d1a473..faec2e251 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Client/Tools/POSIX.py @@ -14,7 +14,12 @@ import os import pwd import shutil import stat +import sys import time +# py3k compatibility +if sys.hexversion >= 0x03000000: + unicode = str + import Bcfg2.Client.Tools import Bcfg2.Options from Bcfg2.Client import XML @@ -55,7 +60,8 @@ def normGid(entry): except: return int(grp.getgrnam(entry.get('group'))[2]) except (OSError, KeyError): - log.error('GID normalization failed for %s' % (entry.get('name'))) + log.error('GID normalization failed for %s. Does group %s exist?' + % (entry.get('name'), entry.get('group'))) return False @@ -70,7 +76,23 @@ def normUid(entry): except: return int(pwd.getpwnam(entry.get('owner'))[2]) except (OSError, KeyError): - log.error('UID normalization failed for %s' % (entry.get('name'))) + log.error('UID normalization failed for %s. Does owner %s exist?' + % (entry.get('name'), entry.get('owner'))) + return False + + +def isString(strng, encoding): + """ + Returns true if the string contains no ASCII control characters + and can be decoded from the specified encoding. + """ + for char in strng: + if ord(char) < 9 or ord(char) > 13 and ord(char) < 32: + return False + try: + strng.decode(encoding) + return True + except: return False @@ -127,7 +149,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): try: content = open(entry.get('name')).read() entry.set('current_bfile', binascii.b2a_base64(content)) - except IOError, error: + except IOError: + error = sys.exc_info()[1] self.logger.error("Failed to read %s: %s" % (error.filename, error.strerror)) @@ -439,12 +462,14 @@ class POSIX(Bcfg2.Client.Tools.Tool): if type(tempdata) == unicode: try: tempdata = tempdata.encode(self.setup['encoding']) - except UnicodeEncodeError, e: + except UnicodeEncodeError: + e = sys.exc_info()[1] self.logger.error("Error encoding file %s:\n %s" % \ (entry.get('name'), e)) try: content = open(entry.get('name')).read() - except IOError, error: + except IOError: + error = sys.exc_info()[1] if error.strerror == "No such file or directory": # print diff for files that don't exist (yet) content = '' @@ -456,12 +481,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): # md5sum so it would be faster for big binary files contentStatus = content == tempdata if not contentStatus: - try: - content.decode('ascii') - isstring = True - except: - isstring = False - if tbin or not isstring: + if tbin or not isString(content, self.setup['encoding']): entry.set('current_bfile', binascii.b2a_base64(content)) nqtext = entry.get('qtext', '') nqtext += '\nBinary file, no printable diff' @@ -491,15 +511,15 @@ class POSIX(Bcfg2.Client.Tools.Tool): difflib.unified_diff(content.split('\n'), \ tempdata.split('\n'))]) try: - eudiff = udiff.encode('ascii') + dudiff = udiff.decode(self.setup['encoding']) except: - eudiff = "Binary file: no diff printed" + dudiff = "Binary file: no diff printed" nqtext = entry.get('qtext', '') if nqtext: nqtext += '\n' - nqtext += eudiff + nqtext += dudiff else: entry.set('current_bfile', binascii.b2a_base64(content)) nqtext = entry.get('qtext', '') @@ -547,8 +567,14 @@ class POSIX(Bcfg2.Client.Tools.Tool): (entry.get('current_exists', 'true') == 'false'): bkupnam = entry.get('name').replace('/', '_') # current list of backups for this file - bkuplist = [f for f in os.listdir(self.ppath) if - f.startswith(bkupnam)] + try: + bkuplist = [f for f in os.listdir(self.ppath) if + f.startswith(bkupnam)] + except OSError: + e = sys.exc_info()[1] + self.logger.error("Failed to create backup list in %s: %s" % + (self.ppath, e.strerror)) + return False bkuplist.sort() while len(bkuplist) >= int(self.max_copies): # remove the oldest backup available @@ -567,7 +593,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): datetime.isoformat(datetime.now()))) self.logger.info("Backup of %s saved to %s" % (entry.get('name'), self.ppath)) - except IOError, e: + except IOError: + e = sys.exc_info()[1] self.logger.error("Failed to create backup file for %s" % \ (entry.get('name'))) self.logger.error(e) @@ -603,7 +630,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): % (entry.get('name'))) return False return True - except (OSError, IOError), err: + except (OSError, IOError): + err = sys.exc_info()[1] if err.errno == errno.EACCES: self.logger.info("Failed to open %s for writing" % (entry.get('name'))) else: @@ -683,7 +711,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): return False try: shutil.rmtree(ename) - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) else: @@ -691,20 +720,63 @@ class POSIX(Bcfg2.Client.Tools.Tool): try: os.rmdir(ename) return True - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) return False try: os.remove(ename) return True - except OSError, e: + except OSError: + e = sys.exc_info()[1] self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) return False def Verifypermissions(self, entry, _): """Verify Path type='permissions' entry""" + if entry.get('perms') == None or \ + entry.get('owner') == None or \ + entry.get('group') == None: + self.logger.error('Entry %s not completely specified. ' + 'Try running bcfg2-lint.' % (entry.get('name'))) + return False + if entry.get('recursive') in ['True', 'true']: + # verify ownership information recursively + owner = normUid(entry) + group = normGid(entry) + + for root, dirs, files in os.walk(entry.get('name')): + for p in dirs + files: + path = os.path.join(root, p) + pstat = os.stat(path) + if owner != pstat.st_uid: + # owner mismatch for path + entry.set('current_owner', str(pstat.st_uid)) + self.logger.debug("%s %s ownership wrong" % \ + (entry.tag, path)) + nqtext = entry.get('qtext', '') + '\n' + nqtext += ("Owner for path %s is incorrect. " + "Current owner is %s but should be %s\n" % \ + (path, pstat.st_uid, entry.get('owner'))) + nqtext += ("\nInstall %s %s: (y/N): " % + (entry.tag, entry.get('name'))) + entry.set('qtext', nqtext) + return False + if group != pstat.st_gid: + # group mismatch for path + entry.set('current_group', str(pstat.st_gid)) + self.logger.debug("%s %s group wrong" % \ + (entry.tag, path)) + nqtext = entry.get('qtext', '') + '\n' + nqtext += ("Group for path %s is incorrect. " + "Current group is %s but should be %s\n" % \ + (path, pstat.st_gid, entry.get('group'))) + nqtext += ("\nInstall %s %s: (y/N): " % + (entry.tag, entry.get('name'))) + entry.set('qtext', nqtext) + return False return self.Verifydirectory(entry, _) def Installpermissions(self, entry): @@ -715,9 +787,23 @@ class POSIX(Bcfg2.Client.Tools.Tool): self.logger.error('Entry %s not completely specified. ' 'Try running bcfg2-lint.' % (entry.get('name'))) return False + plist = [entry.get('name')] + if entry.get('recursive') in ['True', 'true']: + # verify ownership information recursively + owner = normUid(entry) + group = normGid(entry) + + for root, dirs, files in os.walk(entry.get('name')): + for p in dirs + files: + path = os.path.join(root, p) + pstat = os.stat(path) + if owner != pstat.st_uid or group != pstat.st_gid: + # owner mismatch for path + plist.append(path) try: - os.chown(entry.get('name'), normUid(entry), normGid(entry)) - os.chmod(entry.get('name'), calcPerms(S_IFDIR, entry.get('perms'))) + for p in plist: + os.chown(p, normUid(entry), normGid(entry)) + os.chmod(p, calcPerms(S_IFDIR, entry.get('perms'))) return True except (OSError, KeyError): self.logger.error('Permission fixup failed for %s' % \ diff --git a/src/lib/Client/Tools/Pacman.py b/src/lib/Client/Tools/Pacman.py index 082897934..c8c05061c 100644 --- a/src/lib/Client/Tools/Pacman.py +++ b/src/lib/Client/Tools/Pacman.py @@ -78,5 +78,6 @@ class Pacman(Bcfg2.Client.Tools.PkgTool): try: self.logger.debug("Running : %s -S %s" % (self.pkgtool, pkgline)) self.cmd.run("%s -S %s" % (self.pkgtool, pkgline)) - except Exception, e: + except Exception: + e = sys.exc_info()[1] self.logger.error("Error occurred during installation: %s" % e) diff --git a/src/lib/Client/Tools/RPMng.py b/src/lib/Client/Tools/RPMng.py index a1e14b3a6..5376118c2 100644 --- a/src/lib/Client/Tools/RPMng.py +++ b/src/lib/Client/Tools/RPMng.py @@ -2,11 +2,12 @@ __revision__ = '$Revision$' -import ConfigParser import os.path import rpm import rpmtools import Bcfg2.Client.Tools +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: diff --git a/src/lib/Client/Tools/YUM24.py b/src/lib/Client/Tools/YUM24.py index 04d9f5c07..66768fb34 100644 --- a/src/lib/Client/Tools/YUM24.py +++ b/src/lib/Client/Tools/YUM24.py @@ -1,13 +1,14 @@ """This provides bcfg2 support for yum.""" __revision__ = '$Revision: $' -import ConfigParser import copy import os.path import sys import yum import Bcfg2.Client.XML import Bcfg2.Client.Tools.RPMng +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: @@ -76,8 +77,6 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng): __new_gpg_ireq__ = {'Package': ['name'], 'Instance': ['version', 'release']} - conflicts = ['YUMng', 'RPMng'] - def __init__(self, logger, setup, config): Bcfg2.Client.Tools.RPMng.RPMng.__init__(self, logger, setup, config) self.__important__ = self.__important__ + \ @@ -153,7 +152,8 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng): try: pkgDict = dict([(i.name, i) for i in \ self.yb.returnPackagesByDep(entry.get('name'))]) - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error('Yum Error Depsolving for %s: %s' % \ (entry.get('name'), str(e))) pkgDict = {} diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index c9e7aa15e..24605ca44 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -1,9 +1,9 @@ """This provides bcfg2 support for yum.""" __revision__ = '$Revision$' -import ConfigParser import copy import os.path +import sys import yum import yum.packages import yum.rpmtrans @@ -13,6 +13,8 @@ import yum.misc import rpmUtils.arch import Bcfg2.Client.XML import Bcfg2.Client.Tools +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser # Fix for python2.3 try: @@ -141,7 +143,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): 'Path': ['type']} __ireq__ = {'Package': ['name']} - conflicts = ['RPMng'] + conflicts = ['YUM24', 'RPMng'] def __init__(self, logger, setup, config): self.yb = yum.YumBase() @@ -167,10 +169,12 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): self.yb.doConfigSetup() self.yb.doTsSetup() self.yb.doRpmDBSetup() - except yum.Errors.RepoError, e: + except yum.Errors.RepoError: + e = sys.exc_info()[1] self.logger.error("YUMng Repository error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError - except yum.Errors.YumBaseError, e: + except Exception: + e = sys.exc_info()[1] self.logger.error("YUMng error: %s" % e) raise Bcfg2.Client.Tools.toolInstantiationError @@ -447,8 +451,13 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): verify_flags = inst.get('verify_flags', self.verifyFlags) verify_flags = verify_flags.lower().replace(' ', ',').split(',') - if len(POs) == 0: - # Package not installed + if 'arch' in nevra: + # If arch is specified use it to select the package + _POs = [ p for p in POs if p.arch == nevra['arch'] ] + else: + _POs = POs + if len(_POs) == 0: + # Package (name, arch) not installed self.logger.debug(" %s is not installed" % nevraString(nevra)) stat['installed'] = False package_fail = True @@ -485,6 +494,9 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): qtext_versions.append("U(%s)" % str(POs[0])) continue + if self.setup.get('quick', False): + # Passed -q on the command line + continue if not (pkg_verify and \ inst.get('pkg_verify', 'true').lower() == 'true'): continue @@ -502,7 +514,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): try: vResult = self._verifyHelper(_POs[0]) - except Exception, e: + except Exception: + e = sys.exc_info()[1] # Unknown Yum exception self.logger.warning(" Verify Exception: %s" % str(e)) package_fail = True @@ -668,38 +681,58 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): return True def _runYumTransaction(self): + def cleanup(): + self.yb.closeRpmDB() + self.RefreshPackages() + rDisplay = RPMDisplay(self.logger) yDisplay = YumDisplay(self.logger) # Run the Yum Transaction - rescode, restring = self.yb.buildTransaction() + try: + rescode, restring = self.yb.buildTransaction() + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] + self.logger.error("Yum transaction error: %s" % str(e)) + cleanup() + return + self.logger.debug("Initial Yum buildTransaction() run said:") self.logger.debug(" resultcode: %s, msgs: %s" \ % (rescode, restring)) if rescode != 1: # Transaction built successfully, run it - self.yb.processTransaction(callback=yDisplay, - rpmDisplay=rDisplay) - self.logger.info("Single Pass for Install Succeeded") + try: + self.yb.processTransaction(callback=yDisplay, + rpmDisplay=rDisplay) + self.logger.info("Single Pass for Install Succeeded") + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] + self.logger.error("Yum transaction error: %s" % str(e)) + cleanup() + return else: # The yum command failed. No packages installed. # Try installing instances individually. self.logger.error("Single Pass Install of Packages Failed") skipBroken = self.yb.conf.skip_broken self.yb.conf.skip_broken = True - rescode, restring = self.yb.buildTransaction() - if rescode != 1: - self.yb.processTransaction(callback=yDisplay, - rpmDisplay=rDisplay) - self.logger.debug( - "Second pass install did not install all packages") - else: - self.logger.error("Second pass yum install failed.") - self.logger.debug(" %s" % restring) + try: + rescode, restring = self.yb.buildTransaction() + if rescode != 1: + self.yb.processTransaction(callback=yDisplay, + rpmDisplay=rDisplay) + self.logger.debug( + "Second pass install did not install all packages") + else: + self.logger.error("Second pass yum install failed.") + self.logger.debug(" %s" % restring) + except yum.Errors.YumBaseError, e: + self.logger.error("Yum transaction error: %s" % str(e)) + self.yb.conf.skip_broken = skipBroken - self.yb.closeRpmDB() - self.RefreshPackages() + cleanup() def Install(self, packages, states): """ @@ -801,7 +834,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.install(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error installing some packages: %s" % yume) if len(upgrade_pkgs) > 0: @@ -811,7 +845,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.update(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error upgrading some packages: %s" % yume) if len(reinstall_pkgs) > 0: @@ -820,7 +855,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg_arg = self.instance_status[inst].get('pkg').get('name') try: self.yb.reinstall(**build_yname(pkg_arg, inst)) - except yum.Errors.YumBaseError, yume: + except yum.Errors.YumBaseError: + yume = sys.exc_info()[1] self.logger.error("Error upgrading some packages: %s" \ % yume) diff --git a/src/lib/Component.py b/src/lib/Component.py index 88dce906e..b73098d09 100644 --- a/src/lib/Component.py +++ b/src/lib/Component.py @@ -23,8 +23,8 @@ logger = logging.getLogger() class NoExposedMethod (Exception): """There is no method exposed with the given name.""" -def run_component(component_cls, location, daemon, pidfile_name, to_file, - cfile, argv=None, register=True, +def run_component(component_cls, listen_all, location, daemon, pidfile_name, + to_file, cfile, argv=None, register=True, state_name=False, cls_kwargs={}, extra_getopt='', time_out=10, protocol='xmlrpc/ssl', certfile=None, keyfile=None, ca=None): @@ -64,8 +64,13 @@ def run_component(component_cls, location, daemon, pidfile_name, to_file, port = tuple(up[1].split(':')) port = (port[0], int(port[1])) try: - server = XMLRPCServer(port, keyfile=keyfile, certfile=certfile, - register=register, timeout=time_out, ca=ca, + server = XMLRPCServer(listen_all, + port, + keyfile=keyfile, + certfile=certfile, + register=register, + timeout=time_out, + ca=ca, protocol=protocol) except: logger.error("Server startup failed") diff --git a/src/lib/Options.py b/src/lib/Options.py index d5304e696..619b16787 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -173,6 +173,18 @@ def colon_split(c_string): return c_string.split(':') return [] +def get_bool(s): + # these values copied from ConfigParser.RawConfigParser.getboolean + # with the addition of True and False + truelist = ["1", "yes", "True", "true", "on"] + falselist = ["0", "no", "False", "false", "off"] + if s in truelist: + return True + elif s in falselist: + return False + else: + raise ValueError + # General options CFILE = Option('Specify configuration file', DEFAULT_CONFIG_LOCATION, cmd='-C', odesc='<conffile>') @@ -191,8 +203,10 @@ SENDMAIL_PATH = Option('Path to sendmail', cf=('reports', 'sendmailpath'), default='/usr/lib/sendmail') INTERACTIVE = Option('Prompt the user for each change', default=False, cmd='-I', ) -ENCODING = Option('Encoding of cfg files', default=sys.getdefaultencoding(), - cmd='-E', odesc='<encoding>', +ENCODING = Option('Encoding of cfg files', + default='UTF-8', + cmd='-E', + odesc='<encoding>', cf=('components', 'encoding')) PARANOID_PATH = Option('Specify path for paranoid file backups', default='/var/cache/bcfg2', cf=('paranoid', 'path'), @@ -249,6 +263,13 @@ SERVER_MCONNECT = Option('Server Metadata Connector list', cook=list_split, cf=('server', 'connectors'), default=['Probes'], ) SERVER_FILEMONITOR = Option('Server file monitor', cf=('server', 'filemonitor'), default='default', odesc='File monitoring driver') +SERVER_LISTEN_ALL = Option('Listen on all interfaces', + cf=('server', 'listen_all'), + cmd='--listen-all', + default=False, + long_arg=True, + cook=get_bool, + odesc='True|False') SERVER_LOCATION = Option('Server Location', cf=('components', 'bcfg2'), default='https://localhost:6789', cmd='-S', odesc='https://server:port') @@ -302,12 +323,13 @@ CLIENT_BUNDLE = Option('Only configure the given bundle(s)', default=[], cmd='-b', odesc='<bundle:bundle>', cook=colon_split) CLIENT_BUNDLEQUICK = Option('only verify/configure the given bundle(s)', default=False, cmd='-Q') -CLIENT_INDEP = Option('Only configure the given bundle(s)', default=False, +CLIENT_INDEP = Option('Only configure independent entries, ignore bundles', default=False, cmd='-z') CLIENT_KEVLAR = Option('Run in kevlar (bulletproof) mode', default=False, cmd='-k', ) -CLIENT_DLIST = Option('Run client in server decision list mode', default=False, - cmd='-l', odesc='<whitelist|blacklist>') +CLIENT_DLIST = Option('Run client in server decision list mode', default='none', + cf=('client', 'decision'), + cmd='-l', odesc='<whitelist|blacklist|none>') CLIENT_FILE = Option('Configure from a file rather than querying the server', default=False, cmd='-f', odesc='<specification path>') CLIENT_QUICK = Option('Disable some checksum verification', default=False, @@ -316,6 +338,9 @@ CLIENT_USER = Option('The user to provide for authentication', default='root', cmd='-u', cf=('communication', 'user'), odesc='<user>') CLIENT_SERVICE_MODE = Option('Set client service mode', default='default', cmd='-s', odesc='<default|disabled|build>') +CLIENT_TIMEOUT = Option('Set the client XML-RPC timeout', default=90, + cmd='-t', cf=('communication', 'timeout'), + odesc='<timeout>') # APT client tool options CLIENT_APT_TOOLS_INSTALL_PATH = Option('Apt tools install path', diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py index 8a1ad683e..4cb0bbe80 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Proxy.py @@ -47,6 +47,9 @@ __all__ = ["ComponentProxy", class CertificateError(Exception): def __init__(self, commonName): self.commonName = commonName + def __str__(self): + return ("Got unallowed commonName %s from server" + % self.commonName) class RetryMethod(xmlrpclib._Method): @@ -181,7 +184,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): other_side_required = ssl.CERT_NONE self.logger.warning("No ca is specified. Cannot authenticate the server with SSL.") if self.cert and not self.key: - self.logger.warning("SSL cert specfied, but key. Cannot authenticate this client with SSL.") + self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") self.cert = None if self.key and not self.cert: self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") @@ -226,7 +229,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): # authentication to the server ctx.load_cert(self.cert, self.key) elif self.cert: - self.logger.warning("SSL cert specfied, but key. Cannot authenticate this client with SSL.") + self.logger.warning("SSL cert specfied, but no key. Cannot authenticate this client with SSL.") elif self.key: self.logger.warning("SSL key specfied, but no cert. Cannot authenticate this client with SSL.") @@ -332,5 +335,5 @@ def ComponentProxy(url, user=None, password=None, else: newurl = url ssl_trans = XMLRPCTransport(key, cert, ca, - allowedServerCNs, timeout=timeout) + allowedServerCNs, timeout=float(timeout)) return xmlrpclib.ServerProxy(newurl, allow_none=True, transport=ssl_trans) diff --git a/src/lib/SSLServer.py b/src/lib/SSLServer.py index a89beabbb..21bf48d3e 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/SSLServer.py @@ -46,7 +46,12 @@ class XMLRPCDispatcher (SimpleXMLRPCServer.SimpleXMLRPCDispatcher): if '.' not in method: params = (address, ) + params response = self.instance._dispatch(method, params, self.funcs) - response = (response, ) + # py3k compatibility + if isinstance(response, bool) or isinstance(response, str) \ + or isinstance(response, list): + response = (response, ) + else: + response = (response.decode('utf-8'), ) raw_response = xmlrpclib.dumps(response, methodresponse=1, allow_none=self.allow_none, encoding=self.encoding) @@ -79,9 +84,9 @@ class SSLServer (SocketServer.TCPServer, object): allow_reuse_address = True logger = logging.getLogger("Cobalt.Server.TCPServer") - def __init__(self, server_address, RequestHandlerClass, keyfile=None, - certfile=None, reqCert=False, ca=None, timeout=None, - protocol='xmlrpc/ssl'): + def __init__(self, listen_all, server_address, RequestHandlerClass, + keyfile=None, certfile=None, reqCert=False, ca=None, + timeout=None, protocol='xmlrpc/ssl'): """Initialize the SSL-TCP server. @@ -97,9 +102,12 @@ class SSLServer (SocketServer.TCPServer, object): """ - all_iface_address = ('', server_address[1]) + if listen_all: + listen_address = ('', server_address[1]) + else: + listen_address = (server_address[0], server_address[1]) try: - SocketServer.TCPServer.__init__(self, all_iface_address, + SocketServer.TCPServer.__init__(self, listen_address, RequestHandlerClass) except socket.error: self.logger.error("Failed to bind to socket") @@ -188,9 +196,18 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): self.logger.error("No authentication data presented") return False auth_type, auth_content = header.split() - auth_content = base64.standard_b64decode(auth_content) try: - username, password = auth_content.split(":") + # py3k compatibility + auth_content = base64.standard_b64decode(auth_content) + except TypeError: + auth_content = base64.standard_b64decode(bytes(auth_content.encode('ascii'))) + try: + # py3k compatibility + try: + username, password = auth_content.split(":") + except TypeError: + username, pw = auth_content.split(bytes(":", encoding='utf-8')) + password = pw.decode('utf-8') except ValueError: username = auth_content password = "" @@ -231,11 +248,13 @@ class XMLRPCRequestHandler (SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): print("got select timeout") raise chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size)) + L.append(self.rfile.read(chunk_size).decode('utf-8')) size_remaining -= len(L[-1]) data = ''.join(L) response = self.server._marshaled_dispatch(self.client_address, data) + if sys.hexversion >= 0x03000000: + response = response.encode('utf-8') except: try: self.send_response(500) @@ -310,7 +329,7 @@ class XMLRPCServer (SocketServer.ThreadingMixIn, SSLServer, """ - def __init__(self, server_address, RequestHandlerClass=None, + def __init__(self, listen_all, server_address, RequestHandlerClass=None, keyfile=None, certfile=None, ca=None, protocol='xmlrpc/ssl', timeout=10, logRequests=False, @@ -339,8 +358,14 @@ class XMLRPCServer (SocketServer.ThreadingMixIn, SSLServer, """A subclassed request handler to prevent class-attribute conflicts.""" SSLServer.__init__(self, - server_address, RequestHandlerClass, ca=ca, - timeout=timeout, keyfile=keyfile, certfile=certfile, protocol=protocol) + listen_all, + server_address, + RequestHandlerClass, + ca=ca, + timeout=timeout, + keyfile=keyfile, + certfile=certfile, + protocol=protocol) self.logRequests = logRequests self.serve = False self.register = register diff --git a/src/lib/Server/Admin/Client.py b/src/lib/Server/Admin/Client.py index 3af25b15a..81fc4a1b1 100644 --- a/src/lib/Server/Admin/Client.py +++ b/src/lib/Server/Admin/Client.py @@ -27,7 +27,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore): for i in args[2:]: attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address']: + 'location', 'secure', 'address', + 'auth']: print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val @@ -41,7 +42,8 @@ class Client(Bcfg2.Server.Admin.MetadataCore): for i in args[2:]: attr, val = i.split('=', 1) if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address']: + 'location', 'secure', 'address', + 'auth']: print("Attribute %s unknown" % attr) raise SystemExit(1) attr_d[attr] = val diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py index fff8bcd1c..eab030cf8 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Server/Admin/Init.py @@ -103,8 +103,7 @@ plugin_list = ['Account', 'TGenshi'] # Default list of plugins to use -default_plugins = ['Base', - 'Bundler', +default_plugins = ['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', diff --git a/src/lib/Server/Admin/Perf.py b/src/lib/Server/Admin/Perf.py index af1c83072..d03b37d57 100644 --- a/src/lib/Server/Admin/Perf.py +++ b/src/lib/Server/Admin/Perf.py @@ -22,6 +22,7 @@ class Perf(Bcfg2.Server.Admin.Mode): 'password': Bcfg2.Options.SERVER_PASSWORD, 'server': Bcfg2.Options.SERVER_LOCATION, 'user': Bcfg2.Options.CLIENT_USER, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, } setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[2:]) @@ -30,7 +31,8 @@ class Perf(Bcfg2.Server.Admin.Mode): setup['password'], key=setup['key'], cert=setup['certificate'], - ca=setup['ca']) + ca=setup['ca'], + timeout=setup['timeout']) data = proxy.get_statistics() for key, value in list(data.items()): data = tuple(["%.06f" % (item) for item in value[:-1]] + [value[-1]]) diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py index 942477a49..c9f3d3f58 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Server/Admin/Reports.py @@ -257,6 +257,11 @@ class Reports(Bcfg2.Server.Admin.Mode): except (IOError, XMLSyntaxError): self.errExit("StatReports: Failed to parse %s" % (stats_file)) + try: + encoding = self.cfp.get('components', 'encoding') + except: + encoding = 'UTF-8' + if not clientspath: try: clientspath = "%s/Metadata/clients.xml" % \ @@ -271,6 +276,7 @@ class Reports(Bcfg2.Server.Admin.Mode): try: load_stats(clientsdata, statsdata, + encoding, verb, self.log, quick=quick, diff --git a/src/lib/Server/Admin/Xcmd.py b/src/lib/Server/Admin/Xcmd.py index fd5794f88..2cb085346 100644 --- a/src/lib/Server/Admin/Xcmd.py +++ b/src/lib/Server/Admin/Xcmd.py @@ -20,7 +20,8 @@ class Xcmd(Bcfg2.Server.Admin.Mode): 'password': Bcfg2.Options.SERVER_PASSWORD, 'key': Bcfg2.Options.SERVER_KEY, 'certificate': Bcfg2.Options.CLIENT_CERT, - 'ca': Bcfg2.Options.CLIENT_CA + 'ca': Bcfg2.Options.CLIENT_CA, + 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, } setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[2:]) @@ -31,7 +32,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode): key=setup['key'], cert=setup['certificate'], ca=setup['ca'], - timeout=180) + timeout=setup['timeout']) if len(setup['args']) == 0: print("Usage: xcmd <xmlrpc method> <optional arguments>") return diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py index 8915492a3..b34d7108c 100644 --- a/src/lib/Server/Admin/__init__.py +++ b/src/lib/Server/Admin/__init__.py @@ -113,7 +113,8 @@ class MetadataCore(Mode): def __init__(self, configfile, usage, pwhitelist=None, pblacklist=None): Mode.__init__(self, configfile) options = {'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'configfile': Bcfg2.Options.CFILE} + 'configfile': Bcfg2.Options.CFILE, + 'encoding': Bcfg2.Options.ENCODING} setup = Bcfg2.Options.OptionParser(options) setup.hm = usage setup.parse(sys.argv[1:]) @@ -126,7 +127,7 @@ class MetadataCore(Mode): try: self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(), setup['plugins'], - 'foo', 'UTF-8') + 'foo', setup['encoding']) except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] self.errExit("Core load failed because %s" % msg) diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index 8f9d3e746..5adfa5381 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -384,7 +384,7 @@ class Core(Component): # clear dynamic groups self.metadata.cgroups[meta.hostname] = [] try: - xpdata = lxml.etree.XML(probedata) + xpdata = lxml.etree.XML(probedata.encode('utf-8')) except: self.logger.error("Failed to parse probe data from client %s" % \ (address[0])) @@ -433,7 +433,7 @@ class Core(Component): @exposed def RecvStats(self, address, stats): """Act on statistics upload.""" - sdata = lxml.etree.XML(stats) + sdata = lxml.etree.XML(stats.encode('utf-8')) client = self.metadata.resolve_client(address) self.process_statistics(client, sdata) return "<ok/>" diff --git a/src/lib/Server/Hostbase/backends.py b/src/lib/Server/Hostbase/backends.py index aa822409c..bf774f695 100644 --- a/src/lib/Server/Hostbase/backends.py +++ b/src/lib/Server/Hostbase/backends.py @@ -57,12 +57,14 @@ class NISBackend(object): return user - except NISAUTHError, e: + except NISAUTHError: + e = sys.exc_info()[1] return None def get_user(self, user_id): try: return User.objects.get(pk=user_id) - except User.DoesNotExist, e: + except User.DoesNotExist: + e = sys.exc_info()[1] return None diff --git a/src/lib/Server/Hostbase/ldapauth.py b/src/lib/Server/Hostbase/ldapauth.py index 21b462c86..f3db26f67 100644 --- a/src/lib/Server/Hostbase/ldapauth.py +++ b/src/lib/Server/Hostbase/ldapauth.py @@ -69,7 +69,8 @@ class ldapauth(object): None) result_type, result_data = conn.result(result_id, 0) return ('success', 'User profile found', result_data,) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -86,7 +87,8 @@ class ldapauth(object): None) result_type, result_data = conn.result(result_id, 0) return ('success', 'User profile found', result_data,) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -108,7 +110,8 @@ class ldapauth(object): raw_obj = result_data[0][1] distinguishedName = raw_obj['distinguishedName'] return ('success', distinguishedName[0],) - except ldap.LDAPError, e: + except ldap.LDAPError: + e = sys.exc_info()[1] #connection failed return ('error', 'LDAP connect failed', e,) @@ -134,7 +137,8 @@ class ldapauth(object): self.is_superuser = False return - except KeyError, e: + except KeyError: + e = sys.exc_info()[1] raise LDAPAUTHError("Portions of the LDAP User profile not present") def member_of(self): diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Server/Hostbase/settings.py index c44c7bf16..4e641f13c 100644 --- a/src/lib/Server/Hostbase/settings.py +++ b/src/lib/Server/Hostbase/settings.py @@ -1,9 +1,10 @@ -from ConfigParser import ConfigParser, NoSectionError, NoOptionError import os.path +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) -c = ConfigParser() +c = ConfigParser.ConfigParser() #This needs to be configurable one day somehow c.read(['./bcfg2.conf']) diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py index 8e86cc564..09443d4c0 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Server/Lint/Comments.py @@ -70,7 +70,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): props = self.core.plugins['Properties'] for propfile, pdata in props.store.entries.items(): if os.path.splitext(propfile)[1] == ".xml": - self.check_xml(pdata.name, pdata.data, 'properties') + self.check_xml(pdata.name, pdata.xdata, 'properties') def check_metadata(self): """ check metadata files for required headers """ diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py new file mode 100644 index 000000000..1e177acff --- /dev/null +++ b/src/lib/Server/Lint/MergeFiles.py @@ -0,0 +1,71 @@ +import os +from copy import deepcopy +from difflib import SequenceMatcher +import Bcfg2.Options +import Bcfg2.Server.Lint + +class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): + """ find Probes or Cfg files with multiple similar files that + might be merged into one """ + + @Bcfg2.Server.Lint.returnErrors + def Run(self): + if 'Cfg' in self.core.plugins: + self.check_cfg() + if 'Probes' in self.core.plugins: + self.check_probes() + + def check_cfg(self): + for filename, entryset in self.core.plugins['Cfg'].entries.items(): + for mset in self.get_similar(entryset.entries): + self.LintError("merge-cfg", + "The following files are similar: %s. " + "Consider merging them into a single Genshi " + "template." % + ", ".join([os.path.join(filename, p) + for p in mset])) + + def check_probes(self): + probes = self.core.plugins['Probes'].probes.entries + for mset in self.get_similar(probes): + self.LintError("merge-cfg", + "The following probes are similar: %s. " + "Consider merging them into a single probe." % + ", ".join([p for p in mset])) + + def get_similar(self, entries): + if "threshold" in self.config: + # accept threshold either as a percent (e.g., "threshold=75") or + # as a ratio (e.g., "threshold=.75") + threshold = float(self.config['threshold']) + if threshold > 1: + threshold /= 100 + else: + threshold = 0.75 + rv = [] + elist = entries.items() + while elist: + result = self._find_similar(elist.pop(0), deepcopy(elist), + threshold) + if len(result) > 1: + elist = [(fname, fdata) + for fname, fdata in elist + if fname not in result] + rv.append(result) + return rv + + def _find_similar(self, ftuple, others, threshold): + fname, fdata = ftuple + rv = [fname] + while others: + cname, cdata = others.pop(0) + sm = SequenceMatcher(None, fdata.data, cdata.data) + # perform progressively more expensive comparisons + if (sm.real_quick_ratio() > threshold and + sm.quick_ratio() > threshold and + sm.ratio() > threshold): + rv.extend(self._find_similar((cname, cdata), deepcopy(others), + threshold)) + return rv + + diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py index 834608378..8a8406e73 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Server/Lint/Validate.py @@ -1,10 +1,12 @@ +import fnmatch import glob import lxml.etree import os -import fnmatch +from subprocess import Popen, PIPE, STDOUT +import sys + import Bcfg2.Options import Bcfg2.Server.Lint -from subprocess import Popen, PIPE, STDOUT class Validate(Bcfg2.Server.Lint.ServerlessPlugin): """ Ensure that the repo validates """ @@ -14,7 +16,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.filesets = {"metadata:groups":"%s/metadata.xsd", "metadata:clients":"%s/clients.xsd", "info":"%s/info.xsd", - "%s/Bundler/*.{xml,genshi}":"%s/bundle.xsd", + "%s/Bundler/*.xml":"%s/bundle.xsd", + "%s/Bundler/*.genshi":"%s/bundle.xsd", "%s/Pkgmgr/*.xml":"%s/pkglist.xsd", "%s/Base/*.xml":"%s/base.xsd", "%s/Rules/*.xml":"%s/rules.xsd", @@ -33,23 +36,27 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): @Bcfg2.Server.Lint.returnErrors def Run(self): - self.schemadir = self.config['schema'] + schemadir = self.config['schema'] - for schemaname, path in self.filesets.items(): + for path, schemaname in self.filesets.items(): try: filelist = self.filelists[path] except KeyError: filelist = [] - + if filelist: # avoid loading schemas for empty file lists try: schema = lxml.etree.XMLSchema(lxml.etree.parse(schemaname % schemadir)) + except IOError: + e = sys.exc_info()[1] + self.LintError("input-output-error", e.message) + continue except: self.LintError("schema-failed-to-parse", - "Failed to process schema %s", - schemaname % schemadir) + "Failed to process schema %s" % + (schemaname % schemadir)) continue for filename in filelist: self.validate(filename, schemaname % schemadir, diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index 3b89d1f9e..63cb2463b 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -4,6 +4,7 @@ __all__ = ['Bundles', 'Comments', 'Duplicates', 'InfoXML', + 'MergeFiles', 'Pkgmgr', 'RequiredAttrs', 'Validate'] @@ -11,6 +12,7 @@ __all__ = ['Bundles', import logging import os.path from copy import copy +import textwrap import lxml.etree import Bcfg2.Logger @@ -84,7 +86,10 @@ class ErrorHandler (object): "properties-schema-not-found":"warning", "xml-failed-to-parse":"error", "xml-failed-to-read":"error", - "xml-failed-to-verify":"error",} + "xml-failed-to-verify":"error", + "merge-cfg":"warning", + "merge-probes":"warning", + "input-output-error": "error"} def __init__(self, config=None): self.errors = 0 @@ -92,6 +97,9 @@ class ErrorHandler (object): self.logger = logging.getLogger('bcfg2-lint') + self._wrapper = textwrap.TextWrapper(initial_indent = " ", + subsequent_indent = " ") + self._handlers = {} if config is not None: for err, action in config.items(): @@ -116,26 +124,42 @@ class ErrorHandler (object): self._handlers[err](msg) self.logger.debug(" (%s)" % err) else: - self.logger.info("Unknown error %s" % err) + # assume that it's an error, but complain + self.error(msg) + self.logger.warning("Unknown error %s" % err) def error(self, msg): """ log an error condition """ self.errors += 1 - lines = msg.splitlines() - self.logger.error("ERROR: %s" % lines.pop()) - [self.logger.error(" %s" % l) for l in lines] + self._log(msg, self.logger.error, prefix="ERROR: ") def warn(self, msg): """ log a warning condition """ self.warnings += 1 - lines = msg.splitlines() - self.logger.warning("WARNING: %s" % lines.pop()) - [self.logger.warning(" %s" % l) for l in lines] + self._log(msg, self.logger.warning, prefix="WARNING: ") def debug(self, msg): """ log a silent/debug condition """ - lines = msg.splitlines() - [self.logger.debug("%s" % l) for l in lines] + self._log(msg, self.logger.debug) + + def _log(self, msg, logfunc, prefix=""): + # a message may itself consist of multiple lines. wrap() will + # elide them all into a single paragraph, which we don't want. + # so we split the message into its paragraphs and wrap each + # paragraph individually. this means, unfortunately, that we + # lose textwrap's built-in initial indent functionality, + # because we want to only treat the very first line of the + # first paragraph specially. so we do some silliness. + rawlines = msg.splitlines() + firstline = True + for rawline in rawlines: + lines = self._wrapper.wrap(rawline) + for line in lines: + if firstline: + logfunc("%s%s" % (prefix, line.lstrip())) + firstline = False + else: + logfunc(line) class ServerlessPlugin (Plugin): diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index cd2b63656..30c4f9686 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -435,12 +435,13 @@ class XMLFileBacked(FileBacked): def Index(self): """Build local data structures.""" try: - xdata = XML(self.data) + self.xdata = XML(self.data) except XMLSyntaxError: logger.error("Failed to parse %s" % (self.name)) return - self.label = xdata.attrib[self.__identifier__] - self.entries = xdata.getchildren() + self.entries = self.xdata.getchildren() + if self.__identifier__ is not None: + self.label = self.xdata.attrib[self.__identifier__] def __iter__(self): return iter(self.entries) @@ -455,53 +456,51 @@ class SingleXMLFileBacked(XMLFileBacked): class StructFile(XMLFileBacked): """This file contains a set of structure file formatting logic.""" + __identifier__ = None + def __init__(self, name): XMLFileBacked.__init__(self, name) - self.fragments = {} - - def Index(self): - """Build internal data structures.""" - try: - xdata = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - logger.error("Failed to parse file %s" % self.name) - return - self.fragments = {} - work = {lambda x: True: xdata.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - + self.matches = {} + + def _match(self, item, metadata): + """ recursive helper for Match() """ + if isinstance(item, lxml.etree._Comment): + return [] + elif item.tag == 'Group': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') not in metadata.groups) or + (item.get('negate', 'false').lower() == 'false' and + item.get('name') in metadata.groups)): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + elif item.tag == 'Client': + rv = [] + if ((item.get('negate', 'false').lower() == 'true' and + item.get('name') != metadata.hostname) or + (item.get('negate', 'false').lower() == 'false' and + item.get('name') == metadata.hostname)): + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return rv + else: + rv = copy.deepcopy(item) + for child in rv.iterchildren(): + rv.remove(child) + for child in item.iterchildren(): + rv.extend(self._match(child, metadata)) + return [rv] + def Match(self, metadata): """Return matching fragments of independent.""" - matching = [frag for (pred, frag) in list(self.fragments.items()) - if pred(metadata)] - if matching: - return reduce(lambda x, y: x + y, matching) - logger.error("File %s got null match" % (self.name)) - return [] + if metadata.hostname not in self.matches: + rv = [] + for child in self.entries: + rv.extend(self._match(child, metadata)) + logger.debug("File %s got %d match(es)" % (self.name, len(rv))) + self.matches[metadata.hostname] = rv + return self.matches[metadata.hostname] class INode: @@ -644,9 +643,9 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): def BindEntry(self, entry, metadata): """Check package lists of package entries.""" - [src.Cache(metadata) for src in list(self.entries.values())] name = entry.get('name') - if not src.cache: + if False in [src.Cache(metadata) for src in + list(self.entries.values())]: self.logger.error("Called before data loaded") raise PluginExecutionError matching = [src for src in list(self.entries.values()) @@ -693,6 +692,9 @@ class Specificity: self.prio = prio self.delta = delta + def __lt__(self, other): + return self.__cmp__(other) < 0 + def matches(self, metadata): return self.all or \ self.hostname == metadata.hostname or \ diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py index 5e7d89727..5de57a87c 100644 --- a/src/lib/Server/Plugins/Base.py +++ b/src/lib/Server/Plugins/Base.py @@ -21,6 +21,7 @@ class Base(Bcfg2.Server.Plugin.Plugin, __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = Bcfg2.Server.Plugin.StructFile + deprecated = True """Base creates independent clauses based on client metadata.""" def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 998bacc19..23ba0a745 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -4,8 +4,11 @@ __revision__ = '$Revision$' import binascii import logging import lxml +import operator import os +import os.path import re +import stat import sys import tempfile @@ -22,9 +25,10 @@ except: logger = logging.getLogger('Bcfg2.Plugins.Cfg') +# py3k compatibility def u_str(string, encoding): if sys.hexversion >= 0x03000000: - return str(string, encoding) + return string.encode(encoding) else: return unicode(string, encoding) @@ -94,6 +98,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, entry_type, encoding) self.specific = CfgMatcher(path.split('/')[-1]) + path = path def sort_by_specific(self, one, other): return cmp(one.specific, other.specific) @@ -104,7 +109,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): """ matching = [ent for ent in list(self.entries.values()) if \ ent.specific.matches(metadata)] - matching.sort(self.sort_by_specific) + matching.sort(key=operator.attrgetter('specific')) non_delta = [matching.index(m) for m in matching if not m.specific.delta] if not non_delta: @@ -118,6 +123,11 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): self.bind_info_to_entry(entry, metadata) used = self.get_pertinent_entries(metadata) basefile = used.pop(0) + if entry.get('perms').lower() == 'inherit': + # use on-disk permissions + fname = "%s/%s" % (self.path, entry.get('name')) + entry.set('perms', + str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) if entry.tag == 'Path': entry.set('type', 'file') if basefile.name.endswith(".genshi"): @@ -134,9 +144,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): metadata=metadata, path=basefile.name).filter(removecomment) try: - data = stream.render('text', strip_whitespace=False) + data = stream.render('text', encoding=self.encoding, + strip_whitespace=False) except TypeError: - data = stream.render('text') + data = stream.render('text', encoding=self.encoding) if data == '': entry.set('empty', 'true') except Exception: @@ -160,6 +171,13 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) logger.error("Please verify you are using the proper encoding.") raise Bcfg2.Server.Plugin.PluginExecutionError + except ValueError: + e = sys.exc_info()[1] + logger.error("Error in specification for %s" % entry.get('name')) + logger.error("%s" % e) + logger.error("You need to specify base64 encoding for %s." % + entry.get('name')) + raise Bcfg2.Server.Plugin.PluginExecutionError if entry.text in ['', None]: entry.set('empty', 'true') @@ -185,10 +203,15 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): def write_update(self, specific, new_entry, log): if 'text' in new_entry: name = self.build_filename(specific) - if name.endswith(".genshi"): + if os.path.exists("%s.genshi" % name): logger.error("Cfg: Unable to pull data for genshi types") raise Bcfg2.Server.Plugin.PluginExecutionError - open(name, 'w').write(new_entry['text']) + try: + etext = new_entry['text'].encode(self.encoding) + except: + logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding)) + raise Bcfg2.Server.Plugin.PluginExecutionError + open(name, 'w').write(etext) if log: logger.info("Wrote file %s" % name) badattr = [attr for attr in ['owner', 'group', 'perms'] diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py index 5ef1920e1..103fb7353 100644 --- a/src/lib/Server/Plugins/DBStats.py +++ b/src/lib/Server/Plugins/DBStats.py @@ -55,6 +55,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, try: Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata, container, + self.core.encoding, 0, logger, True, diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index ca6e43851..6570f2912 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -766,7 +766,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, (address[0])) return False # populate the session cache - if user != 'root': + if user.decode('utf-8') != 'root': self.session_cache[address] = (time.time(), client) return True diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py index ea2e79ccc..ec0f294dd 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Server/Plugins/Probes.py @@ -1,12 +1,100 @@ import lxml.etree +import operator import re +try: + import json + has_json = True +except ImportError: + try: + import simplejson as json + has_json = True + except ImportError: + has_json = False + +try: + import syck + has_syck = True +except ImportError: + has_syck = False + try: + import yaml + has_yaml = True + except ImportError: + has_yaml = False + import Bcfg2.Server.Plugin specific_probe_matcher = re.compile("(.*/)?(?P<basename>\S+)(.(?P<mode>[GH](\d\d)?)_\S+)") probe_matcher = re.compile("(.*/)?(?P<basename>\S+)") +class ProbeData (object): + """ a ProbeData object emulates a str object, but also has .xdata + and .json properties to provide convenient ways to use ProbeData + objects as XML or JSON data """ + def __init__(self, data): + self.data = data + self._xdata = None + self._json = None + self._yaml = None + + def __str__(self): + return str(self.data) + + def __repr__(self): + return repr(self.data) + + def __getattr__(self, name): + """ make ProbeData act like a str object """ + return getattr(self.data, name) + + def __complex__(self): + return complex(self.data) + + def __int__(self): + return int(self.data) + + def __long__(self): + return long(self.data) + + def __float__(self): + return float(self.data) + + @property + def xdata(self): + if self._xdata is None: + try: + self._xdata = lxml.etree.XML(self.data) + except lxml.etree.XMLSyntaxError: + pass + return self._xdata + + @property + def json(self): + if self._json is None and has_json: + try: + self._json = json.loads(self.data) + except ValueError: + pass + return self._json + + @property + def yaml(self): + if self._yaml is None: + if has_yaml: + try: + self._yaml = yaml.load(self.data) + except yaml.YAMLError: + pass + elif has_syck: + try: + self._yaml = syck.load(self.data) + except syck.error: + pass + return self._yaml + + class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") @@ -27,7 +115,7 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): ret = [] build = dict() candidates = self.get_matching(metadata) - candidates.sort(lambda x, y: cmp(x.specific, y.specific)) + candidates.sort(key=operator.attrgetter('specific')) for entry in candidates: rem = specific_probe_matcher.match(entry.name) if not rem: @@ -80,7 +168,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, cx = lxml.etree.SubElement(top, 'Client', name=client) for probe in sorted(probed): lxml.etree.SubElement(cx, 'Probe', name=probe, - value=self.probedata[client][probe]) + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(cx, "Group", name=group) data = lxml.etree.tostring(top, encoding='UTF-8', @@ -90,7 +178,7 @@ class Probes(Bcfg2.Server.Plugin.Plugin, datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') except IOError: self.logger.error("Failed to write probed.xml") - datafile.write(data) + datafile.write(data.decode('utf-8')) def load_data(self): try: @@ -105,7 +193,8 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.cgroups[client.get('name')] = [] for pdata in client: if (pdata.tag == 'Probe'): - self.probedata[client.get('name')][pdata.get('name')] = pdata.get('value') + self.probedata[client.get('name')][pdata.get('name')] = \ + ProbeData(pdata.get('value')) elif (pdata.tag == 'Group'): self.cgroups[client.get('name')].append(pdata.get('name')) @@ -128,9 +217,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, self.logger.error("Got null response to probe %s from %s" % \ (data.get('name'), client.hostname)) try: - self.probedata[client.hostname].update({data.get('name'): ''}) + self.probedata[client.hostname].update({data.get('name'): + ProbeData('')}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): ''} + self.probedata[client.hostname] = \ + {data.get('name'): ProbeData('')} return dlines = data.text.split('\n') self.logger.debug("%s:probe:%s:%s" % (client.hostname, @@ -141,11 +232,11 @@ class Probes(Bcfg2.Server.Plugin.Plugin, if newgroup not in self.cgroups[client.hostname]: self.cgroups[client.hostname].append(newgroup) dlines.remove(line) - dtext = "\n".join(dlines) + dobj = ProbeData("\n".join(dlines)) try: - self.probedata[client.hostname].update({data.get('name'): dtext}) + self.probedata[client.hostname].update({data.get('name'): dobj}) except KeyError: - self.probedata[client.hostname] = {data.get('name'): dtext} + self.probedata[client.hostname] = {data.get('name'): dobj} def get_additional_groups(self, meta): return self.cgroups.get(meta.hostname, list()) diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py index dea797a10..54c5def57 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Server/Plugins/Properties.py @@ -6,44 +6,7 @@ import Bcfg2.Server.Plugin class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" - def Index(self): - """Build internal data structures.""" - if type(self.data) is not lxml.etree._Element: - try: - self.data = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % - self.name) - - self.fragments = {} - work = {lambda x: True: self.data.getchildren()} - while work: - (predicate, worklist) = work.popitem() - self.fragments[predicate] = \ - [item for item in worklist - if (item.tag != 'Group' and - item.tag != 'Client' and - not isinstance(item, - lxml.etree._Comment))] - for item in worklist: - cmd = None - if item.tag == 'Group': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:'%s' not in x.groups and predicate(x)" - else: - cmd = "lambda x:'%s' in x.groups and predicate(x)" - elif item.tag == 'Client': - if item.get('negate', 'false').lower() == 'true': - cmd = "lambda x:x.hostname != '%s' and predicate(x)" - else: - cmd = "lambda x:x.hostname == '%s' and predicate(x)" - # else, ignore item - if cmd is not None: - newpred = eval(cmd % item.get('name'), - {'predicate':predicate}) - work[newpred] = item.getchildren() - - + pass class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): __child__ = PropertyFile diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py index cf0998aaa..4a33c0cb0 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Server/Plugins/SSHbase.py @@ -39,12 +39,17 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, __author__ = 'bcfg-dev@mcs.anl.gov' pubkeys = ["ssh_host_dsa_key.pub.H_%s", - "ssh_host_rsa_key.pub.H_%s", "ssh_host_key.pub.H_%s"] + "ssh_host_rsa_key.pub.H_%s", + "ssh_host_key.pub.H_%s"] hostkeys = ["ssh_host_dsa_key.H_%s", - "ssh_host_rsa_key.H_%s", "ssh_host_key.H_%s"] - keypatterns = ['ssh_host_dsa_key', 'ssh_host_rsa_key', 'ssh_host_key', - 'ssh_host_dsa_key.pub', 'ssh_host_rsa_key.pub', - 'ssh_host_key.pub'] + "ssh_host_rsa_key.H_%s", + "ssh_host_key.H_%s"] + keypatterns = ["ssh_host_dsa_key", + "ssh_host_rsa_key", + "ssh_host_key", + "ssh_host_dsa_key.pub", + "ssh_host_rsa_key.pub", + "ssh_host_key.pub"] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -74,7 +79,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: - self.__skn = "\n".join([str(value.data) for key, value in \ + self.__skn = "\n".join([value.data.decode() for key, value in \ list(self.entries.items()) if \ key.endswith('.static')]) names = dict() @@ -117,7 +122,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.logger.error("SSHbase: Unknown host %s; ignoring public keys" % hostname) continue self.__skn += "%s %s" % (','.join(names[hostname]), - self.entries[pubkey].data) + self.entries[pubkey].data.decode()) return self.__skn def set_skn(self, value): @@ -206,7 +211,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, hostkeys.sort() for hostkey in hostkeys: entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % ( - self.entries[hostkey].data) + self.entries[hostkey].data.decode()) permdata = {'owner': 'root', 'group': 'root', 'type': 'file', diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py index 8b6bad574..aeb3b9f74 100644 --- a/src/lib/Server/Plugins/Snapshots.py +++ b/src/lib/Server/Plugins/Snapshots.py @@ -28,6 +28,7 @@ datafields = { } +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index b6a3c2599..68774cec6 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -38,7 +38,7 @@ import platform from Bcfg2.Bcfg2Py3k import ConfigParser -def build_reason_kwargs(r_ent): +def build_reason_kwargs(r_ent, encoding, logger): binary_file = False if r_ent.get('current_bfile', False): binary_file = True @@ -54,6 +54,12 @@ def build_reason_kwargs(r_ent): rc_diff = r_ent.get('current_diff') else: rc_diff = '' + if not binary_file: + try: + rc_diff = rc_diff.decode(encoding) + except: + logger.error("Reason isn't %s encoded, cannot decode it" % encoding) + rc_diff = '' return dict(owner=r_ent.get('owner', default=""), current_owner=r_ent.get('current_owner', default=""), group=r_ent.get('group', default=""), @@ -71,7 +77,7 @@ def build_reason_kwargs(r_ent): is_binary=binary_file) -def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): +def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''): clients = {} [clients.__setitem__(c.name, c) \ for c in Client.objects.all()] @@ -129,7 +135,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): for (xpath, type) in pattern: for x in statistics.findall(xpath): counter_fields[type] = counter_fields[type] + 1 - kargs = build_reason_kwargs(x) + kargs = build_reason_kwargs(x, encoding, logger) try: rr = None @@ -270,6 +276,11 @@ if __name__ == '__main__': print("StatReports: Failed to parse %s" % (statpath)) raise SystemExit(1) + try: + encoding = cf.get('components', 'encoding') + except: + encoding = 'UTF-8' + if not clientpath: try: clientspath = "%s/Metadata/clients.xml" % \ @@ -288,6 +299,7 @@ if __name__ == '__main__': update_database() load_stats(clientsdata, statsdata, + encoding, verb, logger, quick=q, diff --git a/src/lib/Server/Reports/reports/templates/base.html b/src/lib/Server/Reports/reports/templates/base.html index 6ef4c9aff..ec9a17468 100644 --- a/src/lib/Server/Reports/reports/templates/base.html +++ b/src/lib/Server/Reports/reports/templates/base.html @@ -87,7 +87,7 @@ <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.2.0pre2</span> + <span>Bcfg2 Version 1.2.0pre3</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py index 291528e2e..2e30125f9 100644 --- a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py +++ b/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py @@ -15,6 +15,7 @@ try: except: colorize = False +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py index 2aa35f1ec..f30c38a05 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Server/Snapshots/model.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import relation, backref from sqlalchemy.ext.declarative import declarative_base +# py3k compatibility def u_str(string): if sys.hexversion >= 0x03000000: return string |