diff options
author | Alexander Sulfrian <asulfrian@zedat.fu-berlin.de> | 2022-01-17 17:36:28 +0100 |
---|---|---|
committer | Alexander Sulfrian <asulfrian@zedat.fu-berlin.de> | 2022-01-23 19:58:36 +0100 |
commit | ba1a18e060a8614b3dcb41b94a7ad37e89f1dfdf (patch) | |
tree | e37d730764210102637c006f2cab4dfed9592b91 /src | |
parent | 3ea270b7583bb13b1234680c4bde4ae03701a109 (diff) | |
parent | ef568d29698c00bf2c02c99a34b98e6b8ca96653 (diff) | |
download | bcfg2-ba1a18e060a8614b3dcb41b94a7ad37e89f1dfdf.tar.gz bcfg2-ba1a18e060a8614b3dcb41b94a7ad37e89f1dfdf.tar.bz2 bcfg2-ba1a18e060a8614b3dcb41b94a7ad37e89f1dfdf.zip |
Merge branch 'packages'
Diffstat (limited to 'src')
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Validate.py | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 39 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Collection.py | 27 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Pac.py | 12 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py | 25 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py | 93 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py | 14 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py | 11 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py | 10 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py | 11 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py | 36 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 74 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py | 25 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/PkgVars.py | 65 |
16 files changed, 408 insertions, 44 deletions
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index d6f18afbb..146f18b0c 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -61,7 +61,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "AWSTags/config.xml": "awstags.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", - "GroupLogic/groups.xml": "grouplogic.xsd" + "GroupLogic/groups.xml": "grouplogic.xsd", + "PkgVars/*.xml": "pkgvars.xsd" } self.filelists = {} diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 956cb9f51..c3a3dc6ad 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -1,7 +1,6 @@ """ APT backend for :mod:`Bcfg2.Server.Plugins.Packages` """ import re -import gzip from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source @@ -68,19 +67,34 @@ class AptSource(Source): #: AptSource sets the ``type`` on Package entries to "deb" ptype = 'deb' + #: Most (3rd-party) debian repositories still only support "gzip". + default_compression = 'gzip' + @property def urls(self): """ A list of URLs to the base metadata file for each repository described by this source. """ + fname = self.build_filename('Packages') + if not self.rawurl: rv = [] for part in self.components: for arch in self.arches: - rv.append("%sdists/%s/%s/binary-%s/Packages.gz" % - (self.url, self.version, part, arch)) + rv.append("%sdists/%s/%s/binary-%s/%s" % + (self.url, self.version, part, arch, fname)) return rv else: - return ["%sPackages.gz" % self.rawurl] + return ["%s%s" % (self.rawurl, fname)] + + def _get_arch(self, fname): + if not self.rawurl: + return [x + for x in fname.split('@') + if x.startswith('binary-')][0][7:] + + # RawURL entries assume that they only have one <Arch></Arch> + # element and that it is the architecture of the source. + return self.arches[0] def read_files(self): # pylint: disable=R0912 bdeps = dict() @@ -89,23 +103,13 @@ class AptSource(Source): self.pkgnames = set() self.essentialpkgs = set() for fname in self.files: - if not self.rawurl: - barch = [x - for x in fname.split('@') - if x.startswith('binary-')][0][7:] - else: - # RawURL entries assume that they only have one <Arch></Arch> - # element and that it is the architecture of the source. - barch = self.arches[0] + barch = self._get_arch(fname) if barch not in bdeps: bdeps[barch] = dict() brecs[barch] = dict() bprov[barch] = dict() - try: - reader = gzip.GzipFile(fname) - except IOError: - self.logger.error("Packages: Failed to read file %s" % fname) - raise + + reader = self.open_file(fname) for line in reader.readlines(): if not isinstance(line, str): line = line.decode('utf-8') @@ -150,5 +154,6 @@ class AptSource(Source): if dname not in bprov[barch]: bprov[barch][dname] = set() bprov[barch][dname].add(pkgname) + reader.close() self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 004e27874..e0d6e1fc3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -289,7 +289,7 @@ class Collection(list, Debuggable): return any(source.is_virtual_package(self.metadata, package) for source in self) - def get_deps(self, package, recs=None): + def get_deps(self, package, recs=None, pinnings=None): """ Get a list of the dependencies of the given package. The base implementation simply aggregates the results of @@ -297,16 +297,35 @@ class Collection(list, Debuggable): :param package: The name of the symbol, but see :ref:`pkg-objects` :type package: string + :param pinnings: Mapping from package names to source names. + :type pinnings: dict :returns: list of strings, but see :ref:`pkg-objects` """ recommended = None if recs and package in recs: recommended = recs[package] + pin_found = False + pin_source = None + if pinnings and package in pinnings: + pin_source = pinnings[package] + for source in self: + if pin_source and source.name not in pin_source: + continue + pin_found = True + if source.is_package(self.metadata, package): return source.get_deps(self.metadata, package, recommended) + if not pin_found: + if pin_source: + self.logger.error("Packages: Source '%s' for package '%s' not found" % + (' or '.join(pin_source), package)) + else: + self.logger.error("Packages: No source found for package '%s'" % + package); + return [] def get_essential(self): @@ -471,12 +490,14 @@ class Collection(list, Debuggable): @track_statistics() def complete(self, packagelist, # pylint: disable=R0912,R0914 - recommended=None): + recommended=None, pinnings=None): """ Build a complete list of all packages and their dependencies. :param packagelist: Set of initial packages computed from the specification. :type packagelist: set of strings, but see :ref:`pkg-objects` + :param pinnings: Mapping from package names to source names. + :type pinnings: dict :returns: tuple of sets - The first element contains a set of strings (but see :ref:`pkg-objects`) describing the complete package list, and the second element is a @@ -535,7 +556,7 @@ class Collection(list, Debuggable): self.debug_log("Packages: handling package requirement %s" % (current,)) packages.add(current) - deps = self.get_deps(current, recommended) + deps = self.get_deps(current, recommended, pinnings) newdeps = set(deps).difference(examined) if newdeps: self.debug_log("Packages: Package %s added requirements %s" diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 6fc084cc4..e3432c934 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -87,6 +87,9 @@ class PacSource(Source): #: PacSource sets the ``type`` on Package entries to "pacman" ptype = 'pacman' + #: The database of pacman repositories is compressed with "gzip" + default_compression = 'gzip' + def __init__(self, basepath, xsource): self.pacgroups = {} @@ -113,9 +116,10 @@ class PacSource(Source): if not self.rawurl: rv = [] for part in self.components: + filename = self.build_filename("%s.db.tar" % part) for arch in self.arches: - rv.append("%s%s/os/%s/%s.db.tar.gz" % - (self.url, part, arch, part)) + rv.append("%s%s/os/%s/%s" % + (self.url, part, arch, filename)) return rv else: raise Exception("PacSource : RAWUrl not supported (yet)") @@ -140,7 +144,8 @@ class PacSource(Source): bprov[barch] = {} try: self.debug_log("Packages: try to read %s" % fname) - tar = tarfile.open(fname, "r") + reader = self.open_file(fname) + tar = tarfile.open(fileobj=reader) except (IOError, tarfile.TarError): self.logger.error("Packages: Failed to read file %s" % fname) raise @@ -185,6 +190,7 @@ class PacSource(Source): self.pacgroups[group].append(pkgname) tar.close() + reader.close() self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 1af046ec0..d982626d0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -106,6 +106,8 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile): source = self.source_from_xml(xsource) if source is not None: self.entries.append(source) + self.entries.sort(key=(lambda source: source.priority), + reverse=True) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """ ``Index`` is responsible for calling :func:`source_from_xml` diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py index 55dd4e488..5248ad896 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py @@ -1,6 +1,5 @@ """ pkgng backend for :mod:`Bcfg2.Server.Plugins.Packages` """ -import lzma import tarfile try: @@ -40,19 +39,30 @@ class PkgngSource(Source): #: PkgngSource sets the ``type`` on Package entries to "pkgng" ptype = 'pkgng' + #: The "packagesite" files of pkgng repositories are compressed + #: with "xz" + default_compression = 'xz' + + def _get_extension(self): + extension = super(PkgngSource, self)._get_extension() + if extension == '': + return 'tar' + return 't%s' % extension + @property def urls(self): """ A list of URLs to the base metadata file for each repository described by this source. """ + fname = self.build_filename("packagesite") if not self.rawurl: rv = [] for part in self.components: for arch in self.arches: - rv.append("%s/freebsd:%s:%s/%s/packagesite.txz" % - (self.url, self.version, arch, part)) + rv.append("%s/freebsd:%s:%s/%s/%s" % + (self.url, self.version, arch, part, fname)) return rv else: - return ["%s/packagesite.txz" % self.rawurl] + return ["%s/%s" % (self.rawurl, fname)] def read_files(self): bdeps = dict() @@ -70,12 +80,13 @@ class PkgngSource(Source): if barch not in bdeps: bdeps[barch] = dict() try: - tar = tarfile.open(fileobj=lzma.LZMAFile(fname)) - reader = tar.extractfile('packagesite.yaml') + reader = self.open_file(fname) + tar = tarfile.open(fileobj=reader) + packagesite = tar.extractfile('packagesite.yaml') except (IOError, tarfile.TarError): self.logger.error("Packages: Failed to read file %s" % fname) raise - for line in reader.readlines(): + for line in packagesite.readlines(): pkg = json.loads(unicode(line, errors='ignore')) pkgname = pkg['name'] self.pkgnames.add(pkgname) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py new file mode 100644 index 000000000..5e0eb55be --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py @@ -0,0 +1,93 @@ +""" +APT backend for :mod:`Bcfg2.Server.Plugins.Packages` using +apt_pkg python bindings. +""" + +import apt_pkg +from Bcfg2.Server.Plugins.Packages.Apt import AptCollection, AptSource + + +class PyaptCollection(AptCollection): + """ Handle collections of PyAPT sources. This is a no-op object + that simply inherits from + :class:`Bcfg2.Server.Plugins.Packages.Apt.AptCollection` and + overrides nothing. + """ + pass + + +class PyaptSource(AptSource): + """ Handle PyAPT sources """ + + def read_files(self): # pylint: disable=R0912 + bdeps = dict() + brecs = dict() + bprov = dict() + bvers = dict() + self.pkgnames = set() + self.essentialpkgs = set() + for fname in self.files: + barch = self._get_arch(fname) + if barch not in bdeps: + bdeps[barch] = dict() + brecs[barch] = dict() + bprov[barch] = dict() + + apt_pkg.init_system() + with apt_pkg.TagFile(fname) as tagfile: + for section in tagfile: + pkgname = section['Package'] + + if pkgname in bvers: + new = section['Version'] + old = bvers[pkgname] + if apt_pkg.version_compare(new, old) <= 0: + continue + + self.pkgnames.add(pkgname) + bvers[pkgname] = section['Version'] + bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] + + if section.find_flag('Essential'): + self.essentialpkgs.add(pkgname) + + for dep_type in ['Depends', 'Pre-Depends', 'Recommends']: + dep_str = section.find(dep_type) + if dep_str is None: + continue + + vindex = 0 + for dep in apt_pkg.parse_depends(dep_str): + if len(dep) > 1: + cdeps = [cdep for (cdep, _, _) in dep] + dyn_dname = "choice-%s-%s-%s" % (pkgname, + barch, + vindex) + vindex += 1 + + if dep_type == 'Recommends': + brecs[barch][pkgname].append(dyn_dname) + else: + bdeps[barch][pkgname].append(dyn_dname) + bprov[barch][dyn_dname] = set(cdeps) + else: + (raw_dep, _, _) = dep[0] + if dep_type == 'Recommends': + brecs[barch][pkgname].append(raw_dep) + else: + bdeps[barch][pkgname].append(raw_dep) + + provides = section.find('Provides') + if provides is not None: + provided_packages = [ + pkg + for group in apt_pkg.parse_depends(provides) + for (pkg, _, _) in group] + for dname in provided_packages: + if dname not in bprov[barch]: + bprov[barch][dname] = set() + bprov[barch][dname].add(pkgname) + + self.process_files(bdeps, bprov, brecs) + read_files.__doc__ = AptSource.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py new file mode 100644 index 000000000..2267197ca --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py @@ -0,0 +1,14 @@ +""" Reader for bzip2 compressed package sources. """ + +import bz2 +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class Bzip2Reader(Reader): + extension = 'bz' + + def _open(self, filename): + return bz2.BZ2File(filename) + + def readable(self): + return True diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py new file mode 100644 index 000000000..8e67e2b33 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py @@ -0,0 +1,11 @@ +""" Reader for gzip compressed package sources. """ + +import gzip +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class GzipReader(Reader): + extension = 'gz' + + def _open(self, filename): + return gzip.GzipFile(filename) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py new file mode 100644 index 000000000..2f7a18d84 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py @@ -0,0 +1,10 @@ +""" Reader for uncompressed package sources. """ + +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class NoneReader(Reader): + extension = '' + + def _open(self, filename): + return open(filename) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py new file mode 100644 index 000000000..39a058767 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py @@ -0,0 +1,11 @@ +""" Reader for lzma compressed package sources. """ + +import lzma +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class XzReader(Reader): + extension = 'xz' + + def _open(self, filename): + return lzma.LZMAFile(filename) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py new file mode 100644 index 000000000..51b4fb79c --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py @@ -0,0 +1,36 @@ +"""This module implements different readers for package files.""" + +from io import IOBase +from Bcfg2.Compat import walk_packages + + +def get_readers(): + """ Return all available packages readers. """ + return [m[1] # pylint: disable=C0103 + for m in walk_packages(path=__path__)] + + +class Reader(IOBase): + extension = None + + def __init__(self, name): + self.name = name + self._file = self._open(name) + + def _open(self, filename): + raise NotImplementedError + + def read(self, size=-1): + return self._file.read(size) + + def readable(self): + return self._file.readable() + + def readline(self, size=-1): + return self._file.readline(size) + + def readlines(self, hint=None): + return self._file.readlines(size) + + def writelines(self, lines): + self._unsupported("writelines") diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 86f7698f7..574dbd851 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -49,6 +49,7 @@ in your ``Source`` subclass. For an example of this kind of import os import re import sys +import Bcfg2.Options from Bcfg2.Logger import Debuggable from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ @@ -56,7 +57,7 @@ from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ from Bcfg2.Server.Statistics import track_statistics -def fetch_url(url): +def fetch_url(url, opts): """ Return the content of the given URL. :param url: The URL to fetch content from. @@ -73,8 +74,14 @@ def fetch_url(url): url = mobj.group(1) + mobj.group(4) auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm()) auth.add_password(None, url, user, passwd) - install_opener(build_opener(auth)) - return urlopen(url).read() + req = build_opener(auth) + else: + req = build_opener() + + if 'user-agent' in opts: + req.addheaders = [('User-Agent', opts['user-agent'])] + + return req.open(url).read() class SourceInitError(Exception): @@ -111,6 +118,12 @@ class Source(Debuggable): # pylint: disable=R0902 #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`. ptype = None + #: The default compression format used by this Source class. This + #: is the file the package metadata files should be loaded. It is + #: used if a source has no custom compression format specified + #: in the :attr:`server_options`. + default_compression = 'None' + def __init__(self, basepath, xsource): # pylint: disable=R0912 """ :param basepath: The base filesystem path under which cache @@ -188,6 +201,12 @@ class Source(Debuggable): # pylint: disable=R0902 #: The "name" attribute from :attr:`xsource` self.name = None + #: The "priority" attribute from :attr:`xsource` + self.priority = xsource.get('priority', 500) + + #: The "pin" attribute from :attr:`xsource` + self.pin = xsource.get('pin', '') + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -250,11 +269,13 @@ class Source(Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch, debsrc=self.debsrc) + arch=arch, debsrc=self.debsrc, + priority=self.priority, pin=self.pin) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, - arch=arch, debsrc=self.debsrc)] + arch=arch, debsrc=self.debsrc, + priority=self.priority, pin=self.pin)] for setting in usettings: if not self.rawurl: @@ -263,6 +284,8 @@ class Source(Debuggable): # pylint: disable=R0902 setting['baseurl'] = self.rawurl setting['url'] = baseurl % setting setting['name'] = self.get_repo_name(setting) + setting['options'] = dict(server=self.server_options, + client=self.client_options) self.url_map.extend(usettings) def _init_attributes(self, xsource): @@ -328,6 +351,38 @@ class Source(Debuggable): # pylint: disable=R0902 self.conditions.append(lambda m, el=el: el.get("name") == m.hostname) + def _get_reader(self): + ctype = self.default_compression + if 'compression' in self.server_options: + ctype = self.server_options['compression'] + + for mod in Bcfg2.Options.setup.packages_readers: + if mod.__name__.endswith(".%s" % ctype.title()): + return getattr(mod, "%sReader" % ctype.title()) + + raise ValueError("Packages: Unknown compression type %s" % ctype) + + def _get_extension(self): + cls = self._get_reader() + if cls.extension is None: + raise ValueError("%s does not define an extension" % + cls.__name__) + return cls.extension + + def build_filename(self, basename): + extension = self._get_extension() + if extension == '': + return basename + return "%s.%s" % (basename, extension) + + def open_file(self, fname): + try: + cls = self._get_reader() + return cls(fname) + except IOError: + self.logger.error("Packages: Failed to read file %s" % fname) + raise + @property def cachekey(self): """ A unique key for this source that will be used to generate @@ -534,6 +589,9 @@ class Source(Debuggable): # pylint: disable=R0902 self.logger.warning("%s provides no packages for %s" % (self, agrp)) continue + if (agrp in self.blacklist or + (len(self.whitelist) != 0 and agrp not in self.whitelist)): + continue for key, value in list(self.provides[agrp].items()): if key not in vdict: vdict[key] = set(value) @@ -686,7 +744,7 @@ class Source(Debuggable): # pylint: disable=R0902 self.logger.info("Packages: Updating %s" % url) fname = self.escape_url(url) try: - open(fname, 'wb').write(fetch_url(url)) + open(fname, 'wb').write(fetch_url(url, self.server_options)) except ValueError: self.logger.error("Packages: Bad url string %s" % url) raise @@ -760,7 +818,9 @@ class Source(Debuggable): # pylint: disable=R0902 :returns: list of strings """ for arch in self.get_arches(metadata): - if package in self.provides[arch]: + if (package in self.provides[arch] and + package not in self.blacklist and + (len(self.whitelist) == 0 or package in self.whitelist)): return self.provides[arch][package] return [] diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 846fb89cd..acb11f1ab 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -850,7 +850,7 @@ class YumCollection(Collection): return new @track_statistics() - def complete(self, packagelist, recommended=None): + def complete(self, packagelist, recommended=None, pinnings=None): """ Build a complete list of all packages and their dependencies. When using the Python yum libraries, this defers to the @@ -868,7 +868,8 @@ class YumCollection(Collection): resolved. """ if not self.use_yum: - return Collection.complete(self, packagelist, recommended) + return Collection.complete(self, packagelist, recommended, + pinnings) lock = FileLock(os.path.join(self.cachefile, "lock")) slept = 0 diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 95b4baa3e..1a9673891 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -14,6 +14,7 @@ from Bcfg2.Compat import urlopen, HTTPError, URLError from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources +from Bcfg2.Server.Plugins.Packages.Readers import get_readers from Bcfg2.Server.Statistics import track_statistics @@ -36,6 +37,12 @@ class PackagesBackendAction(Bcfg2.Options.ComponentAction): fail_silently = True +class PackagesReadersAction(Bcfg2.Options.ComponentAction): + """ ComponentAction to load Packages readers """ + bases = ['Bcfg2.Server.Plugins.Packages.Readers'] + module = True + + class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, @@ -57,7 +64,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, help="Packages backends to load", type=Bcfg2.Options.Types.comma_list, action=PackagesBackendAction, - default=['Yum', 'Apt', 'Pac', 'Pkgng', 'Dummy']), + default=['Yum', 'Apt', 'Pac', 'Pkgng', 'Dummy', 'Pyapt']), Bcfg2.Options.PathOption( cf=("packages", "cache"), dest="packages_cache", help="Path to the Packages cache", @@ -82,7 +89,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin, cf=("packages", "apt_config"), help="The default path for generated apt configs", default="/etc/apt/sources.list.d/" - "bcfg2-packages-generated-sources.list")] + "bcfg2-packages-generated-sources.list"), + Bcfg2.Options.Option( + cf=("packages", "readers"), dest="packages_readers", + help="Packages readers to load", + type=Bcfg2.Options.Types.comma_list, + action=PackagesReadersAction, + default=get_readers())] #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. @@ -315,6 +328,10 @@ class Packages(Bcfg2.Server.Plugin.Plugin, groups = [] recommended = dict() + pinned_src = dict() + if hasattr(metadata, 'PkgVars'): + pinned_src = metadata.PkgVars['pin'] + for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): @@ -356,11 +373,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, base.update(collection.get_essential()) # check for this set of packages in the package cache - pkey = hash(tuple(base)) + pkey = hash((tuple(base), tuple(recommended), tuple(pinned_src))) pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", collection.cachekey) if pkey not in pcache: - pcache[pkey] = collection.complete(base, recommended) + pcache[pkey] = collection.complete(base, recommended, pinned_src) packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) diff --git a/src/lib/Bcfg2/Server/Plugins/PkgVars.py b/src/lib/Bcfg2/Server/Plugins/PkgVars.py new file mode 100644 index 000000000..9a2649d02 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/PkgVars.py @@ -0,0 +1,65 @@ +import os +import re +import sys +import copy +import logging +import lxml.etree +import Bcfg2.Server.Plugin + +logger = logging.getLogger('Bcfg2.Plugins.PkgVars') +vars = ['pin', 'use', 'keywords'] + +class PkgVarsFile(Bcfg2.Server.Plugin.StructFile): + def get_additional_data(self, meta): + data = self.Match(meta) + results = {} + for d in data: + name = d.get('name', '') + + for v in vars: + value = d.get(v, None) + if value: + if v not in results: + results[v] = {} + if name not in results[v]: + results[v][name] = set() + + results[v][name].add(value) + + return results + +class PkgVarsDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): + __child__ = PkgVarsFile + patterns = re.compile(r'.*\.xml$') + + def get_additional_data(self, meta): + results = {} + for v in vars: + results[v] = {} + + for files in self.entries: + new = self.entries[files].get_additional_data(meta) + for x in vars: + if x in new: + results[x].update(new[x]) + + return results + +class PkgVars(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + name = 'PkgVars' + version = '$Revision$' + + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) + Bcfg2.Server.Plugin.Connector.__init__(self) + try: + self.store = PkgVarsDirectoryBacked(self.data) + except OSError: + e = sys.exc_info()[1] + self.logger.error("Error while creating PkgVars store: %s %s" % + (e.strerror, e.filename)) + raise Bcfg2.Server.Plugin.PluginInitError + + def get_additional_data(self, meta): + return self.store.get_additional_data(meta) |