diff options
23 files changed, 818 insertions, 46 deletions
diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index 3a0217aef..12f66f64f 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -729,6 +729,11 @@ control the database connection of the server. port Port for database connections. Not used for sqlite3. + options + Various options for the database connection. The value is + expected as multiple key=value pairs, separated with commas. + The concrete value depends on the database engine. + Reporting options ----------------- diff --git a/doc/server/database.txt b/doc/server/database.txt index b0ec7b571..3c8970f68 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -49,6 +49,12 @@ of ``/etc/bcfg2.conf``. +-------------+------------------------------------------------------------+-------------------------------+ | port | The port to connect to | None | +-------------+------------------------------------------------------------+-------------------------------+ +| options | Extra parameters to use when connecting to the database. | None | +| | Available parameters vary depending on your database | | +| | backend. The parameters are supplied as comma separated | | +| | key=value pairs. | | ++-------------+------------------------------------------------------------+-------------------------------+ + Database Schema Sync ==================== diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index b0db91a5b..85e2f4b98 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,4 +1,4 @@ -.TH "BCFG2.CONF" "5" "March 18, 2013" "1.3" "Bcfg2" +.TH "BCFG2.CONF" "5" "June 19, 2013" "1.3" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 . @@ -771,6 +771,11 @@ Host for database connections. Not used for sqlite3. .TP .B port Port for database connections. Not used for sqlite3. +.TP +.B options +Various options for the database connection. The value is +expected as multiple key=value pairs, separated with commas. +The concrete value depends on the database engine. .UNINDENT .UNINDENT .UNINDENT diff --git a/schemas/packages.xsd b/schemas/packages.xsd index e538bb0e7..e4724fabe 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -14,6 +14,8 @@ <xsd:enumeration value="yum"/> <xsd:enumeration value="apt"/> <xsd:enumeration value="pac"/> + <xsd:enumeration value="portage"/> + <xsd:enumeration value="layman"/> </xsd:restriction> </xsd:simpleType> @@ -211,6 +213,34 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:integer" name="priority"> + <xsd:annotation> + <xsd:documentation> + The priority of the source. This is used to order the + sources. After sorting, the first source, that could + deliver the package, is used. If not supplied the default + priority is 500. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="name"> + <xsd:annotation> + <xsd:documentation> + The name of the source to refer to that specify source + for specific packages. It could be used like pinning under + debian. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="pin"> + <xsd:annotation> + <xsd:documentation> + Extra information for pinning. This information is used + to differ between the sources. Should be used in the + supported format of apt. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> </xsd:complexType> <xsd:complexType name="PackagesGroupType"> diff --git a/schemas/pkgtype.xsd b/schemas/pkgtype.xsd index 18eda88ab..e301f30b6 100644 --- a/schemas/pkgtype.xsd +++ b/schemas/pkgtype.xsd @@ -54,6 +54,15 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute name="recommended" type="xsd:boolean"> + <xsd:annotation> + <xsd:documentation> + Whether also th recommended packages should be installed. + This is currently only used with the :ref:`APT + <client-tools-apt>` driver. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/schemas/pkgvars.xsd b/schemas/pkgvars.xsd new file mode 100644 index 000000000..dbd02726d --- /dev/null +++ b/schemas/pkgvars.xsd @@ -0,0 +1,43 @@ +<xsd:schema xmlns:xsd='http://www.w3.org/2001/XMLSchema' + xmlns:py="http://genshi.edgewall.org/"> + + <xsd:annotation> + <xsd:documentation> + XML-Schema-Definition für PkgVars/*.xml + Alexander Sulfrian + </xsd:documentation> + </xsd:annotation> + + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + + <xsd:complexType name='pkgVarType'> + <xsd:attribute type='xsd:string' name='name'/> + + <xsd:attribute type='xsd:string' name='pin'/> + <xsd:attribute type='xsd:string' name='use'/> + <xsd:attribute type='xsd:string' name='keywords'/> + + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name='containerType'> + <xsd:choice maxOccurs='unbounded'> + <xsd:element name='Package' type='pkgVarType'/> + <xsd:element name='Client' type='containerType'/> + <xsd:element name='Group' type='containerType'/> + </xsd:choice> + <xsd:attribute name='name' type='xsd:string' use='required'/> + <xsd:attribute name='negate' type='xsd:boolean'/> + </xsd:complexType> + + <xsd:complexType name='pkgVarsType'> + <xsd:choice minOccurs='0' maxOccurs='unbounded'> + <xsd:element name='Package' type='pkgVarType'/> + <xsd:element name='Client' type='containerType'/> + <xsd:element name='Group' type='containerType'/> + </xsd:choice> + </xsd:complexType> + + <xsd:element name='PkgVars' type='pkgVarsType'/> +</xsd:schema> diff --git a/schemas/types.xsd b/schemas/types.xsd index 4e3dfd70f..8fff4bdc1 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -386,6 +386,14 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute name='important' type='xsd:boolean' default="false"> + <xsd:annotation> + <xsd:documentation> + Important entries are installed first during client + execution. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 3254da9e9..24b4f409c 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -170,15 +170,8 @@ class Frame(object): return self.__dict__[name] raise AttributeError(name) - def InstallImportant(self): - """Install important entries - - We also process the decision mode stuff here because we want to prevent - non-whitelisted/blacklisted 'important' entries from being installed - prior to determining the decision mode on the client. - """ - # Need to process decision stuff early so that dryrun mode - # works with it + def GenerateActions(self): + """Generate list of all entries that have to be handled""" self.whitelist = [entry for entry in self.states if not self.states[entry]] if not self.setup['file']: @@ -205,6 +198,17 @@ class Frame(object): self.whitelist = [x for x in self.whitelist if x not in b_to_rem] + def InstallImportant(self): + """Install important entries + + We also process the decision mode stuff here because we want to prevent + non-whitelisted/blacklisted 'important' entries from being installed + prior to determining the decision mode on the client. + """ + # Need to process decision stuff early so that dryrun mode + # works with it + self.GenerateActions() + # take care of important entries first if not self.dryrun: parent_map = dict((c, p) @@ -486,6 +490,9 @@ class Frame(object): self.CondDisplayState('initial') self.InstallImportant() self.Decide() + if self.modified: + self.Inventory() + self.GenerateActions() self.Install() self.times['install'] = time.time() self.Remove() @@ -514,7 +521,9 @@ class Frame(object): stats.set('state', 'clean') # List bad elements of the configuration - for (data, ename) in [(self.modified, 'Modified'), + mods = [m for m in self.modified + if (m.tag != 'Action' or m.get('when') != 'always')] + for (data, ename) in [(mods, 'Modified'), (self.extra, "Extra"), (good_entries, "Good"), ([entry for entry in self.states diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index 39816403a..3fc0e310d 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -26,6 +26,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.etc_path = setup.get('apt_etc_path', '/etc') self.debsums = '%s/bin/debsums' % self.install_path self.aptget = '%s/bin/apt-get' % self.install_path + self.aptmark = '%s/bin/apt-mark' % self.install_path self.dpkg = '%s/bin/dpkg' % self.install_path self.__execs__ = [self.debsums, self.aptget, self.dpkg] @@ -69,6 +70,23 @@ class APT(Bcfg2.Client.Tools.Tool): self.logger.info("Failed to initialize APT cache: %s" % e) raise Bcfg2.Client.Tools.ToolInstantiationError self.pkg_cache.update() + # mark dependencies as being automatically installed and vice versa + mark = [] + unmark = [] + try: + installed_pkgs = [p.name for p in self.pkg_cache if p.is_installed] + except AttributeError: + installed_pkgs = [p.name for p in self.pkg_cache if p.isInstalled] + for pkg in self.getSupportedEntries(): + if pkg.get('name') in installed_pkgs: + if pkg.get('origin') == 'Packages': + mark.append(pkg.get('name')) + else: + unmark.append(pkg.get('name')) + if mark: + self.cmd.run("%s markauto %s" % (self.aptmark, (" ".join(mark)))) + if unmark: + self.cmd.run("%s unmarkauto %s" % (self.aptmark, (" ".join(unmark)))) self.pkg_cache = apt.cache.Cache() if 'req_reinstall_pkgs' in dir(self.pkg_cache): self._newapi = True @@ -88,6 +106,11 @@ class APT(Bcfg2.Client.Tools.Tool): type='deb', version=version) \ for (name, version) in extras] + def Inventory(self, states, structures=[]): + # reload pkg cache + self.pkg_cache.open(None) + Bcfg2.Client.Tools.Tool.Inventory(self, states, structures) + def VerifyDebsums(self, entry, modlist): output = \ self.cmd.run("%s -as %s" % @@ -163,7 +186,7 @@ class APT(Bcfg2.Client.Tools.Tool): else: installed_version = pkg.installedVersion candidate_version = pkg.candidateVersion - if entry.get('version') == 'auto': + if entry.get('version').startswith('auto'): if self._newapi: is_upgradable = self.pkg_cache._depcache.is_upgradable(pkg._pkg) else: @@ -172,8 +195,10 @@ class APT(Bcfg2.Client.Tools.Tool): desiredVersion = candidate_version else: desiredVersion = installed_version - elif entry.get('version') == 'any': + entry.set('version', "auto: %s" % desiredVersion) + elif entry.get('version').startswith('any'): desiredVersion = installed_version + entry.set('version', "any: %s" % desiredVersion) else: desiredVersion = entry.get('version') if desiredVersion != installed_version: @@ -226,7 +251,7 @@ class APT(Bcfg2.Client.Tools.Tool): if not self.pkg_cache.has_key(pkg.get('name')): self.logger.error("APT has no information about package %s" % (pkg.get('name'))) continue - if pkg.get('version') in ['auto', 'any']: + if any([pkg.get('version').startswith(v) for v in ['auto', 'any']]): if self._newapi: try: ipkgs.append("%s=%s" % (pkg.get('name'), @@ -262,10 +287,15 @@ class APT(Bcfg2.Client.Tools.Tool): self.logger.error("APT command failed") self.pkg_cache = apt.cache.Cache() self.extra = self.FindExtra() + mark = [] for package in packages: states[package] = self.VerifyPackage(package, [], checksums=False) if states[package]: self.modified.append(package) + if package.get('origin') == 'Packages': + mark.append(package.get('name')) + if mark: + self.cmd.run("%s markauto %s" % (self.aptmark, (" ".join(mark)))) def VerifyPath(self, entry, _): """Do nothing here since we only verify Path type=ignore.""" diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py index 675a4461a..24667d162 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py @@ -11,6 +11,10 @@ class POSIXDirectory(POSIXTool): """ Handle <Path type='directory' ...> entries """ __req__ = ['name', 'mode', 'owner', 'group'] + def __init__(self, logger, setup, config): + super(POSIXDirectory, self).__init__(logger, setup, config) + self.prunes = dict() + def verify(self, entry, modlist): ondisk = self._exists(entry) if not ondisk: @@ -25,10 +29,18 @@ class POSIXDirectory(POSIXTool): if entry.get('prune', 'false').lower() == 'true': # check for any extra entries when prune='true' attribute is set try: + prune_list = list() + if entry.get('name') in self.prunes: + prune_list = self.prunes[entry.get('name')] + else: + self.prunes[entry.get('name')] = list() + extras = [os.path.join(entry.get('name'), ent) for ent in os.listdir(entry.get('name')) - if os.path.join(entry.get('name'), - ent) not in modlist] + if (os.path.join(entry.get('name'), + ent) not in modlist and + os.path.join(entry.get('name'), + ent) not in prune_list)] if extras: prune = False msg = "Directory %s contains extra entries: %s" % \ @@ -37,6 +49,10 @@ class POSIXDirectory(POSIXTool): entry.set('qtext', entry.get('qtext', '') + '\n' + msg) for extra in extras: Bcfg2.Client.XML.SubElement(entry, 'Prune', name=extra) + self.prunes[entry.get('name')] += extras + elif entry.get('name') in self.prunes and \ + len(self.prunes[entry.get('name')]) > 0: + prune = False except OSError: prune = True @@ -71,6 +87,8 @@ class POSIXDirectory(POSIXTool): try: self.logger.debug("POSIX: Removing %s" % pname) self._remove(pent) + if entry.get('name') in self.prunes: + self.prunes[entry.get('name')].remove(pname) except OSError: err = sys.exc_info()[1] self.logger.error("POSIX: Failed to unlink %s: %s" % diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 243c4ed2a..84551a02d 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -319,6 +319,28 @@ def colon_split(c_string): return [] +def dict_split(c_string): + """ split an option string on commans, optionally sourrunded by + whitespace and split the resulting items again on equals signs, + returning a dict """ + result = dict() + if c_string: + items = re.split(r'\s*,\s*', c_string) + for item in items: + if r'=' in item: + key, value = item.split(r'=', 1) + try: + result[key] = get_bool(value) + except ValueError: + try: + result[key] = get_int(value) + except ValueError: + result[key] = value + else: + result[item] = True + return result + + def get_bool(val): """ given a string value of a boolean configuration option, return an actual bool (True or False) """ @@ -652,6 +674,12 @@ DB_PORT = \ cf=('database', 'port'), deprecated_cf=('statistics', 'database_port')) +DB_OPTIONS = \ + Option('Database options', + default=dict(), + cf=('database', 'options'), + cook=dict_split) + # Django options WEB_CFILE = \ Option('Web interface configuration file', @@ -1285,6 +1313,7 @@ DATABASE_COMMON_OPTIONS = dict(web_configfile=WEB_CFILE, db_password=DB_PASSWORD, db_host=DB_HOST, db_port=DB_PORT, + db_options=DB_OPTIONS, time_zone=DJANGO_TIME_ZONE, django_debug=DJANGO_DEBUG, web_prefix=DJANGO_WEB_PREFIX) diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 489682f30..ee8738a0c 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -319,6 +319,8 @@ def determine_client_state(entry): _how_ dirty and adjust the color accordingly. """ if entry.state == 'clean': + if entry.extra_count > 0: + return "extra-lineitem" return "clean-lineitem" bad_percentage = 100 * (float(entry.bad_count) / entry.total_count) diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 09f3f3d25..645870020 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -51,7 +51,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "FileProbes/config.xml": "fileprobes.xsd", "SSLCA/**/cert.xml": "sslca-cert.xsd", "SSLCA/**/key.xml": "sslca-key.xsd", - "GroupLogic/groups.xml": "grouplogic.xsd" + "GroupLogic/groups.xml": "grouplogic.xsd", + "PkgVars/*.xml": "pkgvars.xsd" } self.filelists = {} diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index a82a183d8..009395e8f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -76,10 +76,8 @@ class AptSource(Source): def read_files(self): bdeps = dict() + brecs = dict() bprov = dict() - depfnames = ['Depends', 'Pre-Depends'] - if self.recommended: - depfnames.append('Recommends') for fname in self.files: if not self.rawurl: barch = [x @@ -91,6 +89,7 @@ class AptSource(Source): barch = self.arches[0] if barch not in bdeps: bdeps[barch] = dict() + brecs[barch] = dict() bprov[barch] = dict() try: reader = gzip.GzipFile(fname) @@ -105,9 +104,10 @@ class AptSource(Source): pkgname = words[1].strip().rstrip() self.pkgnames.add(pkgname) bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: self.essentialpkgs.add(pkgname) - elif words[0] in depfnames: + elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']: vindex = 0 for dep in words[1].split(','): if '|' in dep: @@ -118,17 +118,24 @@ class AptSource(Source): barch, vindex) vindex += 1 - bdeps[barch][pkgname].append(dyn_dname) + + if words[0] == 'Recommends': + brecs[barch][pkgname].append(dyn_dname) + else: + bdeps[barch][pkgname].append(dyn_dname) bprov[barch][dyn_dname] = set(cdeps) else: raw_dep = re.sub(r'\(.*\)', '', dep) raw_dep = raw_dep.rstrip().strip() - bdeps[barch][pkgname].append(raw_dep) + if words[0] == 'Recommends': + brecs[barch][pkgname].append(raw_dep) + else: + bdeps[barch][pkgname].append(raw_dep) elif words[0] == 'Provides': for pkg in words[1].split(','): dname = pkg.rstrip().strip() if dname not in bprov[barch]: bprov[barch][dname] = set() bprov[barch][dname].add(pkgname) - self.process_files(bdeps, bprov) + self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index b25cb0fc4..93b398ac4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -308,7 +308,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): return any(source.is_virtual_package(self.metadata, package) for source in self) - def get_deps(self, package): + def get_deps(self, package, recs=None, pinnings=None): """ Get a list of the dependencies of the given package. The base implementation simply aggregates the results of @@ -316,11 +316,31 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): :param package: The name of the symbol, but see :ref:`pkg-objects` :type package: string + :param pinnings: Mapping from package names to source names. + :type pinnings: dict :returns: list of strings, but see :ref:`pkg-objects` """ + recommended = None + if recs and package in recs: + recommended = recs[package] + + pin_found = False + pin_source = None + if pinnings and package in pinnings: + pin_source = pinnings[package] + for source in self: + if pin_source and source.name not in pin_source: + continue + pin_found = True + if source.is_package(self.metadata, package): - return source.get_deps(self.metadata, package) + return source.get_deps(self.metadata, package, recommended) + + if not pin_found: + self.logger.error("Packages: Source '%s' for package '%s' not found" % + (' or '.join(pin_source), package)) + return [] def get_essential(self): @@ -500,12 +520,15 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): return list(complete.difference(initial)) @Bcfg2.Server.Plugin.track_statistics() - def complete(self, packagelist): # pylint: disable=R0912,R0914 + def complete(self, packagelist, recommended=None, + pinnings=None): # pylint: disable=R0912,R0914 """ Build a complete list of all packages and their dependencies. :param packagelist: Set of initial packages computed from the specification. :type packagelist: set of strings, but see :ref:`pkg-objects` + :param pinnings: Mapping from package names to source names. + :type pinnings: dict :returns: tuple of sets - The first element contains a set of strings (but see :ref:`pkg-objects`) describing the complete package list, and the second element is a @@ -564,7 +587,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): self.debug_log("Packages: handling package requirement %s" % (current,)) packages.add(current) - deps = self.get_deps(current) + deps = self.get_deps(current, recommended, pinnings) newdeps = set(deps).difference(examined) if newdeps: self.debug_log("Packages: Package %s added requirements %s" diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Layman.py b/src/lib/Bcfg2/Server/Plugins/Packages/Layman.py new file mode 100644 index 000000000..40358e214 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Layman.py @@ -0,0 +1,115 @@ +import os +import layman +import Bcfg2.Server.Plugin + +class LaymanSource(Bcfg2.Server.Plugin.Debuggable): + basegroups = ['portage', 'gentoo', 'emerge'] + ptype = 'layman' + cclass = 'Portage' + + def __init__(self, basepath, xsource, config): + Bcfg2.Server.Plugin.Debuggable.__init__(self) + self.basepath = basepath + self.xsource = xsource + self.config = config + + self.url = xsource.get('url', 'http://www.gentoo.org/proj/en/overlays/repositories.xml') + self.name = xsource.get('name', '') + self.priority = xsource.get('priority', 0) + self.cachefile = None + self.gpgkeys = [] + self.recommended = False + + # configure layman + base = os.path.join(basepath, 'layman') + config = layman.config.OptionConfig(options = { + 'storage': os.path.join(base, 'overlays'), + 'cache': os.path.join(base, 'cache'), + 'installed': os.path.join(base, 'installed.xml'), + 'local_list': os.path.join(base, 'overlays.xml'), + 'overlays': [url] + }) + self.api = layman.LaymanAPI(config) + + # path + self.dir = os.path.join(basepath, 'overlays', self.name) + + # build the set of conditions to see if this source applies to + # a given set of metadata + self.conditions = [] + self.groups = [] # provided for some limited backwards compat + for el in xsource.iterancestors(): + if el.tag == "Group": + if el.get("negate", "false").lower() == "true": + self.conditions.append(lambda m, el=el: + el.get("name") not in m.groups) + else: + self.groups.append(el.get("name")) + self.conditions.append(lambda m, el=el: + el.get("name") in m.groups) + elif el.tag == "Client": + if el.get("negate", "false").lower() == "true": + self.conditions.append(lambda m, el=el: + el.get("name") != m.hostname) + else: + self.conditions.append(lambda m, el=el: + el.get("name") == m.hostname) + + def save_state(self): + pass + + def load_state(self): + pass + + def filter_unknown(self, unknown): + filtered = set([u for u in unknown if u.startswith('choice')]) + unknown.difference_update(filtered) + + def get_urls(self): + return self.url + urls = property(get_urls) + + def magic_groups_match(self, metadata): + if self.config.getboolean("global", "magic_groups", + default=True) == False: + return True + else: + for group in self.basegroups: + if group in metadata.groups: + return True + return False + + def setup_data(self, force_update=False): + self.api.fetch_remote_list() + if not self.api.is_repo(self.name): + self.logger.error("Packages: Layman overlay '%s' not" + " found" % self.name) + return False + + if not self.api.is_installed(self.name): + self.logger.info("Packages: Adding layman overlay '%s'" % + self.name) + if not self.api.add_repos(name): + self.logger.error("Packages: Failed adding layman" + " overlay '%s'" % self.name) + return False + + if force_update: + if not self.api.sync(name): + self.logger.error("Packages: Failed syncing layman" + " overlay '%s'" % self.name) + return False + + return True + + def applies(self, metadata): + # check base groups + if not self.magic_groups_match(metadata): + return False + + # check Group/Client tags from sources.xml + for condition in self.conditions: + if not condition(metadata): + return False + + return True diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 332f0bbab..e627275c0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -115,6 +115,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, source = self.source_from_xml(xsource) if source is not None: self.entries.append(source) + sorted(self.entries, key=(lambda source: source.priority), reverse=True) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """ ``Index`` is responsible for calling :func:`source_from_xml` diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py b/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py new file mode 100644 index 000000000..8629699ca --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py @@ -0,0 +1,312 @@ +import re +import gzip +import sys +import os +import lxml.etree +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Packages.Collection import Collection +from Bcfg2.Server.Plugins.Packages.Layman import LaymanSource + +_portage_python = '/usr/lib/portage/pym/' + +def _import_portage(caller): + # generate prefix path + caller.prefix = os.path.join(caller.basepath, 'cache', 'portage') + if not os.path.isdir(caller.prefix): + caller.logger.error("Packages: %s is not a dir. " + "Portage will not work. Please " + "remember to setup a EPREFIX there." % + caller.prefix) + # TODO: automatic EPREFIX setup + raise Exception('Invalid EPREFIX') + + os.environ['PORTAGE_OVERRIDE_EPREFIX'] = caller.prefix + + if not os.path.isdir(_portage_python): + self.logger.error("Packages: %s not found. Have you installed " + "the portage python modules?" % _portage_python) + raise Exception('Portage not found') + + sys.path = sys.path + [_portage_python] + portage = __import__('portage', globals(), locals(), + ['_sets', 'dbapi.porttree' ]) + emerge = __import__('_emerge', globals(), locals(), + ['RootConfig', 'depgraph', 'Package', 'actions']) + + # setup profile + if '_setup_profile' in dir(caller): + caller._setup_profile(portage) + + # fix some default settings + portage.settings.unlock() + portage.settings['PORTAGE_RSYNC_INITIAL_TIMEOUT'] = '0' + portage.settings.lock() + + porttree = portage.db[portage.root]['porttree'] + caller._import_portage(portage, emerge, porttree) + + +class PortageCollection(Collection): + def __init__(self, metadata, sources, basepath, debug=False): + Collection.__init__(self, metadata, sources, basepath, debug) + + self.portage = None + self.emerge = None + self.porttree = None + + @property + def cachefiles(self): + return [] + + def complete(self, packagelist, pinnings=None, recommended=None): + if not self.portage: + _import_portage(self) + + # calculate deps + setconfig = self.portage._sets.load_default_config( + self.portage.settings, + self.portage.db[self.portage.root]) + rconfig = self.emerge.RootConfig.RootConfig( + self.portage.settings, + self.portage.db[self.portage.root], + setconfig) + self.portage.db[self.portage.root]['root_config'] = rconfig + tree = self.portage.db[self.portage.root]['porttree'] + + pkgs = ["=" + j.cpv for (i, j) in packagelist if i == 'ok'] + fail = [j for (i, j) in packagelist if i == 'fail'] + + x = self.emerge.depgraph.backtrack_depgraph( + self.portage.settings, + self.portage.db, + {'--pretend': True}, + {'recurse': True}, + 'merge', + pkgs, + None) + + g = x[1]._dynamic_config.digraph + packages = [i for i in g.all_nodes() \ + if isinstance(i, self.emerge.Package.Package)] + + return (set(packages), set(fail)) + + def get_additional_data(self): + return [] + + def get_group(self, group): + self.logger.warning("Packages: Package sets are currently not supported") + return [] + + def packages_from_entry(self, entry): + if not self.portage: + _import_portage(self) + + try: + name = entry.get('name') + pkgs = self.porttree.dep_bestmatch(name) + except self.portage.exception.AmbiguousPackageName as e: + self.logger.error("Packages: AmbiguousPackageName: %s" % e) + pkgs = '' + + if pkgs == '': + return [('fail', name)] + + return [('ok', pkgs)] + + def packages_to_entry(self, pkgs, entry, config): + for pkg in pkgs: + if pkg.slot != '0': + name = "%s:%s" % (pkg.cp, pkg.slot) + else: + name = pkg.cp + + lxml.etree.SubElement(entry, 'BoundPackage', name=name, + version=config.get("packages", "version", + default="auto"), + type=self.ptype, origin='Packages') + + def get_new_packages(self, initial, complete): + new = [] + init = [pkg.cp for status, pkg in initial if status == 'ok'] + + for pkg in complete: + if pkg.cp not in init: + new.append(pkg) + + return new + + def setup_data(self, force_update=False): + pass + + def _setup_profile(self, portage): + if 'gentoo-profile' not in self.metadata.Probes: + raise Exception('Unknown profile.') + + profile = os.path.join(self.prefix, 'usr/portage/profiles/', + self.metadata.Probes['gentoo-profile']) + + env = portage.settings.configdict['backupenv'] + + # add layman overlays + env['PORTDIR_OVERLAY'] = '' + for overlay in self.sources: + if isinstance(overlay, LaymanSource): + env['PORTDIR_OVERLAY'] += ' ' + env['PORTDIR_OVERLAY'] += overlay.dir + + portage.settings = portage.package.ebuild.config.config( + config_root=portage.settings['PORTAGE_CONFIGROOT'], + target_root=portage.settings['ROOT'], + env=env, + eprefix=portage.settings['EPREFIX'], + config_profile_path=profile) + + portage.db[portage.root]['porttree'].settings = portage.settings + newdbapi = portage.dbapi.porttree.portdbapi(mysettings=portage.settings) + portage.db[portage.root]['porttree'].dbapi = newdbapi + + portage.db[portage.root]['vartree'].settings = portage.settings + portage.db[portage.root]['vartree'].dbapi.settings = portage.settings + + def _set_portage_config(self): + # get global use flags + self.portage.settings.unlock() + self.portage.settings['USE'] = '' + if 'gentoo-use-flags' in self.metadata.Probes: + self.portage.settings['USE'] = \ + self.metadata.Probes['gentoo-use-flags'] + + # add package flags (accept_keywords, use) + if hasattr(self.metadata, 'PkgVars'): + for k in self.metadata.PkgVars['keywords']: + keyword = metadata.PkgVars['keywords'][k] + self.portage.settings._keywords_manager.pkeywordsdict[k] = \ + {k: tuple(keyword)} + + + for u in self.metadata.PkgVars['use']: + use = metadata.PkgVars['use'][u] + self.portage.settings._use_manager._pusedict[u] = \ + {u: tuple(use)} + + self.portage.settings.lock() + + def _import_portage(self, portage, emerge, porttree): + self.portage = portage + self.emerge = emerge + self.porttree = porttree + self._set_portage_config() + + for s in self.sources: + if isinstance(s, PortageSource): + s._import_portage(portage, emerge, porttree) + + +class PortageSource(Bcfg2.Server.Plugin.Debuggable): + basegroups = ['portage', 'gentoo', 'emerge'] + ptype = 'ebuild' + + def __init__(self, basepath, xsource, config): + Bcfg2.Server.Plugin.Debuggable.__init__(self) + self.basepath = basepath + self.xsource = xsource + self.config = config + + self.url = xsource.get('url', '') + self.priority = xsource.get('priority', 0) + self.cachefile = None + self.gpgkeys = [] + self.recommended = False + + self.portage = None + self.emerge = None + self.porttree = None + + # build the set of conditions to see if this source applies to + # a given set of metadata + self.conditions = [] + self.groups = [] # provided for some limited backwards compat + for el in xsource.iterancestors(): + if el.tag == "Group": + if el.get("negate", "false").lower() == "true": + self.conditions.append(lambda m, el=el: + el.get("name") not in m.groups) + else: + self.groups.append(el.get("name")) + self.conditions.append(lambda m, el=el: + el.get("name") in m.groups) + elif el.tag == "Client": + if el.get("negate", "false").lower() == "true": + self.conditions.append(lambda m, el=el: + el.get("name") != m.hostname) + else: + self.conditions.append(lambda m, el=el: + el.get("name") == m.hostname) + + def _import_portage(self, portage, emerge, porttree): + self.portage = portage + self.emerge = emerge + self.porttree = porttree + + def save_state(self): + pass + + def load_state(self): + pass + + def filter_unknown(self, unknown): + filtered = set([u for u in unknown if u.startswith('choice')]) + unknown.difference_update(filtered) + + def get_urls(self): + return self.url + urls = property(get_urls) + + def magic_groups_match(self, metadata): + if self.config.getboolean("global", "magic_groups", + default=True) == False: + return True + else: + for group in self.basegroups: + if group in metadata.groups: + return True + return False + + def setup_data(self, force_update=False): + if not self.porttree: + _import_portage(self) + + timestamp = os.path.join(self.porttree.portroot, 'metadata/timestamp.chk') + if not os.path.isfile(timestamp): + self.logger.error("Packages: Timestamp not found; " + "falling back to sync") + force_update = True + + if force_update: + # update sync url + self.portage.settings.unlock() + self.portage.settings['SYNC'] = self.url + self.portage.settings.lock() + + # realy force the sync + if os.path.isfile(timestamp): + os.unlink(timestamp) + + # sync + self.logger.info("Packages: Syncing with %s" % self.url) + self.emerge.actions.action_sync(self.portage.settings, + self.portage.db, None, + {'--quiet': True}, 'sync') + + def applies(self, metadata): + # check base groups + if not self.magic_groups_match(metadata): + return False + + # check Group/Client tags from sources.xml + for condition in self.conditions: + if not condition(metadata): + return False + + return True diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 22073493c..c04611d77 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -209,6 +209,15 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: The "version" attribute from :attr:`xsource` self.version = xsource.get('version', '') + #: The "priority" attribute from :attr:`xsource` + self.priority = xsource.get('priority', 500) + + #: The "name" attribute from :attr:`xsource` + self.name = xsource.get('name', '') + + #: The "priority" attribute from :attr:`xsource` + self.pin = xsource.get('pin', '') + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -256,6 +265,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` self.provides = dict() + #: A dict of ``<package name>`` -> ``<list of recommended + #: symbols>``. This will not necessarily be populated. + self.recommends = dict() + #: The file (or directory) used for this source's cache data self.cachefile = os.path.join(self.basepath, "cache-%s" % self.cachekey) @@ -280,7 +293,8 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch) + arch=arch, priority=self.priority, + name=self.name, pin=self.pin) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, @@ -321,7 +335,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :raises: cPickle.UnpicklingError - If the saved data is corrupt """ data = open(self.cachefile, 'rb') (self.pkgnames, self.deps, self.provides, - self.essentialpkgs) = cPickle.load(data) + self.essentialpkgs, self.recommends) = cPickle.load(data) def save_state(self): """ Save state to :attr:`cachefile`. If caching and @@ -329,7 +343,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 does not need to be implemented. """ cache = open(self.cachefile, 'wb') cPickle.dump((self.pkgnames, self.deps, self.provides, - self.essentialpkgs), cache, 2) + self.essentialpkgs, self.recommends), cache, 2) cache.close() @Bcfg2.Server.Plugin.track_statistics() @@ -524,13 +538,13 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 as its final step.""" pass - def process_files(self, dependencies, provides): + def process_files(self, dependencies, provides, recommends=dict()): """ Given dicts of depends and provides generated by :func:`read_files`, this generates :attr:`deps` and :attr:`provides` and calls :func:`save_state` to save the cached data to disk. - Both arguments are dicts of dicts of lists. Keys are the + All arguments are dicts of dicts of lists. Keys are the arches of packages contained in this source; values are dicts whose keys are package names and values are lists of either dependencies for each package the symbols provided by each @@ -542,14 +556,20 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :param provides: A dict of symbols provided by packages in this repository. :type provides: dict; see above. + :param recommends: A dict of recommended dependencies + found for this source. + :type recommends: dict; see above. """ self.deps['global'] = dict() + self.recommends['global'] = dict() self.provides['global'] = dict() for barch in dependencies: self.deps[barch] = dict() + self.recommends[barch] = dict() self.provides[barch] = dict() for pkgname in self.pkgnames: pset = set() + rset = set() for barch in dependencies: if pkgname not in dependencies[barch]: dependencies[barch][pkgname] = [] @@ -559,6 +579,17 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 else: for barch in dependencies: self.deps[barch][pkgname] = dependencies[barch][pkgname] + + for barch in recommends: + if pkgname not in recommends[barch]: + recommends[barch][pkgname] = [] + rset.add(tuple(recommends[barch][pkgname])) + if len(rset) == 1: + self.recommends['global'][pkgname] = rset.pop() + else: + for barch in recommends: + self.recommends[barch][pkgname] = recommends[barch][pkgname] + provided = set() for bprovided in list(provides.values()): provided.update(set(bprovided)) @@ -667,17 +698,24 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 """ return ['global'] + [a for a in self.arches if a in metadata.groups] - def get_deps(self, metadata, package): + def get_deps(self, metadata, package, recommended=None): """ Get a list of the dependencies of the given package. :param package: The name of the symbol :type package: string :returns: list of strings """ + recs = [] + if ((recommended is None and self.recommended) or + (recommended and recommended.lower() == 'true')): + for arch in self.get_arches(metadata): + if package in self.recommends[arch]: + recs.extend(self.recommends[arch][package]) + for arch in self.get_arches(metadata): if package in self.deps[arch]: - return self.deps[arch][package] - return [] + recs.extend(self.deps[arch][package]) + return recs def get_provides(self, metadata, package): """ Get a list of all symbols provided by the given package. diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 4608bcca5..c2f10b97b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -820,7 +820,7 @@ class YumCollection(Collection): return new @Bcfg2.Server.Plugin.track_statistics() - def complete(self, packagelist): + def complete(self, packagelist, recommended=None, pinnings=None): """ Build a complete list of all packages and their dependencies. When using the Python yum libraries, this defers to the @@ -838,7 +838,8 @@ class YumCollection(Collection): resolved. """ if not self.use_yum: - return Collection.complete(self, packagelist) + return Collection.complete(self, packagelist, recommended, + pinnings) if packagelist: try: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index f82b8a392..3420735dd 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -331,10 +331,19 @@ class Packages(Bcfg2.Server.Plugin.Plugin, initial = set() to_remove = [] groups = [] + recommended = dict() + + pinned_src = dict() + if hasattr(metadata, 'PkgVars'): + pinned_src = metadata.PkgVars['pin'] + for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): initial.update(collection.packages_from_entry(pkg)) + + if pkg.get("recommended"): + recommended[pkg.get("name")] = pkg.get("recommended") elif pkg.get("group"): groups.append((pkg.get("group"), pkg.get("type"))) @@ -362,7 +371,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # essential pkgs are those marked as such by the distribution base.update(collection.get_essential()) - packages, unknown = collection.complete(base) + packages, unknown = collection.complete(base, recommended, pinned_src) if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) @@ -505,20 +514,23 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for source in self.sources.entries: if source.applies(metadata): relevant.append(source) - sclasses.update([source.__class__]) + if 'cclass' in dir(source): + sclasses.update([source.cclass]) + else: + sclass = source.__class__.__name__.replace("Source", "") + sclasses.update([sclass]) if len(sclasses) > 1: self.logger.warning("Packages: Multiple source types found for " "%s: %s" % - ",".join([s.__name__ for s in sclasses])) + (metadata.hostname, ",".join([sclasses]))) cclass = Collection elif len(sclasses) == 0: self.logger.error("Packages: No sources found for %s" % metadata.hostname) cclass = Collection else: - cclass = get_collection_class( - sclasses.pop().__name__.replace("Source", "")) + cclass = get_collection_class(sclasses.pop()) if self.debug_flag: self.logger.error("Packages: Using %s for Collection of sources " diff --git a/src/lib/Bcfg2/Server/Plugins/PkgVars.py b/src/lib/Bcfg2/Server/Plugins/PkgVars.py new file mode 100644 index 000000000..40582bcb3 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/PkgVars.py @@ -0,0 +1,65 @@ +import os +import re +import sys +import copy +import logging +import lxml.etree +import Bcfg2.Server.Plugin + +logger = logging.getLogger('Bcfg2.Plugins.PkgVars') +vars = ['pin', 'use', 'keywords'] + +class PkgVarsFile(Bcfg2.Server.Plugin.StructFile): + def get_additional_data(self, meta): + data = self.Match(meta) + results = {} + for d in data: + name = d.get('name', '') + + for v in vars: + value = d.get(v, None) + if value: + if v not in results: + results[v] = {} + if name not in results[v]: + results[v][name] = set() + + results[v][name].add(value) + + return results + +class PkgVarsDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): + __child__ = PkgVarsFile + patterns = re.compile(r'.*\.xml$') + + def get_additional_data(self, meta): + results = {} + for v in vars: + results[v] = {} + + for files in self.entries: + new = self.entries[files].get_additional_data(meta) + for x in vars: + if x in new: + results[x].update(new[x]) + + return results + +class PkgVars(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + name = 'PkgVars' + version = '$Revision$' + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Connector.__init__(self) + try: + self.store = PkgVarsDirectoryBacked(self.data, core.fam) + except OSError: + e = sys.exc_info()[1] + self.logger.error("Error while creating PkgVars store: %s %s" % + (e.strerror, e.filename)) + raise Bcfg2.Server.Plugin.PluginInitError + + def get_additional_data(self, meta): + return self.store.get_additional_data(meta) diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 9adfd66bf..6e718a079 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -26,6 +26,7 @@ DATABASE_USER = None DATABASE_PASSWORD = None DATABASE_HOST = None DATABASE_PORT = None +DATABASE_OPTIONS = None TIME_ZONE = None @@ -58,8 +59,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): """ read the config file and set django settings based on it """ # pylint: disable=W0602,W0603 global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \ - DATABASE_HOST, DATABASE_PORT, DEBUG, TEMPLATE_DEBUG, TIME_ZONE, \ - MEDIA_URL + DATABASE_HOST, DATABASE_PORT, DATABASE_OPTIONS, DEBUG, \ + TEMPLATE_DEBUG, TIME_ZONE, MEDIA_URL # pylint: enable=W0602,W0603 if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG): @@ -86,7 +87,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): USER=setup['db_user'], PASSWORD=setup['db_password'], HOST=setup['db_host'], - PORT=setup['db_port']) + PORT=setup['db_port'], + OPTIONS=setup['db_options']) if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: DATABASE_ENGINE = setup['db_engine'] @@ -95,6 +97,7 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): DATABASE_PASSWORD = DATABASES['default']['PASSWORD'] DATABASE_HOST = DATABASES['default']['HOST'] DATABASE_PORT = DATABASES['default']['PORT'] + DATABASE_OPTIONS = DATABASES['default']['OPTIONS'] # dropping the version check. This was added in 1.1.2 TIME_ZONE = setup['time_zone'] |