summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--doc/man/bcfg2.conf.txt5
-rw-r--r--doc/server/database.txt6
-rw-r--r--man/bcfg2.conf.57
-rw-r--r--schemas/packages.xsd30
-rw-r--r--schemas/pkgtype.xsd9
-rw-r--r--schemas/pkgvars.xsd43
-rw-r--r--schemas/types.xsd8
-rw-r--r--src/lib/Bcfg2/Client/Frame.py29
-rw-r--r--src/lib/Bcfg2/Client/Tools/APT.py36
-rw-r--r--src/lib/Bcfg2/Client/Tools/POSIX/Directory.py22
-rw-r--r--src/lib/Bcfg2/Options.py29
-rw-r--r--src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py2
-rw-r--r--src/lib/Bcfg2/Server/Lint/Validate.py3
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Apt.py21
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Collection.py31
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Layman.py115
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py1
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Portage.py312
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Source.py54
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/Yum.py5
-rw-r--r--src/lib/Bcfg2/Server/Plugins/Packages/__init__.py22
-rw-r--r--src/lib/Bcfg2/Server/Plugins/PkgVars.py65
-rw-r--r--src/lib/Bcfg2/settings.py9
23 files changed, 818 insertions, 46 deletions
diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt
index 3a0217aef..12f66f64f 100644
--- a/doc/man/bcfg2.conf.txt
+++ b/doc/man/bcfg2.conf.txt
@@ -729,6 +729,11 @@ control the database connection of the server.
port
Port for database connections. Not used for sqlite3.
+ options
+ Various options for the database connection. The value is
+ expected as multiple key=value pairs, separated with commas.
+ The concrete value depends on the database engine.
+
Reporting options
-----------------
diff --git a/doc/server/database.txt b/doc/server/database.txt
index b0ec7b571..3c8970f68 100644
--- a/doc/server/database.txt
+++ b/doc/server/database.txt
@@ -49,6 +49,12 @@ of ``/etc/bcfg2.conf``.
+-------------+------------------------------------------------------------+-------------------------------+
| port | The port to connect to | None |
+-------------+------------------------------------------------------------+-------------------------------+
+| options | Extra parameters to use when connecting to the database. | None |
+| | Available parameters vary depending on your database | |
+| | backend. The parameters are supplied as comma separated | |
+| | key=value pairs. | |
++-------------+------------------------------------------------------------+-------------------------------+
+
Database Schema Sync
====================
diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5
index b0db91a5b..85e2f4b98 100644
--- a/man/bcfg2.conf.5
+++ b/man/bcfg2.conf.5
@@ -1,4 +1,4 @@
-.TH "BCFG2.CONF" "5" "March 18, 2013" "1.3" "Bcfg2"
+.TH "BCFG2.CONF" "5" "June 19, 2013" "1.3" "Bcfg2"
.SH NAME
bcfg2.conf \- Configuration parameters for Bcfg2
.
@@ -771,6 +771,11 @@ Host for database connections. Not used for sqlite3.
.TP
.B port
Port for database connections. Not used for sqlite3.
+.TP
+.B options
+Various options for the database connection. The value is
+expected as multiple key=value pairs, separated with commas.
+The concrete value depends on the database engine.
.UNINDENT
.UNINDENT
.UNINDENT
diff --git a/schemas/packages.xsd b/schemas/packages.xsd
index e538bb0e7..e4724fabe 100644
--- a/schemas/packages.xsd
+++ b/schemas/packages.xsd
@@ -14,6 +14,8 @@
<xsd:enumeration value="yum"/>
<xsd:enumeration value="apt"/>
<xsd:enumeration value="pac"/>
+ <xsd:enumeration value="portage"/>
+ <xsd:enumeration value="layman"/>
</xsd:restriction>
</xsd:simpleType>
@@ -211,6 +213,34 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
+ <xsd:attribute type="xsd:integer" name="priority">
+ <xsd:annotation>
+ <xsd:documentation>
+ The priority of the source. This is used to order the
+ sources. After sorting, the first source, that could
+ deliver the package, is used. If not supplied the default
+ priority is 500.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
+ <xsd:attribute type="xsd:string" name="name">
+ <xsd:annotation>
+ <xsd:documentation>
+ The name of the source to refer to that specify source
+ for specific packages. It could be used like pinning under
+ debian.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
+ <xsd:attribute type="xsd:string" name="pin">
+ <xsd:annotation>
+ <xsd:documentation>
+ Extra information for pinning. This information is used
+ to differ between the sources. Should be used in the
+ supported format of apt.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
</xsd:complexType>
<xsd:complexType name="PackagesGroupType">
diff --git a/schemas/pkgtype.xsd b/schemas/pkgtype.xsd
index 18eda88ab..e301f30b6 100644
--- a/schemas/pkgtype.xsd
+++ b/schemas/pkgtype.xsd
@@ -54,6 +54,15 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
+ <xsd:attribute name="recommended" type="xsd:boolean">
+ <xsd:annotation>
+ <xsd:documentation>
+ Whether also th recommended packages should be installed.
+ This is currently only used with the :ref:`APT
+ &lt;client-tools-apt&gt;` driver.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
<xsd:attributeGroup ref="py:genshiAttrs"/>
</xsd:complexType>
diff --git a/schemas/pkgvars.xsd b/schemas/pkgvars.xsd
new file mode 100644
index 000000000..dbd02726d
--- /dev/null
+++ b/schemas/pkgvars.xsd
@@ -0,0 +1,43 @@
+<xsd:schema xmlns:xsd='http://www.w3.org/2001/XMLSchema'
+ xmlns:py="http://genshi.edgewall.org/">
+
+ <xsd:annotation>
+ <xsd:documentation>
+ XML-Schema-Definition für PkgVars/*.xml
+ Alexander Sulfrian
+ </xsd:documentation>
+ </xsd:annotation>
+
+ <xsd:import namespace="http://genshi.edgewall.org/"
+ schemaLocation="genshi.xsd"/>
+
+ <xsd:complexType name='pkgVarType'>
+ <xsd:attribute type='xsd:string' name='name'/>
+
+ <xsd:attribute type='xsd:string' name='pin'/>
+ <xsd:attribute type='xsd:string' name='use'/>
+ <xsd:attribute type='xsd:string' name='keywords'/>
+
+ <xsd:attributeGroup ref="py:genshiAttrs"/>
+ </xsd:complexType>
+
+ <xsd:complexType name='containerType'>
+ <xsd:choice maxOccurs='unbounded'>
+ <xsd:element name='Package' type='pkgVarType'/>
+ <xsd:element name='Client' type='containerType'/>
+ <xsd:element name='Group' type='containerType'/>
+ </xsd:choice>
+ <xsd:attribute name='name' type='xsd:string' use='required'/>
+ <xsd:attribute name='negate' type='xsd:boolean'/>
+ </xsd:complexType>
+
+ <xsd:complexType name='pkgVarsType'>
+ <xsd:choice minOccurs='0' maxOccurs='unbounded'>
+ <xsd:element name='Package' type='pkgVarType'/>
+ <xsd:element name='Client' type='containerType'/>
+ <xsd:element name='Group' type='containerType'/>
+ </xsd:choice>
+ </xsd:complexType>
+
+ <xsd:element name='PkgVars' type='pkgVarsType'/>
+</xsd:schema>
diff --git a/schemas/types.xsd b/schemas/types.xsd
index 4e3dfd70f..8fff4bdc1 100644
--- a/schemas/types.xsd
+++ b/schemas/types.xsd
@@ -386,6 +386,14 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
+ <xsd:attribute name='important' type='xsd:boolean' default="false">
+ <xsd:annotation>
+ <xsd:documentation>
+ Important entries are installed first during client
+ execution.
+ </xsd:documentation>
+ </xsd:annotation>
+ </xsd:attribute>
<xsd:attributeGroup ref="py:genshiAttrs"/>
</xsd:complexType>
diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py
index 3254da9e9..24b4f409c 100644
--- a/src/lib/Bcfg2/Client/Frame.py
+++ b/src/lib/Bcfg2/Client/Frame.py
@@ -170,15 +170,8 @@ class Frame(object):
return self.__dict__[name]
raise AttributeError(name)
- def InstallImportant(self):
- """Install important entries
-
- We also process the decision mode stuff here because we want to prevent
- non-whitelisted/blacklisted 'important' entries from being installed
- prior to determining the decision mode on the client.
- """
- # Need to process decision stuff early so that dryrun mode
- # works with it
+ def GenerateActions(self):
+ """Generate list of all entries that have to be handled"""
self.whitelist = [entry for entry in self.states
if not self.states[entry]]
if not self.setup['file']:
@@ -205,6 +198,17 @@ class Frame(object):
self.whitelist = [x for x in self.whitelist
if x not in b_to_rem]
+ def InstallImportant(self):
+ """Install important entries
+
+ We also process the decision mode stuff here because we want to prevent
+ non-whitelisted/blacklisted 'important' entries from being installed
+ prior to determining the decision mode on the client.
+ """
+ # Need to process decision stuff early so that dryrun mode
+ # works with it
+ self.GenerateActions()
+
# take care of important entries first
if not self.dryrun:
parent_map = dict((c, p)
@@ -486,6 +490,9 @@ class Frame(object):
self.CondDisplayState('initial')
self.InstallImportant()
self.Decide()
+ if self.modified:
+ self.Inventory()
+ self.GenerateActions()
self.Install()
self.times['install'] = time.time()
self.Remove()
@@ -514,7 +521,9 @@ class Frame(object):
stats.set('state', 'clean')
# List bad elements of the configuration
- for (data, ename) in [(self.modified, 'Modified'),
+ mods = [m for m in self.modified
+ if (m.tag != 'Action' or m.get('when') != 'always')]
+ for (data, ename) in [(mods, 'Modified'),
(self.extra, "Extra"),
(good_entries, "Good"),
([entry for entry in self.states
diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py
index 39816403a..3fc0e310d 100644
--- a/src/lib/Bcfg2/Client/Tools/APT.py
+++ b/src/lib/Bcfg2/Client/Tools/APT.py
@@ -26,6 +26,7 @@ class APT(Bcfg2.Client.Tools.Tool):
self.etc_path = setup.get('apt_etc_path', '/etc')
self.debsums = '%s/bin/debsums' % self.install_path
self.aptget = '%s/bin/apt-get' % self.install_path
+ self.aptmark = '%s/bin/apt-mark' % self.install_path
self.dpkg = '%s/bin/dpkg' % self.install_path
self.__execs__ = [self.debsums, self.aptget, self.dpkg]
@@ -69,6 +70,23 @@ class APT(Bcfg2.Client.Tools.Tool):
self.logger.info("Failed to initialize APT cache: %s" % e)
raise Bcfg2.Client.Tools.ToolInstantiationError
self.pkg_cache.update()
+ # mark dependencies as being automatically installed and vice versa
+ mark = []
+ unmark = []
+ try:
+ installed_pkgs = [p.name for p in self.pkg_cache if p.is_installed]
+ except AttributeError:
+ installed_pkgs = [p.name for p in self.pkg_cache if p.isInstalled]
+ for pkg in self.getSupportedEntries():
+ if pkg.get('name') in installed_pkgs:
+ if pkg.get('origin') == 'Packages':
+ mark.append(pkg.get('name'))
+ else:
+ unmark.append(pkg.get('name'))
+ if mark:
+ self.cmd.run("%s markauto %s" % (self.aptmark, (" ".join(mark))))
+ if unmark:
+ self.cmd.run("%s unmarkauto %s" % (self.aptmark, (" ".join(unmark))))
self.pkg_cache = apt.cache.Cache()
if 'req_reinstall_pkgs' in dir(self.pkg_cache):
self._newapi = True
@@ -88,6 +106,11 @@ class APT(Bcfg2.Client.Tools.Tool):
type='deb', version=version) \
for (name, version) in extras]
+ def Inventory(self, states, structures=[]):
+ # reload pkg cache
+ self.pkg_cache.open(None)
+ Bcfg2.Client.Tools.Tool.Inventory(self, states, structures)
+
def VerifyDebsums(self, entry, modlist):
output = \
self.cmd.run("%s -as %s" %
@@ -163,7 +186,7 @@ class APT(Bcfg2.Client.Tools.Tool):
else:
installed_version = pkg.installedVersion
candidate_version = pkg.candidateVersion
- if entry.get('version') == 'auto':
+ if entry.get('version').startswith('auto'):
if self._newapi:
is_upgradable = self.pkg_cache._depcache.is_upgradable(pkg._pkg)
else:
@@ -172,8 +195,10 @@ class APT(Bcfg2.Client.Tools.Tool):
desiredVersion = candidate_version
else:
desiredVersion = installed_version
- elif entry.get('version') == 'any':
+ entry.set('version', "auto: %s" % desiredVersion)
+ elif entry.get('version').startswith('any'):
desiredVersion = installed_version
+ entry.set('version', "any: %s" % desiredVersion)
else:
desiredVersion = entry.get('version')
if desiredVersion != installed_version:
@@ -226,7 +251,7 @@ class APT(Bcfg2.Client.Tools.Tool):
if not self.pkg_cache.has_key(pkg.get('name')):
self.logger.error("APT has no information about package %s" % (pkg.get('name')))
continue
- if pkg.get('version') in ['auto', 'any']:
+ if any([pkg.get('version').startswith(v) for v in ['auto', 'any']]):
if self._newapi:
try:
ipkgs.append("%s=%s" % (pkg.get('name'),
@@ -262,10 +287,15 @@ class APT(Bcfg2.Client.Tools.Tool):
self.logger.error("APT command failed")
self.pkg_cache = apt.cache.Cache()
self.extra = self.FindExtra()
+ mark = []
for package in packages:
states[package] = self.VerifyPackage(package, [], checksums=False)
if states[package]:
self.modified.append(package)
+ if package.get('origin') == 'Packages':
+ mark.append(package.get('name'))
+ if mark:
+ self.cmd.run("%s markauto %s" % (self.aptmark, (" ".join(mark))))
def VerifyPath(self, entry, _):
"""Do nothing here since we only verify Path type=ignore."""
diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py
index 675a4461a..24667d162 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIX/Directory.py
@@ -11,6 +11,10 @@ class POSIXDirectory(POSIXTool):
""" Handle <Path type='directory' ...> entries """
__req__ = ['name', 'mode', 'owner', 'group']
+ def __init__(self, logger, setup, config):
+ super(POSIXDirectory, self).__init__(logger, setup, config)
+ self.prunes = dict()
+
def verify(self, entry, modlist):
ondisk = self._exists(entry)
if not ondisk:
@@ -25,10 +29,18 @@ class POSIXDirectory(POSIXTool):
if entry.get('prune', 'false').lower() == 'true':
# check for any extra entries when prune='true' attribute is set
try:
+ prune_list = list()
+ if entry.get('name') in self.prunes:
+ prune_list = self.prunes[entry.get('name')]
+ else:
+ self.prunes[entry.get('name')] = list()
+
extras = [os.path.join(entry.get('name'), ent)
for ent in os.listdir(entry.get('name'))
- if os.path.join(entry.get('name'),
- ent) not in modlist]
+ if (os.path.join(entry.get('name'),
+ ent) not in modlist and
+ os.path.join(entry.get('name'),
+ ent) not in prune_list)]
if extras:
prune = False
msg = "Directory %s contains extra entries: %s" % \
@@ -37,6 +49,10 @@ class POSIXDirectory(POSIXTool):
entry.set('qtext', entry.get('qtext', '') + '\n' + msg)
for extra in extras:
Bcfg2.Client.XML.SubElement(entry, 'Prune', name=extra)
+ self.prunes[entry.get('name')] += extras
+ elif entry.get('name') in self.prunes and \
+ len(self.prunes[entry.get('name')]) > 0:
+ prune = False
except OSError:
prune = True
@@ -71,6 +87,8 @@ class POSIXDirectory(POSIXTool):
try:
self.logger.debug("POSIX: Removing %s" % pname)
self._remove(pent)
+ if entry.get('name') in self.prunes:
+ self.prunes[entry.get('name')].remove(pname)
except OSError:
err = sys.exc_info()[1]
self.logger.error("POSIX: Failed to unlink %s: %s" %
diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py
index 243c4ed2a..84551a02d 100644
--- a/src/lib/Bcfg2/Options.py
+++ b/src/lib/Bcfg2/Options.py
@@ -319,6 +319,28 @@ def colon_split(c_string):
return []
+def dict_split(c_string):
+ """ split an option string on commans, optionally sourrunded by
+ whitespace and split the resulting items again on equals signs,
+ returning a dict """
+ result = dict()
+ if c_string:
+ items = re.split(r'\s*,\s*', c_string)
+ for item in items:
+ if r'=' in item:
+ key, value = item.split(r'=', 1)
+ try:
+ result[key] = get_bool(value)
+ except ValueError:
+ try:
+ result[key] = get_int(value)
+ except ValueError:
+ result[key] = value
+ else:
+ result[item] = True
+ return result
+
+
def get_bool(val):
""" given a string value of a boolean configuration option, return
an actual bool (True or False) """
@@ -652,6 +674,12 @@ DB_PORT = \
cf=('database', 'port'),
deprecated_cf=('statistics', 'database_port'))
+DB_OPTIONS = \
+ Option('Database options',
+ default=dict(),
+ cf=('database', 'options'),
+ cook=dict_split)
+
# Django options
WEB_CFILE = \
Option('Web interface configuration file',
@@ -1285,6 +1313,7 @@ DATABASE_COMMON_OPTIONS = dict(web_configfile=WEB_CFILE,
db_password=DB_PASSWORD,
db_host=DB_HOST,
db_port=DB_PORT,
+ db_options=DB_OPTIONS,
time_zone=DJANGO_TIME_ZONE,
django_debug=DJANGO_DEBUG,
web_prefix=DJANGO_WEB_PREFIX)
diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py
index 489682f30..ee8738a0c 100644
--- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py
+++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py
@@ -319,6 +319,8 @@ def determine_client_state(entry):
_how_ dirty and adjust the color accordingly.
"""
if entry.state == 'clean':
+ if entry.extra_count > 0:
+ return "extra-lineitem"
return "clean-lineitem"
bad_percentage = 100 * (float(entry.bad_count) / entry.total_count)
diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py
index 09f3f3d25..645870020 100644
--- a/src/lib/Bcfg2/Server/Lint/Validate.py
+++ b/src/lib/Bcfg2/Server/Lint/Validate.py
@@ -51,7 +51,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin):
"FileProbes/config.xml": "fileprobes.xsd",
"SSLCA/**/cert.xml": "sslca-cert.xsd",
"SSLCA/**/key.xml": "sslca-key.xsd",
- "GroupLogic/groups.xml": "grouplogic.xsd"
+ "GroupLogic/groups.xml": "grouplogic.xsd",
+ "PkgVars/*.xml": "pkgvars.xsd"
}
self.filelists = {}
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
index a82a183d8..009395e8f 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py
@@ -76,10 +76,8 @@ class AptSource(Source):
def read_files(self):
bdeps = dict()
+ brecs = dict()
bprov = dict()
- depfnames = ['Depends', 'Pre-Depends']
- if self.recommended:
- depfnames.append('Recommends')
for fname in self.files:
if not self.rawurl:
barch = [x
@@ -91,6 +89,7 @@ class AptSource(Source):
barch = self.arches[0]
if barch not in bdeps:
bdeps[barch] = dict()
+ brecs[barch] = dict()
bprov[barch] = dict()
try:
reader = gzip.GzipFile(fname)
@@ -105,9 +104,10 @@ class AptSource(Source):
pkgname = words[1].strip().rstrip()
self.pkgnames.add(pkgname)
bdeps[barch][pkgname] = []
+ brecs[barch][pkgname] = []
elif words[0] == 'Essential' and self.essential:
self.essentialpkgs.add(pkgname)
- elif words[0] in depfnames:
+ elif words[0] in ['Depends', 'Pre-Depends', 'Recommends']:
vindex = 0
for dep in words[1].split(','):
if '|' in dep:
@@ -118,17 +118,24 @@ class AptSource(Source):
barch,
vindex)
vindex += 1
- bdeps[barch][pkgname].append(dyn_dname)
+
+ if words[0] == 'Recommends':
+ brecs[barch][pkgname].append(dyn_dname)
+ else:
+ bdeps[barch][pkgname].append(dyn_dname)
bprov[barch][dyn_dname] = set(cdeps)
else:
raw_dep = re.sub(r'\(.*\)', '', dep)
raw_dep = raw_dep.rstrip().strip()
- bdeps[barch][pkgname].append(raw_dep)
+ if words[0] == 'Recommends':
+ brecs[barch][pkgname].append(raw_dep)
+ else:
+ bdeps[barch][pkgname].append(raw_dep)
elif words[0] == 'Provides':
for pkg in words[1].split(','):
dname = pkg.rstrip().strip()
if dname not in bprov[barch]:
bprov[barch][dname] = set()
bprov[barch][dname].add(pkgname)
- self.process_files(bdeps, bprov)
+ self.process_files(bdeps, bprov, brecs)
read_files.__doc__ = Source.read_files.__doc__
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
index b25cb0fc4..93b398ac4 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
@@ -308,7 +308,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable):
return any(source.is_virtual_package(self.metadata, package)
for source in self)
- def get_deps(self, package):
+ def get_deps(self, package, recs=None, pinnings=None):
""" Get a list of the dependencies of the given package.
The base implementation simply aggregates the results of
@@ -316,11 +316,31 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable):
:param package: The name of the symbol, but see :ref:`pkg-objects`
:type package: string
+ :param pinnings: Mapping from package names to source names.
+ :type pinnings: dict
:returns: list of strings, but see :ref:`pkg-objects`
"""
+ recommended = None
+ if recs and package in recs:
+ recommended = recs[package]
+
+ pin_found = False
+ pin_source = None
+ if pinnings and package in pinnings:
+ pin_source = pinnings[package]
+
for source in self:
+ if pin_source and source.name not in pin_source:
+ continue
+ pin_found = True
+
if source.is_package(self.metadata, package):
- return source.get_deps(self.metadata, package)
+ return source.get_deps(self.metadata, package, recommended)
+
+ if not pin_found:
+ self.logger.error("Packages: Source '%s' for package '%s' not found" %
+ (' or '.join(pin_source), package))
+
return []
def get_essential(self):
@@ -500,12 +520,15 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable):
return list(complete.difference(initial))
@Bcfg2.Server.Plugin.track_statistics()
- def complete(self, packagelist): # pylint: disable=R0912,R0914
+ def complete(self, packagelist, recommended=None,
+ pinnings=None): # pylint: disable=R0912,R0914
""" Build a complete list of all packages and their dependencies.
:param packagelist: Set of initial packages computed from the
specification.
:type packagelist: set of strings, but see :ref:`pkg-objects`
+ :param pinnings: Mapping from package names to source names.
+ :type pinnings: dict
:returns: tuple of sets - The first element contains a set of
strings (but see :ref:`pkg-objects`) describing the
complete package list, and the second element is a
@@ -564,7 +587,7 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable):
self.debug_log("Packages: handling package requirement %s" %
(current,))
packages.add(current)
- deps = self.get_deps(current)
+ deps = self.get_deps(current, recommended, pinnings)
newdeps = set(deps).difference(examined)
if newdeps:
self.debug_log("Packages: Package %s added requirements %s"
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Layman.py b/src/lib/Bcfg2/Server/Plugins/Packages/Layman.py
new file mode 100644
index 000000000..40358e214
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Layman.py
@@ -0,0 +1,115 @@
+import os
+import layman
+import Bcfg2.Server.Plugin
+
+class LaymanSource(Bcfg2.Server.Plugin.Debuggable):
+ basegroups = ['portage', 'gentoo', 'emerge']
+ ptype = 'layman'
+ cclass = 'Portage'
+
+ def __init__(self, basepath, xsource, config):
+ Bcfg2.Server.Plugin.Debuggable.__init__(self)
+ self.basepath = basepath
+ self.xsource = xsource
+ self.config = config
+
+ self.url = xsource.get('url', 'http://www.gentoo.org/proj/en/overlays/repositories.xml')
+ self.name = xsource.get('name', '')
+ self.priority = xsource.get('priority', 0)
+ self.cachefile = None
+ self.gpgkeys = []
+ self.recommended = False
+
+ # configure layman
+ base = os.path.join(basepath, 'layman')
+ config = layman.config.OptionConfig(options = {
+ 'storage': os.path.join(base, 'overlays'),
+ 'cache': os.path.join(base, 'cache'),
+ 'installed': os.path.join(base, 'installed.xml'),
+ 'local_list': os.path.join(base, 'overlays.xml'),
+ 'overlays': [url]
+ })
+ self.api = layman.LaymanAPI(config)
+
+ # path
+ self.dir = os.path.join(basepath, 'overlays', self.name)
+
+ # build the set of conditions to see if this source applies to
+ # a given set of metadata
+ self.conditions = []
+ self.groups = [] # provided for some limited backwards compat
+ for el in xsource.iterancestors():
+ if el.tag == "Group":
+ if el.get("negate", "false").lower() == "true":
+ self.conditions.append(lambda m, el=el:
+ el.get("name") not in m.groups)
+ else:
+ self.groups.append(el.get("name"))
+ self.conditions.append(lambda m, el=el:
+ el.get("name") in m.groups)
+ elif el.tag == "Client":
+ if el.get("negate", "false").lower() == "true":
+ self.conditions.append(lambda m, el=el:
+ el.get("name") != m.hostname)
+ else:
+ self.conditions.append(lambda m, el=el:
+ el.get("name") == m.hostname)
+
+ def save_state(self):
+ pass
+
+ def load_state(self):
+ pass
+
+ def filter_unknown(self, unknown):
+ filtered = set([u for u in unknown if u.startswith('choice')])
+ unknown.difference_update(filtered)
+
+ def get_urls(self):
+ return self.url
+ urls = property(get_urls)
+
+ def magic_groups_match(self, metadata):
+ if self.config.getboolean("global", "magic_groups",
+ default=True) == False:
+ return True
+ else:
+ for group in self.basegroups:
+ if group in metadata.groups:
+ return True
+ return False
+
+ def setup_data(self, force_update=False):
+ self.api.fetch_remote_list()
+ if not self.api.is_repo(self.name):
+ self.logger.error("Packages: Layman overlay '%s' not"
+ " found" % self.name)
+ return False
+
+ if not self.api.is_installed(self.name):
+ self.logger.info("Packages: Adding layman overlay '%s'" %
+ self.name)
+ if not self.api.add_repos(name):
+ self.logger.error("Packages: Failed adding layman"
+ " overlay '%s'" % self.name)
+ return False
+
+ if force_update:
+ if not self.api.sync(name):
+ self.logger.error("Packages: Failed syncing layman"
+ " overlay '%s'" % self.name)
+ return False
+
+ return True
+
+ def applies(self, metadata):
+ # check base groups
+ if not self.magic_groups_match(metadata):
+ return False
+
+ # check Group/Client tags from sources.xml
+ for condition in self.conditions:
+ if not condition(metadata):
+ return False
+
+ return True
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
index 332f0bbab..e627275c0 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py
@@ -115,6 +115,7 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile,
source = self.source_from_xml(xsource)
if source is not None:
self.entries.append(source)
+ sorted(self.entries, key=(lambda source: source.priority), reverse=True)
Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """
``Index`` is responsible for calling :func:`source_from_xml`
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py b/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py
new file mode 100644
index 000000000..8629699ca
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Portage.py
@@ -0,0 +1,312 @@
+import re
+import gzip
+import sys
+import os
+import lxml.etree
+import Bcfg2.Server.Plugin
+from Bcfg2.Server.Plugins.Packages.Collection import Collection
+from Bcfg2.Server.Plugins.Packages.Layman import LaymanSource
+
+_portage_python = '/usr/lib/portage/pym/'
+
+def _import_portage(caller):
+ # generate prefix path
+ caller.prefix = os.path.join(caller.basepath, 'cache', 'portage')
+ if not os.path.isdir(caller.prefix):
+ caller.logger.error("Packages: %s is not a dir. "
+ "Portage will not work. Please "
+ "remember to setup a EPREFIX there." %
+ caller.prefix)
+ # TODO: automatic EPREFIX setup
+ raise Exception('Invalid EPREFIX')
+
+ os.environ['PORTAGE_OVERRIDE_EPREFIX'] = caller.prefix
+
+ if not os.path.isdir(_portage_python):
+ self.logger.error("Packages: %s not found. Have you installed "
+ "the portage python modules?" % _portage_python)
+ raise Exception('Portage not found')
+
+ sys.path = sys.path + [_portage_python]
+ portage = __import__('portage', globals(), locals(),
+ ['_sets', 'dbapi.porttree' ])
+ emerge = __import__('_emerge', globals(), locals(),
+ ['RootConfig', 'depgraph', 'Package', 'actions'])
+
+ # setup profile
+ if '_setup_profile' in dir(caller):
+ caller._setup_profile(portage)
+
+ # fix some default settings
+ portage.settings.unlock()
+ portage.settings['PORTAGE_RSYNC_INITIAL_TIMEOUT'] = '0'
+ portage.settings.lock()
+
+ porttree = portage.db[portage.root]['porttree']
+ caller._import_portage(portage, emerge, porttree)
+
+
+class PortageCollection(Collection):
+ def __init__(self, metadata, sources, basepath, debug=False):
+ Collection.__init__(self, metadata, sources, basepath, debug)
+
+ self.portage = None
+ self.emerge = None
+ self.porttree = None
+
+ @property
+ def cachefiles(self):
+ return []
+
+ def complete(self, packagelist, pinnings=None, recommended=None):
+ if not self.portage:
+ _import_portage(self)
+
+ # calculate deps
+ setconfig = self.portage._sets.load_default_config(
+ self.portage.settings,
+ self.portage.db[self.portage.root])
+ rconfig = self.emerge.RootConfig.RootConfig(
+ self.portage.settings,
+ self.portage.db[self.portage.root],
+ setconfig)
+ self.portage.db[self.portage.root]['root_config'] = rconfig
+ tree = self.portage.db[self.portage.root]['porttree']
+
+ pkgs = ["=" + j.cpv for (i, j) in packagelist if i == 'ok']
+ fail = [j for (i, j) in packagelist if i == 'fail']
+
+ x = self.emerge.depgraph.backtrack_depgraph(
+ self.portage.settings,
+ self.portage.db,
+ {'--pretend': True},
+ {'recurse': True},
+ 'merge',
+ pkgs,
+ None)
+
+ g = x[1]._dynamic_config.digraph
+ packages = [i for i in g.all_nodes() \
+ if isinstance(i, self.emerge.Package.Package)]
+
+ return (set(packages), set(fail))
+
+ def get_additional_data(self):
+ return []
+
+ def get_group(self, group):
+ self.logger.warning("Packages: Package sets are currently not supported")
+ return []
+
+ def packages_from_entry(self, entry):
+ if not self.portage:
+ _import_portage(self)
+
+ try:
+ name = entry.get('name')
+ pkgs = self.porttree.dep_bestmatch(name)
+ except self.portage.exception.AmbiguousPackageName as e:
+ self.logger.error("Packages: AmbiguousPackageName: %s" % e)
+ pkgs = ''
+
+ if pkgs == '':
+ return [('fail', name)]
+
+ return [('ok', pkgs)]
+
+ def packages_to_entry(self, pkgs, entry, config):
+ for pkg in pkgs:
+ if pkg.slot != '0':
+ name = "%s:%s" % (pkg.cp, pkg.slot)
+ else:
+ name = pkg.cp
+
+ lxml.etree.SubElement(entry, 'BoundPackage', name=name,
+ version=config.get("packages", "version",
+ default="auto"),
+ type=self.ptype, origin='Packages')
+
+ def get_new_packages(self, initial, complete):
+ new = []
+ init = [pkg.cp for status, pkg in initial if status == 'ok']
+
+ for pkg in complete:
+ if pkg.cp not in init:
+ new.append(pkg)
+
+ return new
+
+ def setup_data(self, force_update=False):
+ pass
+
+ def _setup_profile(self, portage):
+ if 'gentoo-profile' not in self.metadata.Probes:
+ raise Exception('Unknown profile.')
+
+ profile = os.path.join(self.prefix, 'usr/portage/profiles/',
+ self.metadata.Probes['gentoo-profile'])
+
+ env = portage.settings.configdict['backupenv']
+
+ # add layman overlays
+ env['PORTDIR_OVERLAY'] = ''
+ for overlay in self.sources:
+ if isinstance(overlay, LaymanSource):
+ env['PORTDIR_OVERLAY'] += ' '
+ env['PORTDIR_OVERLAY'] += overlay.dir
+
+ portage.settings = portage.package.ebuild.config.config(
+ config_root=portage.settings['PORTAGE_CONFIGROOT'],
+ target_root=portage.settings['ROOT'],
+ env=env,
+ eprefix=portage.settings['EPREFIX'],
+ config_profile_path=profile)
+
+ portage.db[portage.root]['porttree'].settings = portage.settings
+ newdbapi = portage.dbapi.porttree.portdbapi(mysettings=portage.settings)
+ portage.db[portage.root]['porttree'].dbapi = newdbapi
+
+ portage.db[portage.root]['vartree'].settings = portage.settings
+ portage.db[portage.root]['vartree'].dbapi.settings = portage.settings
+
+ def _set_portage_config(self):
+ # get global use flags
+ self.portage.settings.unlock()
+ self.portage.settings['USE'] = ''
+ if 'gentoo-use-flags' in self.metadata.Probes:
+ self.portage.settings['USE'] = \
+ self.metadata.Probes['gentoo-use-flags']
+
+ # add package flags (accept_keywords, use)
+ if hasattr(self.metadata, 'PkgVars'):
+ for k in self.metadata.PkgVars['keywords']:
+ keyword = metadata.PkgVars['keywords'][k]
+ self.portage.settings._keywords_manager.pkeywordsdict[k] = \
+ {k: tuple(keyword)}
+
+
+ for u in self.metadata.PkgVars['use']:
+ use = metadata.PkgVars['use'][u]
+ self.portage.settings._use_manager._pusedict[u] = \
+ {u: tuple(use)}
+
+ self.portage.settings.lock()
+
+ def _import_portage(self, portage, emerge, porttree):
+ self.portage = portage
+ self.emerge = emerge
+ self.porttree = porttree
+ self._set_portage_config()
+
+ for s in self.sources:
+ if isinstance(s, PortageSource):
+ s._import_portage(portage, emerge, porttree)
+
+
+class PortageSource(Bcfg2.Server.Plugin.Debuggable):
+ basegroups = ['portage', 'gentoo', 'emerge']
+ ptype = 'ebuild'
+
+ def __init__(self, basepath, xsource, config):
+ Bcfg2.Server.Plugin.Debuggable.__init__(self)
+ self.basepath = basepath
+ self.xsource = xsource
+ self.config = config
+
+ self.url = xsource.get('url', '')
+ self.priority = xsource.get('priority', 0)
+ self.cachefile = None
+ self.gpgkeys = []
+ self.recommended = False
+
+ self.portage = None
+ self.emerge = None
+ self.porttree = None
+
+ # build the set of conditions to see if this source applies to
+ # a given set of metadata
+ self.conditions = []
+ self.groups = [] # provided for some limited backwards compat
+ for el in xsource.iterancestors():
+ if el.tag == "Group":
+ if el.get("negate", "false").lower() == "true":
+ self.conditions.append(lambda m, el=el:
+ el.get("name") not in m.groups)
+ else:
+ self.groups.append(el.get("name"))
+ self.conditions.append(lambda m, el=el:
+ el.get("name") in m.groups)
+ elif el.tag == "Client":
+ if el.get("negate", "false").lower() == "true":
+ self.conditions.append(lambda m, el=el:
+ el.get("name") != m.hostname)
+ else:
+ self.conditions.append(lambda m, el=el:
+ el.get("name") == m.hostname)
+
+ def _import_portage(self, portage, emerge, porttree):
+ self.portage = portage
+ self.emerge = emerge
+ self.porttree = porttree
+
+ def save_state(self):
+ pass
+
+ def load_state(self):
+ pass
+
+ def filter_unknown(self, unknown):
+ filtered = set([u for u in unknown if u.startswith('choice')])
+ unknown.difference_update(filtered)
+
+ def get_urls(self):
+ return self.url
+ urls = property(get_urls)
+
+ def magic_groups_match(self, metadata):
+ if self.config.getboolean("global", "magic_groups",
+ default=True) == False:
+ return True
+ else:
+ for group in self.basegroups:
+ if group in metadata.groups:
+ return True
+ return False
+
+ def setup_data(self, force_update=False):
+ if not self.porttree:
+ _import_portage(self)
+
+ timestamp = os.path.join(self.porttree.portroot, 'metadata/timestamp.chk')
+ if not os.path.isfile(timestamp):
+ self.logger.error("Packages: Timestamp not found; "
+ "falling back to sync")
+ force_update = True
+
+ if force_update:
+ # update sync url
+ self.portage.settings.unlock()
+ self.portage.settings['SYNC'] = self.url
+ self.portage.settings.lock()
+
+ # realy force the sync
+ if os.path.isfile(timestamp):
+ os.unlink(timestamp)
+
+ # sync
+ self.logger.info("Packages: Syncing with %s" % self.url)
+ self.emerge.actions.action_sync(self.portage.settings,
+ self.portage.db, None,
+ {'--quiet': True}, 'sync')
+
+ def applies(self, metadata):
+ # check base groups
+ if not self.magic_groups_match(metadata):
+ return False
+
+ # check Group/Client tags from sources.xml
+ for condition in self.conditions:
+ if not condition(metadata):
+ return False
+
+ return True
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
index 22073493c..c04611d77 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py
@@ -209,6 +209,15 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
#: The "version" attribute from :attr:`xsource`
self.version = xsource.get('version', '')
+ #: The "priority" attribute from :attr:`xsource`
+ self.priority = xsource.get('priority', 500)
+
+ #: The "name" attribute from :attr:`xsource`
+ self.name = xsource.get('name', '')
+
+ #: The "priority" attribute from :attr:`xsource`
+ self.pin = xsource.get('pin', '')
+
#: A list of predicates that are used to determine if this
#: source applies to a given
#: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata`
@@ -256,6 +265,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
#: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`
self.provides = dict()
+ #: A dict of ``<package name>`` -> ``<list of recommended
+ #: symbols>``. This will not necessarily be populated.
+ self.recommends = dict()
+
#: The file (or directory) used for this source's cache data
self.cachefile = os.path.join(self.basepath,
"cache-%s" % self.cachekey)
@@ -280,7 +293,8 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
for arch in self.arches:
if self.url:
usettings = [dict(version=self.version, component=comp,
- arch=arch)
+ arch=arch, priority=self.priority,
+ name=self.name, pin=self.pin)
for comp in self.components]
else: # rawurl given
usettings = [dict(version=self.version, component=None,
@@ -321,7 +335,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
:raises: cPickle.UnpicklingError - If the saved data is corrupt """
data = open(self.cachefile, 'rb')
(self.pkgnames, self.deps, self.provides,
- self.essentialpkgs) = cPickle.load(data)
+ self.essentialpkgs, self.recommends) = cPickle.load(data)
def save_state(self):
""" Save state to :attr:`cachefile`. If caching and
@@ -329,7 +343,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
does not need to be implemented. """
cache = open(self.cachefile, 'wb')
cPickle.dump((self.pkgnames, self.deps, self.provides,
- self.essentialpkgs), cache, 2)
+ self.essentialpkgs, self.recommends), cache, 2)
cache.close()
@Bcfg2.Server.Plugin.track_statistics()
@@ -524,13 +538,13 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
as its final step."""
pass
- def process_files(self, dependencies, provides):
+ def process_files(self, dependencies, provides, recommends=dict()):
""" Given dicts of depends and provides generated by
:func:`read_files`, this generates :attr:`deps` and
:attr:`provides` and calls :func:`save_state` to save the
cached data to disk.
- Both arguments are dicts of dicts of lists. Keys are the
+ All arguments are dicts of dicts of lists. Keys are the
arches of packages contained in this source; values are dicts
whose keys are package names and values are lists of either
dependencies for each package the symbols provided by each
@@ -542,14 +556,20 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
:param provides: A dict of symbols provided by packages in
this repository.
:type provides: dict; see above.
+ :param recommends: A dict of recommended dependencies
+ found for this source.
+ :type recommends: dict; see above.
"""
self.deps['global'] = dict()
+ self.recommends['global'] = dict()
self.provides['global'] = dict()
for barch in dependencies:
self.deps[barch] = dict()
+ self.recommends[barch] = dict()
self.provides[barch] = dict()
for pkgname in self.pkgnames:
pset = set()
+ rset = set()
for barch in dependencies:
if pkgname not in dependencies[barch]:
dependencies[barch][pkgname] = []
@@ -559,6 +579,17 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
else:
for barch in dependencies:
self.deps[barch][pkgname] = dependencies[barch][pkgname]
+
+ for barch in recommends:
+ if pkgname not in recommends[barch]:
+ recommends[barch][pkgname] = []
+ rset.add(tuple(recommends[barch][pkgname]))
+ if len(rset) == 1:
+ self.recommends['global'][pkgname] = rset.pop()
+ else:
+ for barch in recommends:
+ self.recommends[barch][pkgname] = recommends[barch][pkgname]
+
provided = set()
for bprovided in list(provides.values()):
provided.update(set(bprovided))
@@ -667,17 +698,24 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902
"""
return ['global'] + [a for a in self.arches if a in metadata.groups]
- def get_deps(self, metadata, package):
+ def get_deps(self, metadata, package, recommended=None):
""" Get a list of the dependencies of the given package.
:param package: The name of the symbol
:type package: string
:returns: list of strings
"""
+ recs = []
+ if ((recommended is None and self.recommended) or
+ (recommended and recommended.lower() == 'true')):
+ for arch in self.get_arches(metadata):
+ if package in self.recommends[arch]:
+ recs.extend(self.recommends[arch][package])
+
for arch in self.get_arches(metadata):
if package in self.deps[arch]:
- return self.deps[arch][package]
- return []
+ recs.extend(self.deps[arch][package])
+ return recs
def get_provides(self, metadata, package):
""" Get a list of all symbols provided by the given package.
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
index 4608bcca5..c2f10b97b 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
@@ -820,7 +820,7 @@ class YumCollection(Collection):
return new
@Bcfg2.Server.Plugin.track_statistics()
- def complete(self, packagelist):
+ def complete(self, packagelist, recommended=None, pinnings=None):
""" Build a complete list of all packages and their dependencies.
When using the Python yum libraries, this defers to the
@@ -838,7 +838,8 @@ class YumCollection(Collection):
resolved.
"""
if not self.use_yum:
- return Collection.complete(self, packagelist)
+ return Collection.complete(self, packagelist, recommended,
+ pinnings)
if packagelist:
try:
diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
index f82b8a392..3420735dd 100644
--- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
@@ -331,10 +331,19 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
initial = set()
to_remove = []
groups = []
+ recommended = dict()
+
+ pinned_src = dict()
+ if hasattr(metadata, 'PkgVars'):
+ pinned_src = metadata.PkgVars['pin']
+
for struct in structures:
for pkg in struct.xpath('//Package | //BoundPackage'):
if pkg.get("name"):
initial.update(collection.packages_from_entry(pkg))
+
+ if pkg.get("recommended"):
+ recommended[pkg.get("name")] = pkg.get("recommended")
elif pkg.get("group"):
groups.append((pkg.get("group"),
pkg.get("type")))
@@ -362,7 +371,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
# essential pkgs are those marked as such by the distribution
base.update(collection.get_essential())
- packages, unknown = collection.complete(base)
+ packages, unknown = collection.complete(base, recommended, pinned_src)
if unknown:
self.logger.info("Packages: Got %d unknown entries" % len(unknown))
self.logger.info("Packages: %s" % list(unknown))
@@ -505,20 +514,23 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
for source in self.sources.entries:
if source.applies(metadata):
relevant.append(source)
- sclasses.update([source.__class__])
+ if 'cclass' in dir(source):
+ sclasses.update([source.cclass])
+ else:
+ sclass = source.__class__.__name__.replace("Source", "")
+ sclasses.update([sclass])
if len(sclasses) > 1:
self.logger.warning("Packages: Multiple source types found for "
"%s: %s" %
- ",".join([s.__name__ for s in sclasses]))
+ (metadata.hostname, ",".join([sclasses])))
cclass = Collection
elif len(sclasses) == 0:
self.logger.error("Packages: No sources found for %s" %
metadata.hostname)
cclass = Collection
else:
- cclass = get_collection_class(
- sclasses.pop().__name__.replace("Source", ""))
+ cclass = get_collection_class(sclasses.pop())
if self.debug_flag:
self.logger.error("Packages: Using %s for Collection of sources "
diff --git a/src/lib/Bcfg2/Server/Plugins/PkgVars.py b/src/lib/Bcfg2/Server/Plugins/PkgVars.py
new file mode 100644
index 000000000..40582bcb3
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Plugins/PkgVars.py
@@ -0,0 +1,65 @@
+import os
+import re
+import sys
+import copy
+import logging
+import lxml.etree
+import Bcfg2.Server.Plugin
+
+logger = logging.getLogger('Bcfg2.Plugins.PkgVars')
+vars = ['pin', 'use', 'keywords']
+
+class PkgVarsFile(Bcfg2.Server.Plugin.StructFile):
+ def get_additional_data(self, meta):
+ data = self.Match(meta)
+ results = {}
+ for d in data:
+ name = d.get('name', '')
+
+ for v in vars:
+ value = d.get(v, None)
+ if value:
+ if v not in results:
+ results[v] = {}
+ if name not in results[v]:
+ results[v][name] = set()
+
+ results[v][name].add(value)
+
+ return results
+
+class PkgVarsDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked):
+ __child__ = PkgVarsFile
+ patterns = re.compile(r'.*\.xml$')
+
+ def get_additional_data(self, meta):
+ results = {}
+ for v in vars:
+ results[v] = {}
+
+ for files in self.entries:
+ new = self.entries[files].get_additional_data(meta)
+ for x in vars:
+ if x in new:
+ results[x].update(new[x])
+
+ return results
+
+class PkgVars(Bcfg2.Server.Plugin.Plugin,
+ Bcfg2.Server.Plugin.Connector):
+ name = 'PkgVars'
+ version = '$Revision$'
+
+ def __init__(self, core, datastore):
+ Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
+ Bcfg2.Server.Plugin.Connector.__init__(self)
+ try:
+ self.store = PkgVarsDirectoryBacked(self.data, core.fam)
+ except OSError:
+ e = sys.exc_info()[1]
+ self.logger.error("Error while creating PkgVars store: %s %s" %
+ (e.strerror, e.filename))
+ raise Bcfg2.Server.Plugin.PluginInitError
+
+ def get_additional_data(self, meta):
+ return self.store.get_additional_data(meta)
diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py
index 9adfd66bf..6e718a079 100644
--- a/src/lib/Bcfg2/settings.py
+++ b/src/lib/Bcfg2/settings.py
@@ -26,6 +26,7 @@ DATABASE_USER = None
DATABASE_PASSWORD = None
DATABASE_HOST = None
DATABASE_PORT = None
+DATABASE_OPTIONS = None
TIME_ZONE = None
@@ -58,8 +59,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False):
""" read the config file and set django settings based on it """
# pylint: disable=W0602,W0603
global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \
- DATABASE_HOST, DATABASE_PORT, DEBUG, TEMPLATE_DEBUG, TIME_ZONE, \
- MEDIA_URL
+ DATABASE_HOST, DATABASE_PORT, DATABASE_OPTIONS, DEBUG, \
+ TEMPLATE_DEBUG, TIME_ZONE, MEDIA_URL
# pylint: enable=W0602,W0603
if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG):
@@ -86,7 +87,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False):
USER=setup['db_user'],
PASSWORD=setup['db_password'],
HOST=setup['db_host'],
- PORT=setup['db_port'])
+ PORT=setup['db_port'],
+ OPTIONS=setup['db_options'])
if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2:
DATABASE_ENGINE = setup['db_engine']
@@ -95,6 +97,7 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False):
DATABASE_PASSWORD = DATABASES['default']['PASSWORD']
DATABASE_HOST = DATABASES['default']['HOST']
DATABASE_PORT = DATABASES['default']['PORT']
+ DATABASE_OPTIONS = DATABASES['default']['OPTIONS']
# dropping the version check. This was added in 1.1.2
TIME_ZONE = setup['time_zone']