1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
|
import copy
import logging
import lxml.etree
import Bcfg2.Server.Plugin
logger = logging.getLogger(__name__)
try:
from hashlib import md5
except ImportError:
from md5 import md5
# we have to cache Collection objects so that calling Packages.Refresh
# or .Reload can tell the collection objects to clean up their cache,
# but we don't actually use the cache to return a Collection object
# when one is requested, because that prevents new machines from
# working, since a Collection object gets created by
# get_additional_data(), which is called for all clients at server
# startup. (It would also prevent machines that change groups from
# working properly; e.g., if you reinstall a machine with a new OS,
# then returning a cached Collection object would give the wrong
# sources to that client.)
collections = dict()
class Collection(Bcfg2.Server.Plugin.Debuggable):
def __init__(self, metadata, sources, basepath, debug=False):
""" don't call this directly; use the factory function """
Bcfg2.Server.Plugin.Debuggable.__init__(self)
self.debug_flag = debug
self.metadata = metadata
self.sources = sources
self.basepath = basepath
self.virt_pkgs = dict()
try:
self.config = sources[0].config
self.cachepath = sources[0].basepath
self.ptype = sources[0].ptype
except IndexError:
self.config = None
self.cachepath = None
self.ptype = "unknown"
self.cachefile = None
@property
def cachekey(self):
return md5(self.get_config()).hexdigest()
def get_config(self):
self.logger.error("Packages: Cannot generate config for host with "
"multiple source types (%s)" % self.metadata.hostname)
return ""
def get_relevant_groups(self):
groups = []
for source in self.sources:
groups.extend(source.get_relevant_groups(self.metadata))
return sorted(list(set(groups)))
@property
def basegroups(self):
groups = set()
for source in self.sources:
groups.update(source.basegroups)
return list(groups)
@property
def cachefiles(self):
cachefiles = set([self.cachefile])
for source in self.sources:
cachefiles.add(source.cachefile)
return list(cachefiles)
def get_group(self, group, ptype=None):
for source in self.sources:
pkgs = source.get_group(self.metadata, group, ptype=ptype)
if pkgs:
return pkgs
self.logger.warning("Packages: '%s' is not a valid group" % group)
return []
def is_package(self, package):
for source in self.sources:
if source.is_package(self.metadata, package):
return True
return False
def is_virtual_package(self, package):
for source in self.sources:
if source.is_virtual_package(self.metadata, package):
return True
return False
def get_deps(self, package, pinnings=None, recs=None):
pin_found = False
pin_source = None
if pinnings and package in pinnings:
pin_source = pinnings[package]
recommended = None
if recs and package in recs:
recommended = recs[package]
for source in self.sources:
if pin_source and pin_source != source.name:
continue
pin_found = True
if source.is_package(self.metadata, package):
return source.get_deps(self.metadata, package, recommended)
if not pin_found:
self.logger.error("Packages: Source '%s' for package '%s' not found" %
(pin_source, package))
return []
def get_provides(self, package):
for source in self.sources:
providers = source.get_provides(self.metadata, package)
if providers:
return providers
return []
def get_vpkgs(self):
""" get virtual packages """
vpkgs = dict()
for source in self.sources:
s_vpkgs = source.get_vpkgs(self.metadata)
for name, prov_set in list(s_vpkgs.items()):
if name not in vpkgs:
vpkgs[name] = set(prov_set)
else:
vpkgs[name].update(prov_set)
return vpkgs
def filter_unknown(self, unknown):
for source in self.sources:
source.filter_unknown(unknown)
def magic_groups_match(self):
for source in self.sources:
if source.magic_groups_match(self.metadata):
return True
def build_extra_structures(self, independent):
pass
def get_additional_data(self):
sdata = []
for source in self.sources:
sdata.extend(copy.deepcopy(source.url_map))
return sdata
def setup_data(self, force_update=False):
""" do any collection-level data setup tasks """
pass
def packages_from_entry(self, entry):
""" Given a Package or BoundPackage entry, get a list of the
package(s) described by it in a format appropriate for passing
to :func:`complete`. By default, that's just the name; only
the :mod:`Bcfg2.Server.Plugins.Packages.Yum` backend supports
versions or other extended data. See :ref:`pkg-objects` for
more details.
:param entry: The XML entry describing the package or packages.
:type entry: lxml.etree._Element
:returns: list of strings, but see :ref:`pkg-objects`
"""
return [entry.get("name")]
def packages_to_entry(self, pkglist, entry, config):
""" Given a list of package objects as returned by
:func:`packages_from_entry` or :func:`complete`, return an XML
tree describing the BoundPackage entries that should be
included in the client configuration. See :ref:`pkg-objects`
for more details.
:param pkglist: A list of packages as returned by
:func:`complete`
:type pkglist: list of strings, but see :ref:`pkg-objects`
:param entry: The base XML entry to add all of the Package
entries to. This should be modified in place.
:type entry: lxml.etree._Element
"""
for pkg in pkglist:
lxml.etree.SubElement(entry, 'BoundPackage', name=pkg,
version=config.get("packages", "version",
default="auto"),
type=self.ptype, origin='Packages')
def get_new_packages(self, initial, complete):
""" Compute the difference between the complete package list
(as returned by :func:`complete`) and the initial package list
computed from the specification. This is necessary because
the format may be different between the two lists due to
:func:`packages_to_entry` and :func:`packages_from_entry`. See
:ref:`pkg-objects` for more details.
:param initial: The initial package list
:type initial: set of strings, but see :ref:`pkg-objects`
:param complete: The final package list
:type complete: set of strings, but see :ref:`pkg-objects`
:return: set of strings, but see :ref:`pkg-objects` - the set
of packages that are in ``complete`` but not in
``initial``
"""
return list(complete.difference(initial))
def complete(self, packagelist, pinnings=None, recommended=None):
'''Build the transitive closure of all package dependencies
Arguments:
packageslist - set of package names
pinnings - mapping from package names to source names
returns => (set(packages), set(unsatisfied requirements))
'''
# setup vpkg cache
pgrps = tuple(self.get_relevant_groups())
if pgrps not in self.virt_pkgs:
self.virt_pkgs[pgrps] = self.get_vpkgs()
vpkg_cache = self.virt_pkgs[pgrps]
# unclassified is set of unsatisfied requirements (may be pkg
# for vpkg)
unclassified = set(packagelist)
vpkgs = set()
both = set()
pkgs = set(packagelist)
packages = set()
examined = set()
unknown = set()
final_pass = False
really_done = False
# do while unclassified or vpkgs or both or pkgs
while unclassified or pkgs or both or final_pass:
if really_done:
break
if len(unclassified) + len(pkgs) + len(both) == 0:
# one more pass then exit
really_done = True
while unclassified:
current = unclassified.pop()
examined.add(current)
is_pkg = False
if self.is_package(current):
is_pkg = True
is_vpkg = current in vpkg_cache
if is_pkg and is_vpkg:
both.add(current)
elif is_pkg and not is_vpkg:
pkgs.add(current)
elif is_vpkg and not is_pkg:
vpkgs.add(current)
elif not is_vpkg and not is_pkg:
unknown.add(current)
while pkgs:
# direct packages; current can be added, and all deps
# should be resolved
current = pkgs.pop()
self.debug_log("Packages: handling package requirement %s" %
current)
packages.add(current)
deps = self.get_deps(current, pinnings, recommended)
newdeps = set(deps).difference(examined)
if newdeps:
self.debug_log("Packages: Package %s added requirements %s"
% (current, newdeps))
unclassified.update(newdeps)
satisfied_vpkgs = set()
for current in vpkgs:
# virtual dependencies, satisfied if one of N in the
# config, or can be forced if only one provider
if len(vpkg_cache[current]) == 1:
self.debug_log("Packages: requirement %s satisfied by %s" %
(current, vpkg_cache[current]))
unclassified.update(vpkg_cache[current].difference(examined))
satisfied_vpkgs.add(current)
else:
satisfiers = [item for item in vpkg_cache[current]
if item in packages]
self.debug_log("Packages: requirement %s satisfied by %s" %
(current, satisfiers))
satisfied_vpkgs.add(current)
vpkgs.difference_update(satisfied_vpkgs)
satisfied_both = set()
for current in both:
# packages that are both have virtual providers as
# well as a package with that name. allow use of virt
# through explicit specification, then fall back to
# forcing current on last pass
satisfiers = [item for item in vpkg_cache[current]
if item in packages]
if satisfiers:
self.debug_log("Packages: requirement %s satisfied by %s" %
(current, satisfiers))
satisfied_both.add(current)
elif current in packagelist or final_pass:
pkgs.add(current)
satisfied_both.add(current)
both.difference_update(satisfied_both)
if len(unclassified) + len(pkgs) == 0:
final_pass = True
else:
final_pass = False
self.filter_unknown(unknown)
return packages, unknown
def __len__(self):
return len(self.sources)
def __getitem__(self, item):
return self.sources[item]
def __setitem__(self, item, value):
self.sources[item] = value
def __delitem__(self, item):
del self.sources[item]
def append(self, item):
self.sources.append(item)
def count(self):
return self.sources.count()
def index(self, item):
return self.sources.index(item)
def extend(self, items):
self.sources.extend(items)
def insert(self, index, item):
self.sources.insert(index, item)
def pop(self, index=None):
self.sources.pop(index)
def remove(self, item):
self.sources.remove(item)
def sort(self, cmp=None, key=None, reverse=False):
self.sources.sort(cmp, key, reverse)
def clear_cache():
global collections
collections = dict()
def factory(metadata, sources, basepath, debug=False):
global collections
if not sources.loaded:
# if sources.xml has not received a FAM event yet, defer;
# instantiate a dummy Collection object
return Collection(metadata, [], basepath)
sclasses = set()
relevant = list()
for source in sources:
if source.applies(metadata):
relevant.append(source)
if 'cclass' in dir(source):
sclasses.update([source.cclass])
else:
sclass = source.__class__.__name__.replace("Source", "")
sclasses.update([sclass])
if len(sclasses) > 1:
logger.warning("Packages: Multiple source types found for %s: %s" %
(metadata.hostname, ", ".join(sclasses)))
cclass = Collection
elif len(sclasses) == 0:
# you'd think this should be a warning, but it happens all the
# freaking time if you have a) machines in your clients.xml
# that do not have the proper groups set up yet (e.g., if you
# have multiple Bcfg2 servers and Packages-relevant groups set
# by probes); and b) templates that query all or multiple
# machines (e.g., with metadata.query.all_clients())
if debug:
logger.error("Packages: No sources found for %s" %
metadata.hostname)
cclass = Collection
else:
stype = sclasses.pop()
try:
module = \
getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" %
stype.title()).Server.Plugins.Packages,
stype.title())
cclass = getattr(module, "%sCollection" % stype.title())
except ImportError:
logger.error("Packages: Unknown source type %s" % stype)
except AttributeError:
logger.warning("Packages: No collection class found for %s sources"
% stype)
if debug:
logger.error("Packages: Using %s for Collection of sources for %s" %
(cclass.__name__, metadata.hostname))
collection = cclass(metadata, relevant, basepath, debug=debug)
collections[metadata.hostname] = collection
return collection
|