diff options
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/bcfg2-profile-templates.py | 95 | ||||
-rwxr-xr-x | tools/bcfg2_local.py | 3 | ||||
-rwxr-xr-x | tools/export.py | 3 | ||||
-rwxr-xr-x | tools/posixusers_baseline.py | 4 | ||||
-rwxr-xr-x | tools/selinux_baseline.py | 5 | ||||
-rwxr-xr-x | tools/upgrade/1.3/migrate_dbstats.py | 19 | ||||
-rwxr-xr-x | tools/yum-listpkgs-xml.py | 2 |
7 files changed, 78 insertions, 53 deletions
diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py index 3cd3786f9..93314f1e3 100755 --- a/tools/bcfg2-profile-templates.py +++ b/tools/bcfg2-profile-templates.py @@ -1,25 +1,35 @@ #!/usr/bin/python -Ott +# -*- coding: utf-8 -*- """ Benchmark template rendering times """ -import os import sys import time +import math import logging import operator import Bcfg2.Logger +import Bcfg2.Options import Bcfg2.Server.Core -LOGGER = None + +def stdev(nums): + mean = float(sum(nums)) / len(nums) + return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums))) def main(): - optinfo = \ - dict(client=Bcfg2.Options.Option("Benchmark templates for one client", - cmd="--client", - odesc="<client>", - long_arg=True, - default=None), - ) + optinfo = dict( + client=Bcfg2.Options.Option("Benchmark templates for one client", + cmd="--client", + odesc="<client>", + long_arg=True, + default=None), + runs=Bcfg2.Options.Option("Number of rendering passes per template", + cmd="--runs", + odesc="<runs>", + long_arg=True, + default=5, + cook=int)) optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) @@ -40,12 +50,11 @@ def main(): core = Bcfg2.Server.Core.BaseCore(setup) logger.info("Bcfg2 server core loaded") + core.load_plugins() + logger.debug("Plugins loaded") core.fam.handle_events_in_interval(0.1) logger.debug("Repository events processed") - # how many times to render each template for each client - runs = 5 - if setup['args']: templates = setup['args'] else: @@ -57,41 +66,57 @@ def main(): clients = [core.build_metadata(setup['client'])] times = dict() + client_count = 0 for metadata in clients: - for struct in core.GetStructures(metadata): - logger.info("Rendering templates from structure %s:%s" % - (struct.tag, struct.get("name"))) - for entry in struct.xpath("//Path"): - path = entry.get("name") - logger.info("Rendering %s..." % path) - times[path] = dict() - avg = 0.0 - for i in range(runs): + client_count += 1 + logger.info("Rendering templates for client %s (%s/%s)" % + (metadata.hostname, client_count, len(clients))) + structs = core.GetStructures(metadata) + struct_count = 0 + for struct in structs: + struct_count += 1 + logger.info("Rendering templates from structure %s:%s (%s/%s)" % + (struct.tag, struct.get("name"), struct_count, + len(structs))) + entries = struct.xpath("//Path") + entry_count = 0 + for entry in entries: + entry_count += 1 + if templates and entry.get("name") not in templates: + continue + logger.info("Rendering Path:%s (%s/%s)..." % + (entry.get("name"), entry_count, len(entries))) + ptimes = times.setdefault(entry.get("name"), []) + for i in range(setup['runs']): start = time.time() try: core.Bind(entry, metadata) - avg += (time.time() - start) / runs + ptimes.append(time.time() - start) except: break - if avg: - logger.debug(" %s: %.02f sec" % (metadata.hostname, avg)) - times[path][metadata.hostname] = avg + if ptimes: + avg = sum(ptimes) / len(ptimes) + if avg: + logger.debug(" %s: %.02f sec" % + (metadata.hostname, avg)) # print out per-file results tmpltimes = [] - for tmpl, clients in times.items(): + for tmpl, ptimes in times.items(): try: - avg = sum(clients.values()) / len(clients) + mean = float(sum(ptimes)) / len(ptimes) except ZeroDivisionError: continue - if avg > 0.01 or templates: - tmpltimes.append((tmpl, avg)) - print("%-50s %s" % ("Template", "Average Render Time")) - for tmpl, avg in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): - print("%-50s %.02f" % (tmpl, avg)) - - # TODO: complain about templates that on average were quick but - # for which some clients were slow + ptimes.sort() + median = ptimes[len(ptimes) / 2] + std = stdev(ptimes) + if mean > 0.01 or median > 0.01 or std > 1 or templates: + tmpltimes.append((tmpl, mean, median, std)) + print("%-50s %-9s %-11s %6s" % + ("Template", "Mean Time", "Median Time", "σ")) + for info in reversed(sorted(tmpltimes, key=operator.itemgetter(1))): + print("%-50s %9.02f %11.02f %6.02f" % info) + core.shutdown() if __name__ == "__main__": diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index edb5a7101..8c164e52e 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -19,7 +19,8 @@ class LocalCore(BaseCore): setup['logging'] = None Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) setup['syslog'], setup['logging'] = saved - self.fam.handle_events_in_interval(4) + self.load_plugins() + self.fam.handle_events_in_interval(0.1) def _daemonize(self): return True diff --git a/tools/export.py b/tools/export.py index 716c831d9..0f4724e6b 100755 --- a/tools/export.py +++ b/tools/export.py @@ -227,9 +227,6 @@ E.G. 1.2.0pre1 is a valid version. 'VERSION="%s"\n' % version, startswith=True, dryrun=options.dryrun) - # set new version in setup.py - find_and_replace('setup.py', 'version=', ' version="%s",\n' % version, - dryrun=options.dryrun) # set new version in Bcfg2/version.py find_and_replace('src/lib/Bcfg2/version.py', '__version__ =', diff --git a/tools/posixusers_baseline.py b/tools/posixusers_baseline.py index a4abca42d..c45e54f1a 100755 --- a/tools/posixusers_baseline.py +++ b/tools/posixusers_baseline.py @@ -61,8 +61,8 @@ def main(): if entry.tag == 'POSIXUser': entry.set("group", grp.getgrgid(data[3])[0]) for group in users.user_supplementary_groups(entry): - memberof = lxml.etree.SubElement(entry, "MemberOf") - memberof.text = group[0] + memberof = lxml.etree.SubElement(entry, "MemberOf", + group=group[0]) entry.tag = "Bound" + entry.tag baseline.append(entry) diff --git a/tools/selinux_baseline.py b/tools/selinux_baseline.py index 06f6e6b98..507a16f43 100755 --- a/tools/selinux_baseline.py +++ b/tools/selinux_baseline.py @@ -42,7 +42,10 @@ def main(): baseline.append(lxml.etree.Comment("%s entries" % etype)) extra = handler.FindExtra() for entry in extra: - entry.tag = "Bound%s" % etype + if etype != "SEModule": + entry.tag = "Bound%s" % etype + else: + entry.tag = "%s" % etype baseline.extend(extra) print(lxml.etree.tostring(baseline, pretty_print=True)) diff --git a/tools/upgrade/1.3/migrate_dbstats.py b/tools/upgrade/1.3/migrate_dbstats.py index cbd2a6099..07def2ac8 100755 --- a/tools/upgrade/1.3/migrate_dbstats.py +++ b/tools/upgrade/1.3/migrate_dbstats.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) _our_backend = None + def _quote(value): """ Quote a string to use as a table name or column @@ -44,12 +45,12 @@ def _migrate_perms(): fperms = {} logger.info("Creating FilePerms objects") - for data in ( ('owner', 'group', 'perms'), + for data in (('owner', 'group', 'perms'), ('current_owner', 'current_group', 'current_perms')): for grp in legacy_models.Reason.objects.values_list(*data).distinct(): if grp in fperms: continue - fp = new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2]) + fp = new_models.FilePerms(owner=grp[0], group=grp[1], mode=grp[2]) fp.save() fperms[grp] = fp @@ -60,7 +61,7 @@ def _migrate_perms(): def _migrate_transaction(inter, entries, fperms): """helper""" - logger.debug("Migrating interaction %s for %s" % + logger.debug("Migrating interaction %s for %s" % (inter.id, inter.client.name)) newint = new_models.Interaction(id=inter.id, @@ -107,7 +108,7 @@ def _migrate_transaction(inter, entries, fperms): elif ent.kind == 'Package': act_dict['target_version'] = ei.reason.version act_dict['current_version'] = ei.reason.current_version - logger.debug("Adding package %s %s" % + logger.debug("Adding package %s %s" % (name, act_dict['target_version'])) updates['packages'].append(new_models.PackageEntry.entry_get_or_create(act_dict)) elif ent.kind == 'Path': @@ -116,7 +117,7 @@ def _migrate_transaction(inter, entries, fperms): act_dict['target_perms'] = fperms[( ei.reason.owner, - ei.reason.group, + ei.reason.group, ei.reason.perms )] @@ -141,7 +142,6 @@ def _migrate_transaction(inter, entries, fperms): act_dict['detail_type'] = new_models.PathEntry.DETAIL_PRUNED act_dict['details'] = ei.reason.unpruned - if ei.reason.is_sensitive: act_dict['detail_type'] = new_models.PathEntry.DETAIL_SENSITIVE elif ei.reason.is_binary: @@ -164,7 +164,7 @@ def _migrate_transaction(inter, entries, fperms): for entry_type in updates.keys(): i = 0 while(i < len(updates[entry_type])): - getattr(newint, entry_type).add(*updates[entry_type][i:i+100]) + getattr(newint, entry_type).add(*updates[entry_type][i:i + 100]) i += 100 for perf in inter.performance_items.all(): @@ -220,8 +220,8 @@ def _restructure(): # run any migrations from the previous schema try: - from Bcfg2.Server.Reports.updatefix import update_database - update_database() + from Bcfg2.Server.Reports.updatefix import update_database + update_database() except: logger.error("Failed to run legacy schema updates", exc_info=1) return False @@ -295,4 +295,3 @@ if __name__ == '__main__': Reports(setup).__call__(['update']) _restructure() - diff --git a/tools/yum-listpkgs-xml.py b/tools/yum-listpkgs-xml.py index a052e75af..b4c5f6589 100755 --- a/tools/yum-listpkgs-xml.py +++ b/tools/yum-listpkgs-xml.py @@ -39,5 +39,5 @@ try: sys.argv = [sys.argv[0], '-d', '0', 'list'] yummain.main(sys.argv[1:]) except KeyboardInterrupt: - print("\n\nExiting on user cancel.", file=sys.stderr) + sys.stderr.write("\n\nExiting on user cancel.") sys.exit(1) |