From dab1d03d81c538966d03fb9318a4588a9e803b44 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 24 Mar 2012 11:20:07 -0500 Subject: Allow to run directly from a git checkout (#1037) Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Hostbase.py | 593 +++++++++++++++++++++++++++++++ 1 file changed, 593 insertions(+) create mode 100644 src/lib/Bcfg2/Server/Plugins/Hostbase.py (limited to 'src/lib/Bcfg2/Server/Plugins/Hostbase.py') diff --git a/src/lib/Bcfg2/Server/Plugins/Hostbase.py b/src/lib/Bcfg2/Server/Plugins/Hostbase.py new file mode 100644 index 000000000..e9c1c1cff --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Hostbase.py @@ -0,0 +1,593 @@ +""" +This file provides the Hostbase plugin. +It manages dns/dhcp/nis host information +""" + +import os +os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.Server.Hostbase.settings' +from lxml.etree import Element, SubElement +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugin import PluginExecutionError, PluginInitError +from time import strftime +from sets import Set +from django.template import Context, loader +from django.db import connection +import re +# Compatibility imports +from Bcfg2.Bcfg2Py3k import StringIO + + +class Hostbase(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Structure, + Bcfg2.Server.Plugin.Generator): + """The Hostbase plugin handles host/network info.""" + name = 'Hostbase' + __author__ = 'bcfg-dev@mcs.anl.gov' + filepath = '/my/adm/hostbase/files/bind' + + def __init__(self, core, datastore): + + self.ready = False + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Structure.__init__(self) + Bcfg2.Server.Plugin.Generator.__init__(self) + files = ['zone.tmpl', + 'reversesoa.tmpl', + 'named.tmpl', + 'reverseappend.tmpl', + 'dhcpd.tmpl', + 'hosts.tmpl', + 'hostsappend.tmpl'] + self.filedata = {} + self.dnsservers = [] + self.dhcpservers = [] + self.templates = {'zone': loader.get_template('zone.tmpl'), + 'reversesoa': loader.get_template('reversesoa.tmpl'), + 'named': loader.get_template('named.tmpl'), + 'namedviews': loader.get_template('namedviews.tmpl'), + 'reverseapp': loader.get_template('reverseappend.tmpl'), + 'dhcp': loader.get_template('dhcpd.tmpl'), + 'hosts': loader.get_template('hosts.tmpl'), + 'hostsapp': loader.get_template('hostsappend.tmpl'), + } + self.Entries['ConfigFile'] = {} + self.__rmi__ = ['rebuildState'] + try: + self.rebuildState(None) + except: + raise PluginInitError + + def FetchFile(self, entry, metadata): + """Return prebuilt file data.""" + fname = entry.get('name').split('/')[-1] + if not fname in self.filedata: + raise PluginExecutionError + perms = {'owner': 'root', + 'group': 'root', + 'perms': '644'} + [entry.attrib.__setitem__(key, value) + for (key, value) in list(perms.items())] + entry.text = self.filedata[fname] + + def BuildStructures(self, metadata): + """Build hostbase bundle.""" + if metadata.hostname not in self.dnsservers or metadata.hostname not in self.dhcpservers: + return [] + output = Element("Bundle", name='hostbase') + if metadata.hostname in self.dnsservers: + for configfile in self.Entries['ConfigFile']: + if re.search('/etc/bind/', configfile): + SubElement(output, "ConfigFile", name=configfile) + if metadata.hostname in self.dhcpservers: + SubElement(output, "ConfigFile", name="/etc/dhcp3/dhcpd.conf") + return [output] + + def rebuildState(self, _): + """Pre-cache all state information for hostbase config files + callable as an XMLRPC function. + + """ + self.buildZones() + self.buildDHCP() + self.buildHosts() + self.buildHostsLPD() + self.buildPrinters() + self.buildNetgroups() + return True + + def buildZones(self): + """Pre-build and stash zone files.""" + cursor = connection.cursor() + + cursor.execute("SELECT id, serial FROM hostbase_zone") + zones = cursor.fetchall() + + for zone in zones: + # update the serial number for all zone files + todaydate = (strftime('%Y%m%d')) + try: + if todaydate == str(zone[1])[:8]: + serial = zone[1] + 1 + else: + serial = int(todaydate) * 100 + except (KeyError): + serial = int(todaydate) * 100 + cursor.execute("""UPDATE hostbase_zone SET serial = \'%s\' WHERE id = \'%s\'""" % (str(serial), zone[0])) + + cursor.execute("SELECT * FROM hostbase_zone WHERE zone NOT LIKE \'%%.rev\'") + zones = cursor.fetchall() + + iplist = [] + hosts = {} + + for zone in zones: + zonefile = StringIO() + externalzonefile = StringIO() + cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z + INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id + WHERE z.zone_id = \'%s\'""" % zone[0]) + nameservers = cursor.fetchall() + cursor.execute("""SELECT i.ip_addr FROM hostbase_zone_addresses z + INNER JOIN hostbase_zoneaddress i ON z.zoneaddress_id = i.id + WHERE z.zone_id = \'%s\'""" % zone[0]) + addresses = cursor.fetchall() + cursor.execute("""SELECT m.priority, m.mx FROM hostbase_zone_mxs z + INNER JOIN hostbase_mx m ON z.mx_id = m.id + WHERE z.zone_id = \'%s\'""" % zone[0]) + mxs = cursor.fetchall() + context = Context({ + 'zone': zone, + 'nameservers': nameservers, + 'addresses': addresses, + 'mxs': mxs + }) + zonefile.write(self.templates['zone'].render(context)) + externalzonefile.write(self.templates['zone'].render(context)) + + querystring = """SELECT h.hostname, p.ip_addr, + n.name, c.cname, m.priority, m.mx, n.dns_view + FROM (((((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) + INNER JOIN hostbase_ip p ON i.id = p.interface_id) + INNER JOIN hostbase_name n ON p.id = n.ip_id) + INNER JOIN hostbase_name_mxs x ON n.id = x.name_id) + INNER JOIN hostbase_mx m ON m.id = x.mx_id) + LEFT JOIN hostbase_cname c ON n.id = c.name_id + WHERE n.name LIKE '%%%%%s' + AND h.status = 'active' + ORDER BY h.hostname, n.name, p.ip_addr + """ % zone[1] + cursor.execute(querystring) + zonehosts = cursor.fetchall() + prevhost = (None, None, None, None) + cnames = StringIO() + cnamesexternal = StringIO() + for host in zonehosts: + if not host[2].split(".", 1)[1] == zone[1]: + zonefile.write(cnames.getvalue()) + externalzonefile.write(cnamesexternal.getvalue()) + cnames = StringIO() + cnamesexternal = StringIO() + continue + if not prevhost[1] == host[1] or not prevhost[2] == host[2]: + zonefile.write(cnames.getvalue()) + externalzonefile.write(cnamesexternal.getvalue()) + cnames = StringIO() + cnamesexternal = StringIO() + zonefile.write("%-32s%-10s%-32s\n" % + (host[2].split(".", 1)[0], 'A', host[1])) + zonefile.write("%-32s%-10s%-3s%s.\n" % + ('', 'MX', host[4], host[5])) + if host[6] == 'global': + externalzonefile.write("%-32s%-10s%-32s\n" % + (host[2].split(".", 1)[0], 'A', host[1])) + externalzonefile.write("%-32s%-10s%-3s%s.\n" % + ('', 'MX', host[4], host[5])) + elif not prevhost[5] == host[5]: + zonefile.write("%-32s%-10s%-3s%s.\n" % + ('', 'MX', host[4], host[5])) + if host[6] == 'global': + externalzonefile.write("%-32s%-10s%-3s%s.\n" % + ('', 'MX', host[4], host[5])) + + if host[3]: + try: + if host[3].split(".", 1)[1] == zone[1]: + cnames.write("%-32s%-10s%-32s\n" % + (host[3].split(".", 1)[0], + 'CNAME', host[2].split(".", 1)[0])) + if host[6] == 'global': + cnamesexternal.write("%-32s%-10s%-32s\n" % + (host[3].split(".", 1)[0], + 'CNAME', host[2].split(".", 1)[0])) + else: + cnames.write("%-32s%-10s%-32s\n" % + (host[3] + ".", + 'CNAME', + host[2].split(".", 1)[0])) + if host[6] == 'global': + cnamesexternal.write("%-32s%-10s%-32s\n" % + (host[3] + ".", + 'CNAME', + host[2].split(".", 1)[0])) + + except: + pass + prevhost = host + zonefile.write(cnames.getvalue()) + externalzonefile.write(cnamesexternal.getvalue()) + zonefile.write("\n\n%s" % zone[9]) + externalzonefile.write("\n\n%s" % zone[9]) + self.filedata[zone[1]] = zonefile.getvalue() + self.filedata[zone[1] + ".external"] = externalzonefile.getvalue() + zonefile.close() + externalzonefile.close() + self.Entries['ConfigFile']["%s/%s" % (self.filepath, zone[1])] = self.FetchFile + self.Entries['ConfigFile']["%s/%s.external" % (self.filepath, zone[1])] = self.FetchFile + + cursor.execute("SELECT * FROM hostbase_zone WHERE zone LIKE \'%%.rev\' AND zone <> \'.rev\'") + reversezones = cursor.fetchall() + + reversenames = [] + for reversezone in reversezones: + cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z + INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id + WHERE z.zone_id = \'%s\'""" % reversezone[0]) + reverse_nameservers = cursor.fetchall() + + context = Context({ + 'inaddr': reversezone[1].rstrip('.rev'), + 'zone': reversezone, + 'nameservers': reverse_nameservers, + }) + + self.filedata[reversezone[1]] = self.templates['reversesoa'].render(context) + self.filedata[reversezone[1] + '.external'] = self.templates['reversesoa'].render(context) + self.filedata[reversezone[1]] += reversezone[9] + self.filedata[reversezone[1] + '.external'] += reversezone[9] + + subnet = reversezone[1].split(".") + subnet.reverse() + reversenames.append((reversezone[1].rstrip('.rev'), ".".join(subnet[1:]))) + + for filename in reversenames: + cursor.execute(""" + SELECT DISTINCT h.hostname, p.ip_addr, n.dns_view FROM ((hostbase_host h + INNER JOIN hostbase_interface i ON h.id = i.host_id) + INNER JOIN hostbase_ip p ON i.id = p.interface_id) + INNER JOIN hostbase_name n ON n.ip_id = p.id + WHERE p.ip_addr LIKE '%s%%%%' AND h.status = 'active' ORDER BY p.ip_addr + """ % filename[1]) + reversehosts = cursor.fetchall() + zonefile = StringIO() + externalzonefile = StringIO() + if len(filename[0].split(".")) == 2: + originlist = [] + [originlist.append((".".join([ip[1].split(".")[2], filename[0]]), + ".".join([filename[1], ip[1].split(".")[2]]))) + for ip in reversehosts + if (".".join([ip[1].split(".")[2], filename[0]]), + ".".join([filename[1], ip[1].split(".")[2]])) not in originlist] + for origin in originlist: + hosts = [(host[1].split("."), host[0]) + for host in reversehosts + if host[1].rstrip('0123456789').rstrip('.') == origin[1]] + hosts_external = [(host[1].split("."), host[0]) + for host in reversehosts + if (host[1].rstrip('0123456789').rstrip('.') == origin[1] + and host[2] == 'global')] + context = Context({ + 'hosts': hosts, + 'inaddr': origin[0], + 'fileorigin': filename[0], + }) + zonefile.write(self.templates['reverseapp'].render(context)) + context = Context({ + 'hosts': hosts_external, + 'inaddr': origin[0], + 'fileorigin': filename[0], + }) + externalzonefile.write(self.templates['reverseapp'].render(context)) + else: + originlist = [filename[0]] + hosts = [(host[1].split("."), host[0]) + for host in reversehosts + if (host[1].split("."), host[0]) not in hosts] + hosts_external = [(host[1].split("."), host[0]) + for host in reversehosts + if ((host[1].split("."), host[0]) not in hosts_external + and host[2] == 'global')] + context = Context({ + 'hosts': hosts, + 'inaddr': filename[0], + 'fileorigin': None, + }) + zonefile.write(self.templates['reverseapp'].render(context)) + context = Context({ + 'hosts': hosts_external, + 'inaddr': filename[0], + 'fileorigin': None, + }) + externalzonefile.write(self.templates['reverseapp'].render(context)) + self.filedata['%s.rev' % filename[0]] += zonefile.getvalue() + self.filedata['%s.rev.external' % filename[0]] += externalzonefile.getvalue() + zonefile.close() + externalzonefile.close() + self.Entries['ConfigFile']['%s/%s.rev' % (self.filepath, filename[0])] = self.FetchFile + self.Entries['ConfigFile']['%s/%s.rev.external' % (self.filepath, filename[0])] = self.FetchFile + + ## here's where the named.conf file gets written + context = Context({ + 'zones': zones, + 'reverses': reversenames, + }) + self.filedata['named.conf'] = self.templates['named'].render(context) + self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf'] = self.FetchFile + self.filedata['named.conf.views'] = self.templates['namedviews'].render(context) + self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf.views'] = self.FetchFile + + def buildDHCP(self): + """Pre-build dhcpd.conf and stash in the filedata table.""" + + # fetches all the hosts with DHCP == True + cursor = connection.cursor() + cursor.execute(""" + SELECT hostname, mac_addr, ip_addr + FROM (hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) + INNER JOIN hostbase_ip ip ON i.id = ip.interface_id + WHERE i.dhcp=1 AND h.status='active' AND i.mac_addr <> '' + AND i.mac_addr <> 'float' AND i.mac_addr <> 'unknown' + ORDER BY h.hostname, i.mac_addr + """) + + dhcphosts = cursor.fetchall() + count = 0 + hosts = [] + hostdata = [dhcphosts[0][0], dhcphosts[0][1], dhcphosts[0][2]] + if len(dhcphosts) > 1: + for x in range(1, len(dhcphosts)): + # if an interface has 2 or more ip addresses + # adds the ip to the current interface + if hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0] and hostdata[1] == dhcphosts[x][1]: + hostdata[2] = ", ".join([hostdata[2], dhcphosts[x][2]]) + # if a host has 2 or more interfaces + # writes the current one and grabs the next + elif hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0]: + hosts.append(hostdata) + count += 1 + hostdata = ["-".join([dhcphosts[x][0], str(count)]), dhcphosts[x][1], dhcphosts[x][2]] + # new host found, writes current data to the template + else: + hosts.append(hostdata) + count = 0 + hostdata = [dhcphosts[x][0], dhcphosts[x][1], dhcphosts[x][2]] + #makes sure the last of the data gets written out + if hostdata not in hosts: + hosts.append(hostdata) + + context = Context({ + 'hosts': hosts, + 'numips': len(hosts), + }) + + self.filedata['dhcpd.conf'] = self.templates['dhcp'].render(context) + self.Entries['ConfigFile']['/my/adm/hostbase/files/dhcpd.conf'] = self.FetchFile + + def buildHosts(self): + """Pre-build and stash /etc/hosts file.""" + + append_data = [] + + cursor = connection.cursor() + cursor.execute(""" + SELECT hostname FROM hostbase_host ORDER BY hostname + """) + hostbase = cursor.fetchall() + domains = [host[0].split(".", 1)[1] for host in hostbase] + domains_set = Set(domains) + domain_data = [(domain, domains.count(domain)) for domain in domains_set] + domain_data.sort() + + cursor.execute(""" + SELECT ip_addr FROM hostbase_ip ORDER BY ip_addr + """) + ips = cursor.fetchall() + three_octets = [ip[0].rstrip('0123456789').rstrip('.') \ + for ip in ips] + three_octets_set = Set(three_octets) + three_octets_data = [(octet, three_octets.count(octet)) \ + for octet in three_octets_set] + three_octets_data.sort() + + for three_octet in three_octets_data: + querystring = """SELECT h.hostname, h.primary_user, + p.ip_addr, n.name, c.cname + FROM (((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) + INNER JOIN hostbase_ip p ON i.id = p.interface_id) + INNER JOIN hostbase_name n ON p.id = n.ip_id) + LEFT JOIN hostbase_cname c ON n.id = c.name_id + WHERE p.ip_addr LIKE \'%s.%%%%\' AND h.status = 'active'""" % three_octet[0] + cursor.execute(querystring) + tosort = list(cursor.fetchall()) + tosort.sort(lambda x, y: cmp(int(x[2].split(".")[-1]), int(y[2].split(".")[-1]))) + append_data.append((three_octet, tuple(tosort))) + + two_octets = [ip.rstrip('0123456789').rstrip('.') for ip in three_octets] + two_octets_set = Set(two_octets) + two_octets_data = [(octet, two_octets.count(octet)) + for octet in two_octets_set] + two_octets_data.sort() + + context = Context({ + 'domain_data': domain_data, + 'three_octets_data': three_octets_data, + 'two_octets_data': two_octets_data, + 'three_octets': three_octets, + 'num_ips': len(three_octets), + }) + + self.filedata['hosts'] = self.templates['hosts'].render(context) + + for subnet in append_data: + ips = [] + simple = True + namelist = [name.split('.', 1)[0] for name in [subnet[1][0][3]]] + cnamelist = [] + if subnet[1][0][4]: + cnamelist.append(subnet[1][0][4].split('.', 1)[0]) + simple = False + appenddata = subnet[1][0] + for ip in subnet[1][1:]: + if appenddata[2] == ip[2]: + namelist.append(ip[3].split('.', 1)[0]) + if ip[4]: + cnamelist.append(ip[4].split('.', 1)[0]) + simple = False + appenddata = ip + else: + if appenddata[0] == ip[0]: + simple = False + ips.append((appenddata[2], appenddata[0], Set(namelist), + cnamelist, simple, appenddata[1])) + appenddata = ip + simple = True + namelist = [ip[3].split('.', 1)[0]] + cnamelist = [] + if ip[4]: + cnamelist.append(ip[4].split('.', 1)[0]) + simple = False + ips.append((appenddata[2], appenddata[0], Set(namelist), + cnamelist, simple, appenddata[1])) + context = Context({ + 'subnet': subnet[0], + 'ips': ips, + }) + self.filedata['hosts'] += self.templates['hostsapp'].render(context) + self.Entries['ConfigFile']['/mcs/etc/hosts'] = self.FetchFile + + def buildPrinters(self): + """The /mcs/etc/printers.data file""" + header = """# This file is automatically generated. DO NOT EDIT IT! +# +Name Room User Type Notes +============== ========== ============================== ======================== ==================== +""" + + cursor = connection.cursor() + # fetches all the printers from the database + cursor.execute(""" + SELECT printq, location, primary_user, comments + FROM hostbase_host + WHERE whatami='printer' AND printq <> '' AND status = 'active' + ORDER BY printq + """) + printers = cursor.fetchall() + + printersfile = header + for printer in printers: + # splits up the printq line and gets the + # correct description out of the comments section + temp = printer[3].split('\n') + for printq in re.split(',[ ]*', printer[0]): + if len(temp) > 1: + printersfile += ("%-16s%-12s%-32s%-26s%s\n" % + (printq, printer[1], printer[2], temp[1], temp[0])) + else: + printersfile += ("%-16s%-12s%-32s%-26s%s\n" % + (printq, printer[1], printer[2], '', printer[3])) + self.filedata['printers.data'] = printersfile + self.Entries['ConfigFile']['/mcs/etc/printers.data'] = self.FetchFile + + def buildHostsLPD(self): + """Creates the /mcs/etc/hosts.lpd file""" + + # this header needs to be changed to be more generic + header = """+@machines ++@all-machines +achilles.ctd.anl.gov +raven.ops.anl.gov +seagull.hr.anl.gov +parrot.ops.anl.gov +condor.ops.anl.gov +delphi.esh.anl.gov +anlcv1.ctd.anl.gov +anlvms.ctd.anl.gov +olivia.ctd.anl.gov\n\n""" + + cursor = connection.cursor() + cursor.execute(""" + SELECT hostname FROM hostbase_host WHERE netgroup=\"red\" AND status = 'active' + ORDER BY hostname""") + redmachines = list(cursor.fetchall()) + cursor.execute(""" + SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) + INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id + WHERE netgroup=\"red\" AND n.only=1 AND h.status = 'active' + """) + redmachines.extend(list(cursor.fetchall())) + cursor.execute(""" + SELECT hostname FROM hostbase_host WHERE netgroup=\"win\" AND status = 'active' + ORDER BY hostname""") + winmachines = list(cursor.fetchall()) + cursor.execute(""" + SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) + INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id + WHERE netgroup=\"win\" AND n.only=1 AND h.status = 'active' + """) + winmachines.__add__(list(cursor.fetchall())) + hostslpdfile = header + for machine in redmachines: + hostslpdfile += machine[0] + "\n" + hostslpdfile += "\n" + for machine in winmachines: + hostslpdfile += machine[0] + "\n" + self.filedata['hosts.lpd'] = hostslpdfile + self.Entries['ConfigFile']['/mcs/etc/hosts.lpd'] = self.FetchFile + + def buildNetgroups(self): + """Makes the *-machine files""" + header = """################################################################### +# This file lists hosts in the '%s' machine netgroup, it is +# automatically generated. DO NOT EDIT THIS FILE! +# +# Number of hosts in '%s' machine netgroup: %i +#\n\n""" + + cursor = connection.cursor() + # fetches all the hosts that with valid netgroup entries + cursor.execute(""" + SELECT h.hostname, n.name, h.netgroup, n.only FROM ((hostbase_host h + INNER JOIN hostbase_interface i ON h.id = i.host_id) + INNER JOIN hostbase_ip p ON i.id = p.interface_id) + INNER JOIN hostbase_name n ON p.id = n.ip_id + WHERE h.netgroup <> '' AND h.netgroup <> 'none' AND h.status = 'active' + ORDER BY h.netgroup, h.hostname + """) + nameslist = cursor.fetchall() + # gets the first host and initializes the hash + hostdata = nameslist[0] + netgroups = {hostdata[2]: [hostdata[0]]} + for row in nameslist: + # if new netgroup, create it + if row[2] not in netgroups: + netgroups.update({row[2]: []}) + # if it belongs in the netgroup and has multiple interfaces, put them in + if hostdata[0] == row[0] and row[3]: + netgroups[row[2]].append(row[1]) + hostdata = row + # if its a new host, write the old one to the hash + elif hostdata[0] != row[0]: + netgroups[row[2]].append(row[0]) + hostdata = row + + for netgroup in netgroups: + fileoutput = StringIO() + fileoutput.write(header % (netgroup, netgroup, len(netgroups[netgroup]))) + for each in netgroups[netgroup]: + fileoutput.write(each + "\n") + self.filedata['%s-machines' % netgroup] = fileoutput.getvalue() + fileoutput.close() + self.Entries['ConfigFile']['/my/adm/hostbase/makenets/machines/%s-machines' % netgroup] = self.FetchFile + + cursor.execute(""" + UPDATE hostbase_host SET dirty=0 + """) -- cgit v1.2.3-1-g7c22