Rework SSHbase alias/address support

Rework metadata so that each instance has addresses and aliases instance
attributes containing manually specified addresses and alias names.
Unify pub key resolution loop to process this data once per client.

Change: only clients with active metadata will be included in
ssh_known_hosts data now. Other keys can be stored in *.static, in final
known_hosts file format.

Also, a more exhaustive search for ip addresses is now performed. DNS
resolution for the client hostname, as well as all aliases is performed.
Manually specified addresses are added included as well.

This should also fix some tracebacks reported by Cory and Teknix, as
well as improve performance.

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5319 ce84e21b-d406-0410-9b95-82705330c041

2 files changed, 51 insertions, 54 deletions

diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py
index 88b5092c8..ae717eadf 100644
--- a/src/lib/Server/Plugins/Metadata.py
+++ b/src/lib/Server/Plugins/Metadata.py
@@ -19,10 +19,11 @@ class MetadataRuntimeError(Exception):
 class ClientMetadata(object):
     '''This object contains client metadata'''
     def __init__(self, client, profile, groups, bundles,
-                 addresses, categories, uuid, password, query):
+                 aliases, addresses, categories, uuid, password, query):
         self.hostname = client
         self.profile = profile
         self.bundles = bundles
+        self.aliases = aliases
         self.addresses = addresses
         self.groups = groups
         self.categories = categories
@@ -36,9 +37,9 @@ class ClientMetadata(object):
         return group in self.groups
 
 class MetadataQuery(object):
-    def __init__(self, get_clients, by_groups, by_profiles, all_groups):
+    def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups):
         # resolver is set later
-        self.by_name = None
+        self.by_name = by_name
         self.names_by_groups = by_groups
         self.names_by_profiles = by_profiles
         self.all_clients = get_clients
@@ -64,6 +65,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
     def __init__(self, core, datastore, watch_clients=True):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
         Bcfg2.Server.Plugin.Metadata.__init__(self)
+        Bcfg2.Server.Plugin.Statistics.__init__(self)
         if watch_clients:
             try:
                 core.fam.AddMonitor("%s/%s" % (self.data, "groups.xml"), self)
@@ -94,13 +96,14 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
         self.pdirty = False
         self.extra = {'groups.xml':[], 'clients.xml':[]}
         self.password = core.password
-        self.query = MetadataQuery(lambda:self.clients.keys(),
+        self.query = MetadataQuery(core.build_metadata,
+                                   lambda:self.clients.keys(),
                                    self.get_client_names_by_groups,
                                    self.get_client_names_by_profiles,
                                    self.get_all_group_names)
 
     @classmethod
-    def init_repo(self, repo, groups, os_selection, clients):
+    def init_repo(cls, repo, groups, os_selection, clients):
         Bcfg2.Server.Plugin.Plugin.init_repo(repo)
         open("%s/Metadata/groups.xml" %                              
              repo, "w").write(groups % os_selection)                 
@@ -328,10 +331,12 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
         if dest == 'clients.xml':
             self.clients = {}
             self.aliases = {}
+            self.raliases = {}
             self.bad_clients = {}
             self.secure = []
             self.floating = []
             self.addresses = {}
+            self.raddresses = {}
             self.clientdata_original = xdata_original
             self.clientdata = xdata
             for client in xdata.findall('.//Client'):
@@ -342,6 +347,9 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
                         self.addresses[caddr].append(clname)
                     else:
                         self.addresses[caddr] = [clname]
+                    if clname not in self.raddresses:
+                        self.raddresses[clname] = set()
+                    self.raddresses[clname].add(caddr)
                 if 'auth' in client.attrib:
                     self.auth[client.get('name')] = client.get('auth',
                                                                'cert+password')
@@ -359,10 +367,15 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
                         self.addresses[alias.get('address')].append(clname)
                     else:
                         self.addresses[alias.get('address')] = [clname]
-                    
+                    if clname not in self.raddresses:
+                        self.raddresses[clname] = set()
+                    self.raddresses[clname].add(alias.get('address'))
                 self.clients.update({clname: client.get('profile')})
                 [self.aliases.update({alias.get('name'): clname}) \
                  for alias in client.findall('Alias')]
+                self.raliases[clname] = set()
+                [self.raliases[clname].add(alias.get('name')) for alias \
+                 in client.findall('Alias')]
         elif dest == 'groups.xml':
             self.public = []
             self.profiles = []
@@ -510,20 +523,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
             self.set_profile(client, self.default, (None, None))
             profile = self.default
             [bundles, groups, categories] = self.groups[self.default]
-        '''
-        Handle aliases listed in clients.xml
-        addresses - contains address information for all aliases
-                    mapping is as follows:
-                    {alias: (ip, realname)}
-        '''
-        addresses = {}
-        for alias, host in self.aliases.iteritems():
-            for ip in self.addresses:
-                for name in self.addresses[ip]:
-                    if name == host:
-                        addresses[alias] = (ip, host)
-            if alias not in addresses:
-                addresses[alias] = (None, host)
+        aliases = self.raliases.get(client, set())
+        addresses = self.raddresses.get(client, set())
         newgroups = set(groups)
         newbundles = set(bundles)
         newcategories = {}
@@ -545,8 +546,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin,
             [newbundles.add(b) for b in nbundles if b not in newbundles]
             [newgroups.add(g) for g in ngroups if g not in newgroups]
             newcategories.update(ncategories)
-        return ClientMetadata(client, profile, newgroups, newbundles, addresses,
-                              newcategories, uuid, password, self.query)
+        return ClientMetadata(client, profile, newgroups, newbundles, aliases,
+                              addresses, newcategories, uuid, password, self.query)
 
     def get_all_group_names(self):
         return self.groups.keys()
diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py
index 242aaf580..250882790 100644
--- a/src/lib/Server/Plugins/SSHbase.py
+++ b/src/lib/Server/Plugins/SSHbase.py
@@ -44,6 +44,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
         Bcfg2.Server.Plugin.Generator.__init__(self)
+        Bcfg2.Server.Plugin.PullTarget.__init__(self)
         try:
             Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data,
                                                          self.core.fam)
@@ -66,25 +67,33 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
     def get_skn(self):
         '''build memory cache of the ssh known hosts file'''
         if not self.__skn:
-            static_entries = [key for key in self.entries \
-                              if key.endswith('.static')]
-            if static_entries:
-                self.__skn = "\n".join([self.entries[key].data for \
-                                        key in static_entries])
-            else:
-                self.__skn = ''
+            self.__skn = "\n".join([value.data for key, value in \
+                                    self.entries.iteritems() if \
+                                    key.endswith('.static')])
+            names = dict()
+            # this next part is terrible
+            for cmeta in self.core.metadata.query.all():
+                names[cmeta.hostname] = set([cmeta.hostname])
+                names[cmeta.hostname].update(cmeta.aliases)
+                newnames = set()
+                for name in names[cmeta.hostname]:
+                    newnames.add(name.split('.')[0])
+                    try:
+                        newnames.add(self.get_ipcache_entry(name)[0])
+                    except:
+                        continue
+                names[cmeta.hostname].update(newnames)
+                names[cmeta.hostname].update(cmeta.addresses)
+            # now we have our name cache
             pubkeys = [pubk for pubk in self.entries.keys() \
                        if pubk.find('.pub.H_') != -1]
             pubkeys.sort()
             for pubkey in pubkeys:
                 hostname = pubkey.split('H_')[1]
-                try:
-                    (ipaddr, fqdn) = self.get_ipcache_entry(hostname)
-                except socket.gaierror:
+                if hostname not in names:
                     continue
-                shortname = hostname.split('.')[0]
-                self.__skn += "%s,%s,%s %s" % (shortname, fqdn, ipaddr,
-                                               self.entries[pubkey].data)
+                self.__skn += "%s %s" % (','.join(names[hostname]),
+                                         self.entries[pubkey].data)
         return self.__skn
 
     def set_skn(self, value):
@@ -132,7 +141,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
                 return (ipaddr, client)
             except socket.gaierror:
                 cmd = "getent hosts %s" % client
-                ipaddr = Popen(cmd, shell=True, stdout=PIPE).stdout.read().strip().split()
+                ipaddr = Popen(cmd, shell=True, \
+                               stdout=PIPE).stdout.read().strip().split()
                 if ipaddr:
                     self.ipcache[client] = (ipaddr, client)
                     return (ipaddr, client)
@@ -143,7 +153,6 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
     def build_skn(self, entry, metadata):
         '''This function builds builds a host specific known_hosts file'''
         client = metadata.hostname
-        addresses = metadata.addresses
         entry.text = self.skn
         hostkeys = [keytmpl % client for keytmpl in self.pubkeys \
                         if (keytmpl % client) in self.entries]
@@ -151,20 +160,6 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
         for hostkey in hostkeys:
             entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % (
                 self.entries[hostkey].data)
-        # add entries listed in clients.xml
-        for addr, (ip, host) in addresses.iteritems():
-            shortname = addr.split('.')[0]
-            fqdn = addr
-            if ip == None:
-                ipaddr = self.get_ipcache_entry(addr)[0]
-            else:
-                ipaddr = ip
-            for key in self.entries.keys():
-                if key.find('.pub.H_%s' % host) != -1:
-                    entry.text += "%s,%s,%s %s" % (shortname,
-                                                   fqdn,
-                                                   ipaddr,
-                                                   self.entries[key].data)
         permdata = {'owner':'root', 'group':'root', 'perms':'0644'}
         [entry.attrib.__setitem__(key, permdata[key]) for key in permdata]
 
@@ -207,12 +202,13 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,
                                                      "H_%s" % client])
                 tempdir = tempfile.mkdtemp()
                 temploc = "%s/%s" % (tempdir, hostkey)
-                os.system('ssh-keygen -q -f %s -N "" -t %s -C root@%s < /dev/null' %
-                          (temploc, keytype, client))
+                cmd = 'ssh-keygen -q -f %s -N "" -t %s -C root@%s < /dev/null' 
+                os.system(cmd % (temploc, keytype, client))
                 open(fileloc, 'w').write(open(temploc).read())
                 open(publoc, 'w').write(open("%s.pub" % temploc).read())
                 self.AddEntry(hostkey)
-                self.AddEntry(".".join([hostkey.split('.')[0]]+['pub', "H_%s" % client]))
+                self.AddEntry(".".join([hostkey.split('.')[0]]+['pub', "H_%s" \
+                                                                % client]))
                 try:
                     os.unlink(temploc)
                     os.unlink("%s.pub" % temploc)