last step of repo switches

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@1716 ce84e21b-d406-0410-9b95-82705330c041

1 files changed, 167 insertions, 107 deletions

diff --git a/src/lib/Server/Metadata.py b/src/lib/Server/Metadata.py
index 47bbb3ecb..ecf636476 100644
--- a/src/lib/Server/Metadata.py
+++ b/src/lib/Server/Metadata.py
@@ -1,130 +1,190 @@
 '''This file stores persistent metadata for the BCFG Configuration Repository'''
 __revision__ = '$Revision$'
 
-from lxml.etree import XML, SubElement, Element, _Comment, tostring
 from syslog import syslog, LOG_ERR, LOG_INFO
 
-from Bcfg2.Server.Plugin import SingleXMLFileBacked
+import lxml.etree, os, time, threading
 
 class MetadataConsistencyError(Exception):
     '''This error gets raised when metadata is internally inconsistent'''
     pass
 
-class Metadata(object):
-    '''The Metadata class is a container for all classes of metadata used by Bcfg2'''
-    def __init__(self, all, image, classes, bundles, attributes, hostname, toolset):
-        self.all = all
-        self.image = image
-        self.classes = classes
+class MetadataRuntimeError(Exception):
+    '''This error is raised when the metadata engine is called prior to reading enough data'''
+    pass
+
+class ClientMetadata(object):
+    '''This object contains client metadata'''
+    def __init__(self, client, groups, bundles, toolset):
+        self.hostname = client
         self.bundles = bundles
-        self.attributes = attributes
-        self.hostname = hostname
+        self.groups = groups
         self.toolset = toolset
 
-    def Applies(self, other):
-        '''Check if metadata styled object applies to current metadata'''
-        if (other.all or (other.image and (self.image == other.image)) or
-            (other.classes and (other.classes in self.classes)) or
-            (other.attributes and (other.attributes in self.attributes)) or
-            (other.bundles and (other.bundles in self.bundles)) or
-            (other.hostname and (self.hostname == other.hostname)) or
-            (other.hostname and (self.hostname.split('.')[0] == other.hostname))):
-            return True
-        else:
-            return False
-
-class Profile(object):
-    '''Profiles are configuration containers for sets of classes and attributes'''
-    def __init__(self, xml):
-        object.__init__(self)
-        self.classes = [cls.attrib['name'] for cls in xml.findall("Class")]
-        self.attributes = ["%s.%s" % (attr.attrib['scope'], attr.attrib['name']) for
-                           attr in xml.findall("Attribute")]
-
-class MetadataStore(SingleXMLFileBacked):
-    '''The MetadataStore is a filebacked xml repository that contains all setup info for all clients'''
+class Metadata:
+    '''This class contains data for bcfg2 server metadata'''
+    __name__ = 'Metadata'
+    __version__ = '$Id$'
+    __author__ = 'bcfg-dev@mcs.anl.gov'
 
-    def __init__(self, filename, fam):
-        # initialize Index data to avoid race
-        self.defaults = {}
+    def __init__(self, fam, datastore):
+        self.data = "%s/%s" % (datastore, self.__name__)
+        fam.AddMonitor("%s/%s" % (self.data, "groups.xml"), self)
+        fam.AddMonitor("%s/%s" % (self.data, "clients.xml"), self)
+        self.states = {'groups.xml':False, 'clients.xml':False}
         self.clients = {}
-        self.profiles = {}
-        self.classes = {}
-        self.images = {}
-        self.element = Element("dummy")
-        SingleXMLFileBacked.__init__(self, filename, fam)
-        
-    def Index(self):
-        '''Build data structures for XML data'''
-        self.element = XML(self.data)
-        self.defaults = {}
-        self.clients = {}
-        self.profiles = {}
-        self.classes = {}
-        self.images = {}
-        for prof in self.element.findall("Profile"):
-            self.profiles[prof.attrib['name']] = Profile(prof)
-        for cli in self.element.findall("Client"):
-            self.clients[cli.attrib['name']] = (cli.attrib['image'], cli.attrib['profile'])
-        for cls in self.element.findall("Class"):
-            self.classes[cls.attrib['name']] = [bundle.attrib['name'] for bundle in cls.findall("Bundle")]
-        for img in self.element.findall("Image"):
-            self.images[img.attrib['name']] = img.attrib['toolset']
-        for key in [key[8:] for key in self.element.attrib if key[:8] == 'default_']:
-            self.defaults[key] = self.element.get("default_%s" % key)
+        self.aliases = {}
+        self.groups = {}
+        self.public = []
+        self.profiles = []
+        self.toolsets = {}
+        self.categories = {}
+        self.clientdata = None
+        self.default = None
 
-    def FetchMetadata(self, client, image=None, profile=None):
-        '''Get metadata for client'''
-        if ((image != None) and (profile != None)):
-            # Client asserted profile/image
-            self.clients[client] = (image, profile)
-            syslog(LOG_INFO, "Metadata: Asserted metadata for %s: %s, %s" % (client, image, profile))
-            [self.element.remove(cli) for cli in self.element.findall("Client") if cli.get('name') == client]
-            SubElement(self.element, "Client", name=client, image=image, profile=profile)
-            self.WriteBack()
+    def HandleEvent(self, event):
+        '''Handle update events for data files'''
+        filename = event.filename.split('/')[-1]
+        if filename not in ['groups.xml', 'clients.xml']:
+            return
+        if event.code2str() == 'endExist':
+            return
+        try:
+            xdata = lxml.etree.parse("%s/%s" % (self.data, filename))
+        except lxml.etree.XMLSyntaxError:
+            syslog(LOG_ERR, 'Metadata: Failed to parse %s' % (filename))
+            return
+        if filename == 'clients.xml':
+            self.clients = {}
+            self.aliases = {}
+            self.clientdata = xdata
+            for client in xdata.findall('./Client'):
+                self.clients.update({client.get('name'): client.get('profile')})
+                [self.aliases.update({alias.get('name'): client.get('name')}) for alias in client.findall('Alias')]
         else:
-            # no asserted metadata
-            if self.clients.has_key(client):
-                (image, profile) = self.clients[client]
-            else:
-                # default profile stuff goes here
-                (image, profile) = (self.defaults['image'], self.defaults['profile'])
-                SubElement(self.element, "Client", name=client, profile=profile, image=image)
-                self.WriteBack()
+            self.public = []
+            self.profiles = []
+            self.toolsets = {}
+            self.groups = {}
+            grouptmp = {}
+            self.categories = {}
+            for group in xdata.findall('./Group'):
+                grouptmp[group.get('name')] = tuple([[item.get('name') for item in group.findall(spec)]
+                                                     for spec in ['./Bundle', './Group']])
+                grouptmp[group.get('name')][1].append(group.get('name'))
+                if group.get('default', 'false') == 'true':
+                    self.default = group.get('name')
+                if group.get('profile', 'false') == 'true':
+                    self.profiles.append(group.get('name'))
+                if group.get('public', 'false') == 'true':
+                    self.public.append(group.get('name'))
+                if group.attrib.has_key('toolset'):
+                    self.toolsets[group.get('name')] = group.get('toolset')
+                if group.attrib.has_key('category'):
+                    self.categories[group.get('name')] = group.get('category')
+            for group in grouptmp:
+                self.groups[group] = ([], [])
+                gcategories = []
+                tocheck = [group]
+                while tocheck:
+                    now = tocheck.pop()
+                    if now not in self.groups[group][1]:
+                        self.groups[group][1].append(now)
+                    if grouptmp.has_key(now):
+                        (bundles, groups) = grouptmp[now]
+                        for ggg in [ggg for ggg in groups if ggg not in self.groups[group][1]]:
+                            if not self.categories.has_key(ggg) or (self.categories[ggg] not in gcategories):
+                                self.groups[group][1].append(ggg)
+                                tocheck.append(ggg)
+                            if self.categories.has_key(ggg):
+                                gcategories.append(self.categories[ggg])
+                        [self.groups[group][0].append(bund) for bund in bundles
+                         if bund not in self.groups[group][0]]
+        self.states[filename] = True
+        if False not in self.states.values():
+            # check that all client groups are real and complete
+            real = self.groups.keys()
+            for client in self.clients.keys():
+                if self.clients[client] not in real or self.clients[client] not in self.profiles:
+                    syslog(LOG_ERR, "Metadata: Client %s set as nonexistant or incomplete group %s" \
+                           % (client, self.clients[client]))
+                    syslog(LOG_ERR, "Metadata: Removing client mapping for %s" % (client))
+                    del self.clients[client]
 
-        if not self.profiles.has_key(profile):
-            syslog(LOG_ERR, "Metadata: profile %s not defined" % profile)
-            raise MetadataConsistencyError
-        prof = self.profiles[profile]
-        # should we uniq here? V
-        bundles = reduce(lambda x, y:x + y, [self.classes.get(cls, []) for cls in prof.classes])
-        if not self.images.has_key(image):
-            syslog(LOG_ERR, "Metadata: Image %s not defined" % image)
+    def set_group(self, client, group):
+        '''Set group parameter for provided client'''
+        if False in self.states.values():
+            raise MetadataRuntimeError
+        if group not in self.public:
+            syslog(LOG_ERR, "Metadata: Failed to set client %s to private group %s" % (client,
+                                                                                           group))
             raise MetadataConsistencyError
-        toolset = self.images[image]
-        return Metadata(False, image, prof.classes, bundles, prof.attributes, client, toolset)
+        if self.clients.has_key(client):
+            syslog(LOG_INFO, "Metadata: Changing %s group from %s to %s" % (client,
+                                                                                self.clients[client], group))
+            cli = self.clientdata.xpath('/Clients/Client[@name="%s"]' % (client))
+            cli[0].set('group', group)
+        else:
+            lxml.etree.SubElement(self.clientdata.getroot(), 'Client', name=client, group=group)
+        self.clients[client] = group
+        self.write_back_clients()
+
+    def write_back_clients(self):
+        '''Write changes to client.xml back to disk'''
+        try:
+            datafile = open("%s/%s" % (self.data, 'clients.xml'), 'w')
+        except IOError:
+            syslog(LOG_ERR, "Metadata: Failed to write clients.xml")
+            raise MetadataRuntimeError
+        datafile.write(lxml.etree.tostring(self.clientdata))
+        datafile.close()
 
-    def pretty_print(self, element, level=0):
-        '''Produce a pretty-printed text representation of element'''
-        if isinstance(element, _Comment):
-            return (level * " ") + tostring(element)
-        if element.text:
-            fmt = "%s<%%s %%s>%%s</%%s>" % (level*" ")
-            data = (element.tag, (" ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib])),
-                    element.text, element.tag)
-        numchild = len(element.getchildren())
-        if numchild:
-            fmt = "%s<%%s %%s>\n" % (level*" ",) + (numchild * "%s") + "%s</%%s>\n" % (level*" ")
-            data = (element.tag, ) + (" ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib]),)
-            data += tuple([self.pretty_print(entry, level+2) for entry in element.getchildren()]) + (element.tag, )
+    def find_toolset(self, client):
+        '''Find the toolset for a given client'''
+        tgroups = [self.toolsets[group] for group in self.groups[client][1] if self.toolsets.has_key(group)]
+        if len(tgroups) == 1:
+            return tgroups[0]
+        elif len(tgroups) == 0:
+            syslog(LOG_ERR, "Metadata: Couldn't find toolset for client %s" % (client))
+            raise MetadataConsistencyError
         else:
-            fmt = "%s<%%s %%s/>\n" % (level * " ")
-            data = (element.tag, " ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib]))
-        return fmt % data
+            syslog(LOG_ERR, "Metadata: Got goofy toolset result for client %s" % (client))
+            raise MetadataConsistencyError
 
-    def WriteBack(self):
-        '''Write metadata changes back to persistent store'''
-        fout = open(self.name, 'w')
-        fout.write(self.pretty_print(self.element))
-        fout.close()
+    def get_config_template(self, client):
+        '''Build the configuration header for a client configuration'''
+        return lxml.etree.Element("Configuration", version='2.0', toolset=self.find_toolset(client))
 
+    def get_metadata(self, client):
+        '''Return the metadata for a given client'''
+        if self.aliases.has_key(client):
+            client = self.aliases[client]
+        if self.clients.has_key(client):
+            [bundles, groups] = self.groups[self.clients[client]]
+        else:
+            if self.default == None:
+                syslog(LOG_ERR, "Cannot set group for client %s; no default group set" % (client))
+                raise MetadataConsistencyError
+            [bundles, groups] = self.groups[self.default]
+        toolinfo = [self.toolsets[group] for group in groups if self.toolsets.has_key(group)]
+        if len(toolinfo) > 1:
+            syslog(LOG_ERR, "Metadata: Found multiple toolsets for client %s; choosing one" % (client))
+        elif len(toolinfo) == 0:
+            syslog(LOG_ERR, "Metadata: Cannot determine toolset for client %s" % (client))
+            raise MetadataConsistencyError
+        toolset = toolinfo[0]
+        return ClientMetadata(client, groups, bundles, toolset)
+        
+    def ping_sweep_clients(self):
+        '''Find live and dead clients'''
+        live = {}
+        dead = {}
+        work = self.clients.keys()
+        while work:
+            client = work.pop()
+            rc = os.system("/bin/ping -w 5 -c 1 %s > /dev/null 2>&1" % client)
+            if not rc:
+                live[client] = time.time()
+            else:
+                dead[client] = time.time()
+