Finish up Pull Source and multi-Statistics code

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5012 ce84e21b-d406-0410-9b95-82705330c041

7 files changed, 57 insertions, 63 deletions

diff --git a/doc/plugin-roles b/doc/plugin-roles
index 8cb6fb76c..cc96717b7 100644
--- a/doc/plugin-roles
+++ b/doc/plugin-roles
@@ -2,22 +2,22 @@ This documents available plugin roles.
 
 1) list of plugin roles
 
-| Role          | Class              | Status        |
-|---------------+--------------------+---------------|
-| Metadata      | Metadata           | done          |
-| Connector     | Connector          | done          |
-| Probing       | Probing            | done          |
-| Structure     | Structure          | done          |
-| Structure Val | StructureValidator | done          |
-| Generator     | Generator          | done          |
-| Goals Val     | GoalValidator      | done          |
-| Statistics    | Statistics         | class defined |
-| Pull Source   | PullSource         | class defined |
-| Pull Target   | PullTarget         | done          |
-| Decision      | Decision           | done          |
-| Remote        | Remote             | none          |
-| Syncing       | Syncing            | none          |
-|---------------+--------------------+---------------|
+| Role          | Class              | Status |
+|---------------+--------------------+--------|
+| Metadata      | Metadata           | done   |
+| Connector     | Connector          | done   |
+| Probing       | Probing            | done   |
+| Structure     | Structure          | done   |
+| Structure Val | StructureValidator | done   |
+| Generator     | Generator          | done   |
+| Goals Val     | GoalValidator      | done   |
+| Statistics    | Statistics         | done   |
+| Pull Source   | PullSource         | done   |
+| Pull Target   | PullTarget         | done   |
+| Decision      | Decision           | done   |
+| Remote        | Remote             | none   |
+| Syncing       | Syncing            | none   |
+|---------------+--------------------+--------|
 
 2) Plugin Capabilities
 
@@ -75,5 +75,9 @@ Plugin configuration will change. A single field "plugins" in
 bcfg2.conf will supercede the combination of the "generators" and 
 "structures" fields. 
 
+Default loading of needed plugins is now explicit; this means that
+Statistics (if used) should be listed in the plugins line of
+bcfg2.conf. 
+
 5) Notes
 * Need to fix host specific probe behavior (with basenames) 
diff --git a/src/lib/Server/Admin/Pull.py b/src/lib/Server/Admin/Pull.py
index 2a6432f22..a8113f233 100644
--- a/src/lib/Server/Admin/Pull.py
+++ b/src/lib/Server/Admin/Pull.py
@@ -26,7 +26,6 @@ class Pull(Bcfg2.Server.Admin.MetadataCore):
     def __init__(self, configfile):
         Bcfg2.Server.Admin.MetadataCore.__init__(self, configfile,
                                                  self.__usage__)
-        self.stats = self.bcore.stats
         self.log = False
         self.mode = 'interactive'
         
@@ -49,12 +48,15 @@ class Pull(Bcfg2.Server.Admin.MetadataCore):
     def BuildNewEntry(self, client, etype, ename):
         '''construct a new full entry for given client/entry from statistics'''
         new_entry = {'type':etype, 'name':ename}
-        try:
-            (owner, group, perms, contents) = \
-                    self.stats.GetCurrentEntry(client, etype, ename)
-        except Bcfg2.Server.Plugin.PluginExecutionError:
-            print "Statistics plugin failure; could not fetch current state"
-            raise SystemExit(1)
+        for plugin in self.bcore.pull_sources:
+            try:
+                (owner, group, perms, contents) = \
+                        plugin.GetCurrentEntry(client, etype, ename)
+                break
+            except Bcfg2.Server.Plugin.PluginExecutionError:
+                if plugin == self.bcore.pull_sources[-1]:
+                    print "Pull Source failure; could not fetch current state"
+                    raise SystemExit(1)
 
         data = {'owner':owner, 'group':group, 'perms':perms, 'text':contents}
         for k, v in data.iteritems():
diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py
index d2eaed5dd..6b4df2c35 100644
--- a/src/lib/Server/Core.py
+++ b/src/lib/Server/Core.py
@@ -73,20 +73,6 @@ class Core(object):
                 self.metadata = self.plugins["Metadata"]
                 break
 
-        chk_plugins = self.plugins.values()
-        while True:
-            try:
-                plugin = chk_plugins.pop()
-                if isinstance(plugin, Bcfg2.Server.Plugin.Statistics):
-                    self.stats = plugin
-                    break
-            except:
-                pass
-            if not chk_plugins:
-                self.init_plugins("Statistics")
-                self.stats = self.plugins["Statistics"]
-                break
-
         for plug_names, plug_tname, plug_type, collection in \
             [(structures, 'structure', Bcfg2.Server.Plugin.Structure,
               self.structures),
@@ -106,6 +92,11 @@ class Core(object):
                 else:
                     logger.error("Plugin %s not loaded. Not enabled as a %s" \
                                  % (plugin, plug_tname))
+
+        self.statistics = [plugin for plugin in self.plugins.values() \
+                           if isinstance(plugin, Bcfg2.Server.Plugin.Statistics)]
+        self.pull_sources = [plugin for plugin in self.statistics if \
+                             isinstance(plugin, Bcfg2.Server.Plugin.PullSource)]
     
     def init_plugins(self, plugin):
         try:
@@ -244,10 +235,6 @@ class Core(object):
         count = self.fam.Service()
         if count and self.svn:
             self.read_svn_revision()
-        try:
-            self.stats.WriteBack()
-        except:
-            logger.error("error in Statistics", exc_info=1)
             
     def read_svn_revision(self):
         '''Read svn revision information for the bcfg2 repository'''
@@ -281,3 +268,18 @@ class Core(object):
             self.metadata.merge_additional_metadata(imd, conn.name, grps, data)
         return imd
             
+
+    def process_statistics(self, client_name, statistics):
+        meta = self.build_metadata(client_name)
+        state = statistics.find(".//Statistics")
+        if state.get('version') >= '2.0':
+            for plugin in self.statistics:
+                try:
+                    plugin.process_statistics(meta, statistics)
+                except:
+                    logger.error("Plugin %s failed to process stats from %s" \
+                                 % (plugin.name, metadata.hostname),
+                                 exc_info=1)
+
+        logger.info("Client %s reported state %s" % (client_name,
+                                                     state.get('state')))
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index ccdb9d631..44fbdaac8 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -100,13 +100,9 @@ class Probing(object):
 
 class Statistics(object):
     '''Signal statistics handling capability'''
-    def StoreStatistics(self, client, xdata):
+    def process_statistics(self, client, xdata):
         pass
 
-    def WriteBack(self):
-        pass
-
-
 class PullSource(object):
     def GetExtra(self, client):
         return []
diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py
index 24a030b64..c9d11a1f4 100644
--- a/src/lib/Server/Plugins/DBStats.py
+++ b/src/lib/Server/Plugins/DBStats.py
@@ -27,7 +27,7 @@ class DBStats(Bcfg2.Server.Plugin.Plugin,
             logger.debug(str(inst))
             logger.debug(str(type(inst)))
 
-    def StoreStatistics(self, mdata, xdata):
+    def process_statistics(self, mdata, xdata):
         newstats = xdata.find("Statistics")
         newstats.set('time', time.asctime(time.localtime()))
         e = lxml.etree.Element('Node', name=mdata.hostname)
diff --git a/src/lib/Server/Plugins/Statistics.py b/src/lib/Server/Plugins/Statistics.py
index ca7a3c5b0..cd53d173a 100644
--- a/src/lib/Server/Plugins/Statistics.py
+++ b/src/lib/Server/Plugins/Statistics.py
@@ -22,6 +22,7 @@ class StatisticsStore(object):
 
     def WriteBack(self, force=0):
         '''Write statistics changes back to persistent store'''
+        # FIXME switch to a thread writer
         if (self.dirty and (self.lastwrite + self.__min_write_delay__ <= time()) ) \
                 or force:
             try:
@@ -119,12 +120,9 @@ class Statistics(Bcfg2.Server.Plugin.Plugin,
         fpath = "%s/etc/statistics.xml" % datastore
         self.data_file = StatisticsStore(fpath)
 
-    def StoreStatistics(self, client, xdata):
+    def process_statistics(self, client, xdata):
         self.data_file.updateStats(xdata, client.hostname)
 
-    def WriteBack(self):
-        self.data_file.WriteBack()
-
     def FindCurrent(self, client):
         rt = self.data_file.element.xpath('//Node[@name="%s"]' % client)[0]
         maxtime = max([strptime(stat.get('time')) for stat \
diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server
index fed02bf6b..5d0b9d6bb 100755
--- a/src/sbin/bcfg2-server
+++ b/src/sbin/bcfg2-server
@@ -166,7 +166,8 @@ class Bcfg2Serv(Bcfg2.Component.Component):
         '''Build config for a client'''
         try:
             client = self.Core.metadata.resolve_client(address)
-            return tostring(self.Core.BuildConfiguration(client), encoding='UTF-8', xml_declaration=True)
+            return tostring(self.Core.BuildConfiguration(client),
+                            encoding='UTF-8', xml_declaration=True)
         except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError:
             self.logger.warning("Metadata consistency failure for %s" % (address))
             raise Fault, (6, "Metadata consistency failure")
@@ -174,17 +175,8 @@ class Bcfg2Serv(Bcfg2.Component.Component):
     def Bcfg2RecvStats(self, address, stats):
         '''Act on statistics upload'''
         sdata = XML(stats)
-        state = sdata.find(".//Statistics")
-        # Versioned stats to prevent tied client/server upgrade
-        if state.get('version') >= '2.0':
-            client = self.Core.metadata.resolve_client(address)
-            meta = self.Core.build_metadata(client)
-            
-            # Update statistics
-            self.Core.stats.StoreStatistics(meta, sdata)
-
-        self.logger.info("Client %s reported state %s" % 
-                         (client, state.attrib['state']))
+        client = self.Core.metadata.resolve_client(address)        
+        self.Core.process_statistics(client, sdata)
         return "<ok/>"
 
     def _authenticate_connection(self, _, user, password, address):