Fix nasty core hang, which would manifest itself as a lockup upon shutdown for bcfg2-{server,info,admin}

The root cause of this issue was our use of external threads. Without a channel for data requesting plugin
(and hence thread) shutdown, plugin threads would continue, unabated, forever. This would cause the python
interpreter to hang forever waiting for a futex.

Added a new shutdown method to Bcfg2.Server.Plugin.Plugin which sets a state flag upon shutdown request.
Plugin threads must cycle periodically, even if no work is available, to ensure that the shutdown flag is
honored in a reasonable amount of time.

Also, the shutdown method is triggered using atexit. This means that scripts may need to explicitly sys.exit
or raise SystemExit, depending on circumstances.

This addresses at least one, but probably more tickets, however I am currently offline.

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5472 ce84e21b-d406-0410-9b95-82705330c041

4 files changed, 16 insertions, 3 deletions

diff --git a/src/lib/Server/Admin/Viz.py b/src/lib/Server/Admin/Viz.py
index 3451c56ae..a6dd6682c 100644
--- a/src/lib/Server/Admin/Viz.py
+++ b/src/lib/Server/Admin/Viz.py
@@ -58,6 +58,7 @@ class Viz(Bcfg2.Server.Admin.MetadataCore):
         data = self.Visualize(self.get_repo_path(), hset, bset,
                               kset, outputfile)
         print data
+        raise SystemExit, 0
 
     def Visualize(self, repopath, hosts=False,
                   bundles=False, key=False, output=False):
diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py
index 758241c98..acee2542f 100644
--- a/src/lib/Server/Core.py
+++ b/src/lib/Server/Core.py
@@ -1,6 +1,7 @@
 '''Bcfg2.Server.Core provides the runtime support for bcfg2 modules'''
 __revision__ = '$Revision$'
 
+import atexit
 import logging
 import lxml.etree
 import select
@@ -55,6 +56,7 @@ class Core(Component):
         self.revision = '-1'
         self.password = password
         self.encoding = encoding
+        atexit.register(self.shutdown)
 
         if '' in plugins:
             plugins.remove('')
@@ -132,7 +134,10 @@ class Core(Component):
         except:
             logger.error("Unexpected instantiation failure for plugin %s" % 
                 (plugin), exc_info=1)    
-            
+
+    def shutdown(self):
+        for plugin in self.plugins.values():
+            plugin.shutdown()
 
     def validate_data(self, metadata, data, base_cls):
         for plugin in self.plugins.values():
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index 6c6d5aab3..712f12ea9 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -63,6 +63,7 @@ class Plugin(object):
         self.core = core
         self.data = "%s/%s" % (datastore, self.name)
         self.logger = logging.getLogger('Bcfg2.Plugins.%s' % (self.name))
+        self.running = True
 
     @staticmethod
     def make_path(path):
@@ -79,6 +80,9 @@ class Plugin(object):
         path = "%s/%s" % (repo, cls.name)
         cls.make_path(path)
 
+    def shutdown(self):
+        self.running = False
+
 class Generator(object):
     '''Generator plugins contribute to literal client configurations'''
     def HandlesEntry(self, entry, metadata):
diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py
index 315dc34c0..3b2949e7a 100644
--- a/src/lib/Server/Plugins/Snapshots.py
+++ b/src/lib/Server/Plugins/Snapshots.py
@@ -59,8 +59,11 @@ class Snapshots(Bcfg2.Server.Plugin.Statistics,
         self.loader.start()
 
     def load_snapshot(self):
-        while True:
-            (metadata, data) = self.work_queue.get(block=True)
+        while self.running:
+            try:
+                (metadata, data) = self.work_queue.get(block=True, timeout=5)
+            except:
+                continue
             self.statistics_from_old_stats(metadata, data)
 
     def process_statistics(self, metadata, data):