Packages: Allow soft relaods and use a checksum for cache file.

Use checksum for cache file.
Allow reloads of config.xml and sources without downloading everything.
Merged config.xml and source processing into a single function.

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5997 ce84e21b-d406-0410-9b95-82705330c041

2 files changed, 80 insertions, 41 deletions

diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt
index 75349b3a3..923dafefb 100644
--- a/doc/server/plugins/generators/packages.txt
+++ b/doc/server/plugins/generators/packages.txt
@@ -211,6 +211,16 @@ Once line per file download needed. Packages/config.xml will be reloaded
 at this time, so any source specification changes (new or modified
 sources in this file) will be reflected by the server at this point.
 
+Soft reload
+-----------
+
+.. versionadded:: 1.1.0
+
+A soft reload can be performed to reread the configuration file and download only missing sources.::
+
+    [0:3711] bcfg2-admin xcmd Packages.Reload
+    True
+
 Availability
 ============
 
@@ -241,6 +251,35 @@ need to use :ref:`BoundEntries <boundentries>` like below
 
     <BoundPackage name="mem-agent" priority="1" version="auto" type="yum" verify="false"/>
 
+Generating Client APT/Yum Configurations
+========================================
+
+.. versionadded:: 1.1.0
+
+Client repository information can be generated automatically from software sources using
+:doc:`./tgenshi/index` or :doc:`./tcheetah`.  A list of source urls are exposed in the client's metadata as 
+metadata.Packages.sources.
+
+An example :doc:`./tgenshi/index` APT template::
+
+    # bcfg2 maintained apt
+
+    {% for s in metadata.Packages.sources %}\
+    deb ${s.url}${s.version} ${s.groups[0]} {% for comp in s.components %}$comp {% end %}
+
+    {% end %}\
+
+An example :doc:`./tgenshi/index` YUM template::
+
+    # bcfg2 maintained yum
+
+    % for s in metadata.Packages.sources %}\
+    [${s.groups[0]}_${s.component}]
+    name=${s.groups[0]}_${s.component}
+    baseurl=${s.url}
+
+    {% end %}\
+
 Debugging unexpected behavior
 =============================
 
diff --git a/src/lib/Server/Plugins/Packages.py b/src/lib/Server/Plugins/Packages.py
index 40ddfbb15..c8b1add44 100644
--- a/src/lib/Server/Plugins/Packages.py
+++ b/src/lib/Server/Plugins/Packages.py
@@ -1,12 +1,18 @@
 import cPickle
 import copy
 import gzip
+import glob
 import logging
 import lxml.etree
 import os
 import re
 import urllib2
 
+try:
+    from hashlib import md5
+except ImportError:
+    from md5 import md5
+
 import Bcfg2.Logger
 import Bcfg2.Server.Plugin
 
@@ -79,14 +85,16 @@ class Source(object):
         self.provides = dict()
         self.blacklist = set(blacklist)
         self.whitelist = set(whitelist)
-        self.cachefile = None
+        self.cachefile = '%s/cache-%s' % (self.basepath, md5(cPickle.dumps( \
+                [self.version, self.components, self.url, \
+                self.rawurl, self.groups, self.arches])).hexdigest())
         self.recommended = recommended
         self.url_map = []
 
     def load_state(self):
         pass
 
-    def setup_data(self):
+    def setup_data(self, force_update=False):
         should_read = True
         should_download = False
         if os.path.exists(self.cachefile):
@@ -103,9 +111,12 @@ class Source(object):
                 logger.error("Packages: File read failed; falling back to file download")
                 should_download = True
 
-        if should_download:
-            self.update()
-            self.read_files()
+        if should_download or force_update:
+            try:
+                self.update()
+                self.read_files()
+            except:
+                logger.error("Failed to update source", exc_info=1)
 
     def get_urls(self):
         return []
@@ -263,7 +274,6 @@ class YUMSource(Source):
             self.baseurl = self.url + '%(version)s/%(component)s/%(arch)s/'
         else:
             self.baseurl = self.rawurl
-        self.cachefile = self.escape_url(self.baseurl + '@' + version) + '.data'
         self.packages = dict()
         self.deps = dict([('global', dict())])
         self.provides = dict([('global', dict())])
@@ -420,10 +430,6 @@ class APTSource(Source):
                  rawurl, blacklist, whitelist, recommended):
         Source.__init__(self, basepath, url, version, arches, components, groups,
                         rawurl, blacklist, whitelist, recommended)
-        if not self.rawurl:
-            self.cachefile = self.escape_url(self.url + '@' + self.version) + '.data'
-        else:
-            self.cachefile = self.escape_url(self.rawurl) + '.data'
         self.pkgnames = set()
 
         self.url_map = [{'rawurl': self.rawurl, 'url': self.url, 'version': self.version, \
@@ -545,7 +551,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
     name = 'Packages'
     conflicts = ['Pkgmgr']
     experimental = True
-    __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh']
+    __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload']
 
     def __init__(self, core, datastore):
         Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore)
@@ -553,31 +559,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         Bcfg2.Server.Plugin.Generator.__init__(self)
         Bcfg2.Server.Plugin.Connector.__init__(self)
         self.cachepath = self.data + '/cache'
+        self.sentinels = set()
+        self.sources = []
 
         if not os.path.exists(self.cachepath):
             # create cache directory if needed
             os.makedirs(self.cachepath)
-        try:
-            xdata = lxml.etree.parse(self.data + '/config.xml')
-            xdata.xinclude()
-            xdata = xdata.getroot()
-        except (lxml.etree.XIncludeError, \
-                lxml.etree.XMLSyntaxError), xmlerr:
-            self.logger.error("Package: Error processing xml: %s" % xmlerr)
-            raise Bcfg2.Server.Plugin.PluginInitError
-        except IOError:
-            self.logger.error("Failed to read Packages configuration. Have"
-                              " you created your config.xml file?")
-            raise Bcfg2.Server.Plugin.PluginInitError
-        self.sentinels = set()
-        self.sources = []
-        for s in xdata.findall('.//APTSource'):
-            self.sources.append(APTSource(self.cachepath, **source_from_xml(s)))
-        for s in xdata.findall('.//YUMSource'):
-            self.sources.append(YUMSource(self.cachepath, **source_from_xml(s)))
-        for source in self.sources:
-            source.setup_data()
-            self.sentinels.update(source.basegroups)
+        self._load_config()
 
     def get_matching_sources(self, meta):
         return [s for s in self.sources if s.applies(meta)]
@@ -701,7 +689,22 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
         return pkgnames.difference(redundant), redundant
 
     def Refresh(self):
+        '''Packages.Refresh() => True|False\nReload configuration specification and download sources\n'''
+        self._load_config(force_update=True)
+        return True
+
+    def Reload(self):
         '''Packages.Refresh() => True|False\nReload configuration specification and sources\n'''
+        self._load_config()
+        return True
+
+    def _load_config(self, force_update=False):
+        '''
+        Load the configuration data and setup sources
+
+        Keyword args:
+            force_update    Force downloading repo data
+        '''
         try:
             xdata = lxml.etree.parse(self.data + '/config.xml')
             xdata.xinclude()
@@ -720,17 +723,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin,
             self.sources.append(APTSource(self.cachepath, **source_from_xml(s)))
         for s in xdata.findall('.//YUMSource'):
             self.sources.append(YUMSource(self.cachepath, **source_from_xml(s)))
+        cachefiles = []
         for source in self.sources:
-            source.setup_data()
+            cachefiles.append(source.cachefile)
+            source.setup_data(force_update)
             self.sentinels.update(source.basegroups)
-        for source in self.sources:
-            try:
-                source.update()
-            except:
-                self.logger.error("Failed to update source", exc_info=1)
-                continue
-            source.read_files()
-        return True
+        for cfile in glob.glob("%s/cache-*" % self.cachepath):
+            if cfile not in cachefiles:
+                os.unlink(cfile)
 
     def get_additional_data(self, meta):
         sdata = []