added better common walk_packages implementation for python 2.4 and 2.5

Conflicts:

	src/lib/Bcfg2/Client/Tools/__init__.py

5 files changed, 99 insertions, 77 deletions

diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py
index 6316c749c..a0a8ac9f2 100644
--- a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py
+++ b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py
@@ -4,9 +4,9 @@ import os
 import re
 import sys
 import shutil
-import pkgutil
 from datetime import datetime
 import Bcfg2.Client.Tools
+from Bcfg2.Compat import walk_packages
 try:
     from base import POSIXTool
 except ImportError:
@@ -45,26 +45,12 @@ class POSIX(Bcfg2.Client.Tools.Tool):
         # this must be called at run-time, not at compile-time, or we
         # get wierd circular import issues.
         self._handlers = dict()
-        if hasattr(pkgutil, 'walk_packages'):
-            submodules = pkgutil.walk_packages(path=__path__)
-        else:
-            # python 2.4
-            import glob
-            submodules = []
-            for path in __path__:
-                for submodule in glob.glob(os.path.join(path, "*.py")):
-                    mod = os.path.splitext(os.path.basename(submodule))[0]
-                    if mod not in ['__init__']:
-                        submodules.append((None, mod, True))
-
-        for submodule in submodules:
-            if submodule[1] == 'base':
+        for submodule in walk_packages(path=__path__, prefix=__name__ + "."):
+            mname = submodule[1].rsplit('.', 1)[-1]
+            if mname == 'base':
                 continue
-            module = getattr(__import__("%s.%s" %
-                                        (__name__,
-                                         submodule[1])).Client.Tools.POSIX,
-                             submodule[1])
-            hdlr = getattr(module, "POSIX" + submodule[1])
+            module = getattr(__import__(submodule[1]).Client.Tools.POSIX, mname)
+            hdlr = getattr(module, "POSIX" + mname)
             if POSIXTool in hdlr.__mro__:
                 # figure out what entry type this handler handles
                 etype = hdlr.__name__[5:].lower()
diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py
index b38eca739..b058ae16c 100644
--- a/src/lib/Bcfg2/Client/Tools/__init__.py
+++ b/src/lib/Bcfg2/Client/Tools/__init__.py
@@ -3,25 +3,11 @@ import os
 import sys
 import stat
 import time
-import pkgutil
 from subprocess import Popen, PIPE
-
 import Bcfg2.Client.XML
-from Bcfg2.Compat import input
-
-if hasattr(pkgutil, 'walk_packages'):
-    submodules = pkgutil.walk_packages(path=__path__)
-else:
-    # python 2.4
-    import glob
-    submodules = []
-    for path in __path__:
-        for submodule in glob.glob(os.path.join(path, "*.py")):
-            mod = os.path.splitext(os.path.basename(submodule))[0]
-            if mod not in ['__init__']:
-                submodules.append((None, mod, True))
-
-__all__ = [m[1] for m in submodules]
+from Bcfg2.Compat import input, walk_packages
+
+__all__ = [m[1] for m in walk_packages(path=__path__)]
 drivers = [item for item in __all__ if item not in ['rpmtools']]
 default = [item for item in drivers if item not in ['RPM', 'Yum']]
 
diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py
index be3954f3b..6551ffc40 100644
--- a/src/lib/Bcfg2/Compat.py
+++ b/src/lib/Bcfg2/Compat.py
@@ -132,3 +132,84 @@ class CmpMixin(object):
     
     def __le__(self, other):
         return self.__lt__(other) or self.__eq__(other)
+
+try:
+    from pkgutil import walk_packages
+except ImportError:
+    try:
+        from pkgutil import iter_modules
+        # iter_modules was added in python 2.5; use it to get an exact
+        # re-implementation of walk_packages if possible
+        def walk_packages(path=None, prefix='', onerror=None):
+            def seen(p, m={}):
+                if p in m:
+                    return True
+                m[p] = True
+
+            for importer, name, ispkg in iter_modules(path, prefix):
+                yield importer, name, ispkg
+
+                if ispkg:
+                    try:
+                        __import__(name)
+                    except ImportError:
+                        if onerror is not None:
+                            onerror(name)
+                    except Exception:
+                        if onerror is not None:
+                            onerror(name)
+                        else:
+                            raise
+                    else:
+                        path = getattr(sys.modules[name], '__path__', [])
+
+                        # don't traverse path items we've seen before
+                        path = [p for p in path if not seen(p)]
+
+                        for item in walk_packages(path, name + '.', onerror):
+                            yield item
+    except ImportError:
+        def walk_packages(path=None, prefix='', onerror=None):
+            """ imperfect, incomplete implementation of
+            walk_packages() for python 2.4. Differences:
+            
+            * requires a full path, not a path relative to something
+              in sys.path.  anywhere we care about that shouldn't be
+              an issue
+
+            * the first element of each tuple is None instead of an
+              importer object
+            """
+            def seen(p, m={}):
+                if p in m:
+                    return True
+                m[p] = True
+
+            if path is None:
+                path = sys.path
+            rv = []
+            for mpath in path:
+                for fname in os.listdir(mpath):
+                    fpath = os.path.join(mpath, fname)
+                    if (os.path.isfile(fpath) and fname.endswith(".py") and
+                        fname != '__init__.py'):
+                        yield None, prefix + fname[:-3], False
+                    elif os.path.isdir(fpath):
+                        mname = prefix + fname
+                        if os.path.exists(os.path.join(fpath, "__init__.py")):
+                            yield None, mname, True
+                        try:
+                            __import__(mname)
+                        except ImportError:
+                            if onerror is not None:
+                                onerror(mname)
+                        except Exception:
+                            if onerror is not None:
+                                onerror(mname)
+                            else:
+                                raise
+                        else:
+                            for item in walk_packages([fpath],
+                                                      prefix=mname + '.',
+                                                      onerror=onerror):
+                                yield item
diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
index 1cce36f4b..08b873948 100644
--- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
+++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py
@@ -4,12 +4,11 @@ import re
 import os
 import sys
 import stat
-import pkgutil
 import logging
 import lxml.etree
 import Bcfg2.Options
 import Bcfg2.Server.Plugin
-from Bcfg2.Compat import u_str, unicode, b64encode
+from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages
 import Bcfg2.Server.Lint
 
 logger = logging.getLogger(__name__)
@@ -153,24 +152,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
         global PROCESSORS
         if PROCESSORS is None:
             PROCESSORS = []
-            if hasattr(pkgutil, 'walk_packages'):
-                submodules = pkgutil.walk_packages(path=__path__)
-            else:
-                #python 2.4
-                import glob
-                submodules = []
-                for path in __path__:
-                    for submodule in glob.glob(os.path.join(path, "*.py")):
-                        mod = '.'.join(submodule.split("/")[-1].split('.')[:-1])
-                        if mod != '__init__':
-                            submodules.append((None, mod, True))
-
-            for submodule in submodules:
-                module = getattr(__import__("%s.%s" %
-                                            (__name__,
-                                             submodule[1])).Server.Plugins.Cfg,
-                                 submodule[1])
-                proc = getattr(module, submodule[1])
+            for submodule in walk_packages(path=__path__,
+                                           prefix=__name__ + "."):
+                mname = submodule[1].rsplit('.', 1)[-1]
+                module = getattr(__import__(submodule[1]).Server.Plugins.Cfg,
+                                 mname)
+                proc = getattr(module, mname)
                 if set(proc.__mro__).intersection([CfgInfo, CfgFilter,
                                                    CfgGenerator, CfgVerifier]):
                     PROCESSORS.append(proc)
diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py b/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py
index 79c144811..e7a3191bc 100644
--- a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py
+++ b/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py
@@ -2,12 +2,11 @@ from django.db import connection, DatabaseError
 from django.core.exceptions import ImproperlyConfigured
 import django.core.management
 import logging
-import pkgutil
 import re
 import sys
 import traceback
 
-from Bcfg2.Compat import CmpMixin
+from Bcfg2.Compat import CmpMixin, walk_packages
 from Bcfg2.Server.models import InternalDatabaseVersion
 from Bcfg2.Server.SchemaUpdater.Routines import UpdaterRoutineException, \
                 UpdaterRoutine
@@ -23,18 +22,6 @@ class SchemaTooOldError(UpdaterError):
     pass
 
 
-def _walk_packages(paths):
-    """Python 2.4 lacks this routine"""
-    import glob
-    submodules = []
-    for path in paths:
-        for submodule in glob.glob("%s/*.py" % path):
-            mod = '.'.join(submodule.split("/")[-1].split('.')[:-1])
-            if mod != '__init__':
-                submodules.append((None, mod, False))
-    return submodules
-
-
 def _release_to_version(release):
     """
     Build a release base for a version
@@ -207,12 +194,7 @@ def update_database():
         logger.debug("Verifying database schema")
 
         updaters = []
-        if hasattr(pkgutil, 'walk_packages'):
-            submodules = pkgutil.walk_packages(path=Changes.__path__)
-        else:
-            #python 2.4
-            submodules = _walk_packages(Changes.__path__)
-        for loader, submodule, ispkg in submodules:
+        for loader, submodule, ispkg in walk_packages(path=Changes.__path__):
             if ispkg:
                 continue
             try: