Newest update to RPMng from mbrady

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@3030 ce84e21b-d406-0410-9b95-82705330c041

2 files changed, 395 insertions, 36 deletions

diff --git a/src/lib/Client/Tools/RPMng.py b/src/lib/Client/Tools/RPMng.py
index 926e29c4d..6ada807ee 100644
--- a/src/lib/Client/Tools/RPMng.py
+++ b/src/lib/Client/Tools/RPMng.py
@@ -3,7 +3,12 @@
 __revision__ = '$Revision$'
 
 import Bcfg2.Client.Tools, rpmtools, os.path, rpm, ConfigParser
+#import time, sys
 
+try:
+    set
+except NameError:
+    from sets import Set as set
 
 class RPMng(Bcfg2.Client.Tools.PkgTool):
     '''Support for RPM packages'''
@@ -165,8 +170,11 @@ class RPMng(Bcfg2.Client.Tools.PkgTool):
                                                       pkg.get('gpgkeyid', '')))
                                 self.logger.info('         Disabling signature check.')
 
+                            vp_ts = rpmtools.rpmtransactionset()
                             self.instance_status[inst]['verify'] = \
-                                                      rpmtools.rpm_verify( self.vp_ts, pkg, flags)
+                                                      rpmtools.rpm_verify( vp_ts, pkg, flags)
+                            vp_ts.closeDB()
+                            del vp_ts
 
                     if self.instance_status[inst]['installed'] == False:
                         self.logger.info("        Package %s %s not installed." % \
@@ -209,8 +217,11 @@ class RPMng(Bcfg2.Client.Tools.PkgTool):
                                                           pkg.get('gpgkeyid', '')))
                                     self.logger.info('         Disabling signature check.')
 
+                                vp_ts = rpmtools.rpmtransactionset()
                                 self.instance_status[inst]['verify'] = \
-                                                      rpmtools.rpm_verify( self.vp_ts, pkg, flags )
+                                                      rpmtools.rpm_verify( vp_ts, pkg, flags )
+                                vp_ts.closeDB()
+                                del vp_ts
 
                             else:
                                 # Wrong version installed.
@@ -464,7 +475,7 @@ class RPMng(Bcfg2.Client.Tools.PkgTool):
         # Fix installOnlyPackages
         if len(install_only_pkgs) > 0:
             self.logger.info("Attempting to install 'install only packages'")
-            install_args = " ".join([os.path.join(self.instance_status[inst].get('pkg').get('uri'), \
+            install_args = " ".join([os.path.join(self.instance_status[inst].get('pkg').get('uri'),\
                                                   inst.get('simplefile')) \
                                            for inst in install_only_pkgs])
             self.logger.debug("rpm --install --quiet --oldpackage %s" % install_args)
@@ -507,7 +518,7 @@ class RPMng(Bcfg2.Client.Tools.PkgTool):
                         installed_instances.append(inst)
                     else:
                         self.logger.debug("InstallOnlyPackage %s %s would not install." % \
-                                              (self.instance_status[inst].get('pkg').get('name'),\
+                                              (self.instance_status[inst].get('pkg').get('name'), \
                                                self.str_evra(inst)))
 
                 install_pkg_set = set([self.instance_status[inst].get('pkg') \
@@ -551,7 +562,7 @@ class RPMng(Bcfg2.Client.Tools.PkgTool):
         # Fix upgradeable packages.
         if len(upgrade_pkgs) > 0:
             self.logger.info("Attempting to upgrade packages")
-            upgrade_args = " ".join([os.path.join(self.instance_status[inst].get('pkg').get('uri'), \
+            upgrade_args = " ".join([os.path.join(self.instance_status[inst].get('pkg').get('uri'),\
                                                   inst.get('simplefile')) \
                                            for inst in upgrade_pkgs])
             cmdrc, output = self.cmd.run("rpm --upgrade --quiet --oldpackage --replacepkgs %s" % \
@@ -591,7 +602,7 @@ class RPMng(Bcfg2.Client.Tools.PkgTool):
                         upgraded_instances.append(inst)
                     else:
                         self.logger.debug("Package %s %s would not upgrade." % \
-                                              (self.instance_status[inst].get('pkg').get('name'),\
+                                              (self.instance_status[inst].get('pkg').get('name'), \
                                                self.str_evra(inst)))
 
                 upgrade_pkg_set = set([self.instance_status[inst].get('pkg') \
@@ -749,14 +760,15 @@ class RPMng(Bcfg2.Client.Tools.PkgTool):
                 extra_entry = Bcfg2.Client.XML.Element('Package', name=name, type=self.pkgtype)
                 for installed_inst in instances:
                     self.logger.info("Extra Package %s %s." % \
-                                     (name, self.str_evra(installed_inst)))
-                    Bcfg2.Client.XML.SubElement(extra_entry, \
-                                                'Instance', 
-                                                epoch = str(installed_inst.get('epoch', '')),\
-                                                version = installed_inst.get('version'), \
-                                                release = installed_inst.get('release'), \
-                                                arch = installed_inst.get('arch', ''))
-                    extras.append(extra_entry)
+                                               (name, self.str_evra(installed_inst)))
+                    tmp_entry = Bcfg2.Client.XML.SubElement(extra_entry, 'Instance', \
+                                     version = installed_inst.get('version'), \
+                                     release = installed_inst.get('release'))
+                    if installed_inst.get('epoch', None) != None:
+                        tmp_entry.set('epoch', str(installed_inst.get('epoch')))
+                    if installed_inst.get('arch', None) != None:
+                        tmp_entry.set('arch', installed_inst.get('arch'))
+                extras.append(extra_entry)
         return extras
 
 
@@ -800,35 +812,39 @@ class RPMng(Bcfg2.Client.Tools.PkgTool):
                 if not_found:
                     self.logger.info("Extra Normal Package Instance %s %s" % \
                                                         (name, self.str_evra(installed_inst)))
-                    Bcfg2.Client.XML.SubElement(extra_entry, \
-                                     'Instance', 
-                                     epoch = str(installed_inst.get('epoch', '')),\
+                    tmp_entry = Bcfg2.Client.XML.SubElement(extra_entry, 'Instance', \
                                      version = installed_inst.get('version'), \
-                                     release = installed_inst.get('release'), \
-                                     arch = installed_inst.get('arch', ''))
+                                     release = installed_inst.get('release'))
+                    if installed_inst.get('epoch', None) != None:
+                        tmp_entry.set('epoch', str(installed_inst.get('epoch')))
+                    if installed_inst.get('arch', None) != None:
+                        tmp_entry.set('arch', installed_inst.get('arch'))
 
         if len(extra_entry) == 0:
             extra_entry = None
 
         return extra_entry
 
-    def Inventory(self, structures=[]):
-        '''
-           Wrap the Tool.Inventory() method with its own rpm.TransactionSet() 
-           and an explicit closeDB() as the close wasn't happening and DB4 
-           locks were getting left behind on the RPM database creating a nice 
-           mess.
-
-           ***** Do performance comparison with the transctionset/closeDB
-                 moved into rpmtools, which would mean a transactionset/closeDB
-                 per VerifyPackage() call (meaning one per RPM package) rather 
-                 than one for the whole system.
-        '''
-        self.vp_ts = rpmtools.rpmtransactionset()
-        Bcfg2.Client.Tools.Tool.Inventory(self)
-        # Tool is still an old style class, so super doesn't work. Change it.
-        #super(RPMng, self).Inventory()
-        self.vp_ts.closeDB()
+    #def Inventory(self, structures=[]):
+    #    '''
+    #       Wrap the Tool.Inventory() method with its own rpm.TransactionSet() 
+    #       and an explicit closeDB() as the close wasn't happening and DB4 
+    #       locks were getting left behind on the RPM database creating a nice 
+    #       mess.
+    #
+    #       ***** Do performance comparison with the transctionset/closeDB
+    #             moved into rpmtools, which would mean a transactionset/closeDB
+    #             per VerifyPackage() call (meaning one per RPM package) rather 
+    #             than one for the whole system.
+    #    '''
+    #    self.vp_ts = rpmtools.rpmtransactionset()
+    #    try:
+    #        Bcfg2.Client.Tools.Tool.Inventory(self)
+    #        # Tool is still an old style class, so super doesn't work. Change it.
+    #        #super(RPMng, self).Inventory()
+    #    finally:
+    #        self.vp_ts.closeDB()
+    #        del self.vp_ts
 
     def str_evra(self, instance):
         '''
diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py
new file mode 100644
index 000000000..cdc43b00b
--- /dev/null
+++ b/src/lib/Client/Tools/YUMng.py
@@ -0,0 +1,343 @@
+'''This provides bcfg2 support for yum'''
+__revision__ = '0.1'
+
+import Bcfg2.Client.Tools.RPMng, ConfigParser, sys
+
+try:
+    set
+except NameError:
+    from sets import Set as set
+
+YAD = True
+CP = ConfigParser.ConfigParser()
+try:
+    if '-C' in sys.argv:
+        CP.read([sys.argv[sys.argv.index('-C') + 1]])
+    else:
+        CP.read(['/etc/bcfg2.conf'])
+    if CP.get('YUMng', 'autodep') == 'false':
+        YAD = False
+except:
+    pass
+
+class YUMng(Bcfg2.Client.Tools.RPMng.RPMng):
+    '''Support for Yum packages'''
+    pkgtype = 'yum'
+
+    __name__ = 'YUMng'
+    __execs__ = ['/usr/bin/yum', '/var/lib/rpm']
+    __handles__ = [('Package', 'yum'), ('Package', 'rpm')]
+ 
+    __req__ = {'Package': ['name', 'version']}
+    __ireq__ = {'Package': ['name', 'version']}
+ 
+    __new_req__ = {'Package': ['name'], 'Instance': ['version', 'release', 'arch']}
+    __new_ireq__ = {'Package': ['name', 'uri'], \
+                    'Instance': ['simplefile', 'version', 'release', 'arch']}
+
+    __gpg_req__ = {'Package': ['name', 'version']}
+    __gpg_ireq__ = {'Package': ['name', 'version']}
+
+    __new_gpg_req__ = {'Package': ['name'], 'Instance': ['version', 'release']}
+    __new_gpg_ireq__ = {'Package': ['name'], 'Instance': ['version', 'release']}
+
+    conflicts = ['RPMng']
+
+    def Install(self, packages):
+        '''
+           Try and fix everything that RPMng.VerifyPackages() found wrong for
+           each Package Entry.  This can result in individual RPMs being
+           installed (for the first time), deleted, downgraded
+           or upgraded. 
+
+           NOTE: YUM can not reinstall a package that it thinks is already
+                 installed.
+
+           packages is a list of Package Elements that has
+               self.states[<Package Element>] == False
+
+           The following effects occur:
+           - self.states{} is conditionally updated for each package.
+           - self.installed{} is rebuilt, possibly multiple times.
+           - self.instance_status{} is conditionally updated for each instance
+             of a package.
+           - Each package will be added to self.modified[] if its self.states{}
+             entry is set to True.
+        '''
+        self.logger.info('Runing YUMng.Install()')
+
+        install_pkgs = []
+        gpg_keys = []
+        upgrade_pkgs = []
+
+        # Remove extra instances.
+        # Can not reverify because we don't have a package entry.
+        if len(self.extra_instances) > 0:
+            self.RemovePackages(self.extra_instances)
+
+        # Figure out which instances of the packages actually need something
+        # doing to them and place in the appropriate work 'queue'.
+        for pkg in packages:
+            for inst in [inst for inst in pkg if inst.tag == 'Instance' or inst.tag == 'Package']:
+                if self.instance_status[inst].get('installed', False) == False:
+                    if pkg.get('name') == 'gpg-pubkey':
+                        gpg_keys.append(inst)
+                    else:
+                        install_pkgs.append(inst)
+                elif self.instance_status[inst].get('version_fail', False) == True:
+                    upgrade_pkgs.append(inst)
+
+        # Install GPG keys.
+        # Alternatively specify the required keys using 'gpgkey' in the 
+        # repository definition in yum.conf.  YUM will install the keys 
+        # automatically.
+        if len(gpg_keys) > 0:
+            for inst in gpg_keys:
+                self.logger.info("Installing GPG keys.")
+                key_arg = os.path.join(self.instance_status[inst].get('pkg').get('uri'), \
+                                                     inst.get('simplefile'))
+                cmdrc, output = self.cmd.run("rpm --import %s" % key_arg)
+                if cmdrc != 0:
+                    self.logger.debug("Unable to install %s-%s" % \
+                                              (self.instance_status[inst].get('pkg').get('name'), \
+                                               self.str_evra(inst)))
+                else:
+                    self.logger.debug("Installed %s-%s-%s" % \
+                                              (self.instance_status[inst].get('pkg').get('name'), \
+                                               inst.get('version'), inst.get('release')))
+            self.RefreshPackages()
+            self.gpg_keyids = self.getinstalledgpg()
+            pkg = self.instance_status[gpg_keys[0]].get('pkg')
+            self.states[pkg] = self.VerifyPackage(pkg, [])
+
+        # Install packages.
+        if len(install_pkgs) > 0:
+            self.logger.info("Attempting to install packages")
+
+            if YAD:
+                pkgtool = "/usr/bin/yum -d0 -y install %s"
+            else:
+                pkgtool = "/usr/bin/yum -d0 install %s"
+
+            install_args = []
+            for inst in install_pkgs:
+                pkg_arg = self.instance_status[inst].get('pkg').get('name') + '-'
+                if inst.get('epoch', False):
+                    pkg_arg = pkg_arg + inst.get('epoch') + ':'
+                pkg_arg = pkg_arg + inst.get('version') + '-' + inst.get('release')
+                if inst.get('arch', False):
+                    pkg_arg = pkg_arg + '.' + inst.get('arch')
+                install_args.append(pkg_arg)
+            
+            cmdrc, output = self.cmd.run(pkgtool % " ".join(install_args))
+            if cmdrc == 0:
+                # The yum command succeeded.  All packages installed.
+                self.logger.info("Single Pass for Install Succeded")
+                self.RefreshPackages()
+
+                # Reverify all the packages that we might have just changed.
+                # There may be multiple instances per package, only do the
+                # verification once.
+                install_pkg_set = set([self.instance_status[inst].get('pkg') \
+                                                      for inst in install_pkgs])
+                self.logger.info("Reverifying Installed Packages")
+                for inst in install_pkgs:
+                    pkg_entry = self.instance_status[inst].get('pkg')
+                    if pkg_entry in install_pkg_set:
+                        self.logger.debug("Reverifying Installed %s" % \
+                                                                      (pkg_entry.get('name')))
+                        install_pkg_set.remove(pkg_entry)
+                        self.states[pkg_entry] = self.VerifyPackage(pkg_entry, \
+                                                         self.instance_status[inst].get('modlist'))
+                    else:
+                        # We already reverified this pacakge.
+                        continue
+            else:
+                # The yum command failed.  No packages installed.
+                # Try installing instances individually.
+                self.logger.error("Single Pass Install of Packages Failed")
+                installed_instances = []
+                for inst in install_pkgs:
+                    pkg_arg = self.instance_status[inst].get('pkg').get('name') + '-'
+                    if inst.get('epoch', False):
+                        pkg_arg = pkg_arg + inst.get('epoch') + ':'
+                    pkg_arg = pkg_arg + inst.get('version') + '-' + inst.get('release')
+                    if inst.get('arch', False):
+                        pkg_arg = pkg_arg + '.' + inst.get('arch')
+                    cmdrc, output = self.cmd.run(pkgtool % pkg_arg)
+                    if cmdrc == 0:
+                        installed_instances.append(inst)
+                    else:
+                        self.logger.debug("%s %s would not install." % \
+                                              (self.instance_status[inst].get('pkg').get('name'), \
+                                               self.str_evra(inst)))
+
+                install_pkg_set = set([self.instance_status[inst].get('pkg') \
+                                                      for inst in install_pkgs])
+                self.RefreshPackages()
+                for inst in installed_instances:
+                    pkg = inst.get('pkg')
+                    # Reverify all the packages that we might have just changed.
+                    # There may be multiple instances per package, only do the
+                    # verification once.
+                    if pkg in install_pkg_set:
+                        self.logger.debug("Reverifying Installed Package %s" % \
+                                                                      (pkg_entry.get('name')))
+                        install_pkg_set.remove(pkg)
+                        self.states[pkg_entry] = self.VerifyPackage(pkg, \
+                                                         self.instance_status[inst].get('modlist'))
+                    else:
+                        # We already reverified this pacakge.
+                        continue
+
+        # Fix upgradeable packages.
+        if len(upgrade_pkgs) > 0:
+            self.logger.info("Attempting to upgrade packages")
+
+            if YAD:
+                pkgtool = "/usr/bin/yum -d0 -y update %s"
+            else:
+                pkgtool = "/usr/bin/yum -d0 update %s"
+
+            upgrade_args = []
+            for inst in upgrade_pkgs:
+                pkg_arg = self.instance_status[inst].get('pkg').get('name') + '-'
+                if inst.haskey('epoch'):
+                    pkg_arg = pkg_arg + inst.get('epoch') + ':'
+                pkg_arg = pkg_arg + inst.get('version') + '-' + inst.get('release')
+                if inst.haskey('arch'):
+                    pkg_arg = pkg_arg + '.' + inst.get('arch')
+                upgrade_args.append(pkg_arg)
+            
+            cmdrc, output = self.cmd.run(pkgtool % " ".join(upgrade_args))
+            if cmdrc == 0:
+                # The yum command succeeded.  All packages installed.
+                self.logger.info("Single Pass for Install Succeded")
+                self.RefreshPackages()
+
+                # Reverify all the packages that we might have just changed.
+                # There may be multiple instances per package, only do the
+                # verification once.
+                install_pkg_set = set([self.instance_status[inst].get('pkg') \
+                                                      for inst in upgrade_pkgs])
+                self.logger.info("Reverifying Installed Packages")
+                for inst in upgrade_pkgs:
+                    pkg_entry = self.instance_status[inst].get('pkg')
+                    if pkg_entry in install_pkg_set:
+                        self.logger.debug("Reverifying Installed %s" % \
+                                                                      (pkg_entry.get('name')))
+                        install_pkg_set.remove(pkg_entry)
+                        self.states[pkg_entry] = self.VerifyPackage(pkg_entry, \
+                                                         self.instance_status[inst].get('modlist'))
+                    else:
+                        # We already reverified this pacakge.
+                        continue
+            else:
+                # The yum command failed.  No packages installed.
+                # Try installing instances individually.
+                self.logger.error("Single Pass Install of Packages Failed")
+                installed_instances = []
+                for inst in upgrade_pkgs:
+                    pkg_arg = self.instance_status[inst].get('pkg').get('name') + '-'
+                    if inst.haskey('epoch'):
+                        pkg_arg = pkg_arg + inst.get('epoch') + ':'
+                    pkg_arg = pkg_arg + inst.get('version') + '-' + inst.get('release')
+                    if inst.haskey('arch'):
+                        pkg_arg = pkg_arg + '.' + inst.get('arch')
+                    cmdrc, output = self.cmd.run(pkgtool % pkg_arg)
+                    if cmdrc == 0:
+                        installed_instances.append(inst)
+                    else:
+                        self.logger.debug("%s %s would not install." % \
+                                              (self.instance_status[inst].get('pkg').get('name'), \
+                                               self.str_evra(inst)))
+
+                install_pkg_set = set([self.instance_status[inst].get('pkg') \
+                                                      for inst in upgrade_pkgs])
+                self.RefreshPackages()
+                for inst in installed_instances:
+                    pkg = inst.get('pkg')
+                    # Reverify all the packages that we might have just changed.
+                    # There may be multiple instances per package, only do the
+                    # verification once.
+                    if pkg in install_pkg_set:
+                        self.logger.debug("Reverifying Installed Package %s" % \
+                                                                      (pkg_entry.get('name')))
+                        install_pkg_set.remove(pkg)
+                        self.states[pkg_entry] = self.VerifyPackage(pkg, \
+                                                         self.instance_status[inst].get('modlist'))
+                    else:
+                        # We already reverified this pacakge.
+                        continue
+
+
+        for entry in [ent for ent in packages if self.states[ent]]:
+            self.modified.append(entry)
+
+    def RemovePackages(self, packages):
+        '''
+           Remove specified entries.
+
+           packages is a list of Package Entries with Instances generated
+           by FindExtraPackages().
+        '''
+        self.logger.debug('Running YUMng.RemovePackages()')
+
+        if YAD:
+            pkgtool = "/usr/bin/yum -d0 -y erase %s"
+        else:
+            pkgtool = "/usr/bin/yum -d0 erase %s"
+
+        erase_args = []
+        for pkg in packages:
+            for inst in pkg:
+                if pkg.get('name') != 'gpg-pubkey':
+                    pkg_arg = pkg.get('name') + '-'
+                    if inst.get('epoch', False):
+                        pkg_arg = pkg_arg + inst.get('epoch') + ':'
+                    pkg_arg = pkg_arg + inst.get('version') + '-' + inst.get('release')
+                    if inst.get('arch', False):
+                        pkg_arg = pkg_arg + '.' + inst.get('arch')
+                    erase_args.append(pkg_arg)
+                else:
+                    self.logger.info("WARNING: gpg-pubkey package not in configuration %s %s"\
+                                                 % (pkgspec.get('name'), self.str_evra(pkgspec)))
+                    self.logger.info("         This package will be deleted in a future version of the RPMng driver.")
+
+        cmdrc, output = self.cmd.run(pkgtool % " ".join(erase_args))
+        if cmdrc == 0:
+            self.modified += packages
+            for pkg in erase_args:
+                self.logger.info("Deleted %s" % (pkg))
+        else:
+            self.logger.info("Bulk erase failed with errors:")
+            self.logger.debug("Erase results = %s" % output)
+            self.logger.info("Attempting individual erase for each package.")
+            for pkg in packages:
+                pkg_modified = False
+                for inst in pkg:
+                    if pkg.get('name') != 'gpg-pubkey':
+                        pkg_arg = pkg.get('name') + '-'
+                        if inst.haskey('epoch'):
+                            pkg_arg = pkg_arg + inst.get('epoch') + ':'
+                        pkg_arg = pkg_arg + inst.get('version') + '-' + inst.get('release')
+                        if inst.haskey('arch'):
+                            pkg_arg = pkg_arg + '.' + inst.get('arch')
+                    else:
+                        self.logger.info("WARNING: gpg-pubkey package not in configuration %s %s"\
+                                                 % (pkgspec.get('name'), self.str_evra(pkgspec)))
+                        self.logger.info("         This package will be deleted in a future version of the RPMng driver.")
+
+                    cmdrc, output = self.cmd.run(self.pkgtool % pkg_arg)
+                    if cmdrc == 0:
+                        pkg_modified = True
+                        self.logger.info("Deleted %s" % pkg_arg)
+                    else:
+                        self.logger.error("unable to delete %s" % pkg_arg)
+                        self.logger.debug("Failure = %s" % output)
+                if pkg_modified == True:
+                    self.modified.append(pkg)
+
+
+        self.RefreshPackages()
+        self.extra = self.FindExtraPackages()