RPMng/YUMng driver update from mbrady

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@3055 ce84e21b-d406-0410-9b95-82705330c041

1 files changed, 399 insertions, 0 deletions

diff --git a/tools/pkgmgr_gen.py b/tools/pkgmgr_gen.py
new file mode 100755
index 000000000..d6d55562c
--- /dev/null
+++ b/tools/pkgmgr_gen.py
@@ -0,0 +1,399 @@
+#!/usr/bin/python
+""" Program to generate a bcfg2 Pkgmgr configuration file from a list 
+    of directories that contain RPMS.  
+    
+    All versions or only the latest may be included in the output.
+    rpm.labelCompare is used to compare the package versions, so that 
+    a proper rpm version comparison is done (epoch:version-release).
+
+    The output file may be formated for use with the RPM or Yum 
+    bcfg2 client drivers.  The output can also contain the PackageList
+    and nested group headers.
+"""
+__revision__ = '$Revision: $'
+import sys
+import os
+import rpm
+import optparse
+import datetime
+import glob
+
+def info(object, spacing=10, collapse=1):
+    """Print methods and doc strings.
+       Takes module, class, list, dictionary, or string.
+    """
+    methodList = [method for method in dir(object) 
+                         if callable(getattr(object, method))]
+    processFunc = collapse and (lambda s: " ".join(s.split())) or (lambda s: s)
+    print "\n".join(["%s %s" %
+                      (method.ljust(spacing),
+                       processFunc(str(getattr(object, method).__doc__)))
+                     for method in methodList])
+
+def readRpmHeader(ts, filename):
+    """ 
+        Read an rpm header from an RPM file. 
+    """
+    try:
+        fd = os.open(filename, os.O_RDONLY)
+    except:
+        print 'Failed to open RPM file %s' % filename
+
+    h = ts.hdrFromFdno(fd)
+    os.close(fd)
+    return h
+
+def sortedDictValues(adict):
+    """ 
+        Sort a dictionary by its keys and return the items in sorted key order. 
+    """
+    keys = adict.keys()
+    keys.sort()
+    return map(adict.get, keys)
+
+def cmpRpmHeader(a, b):
+    """
+        cmp() implemetation suitable for use with sort.
+
+        a and b are dictionaries as created by loadRpms().  Comparison is made 
+        by package name and then by the full rpm version (epoch, version, release).  
+        rpm.labelCompare is used for the version part of the comparison.
+    """
+    n1 = str(a['name'])
+    e1 = str(a['epoch'])
+    v1 = str(a['version'])
+    r1 = str(a['release'])
+    n2 = str(b['name'])
+    e2 = str(b['epoch'])
+    v2 = str(b['version'])
+    r2 = str(b['release'])
+
+    ret = cmp(n1, n2)
+    if ret == 0:
+        ret = rpm.labelCompare((e1, v1, r1),(e2, v2, r2))
+    return ret
+
+def loadRpms(dirs):
+    """
+       dirs is a list of directories to search for rpms.
+
+       Builds a dictionary keyed by the package name.  Dictionary item is a list, 
+       one entry per package instance found.
+       
+       The list entries are dictionaries.  Keys are 'filename', 'mtime' 'name', 
+       'arch', 'epoch', 'version' and 'release'.  
+ 
+       e.g.
+
+       packages = {
+       'bcfg2' : [
+           {'filename':'bcfg2-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2', 
+            ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'}
+           {'filename':'bcfg2-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2', 
+            ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}],
+       'bcfg2-server' : [
+           {'filename':'bcfg2-server-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2-server', 
+            ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'}
+           {'filename':'bcfg2-server-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2-server', 
+            ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}],
+       }
+       
+    """
+    packages = {}
+    ts = rpm.TransactionSet()
+    vsflags = 0
+    vsflags |= rpm._RPMVSF_NODIGESTS
+    vsflags |= rpm._RPMVSF_NOSIGNATURES
+    ovsflags = ts.setVSFlags(vsflags)
+    for dir in dirs:
+
+        if options.verbose:
+            print 'Scanning directory: %s' % dir
+
+        for file in [files for files in os.listdir(dir) 
+                           if files.endswith('.rpm')]:
+
+            filename = os.path.join( dir, file )
+             
+            # Get the mtime of the RPM file.
+            file_mtime = datetime.date.fromtimestamp(os.stat(filename).st_mtime)
+
+            # Get the RPM header
+            header = readRpmHeader( ts, filename ) 
+
+            # Get what we are interesting in out of the header.
+            name = header[rpm.RPMTAG_NAME]
+            epoch = header[rpm.RPMTAG_EPOCH]
+            version = header[rpm.RPMTAG_VERSION]
+            release = header[rpm.RPMTAG_RELEASE]
+            subarch = header[rpm.RPMTAG_ARCH]
+
+            # Only load RPMs with subarchitectures as calculated from the --archs option.
+            if subarch in subarchs or 'all' in subarchs:
+                
+                # Store what we want in our structure.
+                if name in packages:
+                    packages[name].append({'filename':file, 'mtime':file_mtime, 'name':name, 'arch':subarch, \
+                                          'epoch':epoch, 'version':version, 'release':release})
+                else:
+                    packages[name] = [{'filename':file, 'mtime':file_mtime, 'name':name, 'arch':subarch, \
+                                      'epoch':epoch, 'version':version, 'release':release}]
+
+                # Print '.' for each package. stdio is line buffered, so have to flush it.
+            if options.verbose:
+                sys.stdout.write('.')
+                sys.stdout.flush()
+        if options.verbose:
+            sys.stdout.write('\n')
+
+    return packages
+
+def printInstance(instance, group_count):
+    """
+        Print the details for a package instance with the appropriate indentation and 
+        in the specified format (rpm or yum).
+    """
+    group_count = group_count + 1
+    name = instance['name']
+    epoch = instance['epoch']
+    version = instance['version']
+    release = instance['release']
+    arch = instance['arch']
+
+    output_line = ''
+    if options.format == 'rpm':
+        output_line = '%s<Instance simplefile=\'%s\' ' % ( indent * group_count , instance['filename'])
+    else:
+        output_line = '%s<Instance ' % (indent * group_count)
+
+    if epoch:
+        output_line += 'epoch=\'%s\' ' % (epoch)
+
+    output_line += 'version=\'%s\' release=\'%s\' arch=\'%s\'/>\n' % ( version, release, arch)
+    output.write(output_line)
+
+def printPackage(entry, group_count):
+    """
+       Print the details of a package with the appropriate indentation.
+       Only the specified (all or latest) release(s) is printed.
+
+       entry is a single package entry as created in loadRpms().
+    """
+    output.write('%s<Package name=\'%s\' type=\'%s\'>\n' \
+                  % ( group_count * indent, entry[0]['name'], options.format) )
+
+    subarch_dict = {}
+    arch_dict = {}
+    # Split instances of this package into subarchitectures.
+    for instance in entry:
+        if instance['arch'] in subarch_dict:
+            subarch_dict[instance['arch']].append(instance)
+        else:
+            subarch_dict[instance['arch']] = [ instance ]
+    
+        # Keep track of the subarchitectures we have found in each architecture.
+        if subarch_mapping[instance['arch']] in arch_dict:
+            if instance['arch'] not in arch_dict[subarch_mapping[instance['arch']]]:
+                arch_dict[subarch_mapping[instance['arch']]].append(instance['arch'])
+        else:
+            arch_dict[subarch_mapping[instance['arch']]] = [ instance['arch'] ]
+    
+    # Only keep the 'highest' subarchitecture in each architecture.
+    for arch in arch_dict.iterkeys():
+        if len(arch_dict[arch]) > 1:
+            arch_dict[arch].sort()
+            for s in arch_dict[arch][:-1]:
+               del subarch_dict[s]
+    
+    # Sort packages within each architecture into version order
+    for arch in subarch_dict:
+        subarch_dict[arch].sort(cmpRpmHeader)
+ 
+        if options.release == 'all':
+            # Output all instances
+            for header in subarch_dict[arch]:
+                printInstance(header, group_count)
+        else:
+            # Output the latest
+            printInstance(subarch_dict[arch][-1], group_count)
+
+    output.write('%s</Package>\n' % ( group_count * indent ) )
+
+def main():
+
+    if options.verbose:
+        print 'Loading package headers'
+
+    package_dict = loadRpms(search_dirs)
+
+    if options.verbose:
+        print 'Processing package headers'
+
+    if options.pkgmgrhdr:
+        if options.format == "rpm":
+            output.write("<PackageList uri='%s' priority='%s' type='rpm'>\n" % ( options.uri, options.priority ))
+        else:
+            output.write("<PackageList priority='%s' type='yum'>\n" %( options.priority ))
+
+    group_count = 1
+    if groups_list:
+        for group in groups_list:
+            output.write("%s<Group name='%s'>\n" % ( indent * group_count , group ) )
+            group_count = group_count + 1
+
+    # Process packages in name order
+    for package_entry in sortedDictValues(package_dict):
+        printPackage(package_entry, group_count)
+
+    if groups_list:
+        group_count = group_count - 1
+        while group_count:
+            output.write( '%s</Group>\n' % ( indent * group_count ) )
+            group_count = group_count - 1
+
+    if options.pkgmgrhdr:
+        output.write( '</PackageList>\n' )
+ 
+    if options.verbose:
+        print '%i package instances were processed' % len(package_dict)
+
+
+if __name__ == "__main__":
+
+    p = optparse.OptionParser()
+
+    p.add_option('--archs', '-a',  action='store', \
+                                   default='all', \
+                                   type='string', \
+                                   help='''Comma separated list of subarchitectures to include. 
+                                           The highest subarichitecture required in an 
+                                           architecture group should specified.   Lower 
+                                           subarchitecture packages will be loaded if that 
+                                           is all that is available. e.g. The higher of i386, 
+                                           i486 and i586 packages will be loaded if -a i586 
+                                           is specified. (Default: all).
+                                        ''')
+
+    p.add_option('--rpmdirs', '-d', action='store', 
+                                   default='.', \
+                                   type='string', \
+                                   help='Comma separated list of directories to scan for RPMS. Wilcards are permitted. (Default: .)')
+    
+    p.add_option('--enddate', '-e', action='store', \
+                                   type='string', \
+                                   help='End date for RPM file selection.')
+
+    p.add_option('--format', '-f', action='store', \
+                                   default='yum', \
+                                   type='choice', \
+                                   choices=('yum','rpm'), \
+                                   help='Format of the Output. Choices are yum or rpm. (Default: yum)')
+
+    p.add_option('--groups', '-g', action='store', \
+                                   type='string', \
+                                   help='List of comma separated groups to nest Package entities in.')
+                                  
+    p.add_option('--indent', '-i', action='store', \
+                                   default=4, \
+                                   type='int', \
+                                   help='Number of leading spaces to indent nested entries in the output. (Default:4)')
+
+    p.add_option('--outfile', '-o', action='store', \
+                                   type='string', \
+                                   help='Output file name.')
+                                  
+    p.add_option('--pkgmgrhdr', '-P', action='store_true', \
+                                   help='Include PackageList header in output.')
+                                  
+    p.add_option('--priority', '-p', action='store', \
+                                   default=0, \
+                                   type='int', \
+                                   help='Value to set priority attribute in the PackageList Tag. (Default: 0)')
+
+    p.add_option('--release', '-r', action='store', \
+                                   default='latest', \
+                                   type='choice', \
+                                   choices=('all','latest'), \
+                                   help='Which releases to include in the output. Choices are all or latest.  (Default: latest).')
+    
+    p.add_option('--startdate', '-s', action='store', \
+                                   type='string', \
+                                   help='Start date for RPM file selection.')
+                                  
+    p.add_option('--uri', '-u',    action='store', \
+                                   type='string', \
+                                   help='URI for PackageList header required for RPM format ouput.')
+
+    p.add_option('--verbose', '-v', action='store_true', \
+                                    help='Enable verbose output.')
+
+    options, arguments = p.parse_args()
+
+    if options.pkgmgrhdr and options.format == 'rpm' and not options.uri:
+        print "Option --uri must be specified to produce a PackageList Tag for rpm formatted files."
+        sys.exit(1)
+
+    # Set up list of directories to search
+    search_dirs = []
+    for d in options.rpmdirs.split(','):
+        search_dirs += glob.glob(d) 
+
+    # Set up list of architectures to include and some mappings
+    # to use later.
+    arch_mapping = {'x86':['i686','i586','i486','i386'], 
+                    'x86_64':['x86_64'], 
+                    'ia64':['ia64'], 
+                    'ppc':['ppc'], 
+                    'ppc64':['ppc64'], 
+                    'sparc':['sparc'], 
+                    'noarch':['noarch']}
+    subarch_mapping = {'i686':'x86', 'i586':'x86', 'i486':'x86', 'i386':'x86', 
+                       'x86_64':'x86_64', 
+                       'ia64':'ia64', 
+                       'ppc':'ppc', 
+                       'ppc64':'ppc64', 
+                       'sparc':'sparc', 
+                       'noarch':'noarch' }
+    commandline_subarchs = options.archs.split(',')
+    arch_list = []
+    subarchs = []
+    if 'all' in commandline_subarchs:
+        subarchs.append('all')
+    else:
+        for s in commandline_subarchs:
+            if s not in subarch_mapping:
+                print 'Error: Invalid subarchitecture specified: ', s
+                sys.exit(1)
+            # Only allow one subarchitecture per architecture to be specified.
+            if s not in arch_list:
+                arch_list.append(s)
+        
+                # Add subarchitectures lower than the one specified to the list.
+                # e.g. If i486 is specified this will add i386 to the list of
+                # subarchitectures to load.
+                i = arch_mapping[subarch_mapping[s]].index(s)
+                #if i != len(arch_mapping[subarch_mapping[s]]):
+                subarchs += arch_mapping[subarch_mapping[s]][i:]
+            else:
+                print 'Error: Multiple subarchitecutes of the same architecture specified.'
+                sys.exit(1)
+
+    if options.verbose:
+        print 'The following directories will be scanned:'
+        for d in search_dirs:
+            print '    %s' % d
+
+    indent = ' ' * options.indent
+
+    if options.groups:
+        groups_list = options.groups.split(',')
+    else:
+        groups_list = None
+    
+    if options.outfile:
+        output = file(options.outfile, "w")
+    else:
+        output = sys.stdout
+
+    main()
+