summaryrefslogtreecommitdiffstats
path: root/tools/create-debian-pkglist.py
diff options
context:
space:
mode:
authorNarayan Desai <desai@mcs.anl.gov>2006-08-17 21:30:15 +0000
committerNarayan Desai <desai@mcs.anl.gov>2006-08-17 21:30:15 +0000
commit8c1d583194f53633beddd419b8ef5987b0785b74 (patch)
treef6f9fb8a8bcc84e0559421d8de9204cd704d2152 /tools/create-debian-pkglist.py
parent286fae75648b2820f37d40a4a98d9c81093f64b6 (diff)
downloadbcfg2-8c1d583194f53633beddd419b8ef5987b0785b74.tar.gz
bcfg2-8c1d583194f53633beddd419b8ef5987b0785b74.tar.bz2
bcfg2-8c1d583194f53633beddd419b8ef5987b0785b74.zip
Add new pkglist repo generator
git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@2072 ce84e21b-d406-0410-9b95-82705330c041
Diffstat (limited to 'tools/create-debian-pkglist.py')
-rwxr-xr-xtools/create-debian-pkglist.py111
1 files changed, 111 insertions, 0 deletions
diff --git a/tools/create-debian-pkglist.py b/tools/create-debian-pkglist.py
new file mode 100755
index 000000000..3d78384e7
--- /dev/null
+++ b/tools/create-debian-pkglist.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+
+'''Build debian/ubuntu package indexes'''
+__revision__ = '$Id: $'
+
+import glob, gzip, lxml.etree, os, re, urllib, cStringIO, sys
+
+def debug(msg):
+ '''print debug messages'''
+ if '-v' in sys.argv:
+ sys.stdout.write(msg)
+
+
+def processSource(prefix, source, dists, archs, prio, groups, packages):
+ '''Build package indices for source'''
+ filename = "%s/%s.xml" % (prefix, source.replace('/', '_'))
+ try:
+ os.stat(filename)
+ filename += '~'
+ except:
+ pass
+ output = open(filename, 'w')
+ depfile = open(filename + '-deps', 'w')
+ groupinfo = "".join(['<Group name="%s">' % group for group in groups])
+ output.write('<PackageList priority="%s" type="deb">%s\n' % (prio, groupinfo))
+ for dist in dists:
+ pkgdata = {}
+ pkgdeps = {}
+ for arch in archs:
+ url = "%s/%s/binary-%s/Packages.gz" % (source, dist, arch)
+ debug("Processing url %s\n" % (url))
+ data = urllib.urlopen(url)
+ buf = cStringIO.StringIO(''.join(data.readlines()))
+ reader = gzip.GzipFile(fileobj=buf)
+ for line in reader.readlines():
+ if line[:8] == 'Package:':
+ pkgname = line.split(' ')[1].strip()
+ elif line[:8] == 'Version:':
+ version = line.split(' ')[1].strip()
+ if pkgdata.has_key(pkgname):
+ pkgdata[pkgname][arch] = version
+ else:
+ pkgdata[pkgname] = {arch:version}
+ elif line[:8] == 'Depends:':
+ deps = re.sub(',', '', re.sub('\(.*\)', '', line)).split()[1:]
+ if pkgdeps.has_key(pkgname):
+ pkgdeps[pkgname][arch] = deps
+ else:
+ pkgdeps[pkgname] = {arch:deps}
+ else:
+ continue
+ coalesced = 0
+ for pkg in pkgdata.keys():
+ data = pkgdata[pkg].values()
+ if data.count(data[0]) == len(data):
+ output.write('<Package name="%s" version="%s"/>\n' % (pkg, data[0]))
+ coalesced += 1
+ del pkgdata[pkg]
+ for pkg in pkgdeps.keys():
+ data = pkgdeps[pkg].values()
+ if data.count(data[0]) == len(data):
+ elt = lxml.etree.Element("Package", name=pkg)
+ [lxml.etree.SubElement(elt, "Package", name=dep) for dep in data[0]]
+ depfile.write(lxml.etree.tostring(elt) + '\n')
+ del pkgdeps[pkg]
+ # now we need to do per-arch entries
+ perarch = 0
+ if pkgdata:
+ for arch in archs:
+ output.write('<Group name="%s">\n' % (arch))
+ for pkg in pkgdata.keys():
+ if pkgdata[pkg].has_key(arch):
+ output.write('<Package name="%s" version="%s"/>\n' % (pkg, pkgdata[pkg][arch]))
+ perarch += 1
+ output.write('</Group>\n')
+ debug("Got %s coalesced, %s per-arch\n" % (coalesced, perarch))
+ closegroup = "".join(['</Group>' for group in groups])
+ output.write('%s</PackageList>\n' % (closegroup))
+ output.close()
+ depfile.close()
+ if filename[-1] == '~':
+ old = open(filename[:-1]).read()
+ new = open(filename).read()
+ if old != new:
+ debug("%s has changed; installing new version\n" % (filename[:-1]))
+ os.rename(filename, filename[:-1])
+ else:
+ debug("%s has not changed\n" % (filename[:-1]))
+
+if __name__ == '__main__':
+ hprefix = '/tmp'
+ rprefix = '/home/desai/data/bcfg2'
+ packages = []
+ for fn in glob.glob("%s/Bundler/*.xml" % rprefix) + glob.glob("%s/Base/*.xml" % rprefix):
+ doc = lxml.etree.parse(fn)
+ [packages.append(pkg.get('name')) for pkg in doc.findall('.//Package') if
+ pkg.get('name') not in packages]
+
+ print len(packages)
+
+ SOURCES = [('http://security.debian.org/dists/stable/updates',
+ ['main', 'contrib', 'non-free'], ['i386'], '10', ['debian-sarge']),
+ ('http://volatile.debian.net/debian-volatile/dists/stable/volatile',
+ ['main', 'contrib', 'non-free'], ['i386'], '20', ['debian-sarge']),
+ ('http://us.archive.ubuntu.com/ubuntu/dists/dapper',
+ ['main', 'multiverse', 'universe'], ['i386', 'amd64'], 0, ['ubuntu-dapper']),
+ ('http://netzero.mcs.anl.gov/disks/distro/ubuntu/dists/dapper',
+ ['main', 'multiverse', 'universe'], ['i386', 'amd64'], 0, ['ubuntu-dapper'])]
+
+ for src, dsts, ars, pri, grps in SOURCES:
+ processSource(hprefix, src, dsts, ars, pri, grps, packages)