diff options
| -rw-r--r-- | tools/create-debian-pkglist.pl | 536 | ||||
| -rw-r--r-- | tools/debian-build-cron.sh | 15 | ||||
| -rw-r--r-- | tools/debian-repo-cleaner | 3 | ||||
| -rw-r--r-- | tools/local-deb-repo-maker | 28 | ||||
| -rw-r--r-- | tools/rsync-debian-repo | 4 |
5 files changed, 586 insertions, 0 deletions
diff --git a/tools/create-debian-pkglist.pl b/tools/create-debian-pkglist.pl index e69de29bb..8cf6409dc 100644 --- a/tools/create-debian-pkglist.pl +++ b/tools/create-debian-pkglist.pl @@ -0,0 +1,536 @@ +#!/usr/bin/perl + +#this is some setup that I have to do to make everything more localized below +#get all the packages/versions from security.debian.org. this is kinda hacky since we don't locally mirror that. I would like to for speed +#issues but since we are low on disk space I am not going to bother. +#these are the files that I will need to use to get the right info. + +system( "wget http://security.debian.org/dists/stable/updates/main/binary-i386/Packages -O /tmp/main.Packages -q" ); +system( "wget http://security.debian.org/dists/stable/updates/contrib/binary-i386/Packages -O /tmp/contrib.Packages -q" ); +system( "wget http://security.debian.org/dists/stable/updates/non-free/binary-i386/Packages -O /tmp/nonfree.Packages -q" ); + +#i now have all the files I need locally so I can do my opens properly. + +open( MAIN, "/cluster/distro/debian/dists/stable/main/binary-i386/Packages" ); +open( CONTRIB, "/cluster/distro/debian/dists/stable/contrib/binary-i386/Packages"); +open( NONFREE, "/cluster/distro/debian/dists/stable/non-free/binary-i386/Packages"); +open( NONMAIN, "/cluster/distro/debian-non-US/dists/stable/non-US/main/binary-i386/Packages" ); +open( NONCONTRIB, "/cluster/distro/debian-non-US/dists/stable/non-US/contrib/binary-i386/Packages"); +open( NONNONFREE, "/cluster/distro/debian-non-US/dists/stable/non-US/non-free/binary-i386/Packages"); +open( SECMAIN, "/tmp/main.Packages"); +open( SECCONTRIB, "/tmp/contrib.Packages"); +open( SECNONFREE, "/tmp/nonfree.Packages"); + +open( LOCAL, "/cluster/debian/woody/Packages"); +open( OUTFILE, ">/cluster/bcfg/images/debian-3.0/pkglist.xml" ); + +print OUTFILE "<PackageList image='debian-stable'>\n"; +print OUTFILE "<Location uri='http://grandpoobah.mcs.anl.gov:8080/disks/cluster/disto/debian'>\n"; + +@mypackages = []; + +#get all the data from the local repo that we maintain. This is the highest precedence in package priority +while( $line = <LOCAL> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + push @mypackages, $basename; + $found=0; + while( !$found ){ + $line = <LOCAL>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + $found =1; + } + } + } +} + +#now i will do all the security packages and add them to the my packages if they aren't already there. If that makes sense. +#basically i am enforceing the priority by processing the package lists inorder. + +$known_package=0; +while( $line = <SECMAIN> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + #print "I already saw: $basename\n"; + $known_package =1; + } + } + $found = 0; + while( !$found ){ + $line = <SECMAIN>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + #print $version."\n"; + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + push @mypackages, $basename; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <SECCONTRIB> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + #print "I already saw: $basename\n"; + $known_package =1; + } + } + $found = 0; + while( !$found ){ + $line = <SECCONTRIB>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + #print $version."\n"; + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + push @mypackages, $basename; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <SECNONFREE> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + #print "I already saw: $basename\n"; + $known_package =1; + } + } + $found = 0; + while( !$found ){ + $line = <SECNONFREE>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + #print $version."\n"; + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + push @mypackages, $basename; + } + $known_package = 0; + $found =1; + } + } + } +} + + +#now for all the regular packages in the mirrored repos + +$known_package=0; +while( $line = <MAIN> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + #print "I already saw: $basename\n"; + $known_package =1; + } + } + $found = 0; + while( !$found ){ + $line = <MAIN>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <CONTRIB> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + $known_package =1; + #print "I already saw: $basename\n"; + } + } + $found=0; + while( !$found ){ + $line = <CONTRIB>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <NONFREE> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + $known_package =1; + #print "I already saw: $basename\n"; + } + } + $found=0; + while( !$found ){ + $line = <NONFREE>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <NONMAIN> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + #print "I already saw: $basename\n"; + $known_package =1; + } + } + $found = 0; + while( !$found ){ + $line = <NONMAIN>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <NONCONTRIB> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + $known_package =1; + #print "I already saw: $basename\n"; + } + } + $found=0; + while( !$found ){ + $line = <NONCONTRIB>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <NONNONFREE> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + $known_package =1; + #print "I already saw: $basename\n"; + } + } + $found=0; + while( !$found ){ + $line = <NONNONFREE>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + } + $known_package = 0; + $found =1; + } + } + } +} + +print OUTFILE "</Location>\n</PackageList>\n"; + + + +close(OUTFILE); +close(MAIN); +close(CONTRIB); +close(NONFREE); +close(NONMAIN); +close(NONCONTRIB); +close(NONNONFREE); +close(SECMAIN); +close(SECCONTRIB); +close(SECNONFREE); +close(LOCAL); + + +#now i will do the same thing but for the sarge build + #i now have all the files I need locally so I can do my opens properly. + +open( MAIN, "/cluster/distro/debian/dists/sarge/main/binary-i386/Packages" ); +open( CONTRIB, "/cluster/distro/debian/dists/sarge/contrib/binary-i386/Packages"); +open( NONFREE, "/cluster/distro/debian/dists/sarge/non-free/binary-i386/Packages"); +open( NONMAIN, "/cluster/distro/debian-non-US/dists/sarge/non-US/main/binary-i386/Packages" ); +open( NONCONTRIB, "/cluster/distro/debian-non-US/dists/sarge/non-US/contrib/binary-i386/Packages"); +open( NONNONFREE, "/cluster/distro/debian-non-US/dists/sarge/non-US/non-free/binary-i386/Packages"); + +open( LOCAL, "/cluster/debian/sarge/Packages"); +open( OUTFILE, ">/cluster/bcfg/images/debian-sarge/pkglist.xml" ); + +print OUTFILE "<PackageList image='debian-sarge'>\n"; +print OUTFILE "<Location uri='http://grandpoobah.mcs.anl.gov:8080/disks/cluster/disto/debian'>\n"; + +@mypackages = []; + +#get all the data from the local repo that we maintain. This is the highest precedence in package priority +while( $line = <LOCAL> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + push @mypackages, $basename; + $found=0; + while( !$found ){ + $line = <LOCAL>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + $found =1; + } + } + } +} + + + +#now for all the regular packages in the mirrored repos + +$known_package=0; +while( $line = <MAIN> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + #print "I already saw: $basename\n"; + $known_package =1; + } + } + $found = 0; + while( !$found ){ + $line = <MAIN>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + push @mypackages, $basename; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <CONTRIB> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + $known_package =1; + #print "I already saw: $basename\n"; + } + } + $found=0; + while( !$found ){ + $line = <CONTRIB>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <NONFREE> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + $known_package =1; + #print "I already saw: $basename\n"; + } + } + $found=0; + while( !$found ){ + $line = <NONFREE>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <NONMAIN> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + #print "I already saw: $basename\n"; + $known_package =1; + } + } + $found = 0; + while( !$found ){ + $line = <NONMAIN>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <NONCONTRIB> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + $known_package =1; + #print "I already saw: $basename\n"; + } + } + $found=0; + while( !$found ){ + $line = <NONCONTRIB>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + } + $known_package = 0; + $found =1; + } + } + } +} + +$known_package=0; +while( $line = <NONNONFREE> ){ + if( $line =~ /^Package:/ ){ + ($filler,$basename)=split( ' ', $line ); + #print $basename."\n"; + foreach $package ( @mypackages ){ + if( $basename eq $package ){ + $known_package =1; + #print "I already saw: $basename\n"; + } + } + $found=0; + while( !$found ){ + $line = <NONNONFREE>; + if( $line =~ /^Version:/ ){ + ($filler,$version)=split( ' ', $line ); + + if ( ! $known_package ){ + print OUTFILE "\t<Package name=\"".$basename."\" version=\"".$version."\"/>\n" ; + } + $known_package = 0; + $found =1; + } + } + } +} + +print OUTFILE "</Location>\n</PackageList>\n"; + + + +close(OUTFILE); +close(MAIN); +close(CONTRIB); +close(NONFREE); +close(NONMAIN); +close(NONCONTRIB); +close(NONNONFREE); +close(LOCAL); + diff --git a/tools/debian-build-cron.sh b/tools/debian-build-cron.sh index e69de29bb..055f5b547 100644 --- a/tools/debian-build-cron.sh +++ b/tools/debian-build-cron.sh @@ -0,0 +1,15 @@ +#!/bin/sh +if /root/bin/rsync-debian-repo ; then + mv /cluster/bcfg/images/debian-3.0/pkglist.xml /cluster/bcfg/images/debian-3.0/pkglist.xml.old; + if /root/bin/create-debian-pkglist.pl ; then + /etc/init.d/bcfgd restart + else + mv /cluster/bcfg/images/debian-3.0/pkglist.xml.old /cluster/bcfg/images/debian-3.0/pkglist.xml; + echo "there was a problem with creating the new pkglist.xml"; + exit 1; + fi +else + echo "repo not synced and bcfg not updates"; + exit 1; +fi +exit 0; diff --git a/tools/debian-repo-cleaner b/tools/debian-repo-cleaner index e69de29bb..3962f6f89 100644 --- a/tools/debian-repo-cleaner +++ b/tools/debian-repo-cleaner @@ -0,0 +1,3 @@ +#!/bin/bash + +for i in alpha arm hppa ia64 m68k mipsel powerpc s390 sparc mips; do find /cluster/distro/debian -name \*${i}.deb | xargs rm -f; done diff --git a/tools/local-deb-repo-maker b/tools/local-deb-repo-maker index e69de29bb..ab0771644 100644 --- a/tools/local-deb-repo-maker +++ b/tools/local-deb-repo-maker @@ -0,0 +1,28 @@ +#!/bin/sh + +HOSTNAME=`/bin/hostname` + +#this is to pull the mplayer/avi stuff. +if [ ${HOSTNAME} == "debian-test" ]; then + cd /tmp + ncftpget ftp://ftp.nerim.net/debian-marillat/dists/stable/main/binary-i386/Packages + grep Package: Packages | awk '{print "ncftpget ftp://ftp.nerim.net/debian-marillat/dists/stable/main/binary-i386/" $2"\\*"}' > marillat-sync + cd /sandbox/woody/ + sh /tmp/marillat-sync + cd /sandbox + dpkg-scanpackages woody /dev/null > woody/Packages + cd woody + gzip -c Packages > Packages.gz +elif [ ${HOSTNAME} == "wonkarific.mcs.anl.gov" ]; then +#this is to pull the testing stuff. + + cd /tmp + ncftpget ftp://ftp.nerim.net/debian-marillat/dists/testing/main/binary-i386/Packages + grep Package: Packages | awk '{print "ncftpget ftp://ftp.nerim.net/debian-marillat/dists/testing/main/binary-i386/" $2"\\*"}' > marillat-sync + cd /sandbox/sarge/ + sh /tmp/marillat-sync + cd /sandbox + dpkg-scanpackages sarge /dev/null > sarge/Packages + cd sarge + gzip -c Packages > Packages.gz +fi diff --git a/tools/rsync-debian-repo b/tools/rsync-debian-repo index e69de29bb..c7902b3ef 100644 --- a/tools/rsync-debian-repo +++ b/tools/rsync-debian-repo @@ -0,0 +1,4 @@ +#!/bin/sh +/usr/bin/rsync -a --exclude="**/*alpha*" --exclude="**/*arm*" --exclude="**/*hppa*" --exclude="**/*ia64*" --exclude="**/*m68k*" --exclude="**/*mipsel*" --exclude="**/*powerpc*" --exclude="**/*s390*" --exclude="**/*sparc*" --exclude="**/*mips*" rsync://mirror.mcs.anl.gov/debian/ /cluster/distro/debian/ +/usr/bin/rsync -a --exclude="**/*alpha*" --exclude="**/*arm*" --exclude="**/*hppa*" --exclude="**/*ia64*" --exclude="**/*m68k*" --exclude="**/*mipsel*" --exclude="**/*powerpc*" --exclude="**/*s390*" --exclude="**/*sparc*" --exclude="**/*mips*" rsync://non-us.debian.org/debian-non-US/ /cluster/distro/debian-non-US/ +rsync -a rsync://debian-test/deb-repo/ /cluster/debian/woody/ |