So my third "git-worthy" project is called the Fedora ARM Image Installer. It will basically allow one to "easily" convert an SD-Card into a bootable ARM media device. It is written in Python (2 & 3 for Linux & Windows) and it is based on the PyQt4 GUI framework giving it a more native feel for each platform. Below are some screenshots and links!
Source Code (Git Hub)
Binary Program (Fedora People)
Fedora Wiki
So Fedora has been disconnecting me a lot lately from my home wifi and I think it’s because I have disabled a lot of background services that I don’t need. Anyway, I’m trying something new:
yum remove NetworkManager
mkdir /etc/wifi wpa_passphrase "your_ssid_here" > /etc/wifi/home.conf
Run this script as root when wanting to connect to wifi (assumes the wlan0 interface):
#!/bin/bash pkil=`which killall` intf=`which ifconfig` scan=`which iwlist` wpas=`which wpa_supplicant` dhcp=`which dhclient` ifce="wlan0" while true do ping -c 1 google.ca > /dev/null 2>&1 if [ $? -eq 0 ] then sleep 10 continue fi for file in /etc/wifi/* do $intf "$ifce" up ssid=`cat $file | grep -i 'ssid' | sed -e 's/^[ \t]*ssid="//g' -e 's/"[ \t]*$//g'` exis=`$scan "$ifce" scan | grep -i "$ssid"` if [ "$exis" != "" ] then echo "[`date`] connecting to [$ssid]..." $pkil -9 wpa_supplicant > /dev/null 2>&1 $pkil -9 dhclient > /dev/null 2>&1 $intf "$ifce" up $wpas -Dwext -i "$ifce" -c "$file" > /dev/null 2>&1 & $dhcp "$ifce" > /dev/null 2>&1 & break fi done sleep 10 done
Just wanted to see how many lines of Python it took me to make a generalized password bruteforce script…
import time
slst = "abcdefghijklmnopqrstuvwxyz"
slen = (len(slst) - 1)
dwrd = [0]
dlen = 1
numb = 0
last = time.time()
wait = 2
while (1):
# process password
i = 0
s = ""
while (i < dlen):
s = (s + slst[dwrd[i]])
i = (i + 1)
# print stats
numb = (numb + 1)
pres = time.time()
if ((pres - last) >= wait):
print("pw=[%s] @ [%d p/s]" % (s, numb / wait))
numb = 0
last = pres
# overflow increase
dwrd[dlen - 1] = (dwrd[dlen - 1] + 1)
i = (dlen - 1)
u = 1
while (i > -1):
if (dwrd[i] > slen):
dwrd[i] = 0
if ((i - 1) > -1):
dwrd[i - 1] = (dwrd[i - 1] + 1)
else:
u = 0
i = (i - 1)
# length expansion
if (u == 1):
i = 0
while (i < dlen):
dwrd[i] = 0
i = (i + 1)
dwrd.append(0)
dlen = (dlen + 1)
Here’s a small shell script based on dd’s output which helps to partition a flat file, split it up into its respective partitions so they can be formatted and put them all back together again.
#!/bin/bash
if [ "${1}" == "c" ]
then
if [ "${3}" == "" ]
then
echo "Usage: $0 $1 <filename> <filesize>"
exit 1
fi
echo "Creating new file [${2}]..."
dd if=/dev/zero of=${2} bs=${3}G count=1 > /dev/null 2>&1
echo "Partition the new flat file ( fdisk ${2} )"
echo
fi
if [ "${1}" == "s" ]
then
if [ "${2}" == "" ]
then
echo "Usage: $0 $1 <filename>"
exit 1
fi
unit=`fdisk -l "${2}" | grep -i '^Units' | sed -e 's/^.* \([0-9][0-9]*\) bytes.*$/\1/g'`
lase="-1"
numb=0
fdisk -l "${2}" | sed -e 's/^[ \t]*//g' -e 's/\*//g' | grep -i "^${2}[0-9][0-9]*" | while read line
do
curb=`echo "${line}" | awk '{ print $2 }'`
cure=`echo "${line}" | awk '{ print $3 }'`
let size="${cure} - ${curb} + 1"
let diff="${curb} - ${lase} - 1"
if [ ${diff} -gt 0 ]
then
let endi="${curb} - 1"
let skip="${lase} + 1"
echo "[${numb}] Unused space [${skip} - ${endi}] [${diff} blocks]..."
dd if=${2} bs=${unit} skip=${skip} count=${diff} of=${2}${numb} > /dev/null 2>&1
let numb="${numb} + 1"
fi
echo "[${numb}] Carving partition [${curb} - ${cure}] [${size} blocks]..."
dd if=${2} bs=${unit} skip=${curb} count=${size} of=${2}${numb} > /dev/null 2>&1
let numb="${numb} + 1"
lase="${cure}"
done
fdisk -l "${2}" | sed -e 's/^[ \t]*//g' -e 's/\*//g' | grep -i "^${2}[0-9][0-9]*" | tail -n 1 | while read line
do
endi=`echo "${line}" | awk '{ print $3 }'`
let rest="${endi} + 1"
numb="9"
echo "[${numb}] Copying ending [${rest} - <end>]..."
dd if=${2} bs=${unit} skip=${rest} of=${2}${numb} > /dev/null 2>&1
done
echo
echo "Create any filesystem needed ( mkfs.* ${2}[0-9] )"
echo "Mount any filesystem needed ( mount -o loop ${2}[0-9] /mnt/tmp[0-9] )"
echo "Copy any files needed ( cp -r source/* /mnt/tmp[0-9]/ )"
echo "Unmount any filesystem needed ( umount /mnt/tmp[0-9] )"
echo
fi
if [ "${1}" == "j" ]
then
if [ "${2}" == "" ]
then
echo "Usage: $0 $1 <filename>"
exit 1
fi
echo "Writing to file [${2}.join]..."
cat ${2}[0-9]* > ${2}.join
echo
fi
So I was trying to get my pc laptop setup for fudcon which meant a triple boot setup between windows, ubuntu and fedora. I had some troubles importing various important information about my music within iTunes.
Here is a script to remove duplicate songs in an iTunes folder:
import hashlib
import os
import re
import sys
def hash(fnam):
fobj = open(fnam, "r")
hobj = hashlib.sha256()
while (1):
data = fobj.read(2**20)
if (not data):
break
hobj.update(data)
return hobj.hexdigest()
fdic = {}
while (1):
file = sys.stdin.readline()
if (not file):
break
file = file.strip()
uniq = hash(file)
if (uniq in fdic.keys()):
print("removing:",file)
os.unlink(file)
else:
print("found:",file)
fdic[uniq] = file
This script attempts to merge an old iTunes library xml file with a newly imported one. The result is a merged iTunes library xml file that can then be re-imported into iTunes thus restoring some various meta-data about your music. Note: Before you re-import the merged xml playlist file, make sure you delete the current one and disable the check-box stating to copy songs into the media folder.
import os
import re
import sys
if (len(sys.argv) < 3):
print("Usage: %s <source> <merge>" % (sys.argv[0]))
sys.exit(0)
def outp(ordr, dict, line):
for item in ordr:
sys.stdout.write(dict[item][1])
sys.stdout.write(line)
sys.stdout.flush()
def xmlr(fnam, sdic={}):
alen = len(sys.argv)
slen = len(sdic.keys())
xord = []; xdic = {}
xobj = open(fnam, "r")
xdat = {}
while (1):
xlin = xobj.readline()
if (not xlin):
break
slin = xlin.strip("\0\t\r\n ")
xreg = re.match("^<key>([^<]+)</key>(<[^>]+>[^<]+<[^>]+>)$", slin)
try:
xkey = (xdic["Artist"][0] + xdic["Album"][0] + xdic["Name"][0])
except:
xkey = ""
if (slin == "<dict>"):
if (slen > 0):
outp(xord, xdic, xlin)
xord = []; xdic = {}
elif (xreg):
if (xreg.group(1) not in xord):
xord.append(xreg.group(1))
xdic[xreg.group(1)] = [xreg.group(2), xlin]
elif (slin == "</dict>"):
if (slen < 1):
xdat[xkey] = xdic
else:
for x in range(3, alen):
try:
xdic[sys.argv[x]] = sdic[xkey][sys.argv[x]]
if (sys.argv[x] not in xord):
xord.append(sys.argv[x])
except:
#print("error:",sys.exc_info())
pass
outp(xord, xdic, xlin)
xord = []; xdic = {}
else:
if (slen > 0):
outp(xord, xdic, xlin)
xord = []; xdic = {}
return xdat
sdat = xmlr(sys.argv[1])
ddat = xmlr(sys.argv[2], sdic=sdat)
Here’s an example of how to use the command:
python.exe /cygdrive/c/Users/jon/Desktop/itml.py /cygdrive/g/tmp/itunes/iTunes\ Music\ Library0.xml /cygdrive/g/tmp/itunes/iTunes\ Music\ Library.xml 'Date Added' 'Play Count' | unix2dos.exe | tee /cygdrive/g/tmp/itunes/iTunes\ Music\ Library2.xml && cp /cygdrive/g/tmp/itunes/iTunes\ Music\ Library2.xml /cygdrive/g/tmp/itunes/iTunes\ Music\ Library3.xml
Here it is, hopefully it works a bit better!
#!/bin/bash export x=1 export l=`find ./f15srpms -type f | grep -i '\.src\.rpm' | wc -l` koji list-tagged dist-f15 > /tmp/pkoji.txt find ./f15srpms -type f | sort -R | grep -i '\.src\.rpm' > /tmp/pkgsl.txt while true do while true do ak list-tagged dist-f15 > /tmp/akoji.txt ak list-tasks --mine --quiet | grep '^[0-9]' | grep -Ei ' (open|free) .* build' > /tmp/tasks.txt #echo "number of our tasks == `cat /tmp/tasks.txt | wc -l`" n=`cat /tmp/tasks.txt | wc -l` if [ $n -ge 10 ] then break fi p=`cat /tmp/pkgsl.txt | head -n "$x" | tail -n 1` q=`basename "$p" | sed -e 's/-[^-]*-[^-]*$//'` let x="($x % $l) + 1" #echo "checking pkg [$p] name [$q] not built on (akoji) and built on (pkoji)" c=`cat /tmp/akoji.txt /tmp/tasks.txt | grep -i "$q"` if [ "$c" != "" ] then continue fi c=`cat /tmp/pkoji.txt | grep -i "$q"` if [ "$c" == "" ] then continue fi echo "queing [$p] position [$x]" ak build dist-f15 "$p" --nowait --background done sleep 60 done
The WordPress.com stats helper monkeys prepared a 2011 annual report for this blog.
Here’s an excerpt:
A San Francisco cable car holds 60 people. This blog was viewed about 1,400 times in 2011. If it were a cable car, it would take about 23 trips to carry that many people.
The U.S. government is taking down websites using DNS poisoning/hijacking methods on pages that are claimed to be infringing on copyright law. I have re-written a *really* basic DNS server which re-implements DNS forwarding/proxying, DNS caching and DNS un-blacklisting of A/IN/IPv4 records. Basically, if one were to run their own DNS server (like the one below) and maintain a simple list of government-poisoned IP addresses for it, the DNS server would be able to provide (previously expired but still good) IPv4 addresses for the blocked site. This could essentially allow one to re-access a now blacklisted website.
Here is a screen shot of my computer using this basic DNS server to visit slashdot:
$ host slashdot.org slashdot.org has address 216.34.181.45 slashdot.org mail is handled by 10 mx.sourceforge.net.
And here is what it would look like if it were taken down via DNS poisoning:
$ host slashdot.org slashdot.org has address 74.81.170.110 slashdot.org has address 216.34.181.45 slashdot.org mail is handled by 10 mx.sourceforge.net.
Once this happens, all one has to do now is append the newly offending IP address to the DNS server’s black list text file like so:
# cat dnsb.txt 74.81.170.110
And if you perform another DNS request to the same (custom) DNS server, you get the last cached answers bringing you back to slashdot:
$ host slashdot.org slashdot.org has address 216.34.181.45 slashdot.org mail is handled by 10 mx.sourceforge.net.
The code below is simply proof-of-concept software and should *NOT* be used for production as I have implemented everything from scratch here and know very little about how DNS really works (i.e. message compression).
import os
import re
import socket
import sqlite3
import sys
import time
sqlo = sqlite3.connect("dnsd.db")
sqlc = sqlo.cursor()
def fixs(inpt):
outp = ""
for letr in inpt:
if (ord(letr) < 32):
outp += "."
else:
outp += letr
return outp
def rslv():
nlst = []
fobj = open("/etc/resolv.conf", "r")
dnsl = fobj.readlines()
for dnse in dnsl:
item = dnse.strip("\0\t\r\n ")
regx = re.match("^nameserver[ \t]*(.*)$", item)
if (regx):
addr = regx.group(1)
nlst.append(addr)
return nlst
def forw(addr, data):
resp = ""
serv = (addr, 53)
nobj = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
nobj.settimeout(5)
nobj.sendto(data, serv)
resp = nobj.recv(2**20)
except:
pass
try:
nobj.close()
except:
pass
return resp
def answ(data, deet=False):
# http://www.ietf.org/rfc/rfc1035.txt
requ = 13
null = (requ + data[requ:].find("\0"))
dtyp = (null + 5)
dlen = (dtyp + 10)
size = len(data)
name = fixs(data[requ:null])
begi = -1
endi = -1
ansl = []
while ((dlen + 1) < size):
dsiz = ((ord(data[dlen]) << 8) + ord(data[dlen + 1]))
dsig = (dtyp + 4)
if (((dlen + 5) < size) and (data[dtyp:dsig] == "\xc0\x0c\x00\x01")):
if (deet == True):
if (begi == -1):
begi = dtyp
else:
endi = (dlen + 1 + dsiz + 1)
addr = ""
for x in range(2, 2 + 4):
addr += ("." + str(ord(data[dlen + x])))
ansl.append([name, addr[1:]])
dtyp = (dlen + 1 + dsiz + 1)
dlen = (dtyp + 10)
if (begi == -1):
begi = (null + 5)
endi = begi
if (deet == False):
return ansl
else:
return [begi, endi]
def sqle(comd, outp=False):
global sqlo
global sqlc
try:
sqlc.execute(comd)
except:
pass
if (outp == False):
sqlo.commit()
else:
return sqlc.fetchall()
def filt():
try:
fobj = open("dnsb.txt", "r")
for line in fobj.readlines():
addr = line.strip("\0\t\r\n ")
sqle("DELETE FROM data WHERE addr = '%s';" % (addr))
fobj.close()
except:
return 0
return 1
def cche(name):
dbdl = sqle("SELECT * FROM data WHERE name = '%s';" % (name), outp=True)
outp = []
for dbdi in dbdl:
adrl = dbdi[1].split(".")
adrs = ""
for adri in adrl:
adrs += chr(int(adri))
outp.append([int(dbdi[2]), adrs])
outp.sort()
outp.reverse()
return outp
def shim(data, adrl):
hedn = [6, 8]
payn = answ(data, deet=True)
lead = (payn[0] + 12)
size = len(adrl)
dlen = (chr((size >> 8) & 0xff) + chr((size >> 0) & 0xff))
head = (data[:hedn[0]] + dlen + data[hedn[1]:payn[0]])
payl = ""
for adri in adrl:
payl += (data[payn[0]:lead] + adri[1])
payl += data[payn[1]:]
outp = (head + payl)
# beg: compression offset bypass
outp = (outp[:8] + chr(0) + chr(0) + outp[10:])
outp = (outp[:10] + chr(0) + chr(0) + outp[12:])
# end: compression offset bypass
# beg: compression offset fix
'''
x = 0
l = len(outp)
s = ((ord(data[6]) << 8) + ord(data[7]))
d = ((size - s) * 16)
if (d > 0):
while ((x + 1) < l):
o = ((ord(outp[x]) << 8) + ord(outp[x + 1]))
if ((o & 0xff00) == 0xc000):
o = (o & 0x00ff)
if (o > 0x0c):
o = (o + d)
outp = (outp[:x] + chr((o >> 8) & 0xff) + chr((o >> 0) & 0xff) + outp[x + 2:])
x = (x + 2)
'''
# end: compression offset fix
return outp
def debg(addr, requ, answ):
adrl = addr.split(".")
adrl[3] = "x"
prio = ""
for letr in answ:
prio += ("." + str(ord(letr)))
print("[%s] requ [%s] answ [%s, ...]" % (".".join(adrl), requ, prio[1:]))
def serv():
snam = ("", 53)
sobj = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sobj.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sobj.bind(snam)
while (1):
(data, addr) = sobj.recvfrom(2**20)
rlvl = rslv()
for rlvi in rlvl:
dnsr = forw(rlvi, data)
if (dnsr):
resl = answ(dnsr)
if (len(resl) > 0):
for resi in resl:
dnsq = re.sub("[^0-9A-Za-z\.\-]", "", resi[0])
dnsa = re.sub("[^0-9\.]", "", resi[1])
epch = str(int(time.time()))
dbdl = sqle("SELECT * FROM data WHERE name = '%s' AND addr = '%s';" % (dnsq, dnsa), outp=True)
if (len(dbdl) < 1):
sqle("INSERT INTO data VALUES ('%s', '%s', '0');" % (dnsq, dnsa))
sqle("UPDATE data SET time = '%s' WHERE name = '%s' AND addr = '%s';" % (epch, dnsq, dnsa))
filt()
dbdl = cche(dnsq)
if (len(dbdl) > 0):
debg(addr[0], resl[0][0], dbdl[0][1])
#a=open("a","w");a.write(dnsr);a.close();
dnsr = shim(dnsr, dbdl)
#b=open("b","w");b.write(dnsr);b.close();
sobj.sendto(dnsr, addr)
break
def priv():
# get username & os.setgid() & os.setuid()
pass
if (__name__ == "__main__"):
sqle("CREATE TABLE data (name VARCHAR(256), addr VARCHAR(256), time VARCHAR(256));")
priv()
serv()
So I haven’t had much to talk about lately due the amount of work given for the last few weeks remaining in school and the building of Fedora 15 for ARMv5 and ARMv7. However, in my spare time, I’ve been trying to contribute positively to the Koji Build System project by submitting mailing list items containing source code unified diff patches. Most of my posts/changes contain feature enhancements requested by other people that I found to be somewhat interesting to code. If you’re interested in viewing the mailing list items, the links can be found below:
Hosts-Page Colour Highlight: [ Ticket ] [ Patch ]
Hosts-Page “Life-Line” Links: [ Ticket ] [ Patch ]
Host-Info Latest-Builds Links: [ Ticket ] [ Patch ]
Koji-Client watch-logs “tail”: [ Ticket ] [ Patch ]
Koji-Hub RPMTAG_SIGSHA2 Hashing: [ Ticket ] [ Patch ]
Note: put this in a cron job as it may take a while to run. Use the output to generate a basic html page instead.
stats.php
<?php
date_default_timezone_set("EST");
$dta = date(DATE_RFC822);
$dis = "dist-f15";
$hua = "http://arm.koji.fedoraproject.org/kojihub";
$hub = "http://koji.fedoraproject.org/kojihub";
function p($n)
{
preg_match("/^(.*)-([^-]+)-([^-]+)$/", $n, $m);
return $m;
}
$a = array();
exec("/usr/bin/koji -s '".$hua."' list-tagged '".$dis."' --latest --quiet | /bin/awk '{ print $1 }'", $a);
$b = array();
exec("/usr/bin/koji -s '".$hub."' list-tagged '".$dis."' --latest --quiet | /bin/awk '{ print $1 }'", $b);
$err = array(); $unb = array(); $dif = array(); $wor = array(); $fin = array();
$t = array();
foreach ($a as $i)
{
$n = p($i);
array_push($t, $n[1]);
}
$u = array();
foreach ($b as $i)
{
$n = p($i);
$c = array_search($n[1], $t);
if ($c === FALSE)
{
array_push($u, $i);
}
else if ($a[$c] != $i)
{
array_push($dif, array($a[$c], $i));
}
else if ($a[$c] == $i)
{
array_push($fin, $i);
}
}
$c = 0; $s = ""; $l = count($u); $v = array();
for ($x = 0; $x < $l; $x += 1)
{
$s = ($s."'".$u[$x]."' ");
$c = ($c + 1);
if (($c >= 256) or ($x == ($l - 1)))
{
exec("/usr/bin/koji -s '".$hua."' buildinfo ".$s." | /bin/grep -Ei '^(genericerror|no such build|build|state): .*$'", $v);
$c = 0; $s = "";
}
}
array_push($v, "");
$b = ""; $s = "";
foreach ($v as $i)
{
$m = array(); $n = array();
if (preg_match("/^state: (.*)$/i", $i, $m))
{
$s = trim($m[1]);
}
else if (preg_match("/^.*: ([^ ]*).*$/i", $i, $n) or ($b != ""))
{
if ($b != "")
{
if ($s == "COMPLETE")
{
array_push($fin, $b);
}
else if ($s == "BUILDING")
{
array_push($wor, $b);
}
else if ($s == "")
{
array_push($unb, $b);
}
else
{
array_push($err, $b);
}
}
$b = trim($n[1]); $s = "";
}
}
$dtb = date(DATE_RFC822);
?>
<html>
<head>
<title>Koji Stats Page</title>
<style>
a
{
color: black;
text-decoration: none;
}
table
{
width: 95%;
}
.errb
{
background-color: #FF6A6A;
}
.unbb
{
background-color: #FFA812;
}
.difb
{
background-color: #FFD700;
}
.work
{
background-color: #33FF33;
}
.buib
{
background-color: #499DF5;
}
.head
{
text-align: left;
width: 95%;
display: none;
}
.butt
{
border: 1px solid black;
padding-left: 10px;
padding-right: 10px;
}
.list
{
border: 1px dotted black;
text-align: left;
padding-top: 10px;
padding-left: 10px;
width: 95%;
height: 512px;
overflow: auto;
}
</style>
<script>
function show(name)
{
var x, objc;
var elem = ["erro", "unbu", "diff", "work", "buil"];
for (x = 0; x < elem.length; ++x)
{
objc = document.getElementById(elem[x]);
objc.style.display = "none";
}
for (x = 0; x < elem.length; ++x)
{
if (name == elem[x])
{
objc = document.getElementById(elem[x]);
objc.style.display = "block";
}
}
}
function init()
{
var objc = document.getElementById("head");
objc.style.display = "block";
}
</script>
</head>
<body onload="init();">
<b><?php print($dis." / ".$dta." / ".$dtb); ?></b>
<br /><br />
<center>
<div id="head" class="head">
<span class="butt errb"><a href="javascript:show('erro');">Errors [<?php echo count($err); ?>]</a></span>
<span class="butt unbb"><a href="javascript:show('unbu');">Unbuilt [<?php echo count($unb); ?>]</a></span>
<span class="butt difb"><a href="javascript:show('diff');">Different [<?php echo count($dif); ?>]</a></span>
<span class="butt work"><a href="javascript:show('work');">Working [<?php echo count($wor); ?>]</a></span>
<span class="butt buib"><a href="javascript:show('buil');">Built [<?php echo count($fin); ?>]</a></span>
</div>
<br />
<div id="erro" class="list"><center><table>
<?php foreach ($err as $i) { print("<tr><td class='butt errb'><a href=\"javascript:logs('".$i."');\">".$i."</a></td></tr>\n"); } ?>
</table></center></div>
<div id="unbu" class="list" style="display: none"><center><table>
<?php foreach ($unb as $i) { print("<tr><td class='butt unbb'><a href=\"javascript:logs('".$i."');\">".$i."</a></td></tr>\n"); } ?>
</table></center></div>
<div id="diff" class="list" style="display: none"><center><table>
<?php foreach ($dif as $i) { print("<tr><td class='butt difb'><a href=\"javascript:logs('".$i[0]."');\">".$i[0]." [".$i[1]."]</a></span></td></tr>\n"); } ?>
</table></center></div>
<div id="work" class="list" style="display: none"><center><table>
<?php foreach ($wor as $i) { print("<tr><td class='butt work'><a href=\"javascript:logs('".$i."');\">".$i."</a></td></tr>\n"); } ?>
</table></center></div>
<div id="buil" class="list" style="display: none"><center><table>
<?php foreach ($fin as $i) { print("<tr><td class='butt buib'><a href=\"javascript:logs('".$i."');\">".$i."</a></td></tr>\n"); } ?>
</table></center></div>
</center>
</body>
</html>
Koji Build Que’r
qu.sh
#!/bin/bash export l=`find ~/Desktop/f15v5.0 -type f | grep -i '\.src\.rpm' | wc -l` export x=1 while true do ak list-tagged dist-f15 > /tmp/t while true do ak list-tasks --mine --quiet | grep '^[0-9]' | grep -Ei ' (open|free) .* build' > /tmp/n #echo "got tasks..." ; cat /tmp/n | wc -l ; echo if [ `cat /tmp/n | wc -l` -ge 10 ] then break fi p=`find ~/Desktop/f15v5.0 -type f | grep -i '\.src\.rpm' | head -n "$x" | tail -n 1` q=`basename "$p" | sed -e 's/[^0-9A-Za-z]/./g' -e 's/\.src\.rpm//g'` #echo "checking pkg [$p] [$q]..." ; echo c=`cat /tmp/n /tmp/t | grep -i "$q"` let x="($x % $l) + 1" if [ "$c" != "" ] then continue fi echo "queing [$p] skipped [$x]" ak build dist-f15 "$p" --nowait let n="$n + 1" done sleep 60 done
and a javascript host highlighter
hosts.js
// ==UserScript==
// @name hosts
// @namespace builds
// @include http://arm.koji.fedoraproject.org/koji/hosts?order=last_update&state=enabled
// ==/UserScript==
var dateob = new Date();
var yerstr = (dateob.getFullYear() + "");
var monstr = ((dateob.getMonth() + 1) + "");
var daystr = (dateob.getDate() + "");
var srcobj = document.getElementById("loginInfo");
var scdate = (parseInt(yerstr + monstr + daystr + srcobj.innerHTML.replace(/[^0-9]/g, "").substring(6)) / 100);
var dsttab = document.getElementsByClassName("data-list");
var dstrow = dsttab[0].getElementsByTagName("tr");
for (var x = 0; x < dstrow.length; ++x)
{
var dstcol = dstrow[x].getElementsByTagName("td");
if (dstcol.length > 5)
{
var chdate = (parseInt(dstcol[5].innerHTML.replace(/[^0-9]/g, "")) / 100);
if ((scdate - 5) >= chdate)
{
var lnklst = dstcol[1].getElementsByTagName("a");
for (var y = 0; y < lnklst.length; ++y)
{
lnklst[y].innerHTML = ("<font color='red'>" + lnklst[y].innerHTML + "</font>");
}
}
}
}
$ ./query.sh "http://australia.proximity.on.ca/fedora-arm/f15v5.0/repo" "pth" pth-2.0.7-10.src.rpm pth-2.0.7-13.fc15.src.rpm *:pth-2.0.7-13.fc15.src.rpm
#!/bin/bash
prim=`curl -s "${1}/repodata/repomd.xml" | tr -d '\t\r\n' | gawk '{ gsub(/<data type=/, "\n<data type="); print; }' | grep -i '<data type="primary">' | sed -e 's@.*<location href="\([^"]*\)"/>.*@\1@g'`
curl -s "${1}/${prim}" > /tmp/repo.data.gz
rm -f /tmp/repo.data 2> /dev/null
gunzip /tmp/repo.data.gz
if [ "${3}" == "f" ]
then
rm -f /tmp/repo.line 2> /dev/null
fi
if [ ! -f /tmp/repo.line ]
then
cat /tmp/repo.data | tr -d '\t\r\n' | gawk '{ gsub(/<package type=/, "\n<data type="); print; }' > /tmp/repo.line
fi
safe=`echo "${2}" | sed -e 's/[^0-9A-Za-z]/./g'`
pkgs=`cat /tmp/repo.line | grep -i "<name>${safe}</name>" | sed -e 's@.*<rpm:sourcerpm>\([^<]*\)</rpm:sourcerpm>.*@\1@g'`
if [ "${pkgs}" == "" ]
then
echo "x:${2}"
exit 1
fi
echo "${pkgs}"
x=2
leng=`echo "${pkgs}" | wc -l`
high=`echo "${pkgs}" | tr '\n' ' ' | awk '{ print $1 }'`
while [ $x -le $leng ]
do
comp=`echo "${pkgs}" | tr '\n' ' ' | awk '{ print $'$x' }'`
rpmdev-vercmp "$high" "$comp" > /dev/null 2>&1
if [ $? -eq 12 ]
then
high="$comp"
fi
let x="$x + 1"
done
echo "*:${high}"
# ko build builds /tmp/nled-2.52-7.fc15.src.rpm 282 build (builds, nled-2.52-7.fc15.src.rpm) completed successfully # ko build builds /tmp/nled-2.52-7.fc15.src.rpm 290 build (builds, nled-2.52-7.fc15.src.rpm) completed successfully # ko build builds /tmp/nled-2.52-7.fc15.src.rpm 298 build (builds, nled-2.52-7.fc15.src.rpm) completed successfully # ko list-tagged builds nled Build Tag Built by ---------------------------------------- -------------------- ---------------- nled-2.52-7.267919.111107.fc15 builds admin nled-2.52-7.496d81.111107.fc15 builds admin nled-2.52-7.fc15 builds admin # ls -l /mnt/koji/packages/nled/2.52 total 12 drwxr-xr-x 6 apache apache 4096 Nov 7 19:43 7.267919.111107.fc15 drwxr-xr-x 6 apache apache 4096 Nov 7 19:52 7.496d81.111107.fc15 drwxr-xr-x 6 apache apache 4096 Nov 7 20:01 7.fc15 ... # ko list-tagged builds --latest nled Build Tag Built by ---------------------------------------- -------------------- ---------------- nled-2.52-7.fc15 builds admin # yum clean all ; yum --disablerepo=* --enablerepo=custom-test list nled nled.x86_64 2.52-7.fc15 custom-test
Code change: ( Don’t forget: import hashlib )
> ./hub/kojihub.py
def new_build(data):
"""insert a new build entry"""
tdic = data.copy()
data = data.copy()
if not data.has_key('pkg_id'):
#see if there's a package name
name = data.get('name')
if not name:
raise koji.GenericError, "No name or package id provided for build"
data['pkg_id'] = new_package(name,strict=False)
for f in ('version','release','epoch'):
if not data.has_key(f):
raise koji.GenericError, "No %s value for build" % f
#provide a few default values
data.setdefault('state',koji.BUILD_STATES['COMPLETE'])
data.setdefault('completion_time', 'NOW')
data.setdefault('owner',context.session.user_id)
data.setdefault('task_id',None)
#check for existing build
# TODO - table lock?
q="""SELECT id,state,task_id FROM build
WHERE pkg_id=%(pkg_id)d AND version=%(version)s AND release=%(release)s
FOR UPDATE"""
row = _fetchSingle(q, data)
if row:
id, state, task_id = row
data['id'] = id
st_desc = koji.BUILD_STATES[state]
if st_desc == 'BUILDING':
koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=state, new=data['state'], info=data)
# check to see if this is the controlling task
if data['state'] == state and data.get('task_id','') == task_id:
#the controlling task must have restarted (and called initBuild again)
return id
raise koji.GenericError, "Build already in progress (task %d)" % task_id
# TODO? - reclaim 'stale' builds (state=BUILDING and task_id inactive)
elif st_desc in ('FAILED','CANCELED'):
koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=state, new=data['state'], info=data)
#should be ok to replace
update = """UPDATE build SET state=%(state)i,task_id=%(task_id)s,
owner=%(owner)s,completion_time=%(completion_time)s,create_event=get_event()
WHERE id = %(id)i"""
_dml(update, data)
koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=state, new=data['state'], info=data)
return id
else:
# BEG METHOD HIJACK
# open the current file
frot = "/mnt/koji/packages"
fpth = ("%s/%s/%s" % (str(data["name"]), str(data["version"]), str(data["release"])))
fnme = ("%s-%s-%s" % (str(data["name"]), str(data["version"]), str(data["release"])))
fsrc = ("%s/%s/src/%s.src.rpm" % (frot, fpth, fnme))
try:
fobj = open(fsrc, "r")
except:
raise koji.GenericError, "Could not open file [%s]" % (fsrc)
# hash the current file
hobj = hashlib.md5()
while (1):
fdta = fobj.read(2**10)
if (not fdta):
break
hobj.update(fdta)
fobj.close()
hash = hobj.hexdigest()
tida = time.strftime("%y%m%d")
# set the new release name
while (1):
rell = str(data["release"]).rsplit(".", 1)
rell.insert(1, hash[0:6] + "." + tida)
reln = ".".join(rell)
foln = ("%s/%s/%s/%s" % (frot, str(data["name"]), str(data["version"]), reln))
if (not os.path.exists(foln)):
break
hobj = hashlib.md5()
hobj.update(hash)
hash = hobj.hexdigest()
data["reln"] = reln
# get a list of old files/folders
fill = []
dirl = ["%s/%s" % (frot, fpth)]
while (len(dirl) > 0):
try:
tmpl = os.listdir(dirl[0])
except:
tmpl = []
parn = dirl[0]
dirl.pop(0)
for tmpi in tmpl:
tpth = ("%s/%s" % (parn, tmpi))
if (os.path.isdir(tpth)):
dirl.append(tpth)
else:
name = re.sub("[^0-9A-Za-z]", ".", str(data["name"]))
vers = re.sub("[^0-9A-Za-z]", ".", str(data["version"]))
rels = re.sub("[^0-9A-Za-z]", ".", str(data["release"]))
robj = re.match("^(.*)%s(.*)%s(.*)%s(.*)$" % (name, vers, rels), tmpi)
if (robj):
newn = (parn + "/" + robj.group(1) + name + robj.group(2) + vers + robj.group(3) + reln + robj.group(4))
fill.append([tpth, newn])
fill.append([frot + "/" + fpth, foln])
# update the sql database
c = context.cnx.cursor()
buil = "UPDATE build SET release=%(reln)s WHERE id = %(id)i"
#_dml(buil, data)
c.execute(buil, data)
rpmi = "UPDATE rpminfo SET release=%(reln)s WHERE build_id = %(id)i"
#_dml(rpmi, data)
c.execute(rpmi, data)
context.cnx.commit()
c.close()
# rename the files/folders now
for item in fill:
os.rename(item[0], item[1])
# carry on the same now
return new_build(tdic)
# END METHOD HIJACK
else:
koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=None, new=data['state'], info=data)
#insert the new data
data['id'] = _singleValue("SELECT nextval('build_id_seq')")
q="""
INSERT INTO build (id,pkg_id,version,release,epoch,state,
task_id,owner,completion_time)
VALUES (%(id)i,%(pkg_id)i,%(version)s,%(release)s,%(epoch)s,
%(state)s,%(task_id)s,%(owner)s,%(completion_time)s)
"""
_dml(q, data)
koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=None, new=data['state'], info=data)
#return build_id
return data['id']
note: this is experimental/proof-of-concept code for timing purposes
This script tries to implement very basic createrepo functionality and is based on a model of forks and pipes for distributed, inter-process communication.
repo.py
import hashlib
import multiprocessing
import os
import re
import select
import sys
def rpmhinfo(p, n):
# file
f = open(n, "r")
d = f.read()
f.close()
o = []
# lead
magic = d[0:4]
vers = d[4:6]
rtype = ((ord(d[6]) << 8) + ord(d[7]))
rarch = d[8:10]
name = d[10:76].strip("\0\t\r\n ")
osnum = d[76:78]
sigt = d[78:80]
resr = d[80:96]
o.append([magic, vers, rtype, rarch, name, osnum, sigt, resr])
if (rtype != 0):
return None
# extra
q = n[len(p):]; o[0].append(q)
s = len(d); o[0].append(s)
h = hashlib.sha256(d).hexdigest(); o[0].append(h)
# loop
i = 1; m = 96
while (d[m:m+3] == "\x8e\xad\xe8"):
# header
magic = d[m:m+3]
vers = d[m+3]
resr = d[m+4:m+8]
numb = ((ord(d[m+8]) << 24) + (ord(d[m+9]) << 16) + (ord(d[m+10]) << 8) + ord(d[m+11]))
size = ((ord(d[m+12]) << 24) + (ord(d[m+13]) << 16) + (ord(d[m+14]) << 8) + ord(d[m+15]))
offs = ((m + 16) + (numb * 16))
o.append([magic, vers, resr, numb, size, offs, m])
# index
x = 0; l = (numb * 16)
while (x < l):
tag = ((ord(d[x+m+16]) << 24) + (ord(d[x+m+17]) << 16) + (ord(d[x+m+18]) << 8) + ord(d[x+m+19]))
rtype = ((ord(d[x+m+20]) << 24) + (ord(d[x+m+21]) << 16) + (ord(d[x+m+22]) << 8) + ord(d[x+m+23]))
offs = ((ord(d[x+m+24]) << 24) + (ord(d[x+m+25]) << 16) + (ord(d[x+m+26]) << 8) + ord(d[x+m+27]))
numb = ((ord(d[x+m+28]) << 24) + (ord(d[x+m+29]) << 16) + (ord(d[x+m+30]) << 8) + ord(d[x+m+31]))
data = ""
y = (o[i][5] + offs); j = 0
while (j < numb):
if (rtype == 3):
data += (str((ord(d[y+0]) << 8) + ord(d[y+1])) + "\n")
y += 2; j += 1
elif (rtype == 4):
data += (str((ord(d[y+0]) << 24) + (ord(d[y+1]) << 16) + (ord(d[y+2]) << 8) + ord(d[y+3])) + "\n")
y += 4; j += 1
elif ((rtype == 6) or (rtype == 8) or (rtype == 9)):
while (d[y] != "\0"):
data += d[y]
y += 1
data += "\n"
y += 1; j += 1
elif (rtype == 7):
data += d[y]
y += 1; j += 1
x += 16
o[i].append([tag, rtype, offs, numb, data])
m = (o[i][4] + o[i][5])
while ((m % 8) != 0):
m += 1
i += 1
# return
return o
def safestri(inptstri):
outpstri = inptstri
outpstri = outpstri.replace("&", "&")
outpstri = outpstri.replace("<", "<")
outpstri = outpstri.replace(">", ">")
outpstri = outpstri.strip()
return outpstri
def makedict(rpmhobjc):
i = (len(rpmhobjc) - 1)
infodict = {"locn":rpmhobjc[0][8], "size":rpmhobjc[0][9], "hash":rpmhobjc[0][10], "poch":0, "pkgr":"", "urln":"", "vndr":"", "heds":rpmhobjc[i][6], "hede":(rpmhobjc[i][4]+rpmhobjc[i][5])}
for x in range(1, len(rpmhobjc)):
for y in range(7, len(rpmhobjc[x])):
#print(rpmhobjc[x][y])
if ((rpmhobjc[x][y][0] == 1000) and (rpmhobjc[x][y][1] == 6)):
infodict["name"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1022) and (rpmhobjc[x][y][1] == 6)):
infodict["arch"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1003) and (rpmhobjc[x][y][1] == 4)):
infodict["poch"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1001) and (rpmhobjc[x][y][1] == 6)):
infodict["vers"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1002) and (rpmhobjc[x][y][1] == 6)):
infodict["rels"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1004) and (rpmhobjc[x][y][1] == 9)):
infodict["summ"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1005) and (rpmhobjc[x][y][1] == 9)):
infodict["desc"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1015) and (rpmhobjc[x][y][1] == 6)):
infodict["pkgr"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1020) and (rpmhobjc[x][y][1] == 6)):
infodict["urln"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1006) and (rpmhobjc[x][y][1] == 4)):
infodict["RPMTAG_BUILDTIME"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1009) and (rpmhobjc[x][y][1] == 4)):
infodict["RPMTAG_SIZE"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1007) and (rpmhobjc[x][y][1] == 4)):
infodict["SIGTAG_PAYLOADSIZE"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1014) and (rpmhobjc[x][y][1] == 6)):
infodict["licn"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1011) and (rpmhobjc[x][y][1] == 6)):
infodict["vndr"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1016) and (rpmhobjc[x][y][1] == 9)):
infodict["grup"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1007) and (rpmhobjc[x][y][1] == 6)):
infodict["host"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1044) and (rpmhobjc[x][y][1] == 6)):
infodict["srpm"] = safestri(rpmhobjc[x][y][4])
elif ((rpmhobjc[x][y][0] == 1047) and (rpmhobjc[x][y][1] == 8)):
infodict["prvn"] = safestri(rpmhobjc[x][y][4]).split("\n")
elif ((rpmhobjc[x][y][0] == 1113) and (rpmhobjc[x][y][1] == 8)):
infodict["prvl"] = rpmhobjc[x][y][4].split("\n")
elif ((rpmhobjc[x][y][0] == 1112) and (rpmhobjc[x][y][1] == 4)):
infodict["prvf"] = rpmhobjc[x][y][4].split("\n")
elif ((rpmhobjc[x][y][0] == 1049) and (rpmhobjc[x][y][1] == 8)):
infodict["reqn"] = safestri(rpmhobjc[x][y][4]).split("\n")
elif ((rpmhobjc[x][y][0] == 1050) and (rpmhobjc[x][y][1] == 8)):
infodict["reql"] = rpmhobjc[x][y][4].split("\n")
elif ((rpmhobjc[x][y][0] == 1048) and (rpmhobjc[x][y][1] == 4)):
infodict["reqf"] = rpmhobjc[x][y][4].split("\n")
elif ((rpmhobjc[x][y][0] == 1054) and (rpmhobjc[x][y][1] == 8)):
infodict["conn"] = safestri(rpmhobjc[x][y][4]).split("\n")
elif ((rpmhobjc[x][y][0] == 1055) and (rpmhobjc[x][y][1] == 8)):
infodict["conl"] = rpmhobjc[x][y][4].split("\n")
elif ((rpmhobjc[x][y][0] == 1053) and (rpmhobjc[x][y][1] == 4)):
infodict["conf"] = rpmhobjc[x][y][4].split("\n")
elif ((rpmhobjc[x][y][0] == 1090) and (rpmhobjc[x][y][1] == 8)):
infodict["obsn"] = safestri(rpmhobjc[x][y][4]).split("\n")
elif ((rpmhobjc[x][y][0] == 1115) and (rpmhobjc[x][y][1] == 8)):
infodict["obsl"] = rpmhobjc[x][y][4].split("\n")
elif ((rpmhobjc[x][y][0] == 1114) and (rpmhobjc[x][y][1] == 4)):
infodict["obsf"] = rpmhobjc[x][y][4].split("\n")
elif ((rpmhobjc[x][y][0] == 1118) and (rpmhobjc[x][y][1] == 8)):
infodict["flep"] = rpmhobjc[x][y][4].split("\n")
elif ((rpmhobjc[x][y][0] == 1117) and (rpmhobjc[x][y][1] == 8)):
infodict["flen"] = rpmhobjc[x][y][4].split("\n")
elif ((rpmhobjc[x][y][0] == 1116) and (rpmhobjc[x][y][1] == 4)):
infodict["flei"] = rpmhobjc[x][y][4].split("\n")
elif ((rpmhobjc[x][y][0] == 1030) and (rpmhobjc[x][y][1] == 3)):
infodict["flet"] = rpmhobjc[x][y][4].split("\n")
return infodict
def procenvr(envrstri):
outpstri = ""
if (envrstri == ""):
return outpstri
verslist = envrstri.rsplit("-", 1)
pochlist = verslist[0].split(":", 1)
if (len(verslist) > 1):
outpstri = ((" rel=\"%s\"" % (verslist[1])) + outpstri)
if (len(pochlist) < 2):
pochlist.insert(0, "0")
outpstri = ((" epoch=\"%s\" ver=\"%s\"" % (pochlist[0], pochlist[1])) + outpstri)
return outpstri
def rpmelist(namelist, verslist, flaglist):
x = len(namelist); y = len(verslist); z = len(flaglist)
d = {"2":"LT", "4":"GT", "8":"EQ", "10":"LE", "12":"GE"}
o = ""
if (x < 1):
return o
if ((x <= y) and (y <= z)):
f = []
for w in range(0, x):
f.append(["", ""])
n = str(int(flaglist[w]) & 0xf)
try:
f[w][0] = (" flags=\"%s\"" % (d[n]))
except:
pass
n = ((int(flaglist[w]) & 0x200) + (int(flaglist[w]) & 0x100))
if (n != 0):
f[w][1] = (" pre=\"1\"")
l = []
for w in range(0, x):
if (re.match("^.*rpmlib.*$", namelist[w], re.I)):
continue
i = procenvr(verslist[w])
n = ("name=\"%s\"%s%s%s" % (namelist[w], f[w][0], i, f[w][1]))
if (n in l):
continue
o += ("\t\t\t<rpm:entry %s/>\n" % (n))
l.append(n)
return o
def makexmld(rpmhdict):
o = ""
o += ("<package type=\"rpm\">\n")
o += ("\t<name>%s</name>\n" % (rpmhdict["name"]))
o += ("\t<arch>%s</arch>\n" % (rpmhdict["arch"]))
o += ("\t<version epoch=\"%s\" ver=\"%s\" rel=\"%s\"/>\n" % (rpmhdict["poch"], rpmhdict["vers"], rpmhdict["rels"]))
o += ("\t<checksum type=\"sha256\" pkgid=\"YES\">%s</checksum>\n" % (rpmhdict["hash"]))
o += ("\t<summary>%s</summary>\n" % (rpmhdict["summ"]))
o += ("\t<description>%s</description>\n" % (rpmhdict["desc"]))
o += ("\t<packager>%s</packager>\n" % (rpmhdict["pkgr"]))
o += ("\t<url>%s</url>\n" % (rpmhdict["urln"]))
o += ("\t<time file=\"%s\" build=\"%s\"/>\n" % (int(rpmhdict["RPMTAG_BUILDTIME"]) - 10, rpmhdict["RPMTAG_BUILDTIME"]))
o += ("\t<size package=\"%s\" installed=\"%s\" archive=\"%s\"/>\n" % (rpmhdict["size"], rpmhdict["RPMTAG_SIZE"], rpmhdict["SIGTAG_PAYLOADSIZE"]))
o += ("\t<location href=\"%s\"/>\n" % (rpmhdict["locn"]))
o += ("\t<format>\n")
o += ("\t\t<rpm:license>%s</rpm:license>\n" % (rpmhdict["licn"]))
o += ("\t\t<rpm:vendor>%s</rpm:vendor>\n" % (rpmhdict["vndr"]))
o += ("\t\t<rpm:group>%s</rpm:group>\n" % (rpmhdict["grup"]))
o += ("\t\t<rpm:buildhost>%s</rpm:buildhost>\n" % (rpmhdict["host"]))
o += ("\t\t<rpm:sourcerpm>%s</rpm:sourcerpm>\n" % (rpmhdict["srpm"]))
o += ("\t\t<rpm:header-range start=\"%s\" end=\"%s\"/>\n" % (rpmhdict["heds"], rpmhdict["hede"]))
try:
t = rpmelist(rpmhdict["prvn"], rpmhdict["prvl"], rpmhdict["prvf"])
o += ("\t\t<rpm:provides>\n%s\t\t</rpm:provides>\n" % (t))
except:
pass
try:
t = rpmelist(rpmhdict["reqn"], rpmhdict["reql"], rpmhdict["reqf"])
o += ("\t\t<rpm:requires>\n%s\t\t</rpm:requires>\n" % (t))
except:
pass
try:
t = rpmelist(rpmhdict["conn"], rpmhdict["conl"], rpmhdict["conf"])
o += ("\t\t<rpm:conflicts>\n%s\t\t</rpm:conflicts>\n" % (t))
except:
pass
try:
t = rpmelist(rpmhdict["obsn"], rpmhdict["obsl"], rpmhdict["obsf"])
o += ("\t\t<rpm:obsoletes>\n%s\t\t</rpm:obsoletes>\n" % (t))
except:
pass
try:
x = len(rpmhdict["flen"])
y = len(rpmhdict["flei"])
z = len(rpmhdict["flet"])
except:
x = -1; y = -2; z = -3
if ((x == y) and (y == z)):
l = []
for u in range(0, z):
if (rpmhdict["flen"][u] == ""):
break
i = int(rpmhdict["flei"][u])
pathname = (rpmhdict["flep"][i].rstrip("/") + "/" + rpmhdict["flen"][u])
if (not re.match("^(/etc.*|.*bin/.*|/usr/lib/sendmail)$", pathname, re.I)):
continue
if (pathname in l):
continue
n = (int(rpmhdict["flet"][u]) & 0x4000)
if (n != 0):
o += ("\t\t<file type=\"dir\">%s</file>\n" % (pathname))
else:
o += ("\t\t<file>%s</file>\n" % (pathname))
l.append(pathname)
o += ("\t</format>\n")
o += ("</package>\n")
return [o]
def findfile(foldname):
x = 0
filelist = [foldname]
remolist = []
while (x < len(filelist)):
if (os.path.isdir(filelist[x])):
templist = os.listdir(filelist[x])
for tempitem in templist:
filelist.append(filelist[x] + "/" + tempitem)
remolist.append(filelist[x])
else:
if (not re.match("^.*\.rpm$", filelist[x], re.I)):
remolist.append(filelist[x])
elif (re.match("^(%s/repodata/.*|.*debuginfo.*|.*\.src\.rpm)$" % (foldname), filelist[x], re.I)):
remolist.append(filelist[x])
x += 1
for remoitem in remolist:
while (filelist.count(remoitem) > 0):
filelist.remove(remoitem)
return filelist
def main():
if (len(sys.argv) < 2):
print("Usage: %s <repo-fold>" % (sys.argv[0]))
sys.exit(0)
repofold = sys.argv[1].rstrip("/")
repolist = findfile(repofold)
repoleng = len(repolist)
print("found [%d] rpm files..." % (repoleng))
childnum = 3
destfold = "/tmp/repodata"
pipelist = []; proclist = []
try:
os.mkdir(destfold)
except:
pass
print("beg primary.xml...")
for x in range(0, childnum):
(parnpipe, chilpipe) = multiprocessing.Pipe(True)
procnumb = os.fork()
if (procnumb == 0):
parnpipe.close()
fileobjc = open("%s/primary.%d.xml" % (destfold, x), "w")
while (1):
chilpipe.send("more")
filename = chilpipe.recv()
if (not filename):
break
print("[%d] : %s" % (x, filename))
headobjc = rpmhinfo(repofold + "/", filename)
try:
headdict = makedict(headobjc)
xmldobjc = makexmld(headdict)
fileobjc.write(xmldobjc[0])
except:
sys.stderr.write("erro:" + str(sys.exc_info()) + "\n")
pass
fileobjc.close()
sys.exit(0)
else:
chilpipe.close()
pipelist.append(parnpipe)
proclist.append(procnumb)
while (len(pipelist) > 0):
(readlist, outplist, errolist) = select.select(pipelist, [], [])
for pipeitem in pipelist:
if (pipeitem not in readlist):
continue
try:
dumydata = pipeitem.recv()
if (len(repolist) > 0):
pipeitem.send(repolist[0])
repolist.pop(0)
else:
pipeitem.send("")
except:
pipelist.remove(pipeitem)
for procitem in proclist:
try:
os.waitpid(procitem, 0)
except:
pass
print("end primary.xml...")
primoutp = ""
for x in range(0, childnum):
try:
fileobjc = open("%s/primary.%d.xml" % (destfold, x), "r")
primoutp += fileobjc.read()
fileobjc.close()
os.unlink("%s/primary.%d.xml" % (destfold, x))
except:
pass
fileobjc = open("%s/primary.xml" % (destfold), "w")
fileobjc.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
fileobjc.write("<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"%d\">\n" % (repoleng))
fileobjc.write(primoutp)
fileobjc.write("</metadata>\n")
fileobjc.close()
fileobjc = open("%s/repomd.xml" % (destfold), "w")
fileobjc.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
fileobjc.write("<repomd xmlns=\"http://linux.duke.edu/metadata/repo\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\">\n")
fileobjc.write("<data type=\"primary\">\n")
fileobjc.write("\t<checksum type=\"sha256\">%s</checksum>\n" % (hashlib.sha256(primoutp).hexdigest()))
fileobjc.write("\t<location href=\"repodata/primary.xml\"/>\n")
fileobjc.write("\t<size>%d</size>\n" % (len(primoutp)))
fileobjc.write("</data>\n")
fileobjc.write("</repomd>\n")
fileobjc.close()
if (__name__ == "__main__"):
main()
So I wanted to see if I could speed up common mock tasks without using their built-in caching methods as the buildroot package set is changing and it may not pick up those changes. The mock source code is located here (https://fedorahosted.org/mock/) and the results below show that on an x64 machine, the nled build went 22.5% faster! The downside is that you would have to set aside a dedicate builder to constantly create and tar the updated buildroots for the builders running the modified mock. Also note that the speed-up shown for the modified mock did not have to download the tar as it was locally stored.
Regular mock
# rm -rf /tmp/*mock.output* /var/lib/mock/* /var/cache/mock/* /tmp/buildroot.flag ; time mock -vr fedora-15-x86_64 --resultdir=/tmp/mock.output /tmp/nled-2.52-7.fc15.src.rpm INFO: mock.py version 1.1.9 starting... ... INFO: Done(/tmp/nled-2.52-7.fc15.src.rpm) Config(fedora-15-x86_64) 1 minutes 36 seconds ... DEBUG: kill orphans real 1m37.880s user 1m3.519s sys 0m4.444s
Modified mock
# rm -rf /tmp/*mock.output* /var/lib/mock/* /var/cache/mock/* /tmp/buildroot.flag ; time ./py/mock.py -vr fedora-15-x86_64 --resultdir=/tmp/mock.output /tmp/nled-2.52-7.fc15.src.rpm INFO: mock.py version unreleased_version starting... ... INFO: Done(/tmp/nled-2.52-7.fc15.src.rpm) Config(fedora-15-x86_64) 0 minutes 21 seconds ... DEBUG: kill orphans real 0m22.001s user 0m44.526s sys 0m2.925s
./py/mockbuild/backend.py
155: decorate(traceLog())
156: def _unlock_and_rm_chroot(self):
157: if (os.path.exists("/tmp/buildroot.flag")):
158: print("CUSTOM: HIJACK SAVING [%s]" % (self.makeChrootPath()))
159: os.system("tar -cf \"/tmp/buildroot.tar\" -C '%s' ." % (self.makeChrootPath()))
160: try:
161: os.unlink("/tmp/buildroot.flag")
162: except:
163: pass
...
826: decorate(traceLog())
827: def _yum(self, cmd, returnOutput=0):
828: """use yum to install packages/package groups into the chroot"""
829: flag=0
830: try:
831: if ((cmd[0] == "groupinstall") and (cmd[1] == "buildsys-build")):
832: flag=1
833: if (os.path.exists("/tmp/buildroot.tar")):
834: os.system("tar -xvf /tmp/buildroot.tar -C '%s'" % (self.makeChrootPath()))
835: return ""
836: except:
837: pass
...
853: try:
854: self._callHooks("preyum")
855: output = mockbuild.util.do(yumcmd, returnOutput=returnOutput)
856: self._callHooks("postyum")
857: if (flag == 1):
858: os.system("touch /tmp/buildroot.flag")
859: raise mockbuild.exception.YumError, "CUSTOM: HIJACK YUM ERROR"
860: else:
861: return output
862: except mockbuild.exception.Error, e:
863: raise mockbuild.exception.YumError, str(e)
Jon's FOSS Blog 