Some Koji Scripts (The Push For Primary)

The push for ARM to become primary arch is not as easy as one might think. Everyone in the community is working hard to prove how ARM is as capable in most areas to that of the machines in primary architecture. Anyway, here are a couple of scripts one could use to help determine some basic stats about ones current build process.

To find out the current error counts and types (and unbuilt packages) of a given tag:

(could help to show how far off one is in keeping up with primary and where the errors are)

#!/bin/bash

tagname="f17"
tagnumb=`echo "${tagname}" | sed -e 's/[^0-9]//g'`

if [ ! -f pripkgs.txt -o "$1" == "f" ]
then
	koji --server=http://koji.fedoraproject.org/kojihub list-tagged ${tagname} --inherit --latest --quiet | awk '{ print $1 }' | sed -e 's/-[^-]*-[^-]*$//g' > pripkgs.txt
fi

if [ ! -f armpkgs.txt -o "$1" == "f" ]
then
	koji --server=http://arm.koji.fedoraproject.org/kojihub list-tagged ${tagname} --inherit --latest --quiet | awk '{ print $1 }' | sed -e 's/-[^-]*-[^-]*$//g' > armpkgs.txt
fi

builtnum=0; builtstr=""
archnumb=0; archstri=""
depsnumb=0; depsstri=""
reqsnumb=0; reqsstri=""
confnumb=0; confstri=""
buildnum=0; buildstr=""
makenumb=0; makestri=""
unknnumb=0; unknstri=""
ubuilnum=0; ubuilstr=""

outpstri=""

while read pkgname
do
	pkgtagged=`cat armpkgs.txt | grep "${pkgname}"`
	
	if [ "${pkgtagged}" == "" ]
	then
		echo "[p] ${pkgname}" 1>&2
		
		pkgstat=`curl -Ls "http://arm.koji.fedoraproject.org/koji/search?match=glob&type=package&terms=${pkgname}" | tr -d '\t\r\n' | sed -e 's/<tr/\t<tr/g' | tr '\t' '\n' | grep -i "^<tr.*buildinfo.buildID" | sed -e 's/<a[^>]*buildinfo.buildID=\([0-9]*\)[^>]*>/ \1 /g' -e 's/<img[^>]*alt=.\([a-zA-Z]*\).[^>]*>/ \1 /g' -e 's/<[^>]*>/ /g' | grep -i "fc${tagnumb}" | sed -e 's/^[ \t]*//g' -e 's/[ \t]*$//g'`
		builtstat=`echo "${pkgstat}" | grep -i "complete$" | head -n 1`
		failstat=`echo "${pkgstat}" | grep -i "failed$" | head -n 1`
		
		if [ "${builtstat}" != "" ]
		then
			if [ "$2" == "p" ]
			then
				let builtnum="${builtnum} + 1"
				builtstr="${builtstr}${pkgname} http://arm.koji.fedoraproject.org/koji/search?match=glob&type=package&terms=${pkgname}\n"
			
			else
				outpstri="${outpstri}\n\"${pkgname}\":[\"complete\", \"\", 0]"
			fi
		
		elif [ "${failstat}" != "" ]
		then
			pkgnumb=`echo "${failstat}" | awk '{ print $1 }'`
			tasknumb=`curl -Ls "http://arm.koji.fedoraproject.org/koji/buildinfo?buildID=${pkgnumb}" | tr -d '\t\r\n' | sed -e 's/<a/\t<a/g' | tr '\t' '\n' | grep -i "^<a.*taskinfo.taskID" | sed -e 's/<a[^>]*taskinfo.taskID=\([0-9]*\)[^>]*>/ \1 /g' | awk '{ print $1 }' | head -n 1`
			failnumb=`curl -Ls "http://arm.koji.fedoraproject.org/koji/taskinfo?taskID=${tasknumb}" | tr -d '\t\r\n' | sed -e 's/<a/\t<a/g' | tr '\t' '\n' | grep -i "^<a.*taskinfo.taskID.*failed" | sed -e 's/<a[^>]*taskinfo.taskID=\([0-9]*\)[^>]*>/ \1 /g' | awk '{ print $1 }' | head -n 1`
			logsoutp=`koji --server=http://arm.koji.fedoraproject.org/kojihub watch-logs "${failnumb}"`
			
			archerro=`echo "${logsoutp}" | grep -i 'error: architecture is excluded' | head -n 1`
			depserro=`echo "${logsoutp}" | grep -i 'error: no package found for' | head -n 1`
			reqserro=`echo "${logsoutp}" | grep -i 'error: package' -A 1 | head -n 2 | tr -d '\t\r\n' | tr -s ' '`
			conferro=`echo "${logsoutp}" | grep -i 'error: bad exit status from' | head -n 1`
			builderr=`echo "${logsoutp}" | grep -i 'error: file not found' | head -n 1`
			makeerro=`echo "${logsoutp}" | grep -i 'make: .* error' | head -n 1`
			
			if [ "${archerro}" != "" ]
			then
				if [ "$2" == "p" ]
				then
					let archnumb="${archnumb} + 1"
					archstri="${archstri}${pkgname} http://arm.koji.fedoraproject.org/koji/taskinfo?taskID=${failnumb}\n"
				
				else
					outpstri="${outpstri}\"${pkgname}\":[\"arch-error\", \"${archerro}\", ${failnumb}]\n"
				fi
			
			elif [ "${depserro}" != "" ]
			then
				if [ "$2" == "p" ]
				then
					let depsnumb="${depsnumb} + 1"
					depsstri="${depsstri}${pkgname} http://arm.koji.fedoraproject.org/koji/taskinfo?taskID=${failnumb}\n"
				
				else
					outpstri="${outpstri}\"${pkgname}\":[\"dep-error\", \"${depserro}\", ${failnumb}]\n"
				fi
			
			elif [ "${reqserro}" != "" ]
			then
				if [ "$2" == "p" ]
				then
					let reqsnumb="${reqsnumb} + 1"
					reqsstri="${reqsstri}${pkgname} http://arm.koji.fedoraproject.org/koji/taskinfo?taskID=${failnumb}\n"
				
				else
					outpstri="${outpstri}\"${pkgname}\":[\"req-error\", \"${reqserro}\", ${failnumb}]\n"
				fi
			
			elif [ "${builderr}" != "" ]
			then
				if [ "$2" == "p" ]
				then
					let buildnum="${buildnum} + 1"
					buildstr="${buildstr}${pkgname} http://arm.koji.fedoraproject.org/koji/taskinfo?taskID=${failnumb}\n"
				
				else
					outpstri="${outpstri}\"${pkgname}\":[\"build-error\", \"${builderr}\", ${failnumb}]\n"
				fi
			
			elif [ "${makeerro}" != "" ]
			then
				if [ "$2" == "p" ]
				then
					let makenumb="${makenumb} + 1"
					makestri="${makestri}${pkgname} http://arm.koji.fedoraproject.org/koji/taskinfo?taskID=${failnumb}\n"
				
				else
					outpstri="${outpstri}\"${pkgname}\":[\"make-error\", \"${makeerro}\", ${failnumb}]\n"
				fi
			
			elif [ "${conferro}" != "" ]
			then
				if [ "$2" == "p" ]
				then
					let confnumb="${confnumb} + 1"
					confstri="${confstri}${pkgname} http://arm.koji.fedoraproject.org/koji/taskinfo?taskID=${failnumb}\n"
				
				else
					outpstri="${outpstri}\"${pkgname}\":[\"config-error\", \"${conferro}\", ${failnumb}]\n"
				fi
			
			else
				if [ "$2" == "p" ]
				then
					let unknnumb="${unknnumb} + 1"
					unknstri="${unknstri}${pkgname} http://arm.koji.fedoraproject.org/koji/taskinfo?taskID=${failnumb}\n"
				
				else
					outpstri="${outpstri}\"${pkgname}\":[\"unknown-error\", \"Error could not be interpreted.\", ${failnumb}]\n"
				fi
			fi
		
		else
			if [ "$2" == "p" ]
			then
				let ubuilnum="${ubuilnum} + 1"
				ubuilstr="${ubuilstr}${pkgname}\n"
			
			else
				outpstri="${outpstri}\"${pkgname}\":[\"unbuilt\", \"\", 0]\n"
			fi
		fi
	fi
done < pripkgs.txt

extranum=0; extrastr=""

while read pkgname
do
	pkgtagged=`cat pripkgs.txt | grep "${pkgname}"`
	
	if [ "${pkgtagged}" == "" ]
	then
		echo "[s] ${pkgname}" 1>&2
		
		if [ "$2" == "p" ]
		then
			let extranum="${extranum} + 1"
			extrastr="${extrastr}${pkgname}\n"
		
		else
			outpstri="${outpstri}\"${pkgname}\":[\"extra\", \"\", 0]\n"
		fi
	fi
done < armpkgs.txt

if [ "$2" == "p" ]
then
	printf '%10s|%10s|%10s|%10s|%10s|%10s|%10s\n' "Arch" "Deps" "Reqs" "Config" "Build" "Make" "Unknown"
	for i in `seq 1 80` ; do printf '-' ; done ; printf '\n'
	printf '%10s|%10s|%10s|%10s|%10s|%10s|%10s\n' "${archnumb}" "${depsnumb}" "${reqsnumb}" "${confnumb}" "${buildnum}" "${makenumb}" "${unknnumb}"
	echo
	if [ "${archstri}" != "" ] ; then echo -e "Arch-Errors:" ; echo -e "${archstri}" ; fi
	if [ "${depsstri}" != "" ] ; then echo -e "Dep-Errors:" ; echo -e "${depsstri}" ; fi
	if [ "${reqsstri}" != "" ] ; then echo -e "Require-Errors:" ; echo -e "${reqsstri}" ; fi
	if [ "${confstri}" != "" ] ; then echo -e "Config-Errors:" ; echo -e "${confstri}" ; fi
	if [ "${buildstr}" != "" ] ; then echo -e "Build-Errors:" ; echo -e "${buildstr}" ; fi
	if [ "${makestri}" != "" ] ; then echo -e "Make-Errors:" ; echo -e "${makestri}" ; fi
	if [ "${unknstri}" != "" ] ; then echo -e "Unknown-Errors:" ; echo -e "${unknstri}" ; fi
	if [ "${builtstr}" != "" ] ; then echo -e "Built but un-tagged/blocked pkgs = ${builtnum}" ; echo -e "${builtstr}" ; fi
	echo -e "Number of unbuilt/cancelled/building pkgs = ${ubuilnum}\n"
	if [ "${extrastr}" != "" ] ; then echo -e "Pkgs tagged in arm and not primary = ${extranum}" ; echo -e "${extrastr}" ; fi
	echo

else
	echo -e "${outpstri}"
	echo
fi

This script can help to compare build times between your various machines and primary:

(could help to compare how quickly ones build farm could keep up with primaries)

#!/bin/bash

function taskinfo
{
	roottask=`koji --server="$1/kojihub" buildinfo "$2" | grep -i '^task:' | awk '{ print $2 }'`
	curl -Ls "$1/koji/taskinfo?taskID=${roottask}" | tr -d '\t\r\n' | sed -e 's/<a/\t<a/g' | tr '\t' '\n' | grep -i 'taskinfo.taskID=.*buildArch' | sed -e 's/^.*taskinfo.taskID=\([0-9]*\).*$/ \1 /g' > /tmp/koji.task.$4.txt
	
	while read tasknumb
	do
		taskinfo=`curl -Ls "$1/koji/taskinfo?taskID=${tasknumb}" | tr -d '\t\r\n' | sed -e 's/<tr/\t<tr/g' | tr '\t' '\n'`
		begtime=`echo "${taskinfo}" | grep -i '^<tr>.*<th>.*Started.*</th>' | sed -e 's/^.*<td>//g' -e 's/<[^>]*>/ /g' -e 's/^[ \t]*//g' -e 's/[ \t]*$//g'`
		endtime=`echo "${taskinfo}" | grep -i '^<tr>.*<th>.*Completed.*</th>' | sed -e 's/^.*<td>//g' -e 's/<[^>]*>/ /g' -e 's/^[ \t]*//g' -e 's/[ \t]*$//g'`
		archname=`echo "${taskinfo}" | grep -i "^<tr>.*<th>.*Arch.*</th>" | sed -e 's/^.*<td>//g' -e 's/<[^>]*>/ /g' -e 's/^[ \t]*//g' -e 's/[ \t]*$//g'`
		hostname=`echo "${taskinfo}" | grep -i "^<tr>.*<th>.*Host.*</th>.*<td>.*$3.*</td>" | sed -e 's/^.*<td>//g' -e 's/<[^>]*>/ /g' -e 's/^[ \t]*//g' -e 's/[ \t]*$//g'`
		
		if [ "${begtime}" != "" -a "${endtime}" != "" -a "${archname}" != "" -a "${hostname}" != "" ]
		then
			begepoch=`date --date="${begtime}" '+%s'`
			endepoch=`date --date="${endtime}" '+%s'`
			
			hourdiff=`echo "(${endepoch} - ${begepoch}) / 3600" | bc`
			if [ ${hourdiff} -lt 10 ] ; then hourdiff="0${hourdiff}" ; fi
			
			mindiff=`echo "((${endepoch} - ${begepoch}) / 60) % 60" | bc`
			if [ ${mindiff} -lt 10 ] ; then mindiff="0${mindiff}" ; fi
			
			echo "${archname} ${hourdiff}:${mindiff}"
		fi
	done < /tmp/koji.task.$4.txt
}

tagname="f17"
tagnumb=`echo "${tagname}" | sed -e 's/[^0-9]//g'`

if [ ! -f primary.built.txt ]
then
	koji --server=http://koji.fedoraproject.org/kojihub list-tagged ${tagname} --inherit --latest --quiet | awk '{ print $1 }' > primary.built.txt
fi

tempfile="$1"
uniqname=`head -c 1024 /dev/urandom | md5sum | awk '{ print $1 }'`

if [ "${tempfile}" == "" ]
then
	koji --server=http://arm.koji.fedoraproject.org/kojihub list-tagged ${tagname} --inherit --latest --quiet | awk '{ print $1 }' > arm.built.txt
	tempfile="arm.built.txt"
fi

builder="$2"

if [ "${builder}" == "" ]
then
	builder="cdot.trimslice"
fi

printf '%-40s | %10s | %10s | %10s | %10s | %20s\n' "Package Name" "x86" "x64" "ARMvSFP" "ARMvHFP" "(Time = HH:MM)"
for i in `seq 1 95` ; do printf '-' ; done ; printf '\n'

while read srpmname
do
	pkgname=`echo "${srpmname}" | sed -e 's/-[^-]*-[^-]*$//g'`
	primpkg=`cat primary.built.txt | grep "^${pkgname}-[0-9\.][0-9\.]*-" | head -n 1`
	
	arminfo=`taskinfo "http://arm.koji.fedoraproject.org" "${srpmname}" "${builder}" "${uniqname}"`
	
	if [ "${arminfo}" != "" ]
	then
		priminfo=`taskinfo "http://koji.fedoraproject.org" "${primpkg}" ".*" "${uniqname}"`
		
		tttime=`echo "${priminfo}" | grep -i '^i386 ' | awk '{ print $2 }' | head -n 1`
		sftime=`echo "${priminfo}" | grep -i '^x86_64 ' | awk '{ print $2 }' | head -n 1`
		astime=`echo "${arminfo}" | grep -i '^arm ' | awk '{ print $2 }' | head -n 1`
		ahtime=`echo "${arminfo}" | grep -i '^armhfp ' | awk '{ print $2 }' | head -n 1`
		
		printf '%-40s | %10s | %10s | %10s | %10s |\n' "${pkgname}" "${tttime}" "${sftime}" "${astime}" "${ahtime}"
	fi
done < ${tempfile}

Advertisements
Some Koji Scripts (The Push For Primary)

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s