1#!/bin/bash
2#
3# SPDX-License-Identifier: GPL-2.0-only
4#
5# Used to compare sstate checksums between MACHINES.
6# Execute script and compare generated list.M files.
7# Using bash to have PIPESTATUS variable.
8
9# It's also usefull to keep older sstate checksums
10# to be able to find out why something is rebuilding
11# after updating metadata
12
13# $ diff \
14#     sstate-diff/1349348392/fake-cortexa8/list.M \
15#     sstate-diff/1349348392/fake-cortexa9/list.M \
16#     | wc -l
17# 538
18
19# Then to compare sigdata use something like:
20# $ ls sstate-diff/1349348392/*/armv7a-vfp-neon*/linux-libc-headers/*do_configure*sigdata*
21#   sstate-diff/1349348392/fake-cortexa8/armv7a-vfp-neon-oe-linux-gnueabi/linux-libc-headers/3.4.3-r0.do_configure.sigdata.cb73b3630a7b8191e72fc469c5137025
22#   sstate-diff/1349348392/fake-cortexa9/armv7a-vfp-neon-oe-linux-gnueabi/linux-libc-headers/3.4.3-r0.do_configure.sigdata.f37ada177bf99ce8af85914df22b5a0b
23# $ bitbake-diffsigs stamps.1349348392/*/armv7a-vfp-neon*/linux-libc-headers/*do_configure*sigdata*
24#   basehash changed from 8d0bd67bb1da6f68717760fc3ef43171 to e869fa61426e88e9c30726ba88a1216a
25#   Variable TUNE_CCARGS value changed from  -march=armv7-a     -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a8 to  -march=armv7-a     -mthumb-interwork -mfloat-abi=softfp -mfpu=neon -mtune=cortex-a9
26
27# Global vars
28tmpdir=
29machines=
30targets=
31default_machines="qemuarm qemux86 qemux86-64"
32default_targets="core-image-base"
33analyze="N"
34
35usage () {
36  cat << EOF
37Welcome to utility to compare sstate checksums between different MACHINEs.
38$0 <OPTION>
39
40Options:
41  -h, --help
42        Display this help and exit.
43
44  --tmpdir=<tmpdir>
45        Specify tmpdir, will use the environment variable TMPDIR if it is not specified.
46        Something like /OE/oe-core/tmp-eglibc (no / at the end).
47
48  --machines=<machines>
49        List of MACHINEs separated by space, will use the environment variable MACHINES if it is not specified.
50        Default value is "qemuarm qemux86 qemux86-64".
51
52  --targets=<targets>
53        List of targets separated by space, will use the environment variable TARGETS if it is not specified.
54        Default value is "core-image-base".
55
56  --analyze
57        Show the differences between MACHINEs. It assumes:
58        * First 2 MACHINEs in --machines parameter have the same TUNE_PKGARCH
59        * Third optional MACHINE has different TUNE_PKGARCH - only native and allarch recipes are compared).
60        * Next MACHINEs are ignored
61EOF
62}
63
64# Print error information and exit.
65echo_error () {
66  echo "ERROR: $1" >&2
67  exit 1
68}
69
70while [ -n "$1" ]; do
71  case $1 in
72    --tmpdir=*)
73      tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e`
74      [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir"
75      shift
76        ;;
77    --machines=*)
78      machines=`echo $1 | sed -e 's#^--machines="*\([^"]*\)"*#\1#'`
79      shift
80        ;;
81    --targets=*)
82      targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'`
83      shift
84        ;;
85    --analyze)
86      analyze="Y"
87      shift
88        ;;
89    --help|-h)
90      usage
91      exit 0
92        ;;
93    *)
94      echo "Invalid arguments $*"
95      echo_error "Try '$0 -h' for more information."
96        ;;
97  esac
98done
99
100# tmpdir directory, use environment variable TMPDIR
101# if it was not specified, otherwise, error.
102[ -n "$tmpdir" ] || tmpdir=$TMPDIR
103[ -n "$tmpdir" ] || echo_error "No tmpdir found!"
104[ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\""
105[ -n "$machines" ] || machines=$MACHINES
106[ -n "$machines" ] || machines=$default_machines
107[ -n "$targets" ] || targets=$TARGETS
108[ -n "$targets" ] || targets=$default_targets
109
110OUTPUT=${tmpdir}/sstate-diff/`date "+%s"`
111declare -i RESULT=0
112
113for M in ${machines}; do
114  [ -d ${tmpdir}/stamps/ ] && find ${tmpdir}/stamps/ -name \*sigdata\* | xargs rm -f
115  mkdir -p ${OUTPUT}/${M}
116  export MACHINE=${M}
117  bitbake -S none ${targets} 2>&1 | tee -a ${OUTPUT}/${M}/log;
118  RESULT+=${PIPESTATUS[0]}
119  if ls ${tmpdir}/stamps/* >/dev/null 2>/dev/null ; then
120    cp -ra ${tmpdir}/stamps/* ${OUTPUT}/${M}
121    find ${OUTPUT}/${M} -name \*sigdata\* | sed "s#${OUTPUT}/${M}/##g" | sort > ${OUTPUT}/${M}/list
122    M_UNDERSCORE=`echo ${M} | sed 's/-/_/g'`
123    sed "s/^${M_UNDERSCORE}-/MACHINE/g" ${OUTPUT}/${M}/list | sort > ${OUTPUT}/${M}/list.M
124    find ${tmpdir}/stamps/ -name \*sigdata\* | xargs rm -f
125  else
126    printf "ERROR: no sigdata files were generated for MACHINE $M in ${tmpdir}/stamps\n";
127  fi
128done
129
130function compareSignatures() {
131  MACHINE1=$1
132  MACHINE2=$2
133  PATTERN="$3"
134  PRE_PATTERN=""
135  [ -n "${PATTERN}" ] || PRE_PATTERN="-v"
136  [ -n "${PATTERN}" ] || PATTERN="MACHINE"
137  for TASK in do_configure.sigdata do_populate_sysroot.sigdata do_package_write_ipk.sigdata; do
138    printf "\n\n === Comparing signatures for task ${TASK} between ${MACHINE1} and ${MACHINE2} ===\n" | tee -a ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log
139    diff ${OUTPUT}/${MACHINE1}/list.M ${OUTPUT}/${MACHINE2}/list.M | grep ${PRE_PATTERN} "${PATTERN}" | grep ${TASK} > ${OUTPUT}/signatures.${MACHINE2}.${TASK}
140    for i in `cat ${OUTPUT}/signatures.${MACHINE2}.${TASK} | sed 's#[^/]*/\([^/]*\)/.*#\1#g' | sort -u | xargs`; do
141      [ -e ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ] || echo "INFO: ${i} task ${TASK} doesn't exist in ${MACHINE1}" >&2
142      [ -e ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ] || continue
143      [ -e ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}* ] || echo "INFO: ${i} task ${TASK} doesn't exist in ${MACHINE2}" >&2
144      [ -e ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}* ] || continue
145      printf "ERROR: $i different signature for task ${TASK} between ${MACHINE1} and ${MACHINE2}\n";
146      bitbake-diffsigs ${OUTPUT}/${MACHINE1}/*/$i/*${TASK}* ${OUTPUT}/${MACHINE2}/*/$i/*${TASK}*;
147      echo "$i" >> ${OUTPUT}/failed-recipes.log
148      echo
149    done | tee -a ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log
150    # don't create empty files
151    ERRORS=`grep "^ERROR.*" ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log | wc -l`
152    if [ "${ERRORS}" != "0" ] ; then
153      echo "ERROR: ${ERRORS} errors found in ${OUTPUT}/signatures.${MACHINE2}.${TASK}.log"
154      RESULT+=${ERRORS}
155    fi
156  done
157}
158
159function compareMachines() {
160  [ "$#" -ge 2 ] && compareSignatures $1 $2
161  [ "$#" -ge 3 ] && compareSignatures $1 $3 "\(^< all\)\|\(^< x86_64-linux\)\|\(^< i586-linux\)"
162}
163
164if [ "${analyze}" = "Y" ] ; then
165  compareMachines ${machines}
166fi
167
168if [ "${RESULT}" != "0" -a -f ${OUTPUT}/failed-recipes.log ] ; then
169  cat ${OUTPUT}/failed-recipes.log | sort -u >${OUTPUT}/failed-recipes.log.u && mv ${OUTPUT}/failed-recipes.log.u ${OUTPUT}/failed-recipes.log
170  echo "ERROR: ${RESULT} issues were found in these recipes: `cat ${OUTPUT}/failed-recipes.log | xargs`"
171fi
172
173echo "INFO: Output written in: ${OUTPUT}"
174exit ${RESULT}
175