1#!/bin/bash
2
3# Copyright (c) 2011 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7# This script can be used by waterfall sheriffs to fetch the status
8# of Valgrind bots on the memory waterfall and test if their local
9# suppressions match the reports on the waterfall.
10
11set -e
12
13THISDIR=$(dirname "${0}")
14LOGS_DIR=$THISDIR/waterfall.tmp
15WATERFALL_PAGE="http://build.chromium.org/p/chromium.memory/builders"
16WATERFALL_FYI_PAGE="http://build.chromium.org/p/chromium.memory.fyi/builders"
17
18download() {
19  # Download a file.
20  # $1 = URL to download
21  # $2 = Path to the output file
22  # {{{1
23  if [ "$(which curl)" != "" ]
24  then
25    if ! curl -s -o "$2" "$1"
26    then
27      echo
28      echo "Failed to download '$1'... aborting"
29      exit 1
30    fi
31  elif [ "$(which wget)" != "" ]
32  then
33    if ! wget "$1" -O "$2" -q
34    then
35      echo
36      echo "Failed to download '$1'... aborting"
37      exit 1
38    fi
39  else
40    echo "Need either curl or wget to download stuff... aborting"
41    exit 1
42  fi
43  # }}}
44}
45
46fetch_logs() {
47  # Fetch Valgrind logs from the waterfall {{{1
48
49  # TODO(timurrrr,maruel): use JSON, see
50  # http://build.chromium.org/p/chromium.memory/json/help
51
52  rm -rf "$LOGS_DIR" # Delete old logs
53  mkdir "$LOGS_DIR"
54
55  echo "Fetching the list of builders..."
56  download $1 "$LOGS_DIR/builders"
57  SLAVES=$(grep "<a href=\"builders\/" "$LOGS_DIR/builders" | \
58           grep 'td class="box"' | \
59           sed "s/.*<a href=\"builders\///" | sed "s/\".*//" | \
60           sort | uniq)
61
62  for S in $SLAVES
63  do
64    SLAVE_URL=$1/$S
65    SLAVE_NAME=$(echo $S | sed -e "s/%20/ /g" -e "s/%28/(/g" -e "s/%29/)/g")
66    echo -n "Fetching builds by slave '${SLAVE_NAME}'"
67    download $SLAVE_URL?numbuilds=${NUMBUILDS} "$LOGS_DIR/slave_${S}"
68
69    # We speed up the 'fetch' step by skipping the builds/tests which succeeded.
70    # TODO(timurrrr): OTOH, we won't be able to check
71    # if some suppression is not used anymore.
72    #
73    # The awk script here joins the lines ending with </td> to make it possible
74    # to find the failed builds.
75    LIST_OF_BUILDS=$(cat "$LOGS_DIR/slave_$S" | \
76                     awk 'BEGIN { buf = "" }
77                          {
78                            if ($0 ~ /<\/td>/) { buf = (buf $0); }
79                            else {
80                              if (buf) { print buf; buf="" }
81                              print $0
82                            }
83                          }
84                          END {if (buf) print buf}' | \
85                     grep "success\|failure" | \
86                     head -n $NUMBUILDS | \
87                     grep "failure" | \
88                     grep -v "failed compile" | \
89                     sed "s/.*\/builds\///" | sed "s/\".*//")
90
91    for BUILD in $LIST_OF_BUILDS
92    do
93      # We'll fetch a few tiny URLs now, let's use a temp file.
94      TMPFILE=$(mktemp -t memory_waterfall.XXXXXX)
95      download $SLAVE_URL/builds/$BUILD "$TMPFILE"
96
97      REPORT_FILE="$LOGS_DIR/report_${S}_${BUILD}"
98      rm -f $REPORT_FILE 2>/dev/null || true  # make sure it doesn't exist
99
100      REPORT_URLS=$(grep -o "[0-9]\+/steps/\(memory\|heapcheck\).*/logs/[0-9A-F]\{16\}" \
101                    "$TMPFILE" \
102                    || true)  # `true` is to succeed on empty output
103      FAILED_TESTS=$(grep -o "[0-9]\+/steps/\(memory\|heapcheck\).*/logs/[A-Za-z0-9_.]\+" \
104                     "$TMPFILE" | grep -v "[0-9A-F]\{16\}" \
105                     | grep -v "stdio" || true)
106
107      for REPORT in $REPORT_URLS
108      do
109        download "$SLAVE_URL/builds/$REPORT/text" "$TMPFILE"
110        echo "" >> "$TMPFILE"  # Add a newline at the end
111        cat "$TMPFILE" | tr -d '\r' >> "$REPORT_FILE"
112      done
113
114      for FAILURE in $FAILED_TESTS
115      do
116        echo -n "FAILED:" >> "$REPORT_FILE"
117        echo "$FAILURE" | sed -e "s/.*\/logs\///" -e "s/\/.*//" \
118          >> "$REPORT_FILE"
119      done
120
121      rm "$TMPFILE"
122      echo $SLAVE_URL/builds/$BUILD >> "$REPORT_FILE"
123    done
124    echo " DONE"
125  done
126  # }}}
127}
128
129match_suppressions() {
130  PYTHONPATH=$THISDIR/../python/google \
131             python "$THISDIR/test_suppressions.py" $@ "$LOGS_DIR/report_"*
132}
133
134match_gtest_excludes() {
135  for PLATFORM in "Linux" "Chromium%20Mac" "Chromium%20OS"
136  do
137    echo
138    echo "Test failures on ${PLATFORM}:" | sed "s/%20/ /"
139    grep -h -o "^FAILED:.*" -R "$LOGS_DIR"/*${PLATFORM}* | \
140         grep -v "FAILS\|FLAKY" | sort | uniq | \
141         sed -e "s/^FAILED://" -e "s/^/  /"
142    # Don't put any operators between "grep | sed" and "RESULT=$PIPESTATUS"
143    RESULT=$PIPESTATUS
144
145    if [ "$RESULT" == 1 ]
146    then
147      echo "  None!"
148    else
149      echo
150      echo "  Note: we don't check for failures already excluded locally yet"
151      echo "  TODO(timurrrr): don't list tests we've already excluded locally"
152    fi
153  done
154  echo
155  echo "Note: we don't print FAILS/FLAKY tests and 1200s-timeout failures"
156}
157
158usage() {
159  cat <<EOF
160usage: $0 fetch|match options
161
162This script can be used by waterfall sheriffs to fetch the status
163of Valgrind bots on the memory waterfall and test if their local
164suppressions match the reports on the waterfall.
165
166OPTIONS:
167   -h      Show this message
168   -n N    Fetch N builds from each slave.
169
170COMMANDS:
171  fetch    Fetch Valgrind logs from the memory waterfall
172  match    Test the local suppression files against the downloaded logs
173
174EOF
175}
176
177NUMBUILDS=3
178
179CMD=$1
180if [ $# != 0 ]; then
181  shift
182fi
183
184# Arguments for "match" are handled in match_suppressions
185if [ "$CMD" != "match" ]; then
186  while getopts “hn:” OPTION
187  do
188    case $OPTION in
189      h)
190        usage
191        exit
192        ;;
193      n)
194        NUMBUILDS=$OPTARG
195        ;;
196      ?)
197        usage
198        exit
199        ;;
200    esac
201  done
202  shift $((OPTIND-1))
203  if [ $# != 0 ]; then
204    usage
205    exit 1
206  fi
207fi
208
209if [ "$CMD" = "fetch" ]; then
210  echo "Fetching $NUMBUILDS builds"
211  fetch_logs $WATERFALL_PAGE
212  fetch_logs $WATERFALL_FYI_PAGE
213elif [ "$CMD" = "match" ]; then
214  match_suppressions $@
215  match_gtest_excludes
216elif [ "$CMD" = "blame" ]; then
217  echo The blame command died of bitrot. If you need it, please reimplement it.
218  echo Reimplementation is blocked on http://crbug.com/82688
219else
220  usage
221  exit 1
222fi
223