ci: Add known-flake handling for the IRC flake reports
Now, flakes that aren't in the *-flakes.txt get a "NEW" in their report so I can watch for them. The bash was unwieldy and made debugging hard, so I switched to python. Reviewed-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11020>
This commit is contained in:
Emma Anholt
Marge Bot
parent
5515d3aec5
commit
e414718aef
|
|
@ -91,9 +91,8 @@ if [ -e "$INSTALL/deqp-$GPU_VERSION-fails.txt" ]; then
|
|||
DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --baseline $INSTALL/deqp-$GPU_VERSION-fails.txt"
|
||||
fi
|
||||
|
||||
if [ -e "$INSTALL/deqp-$GPU_VERSION-flakes.txt" ]; then
|
||||
DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt"
|
||||
fi
|
||||
# Default to an empty known flakes file if it doesn't exist.
|
||||
touch $INSTALL/deqp-$GPU_VERSION-flakes.txt
|
||||
|
||||
if [ -e "$INSTALL/deqp-$GPU_VERSION-skips.txt" ]; then
|
||||
DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --skips $INSTALL/deqp-$GPU_VERSION-skips.txt"
|
||||
|
|
@ -125,6 +124,7 @@ run_cts() {
|
|||
--deqp $deqp \
|
||||
--output $RESULTS \
|
||||
--caselist $caselist \
|
||||
--flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \
|
||||
--testlog-to-xml /deqp/executor/testlog-to-xml \
|
||||
$JOB \
|
||||
$SUMMARY_LIMIT \
|
||||
|
|
@ -133,49 +133,6 @@ run_cts() {
|
|||
$DEQP_OPTIONS
|
||||
}
|
||||
|
||||
report_flakes() {
|
||||
flakes=`grep ",Flake" $1 | sed 's|,Flake.*||g'`
|
||||
if [ -z "$flakes" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ -z "$FLAKES_CHANNEL" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# The nick needs to be something unique so that multiple runners
|
||||
# connecting at the same time don't race for one nick and get blocked.
|
||||
# freenode has a 16-char limit on nicks (9 is the IETF standard, but
|
||||
# various servers extend that). So, trim off the common prefixes of the
|
||||
# runner name, and append the job ID so that software runners with more
|
||||
# than one concurrent job (think swrast) don't collide. For freedreno,
|
||||
# that gives us a nick as long as db410c-N-JJJJJJJJ, and it'll be a while
|
||||
# before we make it to 9-digit jobs (we're at 7 so far).
|
||||
runner=`echo $CI_RUNNER_DESCRIPTION | sed 's|mesa-||' | sed 's|google-freedreno-||g'`
|
||||
bot="$runner-$CI_JOB_ID"
|
||||
channel="$FLAKES_CHANNEL"
|
||||
(
|
||||
echo NICK $bot
|
||||
echo USER $bot unused unused :Gitlab CI Notifier
|
||||
sleep 10
|
||||
echo "JOIN $channel"
|
||||
sleep 1
|
||||
desc="Flakes detected in job: $CI_JOB_URL on $CI_RUNNER_DESCRIPTION"
|
||||
if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then
|
||||
desc="$desc on branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ($CI_MERGE_REQUEST_TITLE)"
|
||||
elif [ -n "$CI_COMMIT_BRANCH" ]; then
|
||||
desc="$desc on branch $CI_COMMIT_BRANCH ($CI_COMMIT_TITLE)"
|
||||
fi
|
||||
echo "PRIVMSG $channel :$desc"
|
||||
for flake in $flakes; do
|
||||
echo "PRIVMSG $channel :$flake"
|
||||
done
|
||||
echo "PRIVMSG $channel :See $CI_JOB_URL/artifacts/browse/results/"
|
||||
echo "QUIT"
|
||||
) | nc irc.freenode.net 6667 > /dev/null
|
||||
|
||||
}
|
||||
|
||||
parse_renderer() {
|
||||
RENDERER=`grep -A1 TestCaseResult.\*info.renderer $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
|
||||
VERSION=`grep -A1 TestCaseResult.\*info.version $RESULTS/deqp-info.qpa | grep '<Text' | sed 's|.*<Text>||g' | sed 's|</Text>||g'`
|
||||
|
|
@ -281,6 +238,18 @@ deqp-runner junit \
|
|||
--template "See https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/results/{{testcase}}.xml"
|
||||
|
||||
# Report the flakes to the IRC channel for monitoring (if configured):
|
||||
quiet report_flakes $RESULTS_CSV
|
||||
if [ -n "$FLAKES_CHANNEL" ]; then
|
||||
python3 $INSTALL/report-flakes.py \
|
||||
--host irc.freenode.net \
|
||||
--port 6667 \
|
||||
--results $RESULTS_CSV \
|
||||
--known-flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \
|
||||
--channel "$FLAKES_CHANNEL" \
|
||||
--runner "$CI_RUNNER_DESCRIPTION" \
|
||||
--job "$CI_JOB_ID" \
|
||||
--url "$CI_JOB_URL" \
|
||||
--branch "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-$CI_COMMIT_BRANCH}" \
|
||||
--branch-title "${CI_MERGE_REQUEST_TITLE:-$CI_COMMIT_TITLE}"
|
||||
fi
|
||||
|
||||
exit $DEQP_EXITCODE
|
||||
|
|
|
|||
|
|
@ -32,9 +32,8 @@ if [ -e "$INSTALL/piglit-$GPU_VERSION-fails.txt" ]; then
|
|||
PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --baseline $INSTALL/piglit-$GPU_VERSION-fails.txt"
|
||||
fi
|
||||
|
||||
if [ -e "$INSTALL/piglit-$GPU_VERSION-flakes.txt" ]; then
|
||||
PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --flakes $INSTALL/piglit-$GPU_VERSION-flakes.txt"
|
||||
fi
|
||||
# Default to an empty known flakes file if it doesn't exist.
|
||||
touch $INSTALL/piglit-$GPU_VERSION-flakes.txt
|
||||
|
||||
if [ -e "$INSTALL/piglit-$GPU_VERSION-skips.txt" ]; then
|
||||
PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --skips $INSTALL/piglit-$GPU_VERSION-skips.txt"
|
||||
|
|
@ -50,59 +49,6 @@ else
|
|||
PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --jobs 4"
|
||||
fi
|
||||
|
||||
report_flakes() {
|
||||
# Replace spaces in test names with _ to make the channel reporting not
|
||||
# split it across lines, even though it makes it so you can't copy and
|
||||
# paste from IRC into your flakes list.
|
||||
flakes=`grep ",Flake" $1 | sed 's|,Flake.*||g' | sed 's| |_|g'`
|
||||
if [ -z "$flakes" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ -z "$FLAKES_CHANNEL" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# The nick needs to be something unique so that multiple runners
|
||||
# connecting at the same time don't race for one nick and get blocked.
|
||||
# freenode has a 16-char limit on nicks (9 is the IETF standard, but
|
||||
# various servers extend that). So, trim off the common prefixes of the
|
||||
# runner name, and append the job ID so that software runners with more
|
||||
# than one concurrent job (think swrast) don't collide. For freedreno,
|
||||
# that gives us a nick as long as db410c-N-JJJJJJJJ, and it'll be a while
|
||||
# before we make it to 9-digit jobs (we're at 7 so far).
|
||||
runner=`echo $CI_RUNNER_DESCRIPTION | sed 's|mesa-||' | sed 's|google-freedreno-||g'`
|
||||
bot="$runner-$CI_JOB_ID"
|
||||
channel="$FLAKES_CHANNEL"
|
||||
(
|
||||
echo NICK $bot
|
||||
echo USER $bot unused unused :Gitlab CI Notifier
|
||||
sleep 10
|
||||
echo "JOIN $channel"
|
||||
sleep 1
|
||||
desc="Flakes detected in job: $CI_JOB_URL on $CI_RUNNER_DESCRIPTION"
|
||||
if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then
|
||||
desc="$desc on branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ($CI_MERGE_REQUEST_TITLE)"
|
||||
elif [ -n "$CI_COMMIT_BRANCH" ]; then
|
||||
desc="$desc on branch $CI_COMMIT_BRANCH ($CI_COMMIT_TITLE)"
|
||||
fi
|
||||
echo "PRIVMSG $channel :$desc"
|
||||
for flake in $flakes; do
|
||||
echo "PRIVMSG $channel :$flake"
|
||||
done
|
||||
echo "PRIVMSG $channel :See $CI_JOB_URL/artifacts/browse/results/"
|
||||
echo "QUIT"
|
||||
) | nc irc.freenode.net 6667 > /dev/null
|
||||
|
||||
}
|
||||
|
||||
# wrapper to supress +x to avoid spamming the log
|
||||
quiet() {
|
||||
set +x
|
||||
"$@"
|
||||
set -x
|
||||
}
|
||||
|
||||
RESULTS_CSV=$RESULTS/results.csv
|
||||
FAILURES_CSV=$RESULTS/failures.csv
|
||||
|
||||
|
|
@ -112,6 +58,7 @@ export LD_PRELOAD=$TEST_LD_PRELOAD
|
|||
run \
|
||||
--piglit-folder /piglit \
|
||||
--output $RESULTS \
|
||||
--flakes $INSTALL/piglit-$GPU_VERSION-flakes.txt \
|
||||
--profile $PIGLIT_PROFILES \
|
||||
--process-isolation \
|
||||
$PIGLIT_RUNNER_OPTIONS \
|
||||
|
|
@ -129,6 +76,18 @@ deqp-runner junit \
|
|||
--template "See https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/results/{{testcase}}.xml"
|
||||
|
||||
# Report the flakes to the IRC channel for monitoring (if configured):
|
||||
quiet report_flakes $RESULTS_CSV
|
||||
if [ -n "$FLAKES_CHANNEL" ]; then
|
||||
python3 $INSTALL/report-flakes.py \
|
||||
--host irc.freenode.net \
|
||||
--port 6667 \
|
||||
--results $RESULTS_CSV \
|
||||
--known-flakes $INSTALL/piglit-$GPU_VERSION-flakes.txt \
|
||||
--channel "$FLAKES_CHANNEL" \
|
||||
--runner "$CI_RUNNER_DESCRIPTION" \
|
||||
--job "$CI_JOB_ID" \
|
||||
--url "$CI_JOB_URL" \
|
||||
--branch "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-$CI_COMMIT_BRANCH}" \
|
||||
--branch-title "${CI_MERGE_REQUEST_TITLE:-$CI_COMMIT_TITLE}"
|
||||
fi
|
||||
|
||||
exit $PIGLIT_EXITCODE
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ cp -Rp .gitlab-ci/fossils install/
|
|||
cp -Rp .gitlab-ci/fossilize-runner.sh install/
|
||||
cp -Rp .gitlab-ci/deqp-runner.sh install/
|
||||
cp -Rp .gitlab-ci/deqp-*.txt install/
|
||||
cp -Rp .gitlab-ci/report-flakes.py install/
|
||||
cp -Rp .gitlab-ci/vkd3d-proton install/
|
||||
find . -path \*/ci/\*.txt \
|
||||
-o -path \*/ci/\*traces\*.yml \
|
||||
|
|
|
|||
|
|
@ -0,0 +1,151 @@
|
|||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright © 2021 Google LLC
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice (including the next
|
||||
# paragraph) shall be included in all copies or substantial portions of the
|
||||
# Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
# IN THE SOFTWARE.
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
|
||||
|
||||
class Connection:
|
||||
def __init__(self, host, port, verbose):
|
||||
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self.s.connect((host, port))
|
||||
self.s.setblocking(0)
|
||||
self.verbose = verbose
|
||||
|
||||
def send_line(self, line):
|
||||
if self.verbose:
|
||||
print(f"IRC: sending {line}")
|
||||
self.s.sendall((line + '\n').encode())
|
||||
|
||||
def wait(self, secs):
|
||||
for i in range(secs):
|
||||
if self.verbose:
|
||||
while True:
|
||||
try:
|
||||
data = self.s.recv(1024)
|
||||
except io.BlockingIOError:
|
||||
break
|
||||
if data == "":
|
||||
break
|
||||
for line in data.decode().split('\n'):
|
||||
print(f"IRC: received {line}")
|
||||
time.sleep(1)
|
||||
|
||||
def quit(self):
|
||||
self.send_line("QUIT")
|
||||
self.s.shutdown(socket.SHUT_WR)
|
||||
self.s.close()
|
||||
|
||||
|
||||
def read_flakes(results):
|
||||
flakes = []
|
||||
csv = re.compile("(.*),(.*),(.*)")
|
||||
for line in open(results, 'r').readlines():
|
||||
match = csv.match(line)
|
||||
if match.group(2) == "Flake":
|
||||
flakes.append(match.group(1))
|
||||
return flakes
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--host', type=str,
|
||||
help='IRC server hostname', required=True)
|
||||
parser.add_argument('--port', type=int,
|
||||
help='IRC server port', required=True)
|
||||
parser.add_argument('--results', type=str,
|
||||
help='results.csv file from deqp-runner or piglit-runner', required=True)
|
||||
parser.add_argument('--known-flakes', type=str,
|
||||
help='*-flakes.txt file passed to deqp-runner or piglit-runner', required=True)
|
||||
parser.add_argument('--channel', type=str,
|
||||
help='Known flakes report channel', required=True)
|
||||
parser.add_argument('--url', type=str,
|
||||
help='$CI_JOB_URL', required=True)
|
||||
parser.add_argument('--runner', type=str,
|
||||
help='$CI_RUNNER_DESCRIPTION', required=True)
|
||||
parser.add_argument('--branch', type=str,
|
||||
help='optional branch name')
|
||||
parser.add_argument('--branch-title', type=str,
|
||||
help='optional branch title')
|
||||
parser.add_argument('--job', type=str,
|
||||
help='$CI_JOB_ID', required=True)
|
||||
parser.add_argument('--verbose', "-v", action="store_true",
|
||||
help='log IRC interactions')
|
||||
args = parser.parse_args()
|
||||
|
||||
flakes = read_flakes(args.results)
|
||||
if not flakes:
|
||||
exit(0)
|
||||
|
||||
known_flakes = []
|
||||
for line in open(args.known_flakes).readlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
known_flakes.append(re.compile(line))
|
||||
|
||||
irc = Connection(args.host, args.port, args.verbose)
|
||||
|
||||
# The nick needs to be something unique so that multiple runners
|
||||
# connecting at the same time don't race for one nick and get blocked.
|
||||
# freenode has a 16-char limit on nicks (9 is the IETF standard, but
|
||||
# various servers extend that). So, trim off the common prefixes of the
|
||||
# runner name, and append the job ID so that software runners with more
|
||||
# than one concurrent job (think swrast) don't collide. For freedreno,
|
||||
# that gives us a nick as long as db410c-N-JJJJJJJJ, and it'll be a while
|
||||
# before we make it to 9-digit jobs (we're at 7 so far).
|
||||
nick = args.runner
|
||||
nick = nick.replace('mesa-', '')
|
||||
nick = nick.replace('google-freedreno-', '')
|
||||
nick += f'-{args.job}'
|
||||
irc.send_line(f"NICK {nick}")
|
||||
irc.send_line(f"USER {nick} unused unused: Gitlab CI Notifier")
|
||||
irc.wait(10)
|
||||
irc.send_line(f"JOIN {args.channel}")
|
||||
irc.wait(1)
|
||||
|
||||
branchinfo = ""
|
||||
if args.branch:
|
||||
branchinfo = f" on branch {args.branch} ({args.branch_title})"
|
||||
irc.send_line(
|
||||
f"PRIVMSG {args.channel} :Flakes detected in job {args.url} on {args.runner}{branchinfo}:")
|
||||
|
||||
for flake in flakes:
|
||||
status = "NEW "
|
||||
for known in known_flakes:
|
||||
if known.match(flake):
|
||||
status = ""
|
||||
break
|
||||
|
||||
irc.send_line(f"PRIVMSG {args.channel} :{status}{flake}")
|
||||
|
||||
irc.send_line(
|
||||
f"PRIVMSG {args.channel} :See {args.url}/artifacts/browse/results/")
|
||||
|
||||
irc.quit()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue