diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh index 8fc64f20827..35ba6f4b89e 100755 --- a/.gitlab-ci/deqp-runner.sh +++ b/.gitlab-ci/deqp-runner.sh @@ -91,9 +91,8 @@ if [ -e "$INSTALL/deqp-$GPU_VERSION-fails.txt" ]; then DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --baseline $INSTALL/deqp-$GPU_VERSION-fails.txt" fi -if [ -e "$INSTALL/deqp-$GPU_VERSION-flakes.txt" ]; then - DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt" -fi +# Default to an empty known flakes file if it doesn't exist. +touch $INSTALL/deqp-$GPU_VERSION-flakes.txt if [ -e "$INSTALL/deqp-$GPU_VERSION-skips.txt" ]; then DEQP_RUNNER_OPTIONS="$DEQP_RUNNER_OPTIONS --skips $INSTALL/deqp-$GPU_VERSION-skips.txt" @@ -125,6 +124,7 @@ run_cts() { --deqp $deqp \ --output $RESULTS \ --caselist $caselist \ + --flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \ --testlog-to-xml /deqp/executor/testlog-to-xml \ $JOB \ $SUMMARY_LIMIT \ @@ -133,49 +133,6 @@ run_cts() { $DEQP_OPTIONS } -report_flakes() { - flakes=`grep ",Flake" $1 | sed 's|,Flake.*||g'` - if [ -z "$flakes" ]; then - return 0 - fi - - if [ -z "$FLAKES_CHANNEL" ]; then - return 0 - fi - - # The nick needs to be something unique so that multiple runners - # connecting at the same time don't race for one nick and get blocked. - # freenode has a 16-char limit on nicks (9 is the IETF standard, but - # various servers extend that). So, trim off the common prefixes of the - # runner name, and append the job ID so that software runners with more - # than one concurrent job (think swrast) don't collide. For freedreno, - # that gives us a nick as long as db410c-N-JJJJJJJJ, and it'll be a while - # before we make it to 9-digit jobs (we're at 7 so far). - runner=`echo $CI_RUNNER_DESCRIPTION | sed 's|mesa-||' | sed 's|google-freedreno-||g'` - bot="$runner-$CI_JOB_ID" - channel="$FLAKES_CHANNEL" - ( - echo NICK $bot - echo USER $bot unused unused :Gitlab CI Notifier - sleep 10 - echo "JOIN $channel" - sleep 1 - desc="Flakes detected in job: $CI_JOB_URL on $CI_RUNNER_DESCRIPTION" - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then - desc="$desc on branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ($CI_MERGE_REQUEST_TITLE)" - elif [ -n "$CI_COMMIT_BRANCH" ]; then - desc="$desc on branch $CI_COMMIT_BRANCH ($CI_COMMIT_TITLE)" - fi - echo "PRIVMSG $channel :$desc" - for flake in $flakes; do - echo "PRIVMSG $channel :$flake" - done - echo "PRIVMSG $channel :See $CI_JOB_URL/artifacts/browse/results/" - echo "QUIT" - ) | nc irc.freenode.net 6667 > /dev/null - -} - parse_renderer() { RENDERER=`grep -A1 TestCaseResult.\*info.renderer $RESULTS/deqp-info.qpa | grep '||g' | sed 's|||g'` VERSION=`grep -A1 TestCaseResult.\*info.version $RESULTS/deqp-info.qpa | grep '||g' | sed 's|||g'` @@ -281,6 +238,18 @@ deqp-runner junit \ --template "See https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/results/{{testcase}}.xml" # Report the flakes to the IRC channel for monitoring (if configured): -quiet report_flakes $RESULTS_CSV +if [ -n "$FLAKES_CHANNEL" ]; then + python3 $INSTALL/report-flakes.py \ + --host irc.freenode.net \ + --port 6667 \ + --results $RESULTS_CSV \ + --known-flakes $INSTALL/deqp-$GPU_VERSION-flakes.txt \ + --channel "$FLAKES_CHANNEL" \ + --runner "$CI_RUNNER_DESCRIPTION" \ + --job "$CI_JOB_ID" \ + --url "$CI_JOB_URL" \ + --branch "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-$CI_COMMIT_BRANCH}" \ + --branch-title "${CI_MERGE_REQUEST_TITLE:-$CI_COMMIT_TITLE}" +fi exit $DEQP_EXITCODE diff --git a/.gitlab-ci/piglit/piglit-runner.sh b/.gitlab-ci/piglit/piglit-runner.sh index 384a617a84c..c531b2fb030 100755 --- a/.gitlab-ci/piglit/piglit-runner.sh +++ b/.gitlab-ci/piglit/piglit-runner.sh @@ -32,9 +32,8 @@ if [ -e "$INSTALL/piglit-$GPU_VERSION-fails.txt" ]; then PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --baseline $INSTALL/piglit-$GPU_VERSION-fails.txt" fi -if [ -e "$INSTALL/piglit-$GPU_VERSION-flakes.txt" ]; then - PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --flakes $INSTALL/piglit-$GPU_VERSION-flakes.txt" -fi +# Default to an empty known flakes file if it doesn't exist. +touch $INSTALL/piglit-$GPU_VERSION-flakes.txt if [ -e "$INSTALL/piglit-$GPU_VERSION-skips.txt" ]; then PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --skips $INSTALL/piglit-$GPU_VERSION-skips.txt" @@ -50,59 +49,6 @@ else PIGLIT_RUNNER_OPTIONS="$PIGLIT_RUNNER_OPTIONS --jobs 4" fi -report_flakes() { - # Replace spaces in test names with _ to make the channel reporting not - # split it across lines, even though it makes it so you can't copy and - # paste from IRC into your flakes list. - flakes=`grep ",Flake" $1 | sed 's|,Flake.*||g' | sed 's| |_|g'` - if [ -z "$flakes" ]; then - return 0 - fi - - if [ -z "$FLAKES_CHANNEL" ]; then - return 0 - fi - - # The nick needs to be something unique so that multiple runners - # connecting at the same time don't race for one nick and get blocked. - # freenode has a 16-char limit on nicks (9 is the IETF standard, but - # various servers extend that). So, trim off the common prefixes of the - # runner name, and append the job ID so that software runners with more - # than one concurrent job (think swrast) don't collide. For freedreno, - # that gives us a nick as long as db410c-N-JJJJJJJJ, and it'll be a while - # before we make it to 9-digit jobs (we're at 7 so far). - runner=`echo $CI_RUNNER_DESCRIPTION | sed 's|mesa-||' | sed 's|google-freedreno-||g'` - bot="$runner-$CI_JOB_ID" - channel="$FLAKES_CHANNEL" - ( - echo NICK $bot - echo USER $bot unused unused :Gitlab CI Notifier - sleep 10 - echo "JOIN $channel" - sleep 1 - desc="Flakes detected in job: $CI_JOB_URL on $CI_RUNNER_DESCRIPTION" - if [ -n "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ]; then - desc="$desc on branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME ($CI_MERGE_REQUEST_TITLE)" - elif [ -n "$CI_COMMIT_BRANCH" ]; then - desc="$desc on branch $CI_COMMIT_BRANCH ($CI_COMMIT_TITLE)" - fi - echo "PRIVMSG $channel :$desc" - for flake in $flakes; do - echo "PRIVMSG $channel :$flake" - done - echo "PRIVMSG $channel :See $CI_JOB_URL/artifacts/browse/results/" - echo "QUIT" - ) | nc irc.freenode.net 6667 > /dev/null - -} - -# wrapper to supress +x to avoid spamming the log -quiet() { - set +x - "$@" - set -x -} - RESULTS_CSV=$RESULTS/results.csv FAILURES_CSV=$RESULTS/failures.csv @@ -112,6 +58,7 @@ export LD_PRELOAD=$TEST_LD_PRELOAD run \ --piglit-folder /piglit \ --output $RESULTS \ + --flakes $INSTALL/piglit-$GPU_VERSION-flakes.txt \ --profile $PIGLIT_PROFILES \ --process-isolation \ $PIGLIT_RUNNER_OPTIONS \ @@ -129,6 +76,18 @@ deqp-runner junit \ --template "See https://$CI_PROJECT_ROOT_NAMESPACE.pages.freedesktop.org/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts/results/{{testcase}}.xml" # Report the flakes to the IRC channel for monitoring (if configured): -quiet report_flakes $RESULTS_CSV +if [ -n "$FLAKES_CHANNEL" ]; then + python3 $INSTALL/report-flakes.py \ + --host irc.freenode.net \ + --port 6667 \ + --results $RESULTS_CSV \ + --known-flakes $INSTALL/piglit-$GPU_VERSION-flakes.txt \ + --channel "$FLAKES_CHANNEL" \ + --runner "$CI_RUNNER_DESCRIPTION" \ + --job "$CI_JOB_ID" \ + --url "$CI_JOB_URL" \ + --branch "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-$CI_COMMIT_BRANCH}" \ + --branch-title "${CI_MERGE_REQUEST_TITLE:-$CI_COMMIT_TITLE}" +fi exit $PIGLIT_EXITCODE diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh index 9acbd023c58..005cfe590bd 100755 --- a/.gitlab-ci/prepare-artifacts.sh +++ b/.gitlab-ci/prepare-artifacts.sh @@ -33,6 +33,7 @@ cp -Rp .gitlab-ci/fossils install/ cp -Rp .gitlab-ci/fossilize-runner.sh install/ cp -Rp .gitlab-ci/deqp-runner.sh install/ cp -Rp .gitlab-ci/deqp-*.txt install/ +cp -Rp .gitlab-ci/report-flakes.py install/ cp -Rp .gitlab-ci/vkd3d-proton install/ find . -path \*/ci/\*.txt \ -o -path \*/ci/\*traces\*.yml \ diff --git a/.gitlab-ci/report-flakes.py b/.gitlab-ci/report-flakes.py new file mode 100644 index 00000000000..ed7009c9bd6 --- /dev/null +++ b/.gitlab-ci/report-flakes.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# +# Copyright © 2021 Google LLC +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice (including the next +# paragraph) shall be included in all copies or substantial portions of the +# Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import argparse +import io +import re +import socket +import time + + +class Connection: + def __init__(self, host, port, verbose): + self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.s.connect((host, port)) + self.s.setblocking(0) + self.verbose = verbose + + def send_line(self, line): + if self.verbose: + print(f"IRC: sending {line}") + self.s.sendall((line + '\n').encode()) + + def wait(self, secs): + for i in range(secs): + if self.verbose: + while True: + try: + data = self.s.recv(1024) + except io.BlockingIOError: + break + if data == "": + break + for line in data.decode().split('\n'): + print(f"IRC: received {line}") + time.sleep(1) + + def quit(self): + self.send_line("QUIT") + self.s.shutdown(socket.SHUT_WR) + self.s.close() + + +def read_flakes(results): + flakes = [] + csv = re.compile("(.*),(.*),(.*)") + for line in open(results, 'r').readlines(): + match = csv.match(line) + if match.group(2) == "Flake": + flakes.append(match.group(1)) + return flakes + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--host', type=str, + help='IRC server hostname', required=True) + parser.add_argument('--port', type=int, + help='IRC server port', required=True) + parser.add_argument('--results', type=str, + help='results.csv file from deqp-runner or piglit-runner', required=True) + parser.add_argument('--known-flakes', type=str, + help='*-flakes.txt file passed to deqp-runner or piglit-runner', required=True) + parser.add_argument('--channel', type=str, + help='Known flakes report channel', required=True) + parser.add_argument('--url', type=str, + help='$CI_JOB_URL', required=True) + parser.add_argument('--runner', type=str, + help='$CI_RUNNER_DESCRIPTION', required=True) + parser.add_argument('--branch', type=str, + help='optional branch name') + parser.add_argument('--branch-title', type=str, + help='optional branch title') + parser.add_argument('--job', type=str, + help='$CI_JOB_ID', required=True) + parser.add_argument('--verbose', "-v", action="store_true", + help='log IRC interactions') + args = parser.parse_args() + + flakes = read_flakes(args.results) + if not flakes: + exit(0) + + known_flakes = [] + for line in open(args.known_flakes).readlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + known_flakes.append(re.compile(line)) + + irc = Connection(args.host, args.port, args.verbose) + + # The nick needs to be something unique so that multiple runners + # connecting at the same time don't race for one nick and get blocked. + # freenode has a 16-char limit on nicks (9 is the IETF standard, but + # various servers extend that). So, trim off the common prefixes of the + # runner name, and append the job ID so that software runners with more + # than one concurrent job (think swrast) don't collide. For freedreno, + # that gives us a nick as long as db410c-N-JJJJJJJJ, and it'll be a while + # before we make it to 9-digit jobs (we're at 7 so far). + nick = args.runner + nick = nick.replace('mesa-', '') + nick = nick.replace('google-freedreno-', '') + nick += f'-{args.job}' + irc.send_line(f"NICK {nick}") + irc.send_line(f"USER {nick} unused unused: Gitlab CI Notifier") + irc.wait(10) + irc.send_line(f"JOIN {args.channel}") + irc.wait(1) + + branchinfo = "" + if args.branch: + branchinfo = f" on branch {args.branch} ({args.branch_title})" + irc.send_line( + f"PRIVMSG {args.channel} :Flakes detected in job {args.url} on {args.runner}{branchinfo}:") + + for flake in flakes: + status = "NEW " + for known in known_flakes: + if known.match(flake): + status = "" + break + + irc.send_line(f"PRIVMSG {args.channel} :{status}{flake}") + + irc.send_line( + f"PRIVMSG {args.channel} :See {args.url}/artifacts/browse/results/") + + irc.quit() + + +if __name__ == '__main__': + main()