ci/lava: Flexibilize section marker regexes

In some jobs, such as
https://gitlab.freedesktop.org/gallo/mesa/-/jobs/24904100, the kmsg is
interleaved with stderr/stdout in serial console, making it difficult to
confidently find the log messages to detect when the DUT is booting,
when the DUT is running etc.

Luckily, LAVA sends redundant messages about their signals. We can use
them to mitigate the chance of missing an interleaved message by being
more open to different messages, using the regex on both `debug` and
`target` LAVA log levels.

Signed-off-by: Guilherme Gallo <guilherme.gallo@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/16323>
This commit is contained in:
Guilherme Gallo 2022-07-04 17:34:17 -03:00 committed by Marge Bot
parent 24f368d652
commit 6ba2b33a8c
2 changed files with 37 additions and 18 deletions

View File

@ -145,7 +145,7 @@ class GitlabSection:
@dataclass(frozen=True) @dataclass(frozen=True)
class LogSection: class LogSection:
regex: Union[Pattern, str] regex: Union[Pattern, str]
level: str levels: tuple[str]
section_id: str section_id: str
section_header: str section_header: str
section_type: LogSectionType section_type: LogSectionType
@ -154,36 +154,38 @@ class LogSection:
def from_log_line_to_section( def from_log_line_to_section(
self, lava_log_line: dict[str, str] self, lava_log_line: dict[str, str]
) -> Optional[GitlabSection]: ) -> Optional[GitlabSection]:
if lava_log_line["lvl"] == self.level: if lava_log_line["lvl"] not in self.levels:
if match := re.match(self.regex, lava_log_line["msg"]): return
section_id = self.section_id.format(*match.groups())
section_header = self.section_header.format(*match.groups()) if match := re.search(self.regex, lava_log_line["msg"]):
return GitlabSection( section_id = self.section_id.format(*match.groups())
id=section_id, section_header = self.section_header.format(*match.groups())
header=section_header, return GitlabSection(
type=self.section_type, id=section_id,
start_collapsed=self.collapsed, header=section_header,
) type=self.section_type,
start_collapsed=self.collapsed,
)
LOG_SECTIONS = ( LOG_SECTIONS = (
LogSection( LogSection(
regex=re.compile(r".*<STARTTC> (.*)"), regex=re.compile(r"<?STARTTC>? ([^>]*)"),
level="debug", levels=("target", "debug"),
section_id="{}", section_id="{}",
section_header="test_case {}", section_header="test_case {}",
section_type=LogSectionType.TEST_CASE, section_type=LogSectionType.TEST_CASE,
), ),
LogSection( LogSection(
regex=re.compile(r".*<STARTRUN> (\S*)"), regex=re.compile(r"<?STARTRUN>? ([^>]*)"),
level="debug", levels=("target", "debug"),
section_id="{}", section_id="{}",
section_header="test_suite {}", section_header="test_suite {}",
section_type=LogSectionType.TEST_SUITE, section_type=LogSectionType.TEST_SUITE,
), ),
LogSection( LogSection(
regex=re.compile(r"^<LAVA_SIGNAL_ENDTC ([^>]+)"), regex=re.compile(r"ENDTC>? ([^>]+)"),
level="target", levels=("target", "debug"),
section_id="post-{}", section_id="post-{}",
section_header="Post test_case {}", section_header="Post test_case {}",
collapsed=True, collapsed=True,
@ -208,7 +210,7 @@ class LogFollower:
) )
assert ( assert (
section_is_created == section_has_started section_is_created == section_has_started
), "Can't follow logs beginning from uninitalized GitLab sections." ), "Can't follow logs beginning from uninitialized GitLab sections."
def __enter__(self): def __enter__(self):
return self return self
@ -240,6 +242,10 @@ class LogFollower:
self.current_section = None self.current_section = None
def update_section(self, new_section: GitlabSection): def update_section(self, new_section: GitlabSection):
# Sections can have redundant regex to find them to mitigate LAVA
# interleaving kmsg and stderr/stdout issue.
if self.current_section and self.current_section.id == new_section.id:
return
self.clear_current_section() self.clear_current_section()
self.current_section = new_section self.current_section = new_section
self._buffer.append(new_section.start()) self._buffer.append(new_section.start())

View File

@ -86,11 +86,24 @@ def test_gl_sections():
"lvl": "debug", "lvl": "debug",
"msg": "Received signal: <STARTRUN> 0_mesa 5971831_1.3.2.3.1", "msg": "Received signal: <STARTRUN> 0_mesa 5971831_1.3.2.3.1",
}, },
# Redundant log message which triggers the same Gitlab Section, it
# should be ignored, unless the id is different
{
"dt": datetime.now(),
"lvl": "target",
"msg": "[ 7.778836] <LAVA_SIGNAL_STARTRUN 0_mesa 5971831_1.3.2.3.1>",
},
{ {
"dt": datetime.now(), "dt": datetime.now(),
"lvl": "debug", "lvl": "debug",
"msg": "Received signal: <STARTTC> mesa-ci_iris-kbl-traces", "msg": "Received signal: <STARTTC> mesa-ci_iris-kbl-traces",
}, },
# Another redundant log message
{
"dt": datetime.now(),
"lvl": "target",
"msg": "[ 16.997829] <LAVA_SIGNAL_STARTTC mesa-ci_iris-kbl-traces>",
},
{ {
"dt": datetime.now(), "dt": datetime.now(),
"lvl": "target", "lvl": "target",