forked from cockpit-project/bots
-
Notifications
You must be signed in to change notification settings - Fork 0
/
flakes-refresh
executable file
·193 lines (150 loc) · 6.05 KB
/
flakes-refresh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
#!/usr/bin/env python3
# This file is part of Cockpit.
#
# Copyright (C) 2018 Red Hat, Inc.
#
# Cockpit is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Cockpit is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Cockpit; If not, see <http://www.gnu.org/licenses/>.
import sys
import time
import os
import urllib
import json
import re
import ssl
sys.dont_write_bytecode = True
import task
from machine.machine_core.directories import BOTS_DIR
NUMBER_OPEN_ISSUES = 7 # How many issues do we want to have open at a given time?
# How far back does our data go? If a flake gets fixed but is still
# flaky after this many days, the bots open another issue.
WINDOW_DAYS = 21
# This parses the output JSONL format discussed here, where various
# values are grouped:
#
# https://github.com/cockpit-project/cockpituous/blob/master/learn/README.md
# Here we're looking for a field in a record that only has one value
def value(record, field):
values = record.get(field, [])
if len(values) == 1:
return values[0][0] or ""
return None
# Here we're looking for the count of a specific field/value in the record
def count(record, field, only):
values = record.get(field, [])
for value, count in values:
if value != only:
continue
return count
return 0
# For linking flakes to test logs
def slurp_one(url, n, logs):
items_url = url + str(n) + "/items.jsonl"
try:
with urllib.request.urlopen(items_url) as f:
for line in f.readlines():
try:
record = json.loads(line.decode('utf-8'))
logs.setdefault(record["test"], []).append(record["url"])
except ValueError as ex:
sys.stderr.write("{0}: {1}\n".format(url, ex))
except urllib.error.URLError as ex:
if ex.code == 404:
return False
raise
return True
def slurp_failure_logs(url):
logs = {}
n = 0
while slurp_one(url, n, logs):
n = n + 1
return logs
def get_failure_logs(failure_logs, name, context):
match = context.replace("/", "-")
return sorted(filter(lambda url: match in url, failure_logs[name]), reverse=True)[0:10]
# Main
def run(context, verbose=False, **kwargs):
api = task.github.GitHub()
open_issues = api.issues(labels=["flake"])
create_count = NUMBER_OPEN_ISSUES - len(open_issues)
if create_count <= 0:
return 0
if verbose:
sys.stderr.write("Going to create %s new flake issue(s)\n" % create_count)
host = os.environ.get("COCKPIT_LEARN_SERVICE_HOST", "learn-cockpit.apps.ci.centos.org")
port = os.environ.get("COCKPIT_LEARN_SERVICE_PORT", "443")
url = "{0}://{1}:{2}/active/".format("https" if port == "443" else "http", host, port)
cafile = os.path.join(BOTS_DIR, "images", "files", "ca.pem")
context = ssl.create_default_context(cafile=cafile)
failure_logs = slurp_failure_logs(url)
# Retrieve the URL
statistics = []
with urllib.request.urlopen(url + "statistics.jsonl", context=context) as f:
for line in f.readlines():
try:
record = json.loads(line.decode('utf-8'))
statistics.append(record)
except ValueError as ex:
sys.stderr.write("{0}: {1}\n".format(url, ex))
tests = {}
for record in statistics:
test = value(record, "test")
context = value(record, "context")
status = value(record, "status")
tracker = value(record, "tracker")
# Flaky tests only score on those that fail and are not tracked
if test is not None and status == "failure" and not tracker:
merged = count(record, "merged", True)
not_merged = count(record, "merged", False)
null_merged = count(record, "merged", None)
total = merged + not_merged + null_merged
# And the key is that they were merged anyway
if total > 10:
tests.setdefault(test, []).append((merged / total, context, record))
scores = []
for n, t in tests.items():
scores.append((sum(map(lambda f: f[0], t)) / len(t), n, t))
closed_issues = api.issues(labels=["flake"], state="closed", since=(time.time() - (WINDOW_DAYS * 86400)))
def find_in_issues(issues, name):
for issue in issues:
if name in issue["title"]:
return True
return False
def url_desc(url):
m = re.search("pull-[0-9]+", url)
return m.group(0) if m else url
def failure_description(name, f, logs):
return ("%s%% on %s\n" % (int(f[0] * 100), f[1]) +
"".join(map(lambda url: " - [%s](%s)\n" % (url_desc(url), url),
get_failure_logs(logs, name, f[1]))))
scores.sort(reverse=True)
for score, name, failures in scores:
if find_in_issues(open_issues, name) or find_in_issues(closed_issues, name):
continue
if verbose:
sys.stderr.write("Opening issue for %s\n" % name)
source = "<details><summary>Source material</summary>\n\n```json\n%s\n```\n</details>\n" % "\n".join(
map(lambda f: json.dumps(f[2], indent=2), failures))
data = {
"title": "%s is flaky" % name,
"body": ("\n".join(map(lambda f: failure_description(name, f, failure_logs), failures)) +
"\n\n" + source),
"labels": ["flake"]
}
api.post("issues", data)
create_count -= 1
if create_count == 0:
break
return 0
if __name__ == '__main__':
task.main(function=run, title="Create issues for test flakes")