Codebase list rust-serde-xml-rs / 60c1369
Add a script to analyze CI test failures Ximin Luo 4 years ago
4 changed file(s) with 113 addition(s) and 6 deletion(s). Raw diff Collapse all Expand all
+2
-0
.gitignore less more
00 /build
1 /ci.debian.net
12 /src/*/*
23 !/src/*/debian
34
45 /excuses.yaml
56 /rust-excuses*.*
7 /rust-regressions.list
+3
-3
dev/rust-excuses.mk less more
1212 # Usage:
1313 # $ dev/rust-excuses.mk refresh all
1414
15 DST = rust-excuses.png rust-excuses-arch.png
15 DST = rust-excuses.png rust-excuses-arch.png rust-regressions.list
1616 DOWNLOAD = wget -N --no-use-server-timestamps https://release.debian.org/britney/excuses.yaml
1717
1818 all: $(DST)
2323 excuses.yaml:
2424 $(DOWNLOAD)
2525
26 rust-excuses.dot rust-excuses-arch.dot: excuses.yaml dev/rust-excuses.py
27 dev/rust-excuses.py rust-excuses.dot rust-excuses-arch.dot
26 rust-excuses.dot rust-excuses-arch.dot rust-regressions.list: excuses.yaml dev/rust-excuses.py
27 dev/rust-excuses.py rust-excuses.dot rust-excuses-arch.dot rust-regressions.list
2828
2929 %.png: %.dot
3030 unflatten -c 10 "$<" | dot -Tpng > "$@"
+12
-3
dev/rust-excuses.py less more
88 import sys
99 import yaml
1010
11 if len(sys.argv) != 3:
11 if len(sys.argv) != 4:
1212 print(
1313 """Generates dot files to show the migration deps
14 Usage: %s excuses.dot excuses_arch.dot
14 Usage: %s excuses.dot excuses_arch.dot regressions.list
1515
1616 Expects excuses.yaml in the current dir
1717 """
3030 print("generating dot files...", file=sys.stderr)
3131 rust_excuses = open(sys.argv[1], "w")
3232 rust_excuses_arch = open(sys.argv[2], "w")
33 rust_regressions = open(sys.argv[3], "w")
3334
3435 already_seen = set()
3536
107108 failed = [k for (k, v) in policy.items() if v.get("verdict", "") != "PASS"]
108109 attrs = {"shape": "box"}
109110 if "age" in failed:
110 failed = failed.remove("age")
111 failed.remove("age")
111112 attrs.update({ "fillcolor": BG_TOO_NEW, "style": "filled" })
113 if "autopkgtest" in failed:
114 for k, v in policy["autopkgtest"].items():
115 if k == "verdict": continue
116 for uu in v.values():
117 if "REGRESSION" in uu:
118 for u in uu:
119 if u and u.startswith("https://ci.debian.net/data/autopkgtest/testing"):
120 print(u, file=rust_regressions)
112121 if failed:
113122 attrs.update({ "label": "\\N\\nfailed: %s" % ",".join(failed) })
114123 attrs.update({ "fillcolor": BG_MISC_FAIL, "style": "filled" })
+96
-0
dev/rust-regressions.sh less more
0 #!/bin/bash
1 # Analyze debian CI test failures for rust packages against some known failures.
2 #
3 # Usage:
4 # $ dev/rust-regressions.sh # analyse logs mentioned in rust-regressions.list
5 # $ dev/rust-regressions.sh [<test logs>] # analyse logs downloaded manually
6 #
7
8 set -e
9 #set -x
10 shopt -s lastpipe
11
12 declare -A results
13
14 classify() {
15 local c=0
16 local url="$1"
17 local name="${url#ci.debian.net/data/autopkgtest/testing/*/r/}"
18 name="${name%%/*}"
19 if zegrep -q '^error\[E0554\]' "$url"; then
20 results["falsepositive_unstable"]+="$name"$'\n'
21 c=$((c+1))
22 fi
23 if zgrep -q "^crate directory not found" "$url"; then
24 results["falsepositive_autopkgtest_version_bug"]+="$name"$'\n'
25 c=$((c+1))
26 fi
27 if zgrep -q "^error: no matching package named" "$url"; then
28 results["falsepositive_debcargo_945560"]+="$name"$'\n'
29 c=$((c+1))
30 fi
31 if zgrep -q "^error.* No such file or directory (os error 2)$" "$url"; then
32 results["maybetestbroken_filemissing"]+="$name"$'\n'
33 c=$((c+1))
34 fi
35 if [ "$c" = 0 ]; then
36 results["UNKNOWN"]+="$url"$'\n'
37 results["UNKNOWN"]+="$(zgrep ^error "$url" | head -n3 || true)"
38 results["UNKNOWN"]+=$'\n\n'
39 fi
40 }
41
42 declare -A action explain
43
44 action["UNKNOWN"]="Analyze the relevant test log and deal with it. Ask in #debian-rust for help if you get stuck."
45 explain["UNKNOWN"]="We weren't able to automatically guess the reason behind the test failure"
46
47 action["falsepositive_unstable"]="Re-run the test via the ci.debian.net web interface. No package upload is necessary."
48 explain["falsepositive_unstable"]="dh-cargo since version 23 will automatically detect this error and re-run the test with RUSTC_BOOTSTRAP=1"
49
50 action["falsepositive_autopkgtest_version_bug"]="Either wait for the bug to be fixed, or help to fix the bug. Don't change the rust packaging."
51 explain["falsepositive_autopkgtest_version_bug"]="Autopkgtest has a bug, see https://alioth-lists.debian.net/pipermail/pkg-rust-maintainers/2020-January/009512.html"
52
53 action["falsepositive_debcargo_945560"]="regenerate your crate packaging using debcargo 2.4.2 or later"
54 explain["falsepositive_debcargo_945560"]="cargo requires more dependencies than necessary, and debcargo did not work around this until 2.4.2"
55
56 action["maybetestbroken_filemissing"]="Mark the relevant tests as test_is_broken=true in debcargo.toml, re-upload the package, and file a bug upstream for them to fix it."
57 explain["maybetestbroken_filemissing"]="Carge does not enforce that tests pass per-crate, so sometimes crate authors run tests on a per-workspace basis and have crate tests depend on files in the parent workspace, which are not part of the crate when uploaded to crates.io."
58
59 if [ -z "$*" ]; then
60 if ! [ -f rust-regressions.list ]; then
61 echo >&2 "rust-regressions.list does not exist, run \`dev/rust-excuses.mk refresh all\` to update it"
62 elif [ $(($(date +%s) - $(stat -c %Y rust-regressions.list))) -gt 7200 ]; then
63 echo >&2 "rust-regressions.list is a bit old, run \`dev/rust-excuses.mk refresh all\` to update it"
64 fi
65 if [ -z "$DEB_RUST_SKIP_CI_DL" ]; then
66 echo >&2 "Downloading CI test logs. If you did this recently and want to skip, re-run me with DEB_RUST_SKIP_CI_DL=1"
67 wget -nv --mirror $(< rust-regressions.list)
68 fi
69 cat rust-regressions.list | while read url; do
70 classify "${url#https://}"
71 done
72 else
73 for i in "$@"; do classify "$i"; done
74 fi
75
76 for p in "${!results[@]}"; do
77 echo "$p"
78 done | LC_COLLATE=C.UTF-8 sort | while read p; do
79 echo "$p"
80 echo "$p" | sed -e 's/./=/g'
81 echo
82 case "$p" in
83 UNKNOWN)
84 printf "%s" "${results[$p]}"
85 ;;
86 *)
87 printf "%s" "${results[$p]}" | sort -u | tr '\n' ','
88 echo
89 ;;
90 esac
91 echo
92 echo "You should: ${action[$p]}"
93 echo "Explanation: ${explain[$p]}"
94 echo
95 done