From 740cfc7f8a15847cfe8af017f841bd893bf25804 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 1/30] Add script to manage FTBFS/FTI bugs


Signed-off-by: Igor Raits <ignatenkobrain@fedoraproject.org>
Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/close-fti.j2 b/scripts/ftbfs-fti/close-fti.j2
new file mode 100644
index 0000000..5b5ec2e
--- /dev/null
+++ b/scripts/ftbfs-fti/close-fti.j2
@@ -0,0 +1,5 @@
+{% include "header.j2" %}
+
+All subpackages of a package agaisnt which this bug was filled are installable now in Fedora 33.
+
+Thanks for taking care of it!
diff --git a/scripts/ftbfs-fti/create-footer.j2 b/scripts/ftbfs-fti/create-footer.j2
new file mode 100644
index 0000000..d0bb5b9
--- /dev/null
+++ b/scripts/ftbfs-fti/create-footer.j2
@@ -0,0 +1,7 @@
+According to a policy for FTBFS/FTI bugs (https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/), your package may be orphaned in 8+ weeks if you won't reply to this bug.
+
+P.S. The data was generated solely from koji buildroot, so it might be newer than the latest rawhide compose.
+
+P.P.S. If this bug has been reported in the middle of upgrading multiple dependent packages, please consider using side tags: https://docs.fedoraproject.org/en-US/rawhide-gating/multi-builds/
+
+Thanks!
diff --git a/scripts/ftbfs-fti/create-fti.j2 b/scripts/ftbfs-fti/create-fti.j2
new file mode 100644
index 0000000..926bce1
--- /dev/null
+++ b/scripts/ftbfs-fti/create-fti.j2
@@ -0,0 +1,12 @@
+{% include "header.j2" %}
+
+Your package ({{ src }}) Fails To Install in Fedora 33:
+
+{% for pkg, problems in pkg_problems.items() -%}
+can't install {{ pkg }}:
+  {% for problem in problems -%}
+    - {{ problem }}
+  {% endfor %}
+{% endfor -%}
+
+{% include "create-footer.j2" %}
diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
new file mode 100755
index 0000000..9ab262e
--- /dev/null
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -0,0 +1,272 @@
+#!/usr/bin/python3
+
+import collections
+import datetime
+import os
+import re
+import sys
+
+import bugzilla
+import click
+import jinja2
+import solv
+
+TEMPLATE_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+def _bzdate_to_python(date):
+    return datetime.datetime.strptime(str(date), "%Y%m%dT%H:%M:%S")
+
+
+def find_broken_packages(pool):
+    solver = pool.Solver()
+    solver.set_flag(solv.Solver.SOLVER_FLAG_IGNORE_RECOMMENDED, True)
+    # Check for packages installability
+    candq = set(pool.solvables)
+    while candq:
+        jobs = [
+            pool.Job(
+                solv.Job.SOLVER_SOLVABLE
+                | solv.Job.SOLVER_INSTALL
+                | solv.Job.SOLVER_WEAK,
+                p.id,
+            )
+            for p in candq
+        ]
+        solver.solve(jobs)
+        candq_n = candq - set(pool.id2solvable(s) for s in solver.raw_decisions(1))
+        if candq == candq_n:
+            # No more packages is possible to resolve
+            break
+        candq = candq_n
+
+    ftbfs = {}
+    fti = collections.defaultdict(dict)
+
+    if not candq:
+        return ftbfs, fti
+
+    for s in candq:
+        problems = solver.solve(
+            [pool.Job(solv.Job.SOLVER_SOLVABLE | solv.Job.SOLVER_INSTALL, s.id)]
+        )
+        if not problems:
+            continue
+        elif len(problems) > 1:
+            raise AssertionError
+        problem = problems[0]
+
+        if s.arch in {"src", "nosrc"}:
+            srcname = s.name
+            tmp = ftbfs[s.name] = []
+        else:
+            srcname = s.lookup_sourcepkg().rsplit("-", 2)[0]
+            tmp = fti[srcname][s.name] = []
+
+        for rule in problem.findallproblemrules():
+            if rule.type != solv.Solver.SOLVER_RULE_PKG:
+                raise NotImplementedError(f"Unsupported rule type: {rule.type}")
+            tmp.append(
+                [
+                    {
+                        "type": info.type,
+                        "dep": info.dep,
+                        "solvable": info.solvable,
+                        "othersolvable": info.othersolvable,
+                        "str": info.problemstr(),
+                    }
+                    for info in rule.allinfos()
+                ]
+            )
+
+    return ftbfs, fti
+
+
+@click.command()
+def follow_policy():
+    pool = solv.Pool()
+    pool.setarch()
+
+    for r in ("koji",):  # "koji-source"):
+        repo = pool.add_repo(r)
+        f = solv.xfopen(f"/var/cache/dnf/{r}.solv")
+        repo.add_solv(f)
+        f.close()
+
+    pool.addfileprovides()
+    pool.createwhatprovides()
+
+    ftbfs, fti = find_broken_packages(pool)
+
+    bz = bugzilla.Bugzilla("https://bugzilla.redhat.com")
+    # ftbfsbug = bz.getbug("F33FTBFS")
+
+    ftibug = bz.getbug("F33FailsToInstall")
+    fti_report = {}
+    # TODO: report obsoleted packages
+    for src, pkg_rules in fti.items():
+        problems = collections.defaultdict(list)
+        for pkg, rules in pkg_rules.items():
+            if re.search(r"^rust-.*-devel$", pkg):
+                continue
+
+            # All direct problems will be in the first rule
+            for info in rules[0]:
+                # TODO: add support for reporting conflicts and such
+                if (
+                    info["solvable"].name != pkg
+                    or info["type"] != solv.Solver.SOLVER_RULE_PKG_NOTHING_PROVIDES_DEP
+                ):
+                    # Only interested in missing providers of a package itself
+                    continue
+                # Skip hacky multilib packages
+                if "(x86-32)" in str(info["dep"]):
+                    continue
+                problems[pkg].append(info["str"])
+
+        if problems:
+            fti_report[src] = problems
+
+    ftibug = bz.getbug("F33FailsToInstall")
+    query_fti = bz.build_query(
+        product="Fedora",
+        include_fields=[
+            "id",
+            "status",
+            "component",
+            "creation_time",
+            "assigned_to",
+            "flags",
+        ],
+    )
+    query_fti["blocks"] = ftibug.id
+    current_ftis = {b.component: b for b in bz.query(query_fti) if b.status != "CLOSED"}
+
+    env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_DIR))
+
+    fti_template = env.get_template("create-fti.j2")
+
+    for src, pkgs in sorted(fti_report.items()):
+        if src in current_ftis:
+            print(
+                f"Skipping {src} because bug already exists: {current_ftis[src].id}",
+                file=sys.stderr,
+            )
+            continue
+
+        description = fti_template.render(src=src, pkg_problems=pkgs)
+        create_fti_info = bz.build_createbug(
+            product="Fedora",
+            version="rawhide",
+            component=src,
+            summary=f"FTI: {src}: {', '.join(pkgs)}",
+            description=description,
+            blocks=ftibug.id,
+        )
+        print(description)
+        bz.createbug(create_fti_info)
+
+    fixed_ftis = {src: b for src, b in current_ftis.items() if src not in fti_report}
+    if fixed_ftis:
+        close_fti_update = bz.build_update(
+            comment=env.get_template("close-fti.j2").render(),
+            status="CLOSED",
+            resolution="RAWHIDE",
+        )
+        print(f"Closing FTI bugs for fixed components: {', '.join(fixed_ftis.keys())}")
+        bz.update_bugs([b.id for b in fixed_ftis.values()], close_fti_update)
+        current_ftis = {
+            src: b for src, b in current_ftis.items() if src not in fixed_ftis
+        }
+    else:
+        print("No FTI bugs to close, everything is still broken", file=sys.stderr)
+
+    # Now we care only about bugs in NEW state
+    current_ftis = {src: b for src, b in current_ftis.items() if b.status == "NEW"}
+    now = datetime.datetime.now()
+    for src, b in current_ftis.items():
+        print(f"Checking {b.id} ({src})…")
+        creation_time = _bzdate_to_python(b.creation_time)
+        diff = now - creation_time
+        if diff < datetime.timedelta(weeks=1):
+            print("Skipping because week did not pass yet", file=sys.stderr)
+            continue
+
+        # Only reliable way to get whether needinfos were set is go through history
+        history = b.get_history_raw()["bugs"][0]["history"]
+        needinfos = [
+            u
+            for u in history
+            for c in u["changes"]
+            if f"needinfo?({b.assigned_to})" in c["added"]
+        ]
+        bzupdate = None
+        flag = {"name": "needinfo", "status": "?", "requestee": b.assigned_to}
+        if not needinfos:
+            print("Asking for a first needinfo")
+            bzupdate = bz.build_update(
+                comment="""Hello,
+
+This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).""",
+                flags=[flag],
+            )
+        else:
+            try:
+                needinfo_after_week = next(
+                    _bzdate_to_python(n["when"])
+                    for n in needinfos
+                    if now - _bzdate_to_python(n["when"]) >= datetime.timedelta(weeks=1)
+                )
+            except StopIteration:
+                print("No needinfo older than 1 week", file=sys.stderr)
+                continue
+            try:
+                needinfo_after_four_weeks = next(
+                    _bzdate_to_python(n["when"])
+                    for n in needinfos
+                    if _bzdate_to_python(n["when"]) - needinfo_after_week
+                    >= datetime.timedelta(weeks=3)
+                )
+            except StopIteration:
+                print(
+                    f"No needinfo older than 3 weeks starting from first needinfo ({needinfo_after_week})",
+                    file=sys.stderr,
+                )
+                if now - needinfo_after_week >= datetime.timedelta(weeks=3):
+                    print("Asking for another needinfo")
+                    # RHBZ can have multiple needinfo flags, but python-bugzilla does not like it much
+                    # https://github.com/python-bugzilla/python-bugzilla/issues/118
+                    flags = [f for f in b.flags if f["name"] == "needinfo"]
+                    if flags:
+                        # If there are any needinfos, we will drop all of them and then create a new one
+                        bz.update_bugs(
+                            [b.id],
+                            bz.build_update(
+                                flags=[
+                                    {"name": "needinfo", "id": f["id"], "status": "X"}
+                                    for f in flags
+                                ]
+                            ),
+                        )
+                    bzupdate = bz.build_update(
+                        comment="""Hello,
+
+This is the second reminder (step 4 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).""",
+                        flags=[flag],
+                    )
+                    # TODO: Implement
+            if now - needinfo_after_four_weeks >= datetime.timedelta(weeks=4):
+                print("Opening releng ticket")
+                # TODO: Implement
+            else:
+                print(
+                    f"No needinfo older than 4 weeks starting from second needinfo ({needinfo_after_four_weeks})",
+                    file=sys.stderr,
+                )
+
+        if bzupdate is not None:
+            bz.update_bugs([b.id], bzupdate)
+
+
+if __name__ == "__main__":
+    follow_policy()
diff --git a/scripts/ftbfs-fti/header.j2 b/scripts/ftbfs-fti/header.j2
new file mode 100644
index 0000000..a85a07e
--- /dev/null
+++ b/scripts/ftbfs-fti/header.j2
@@ -0,0 +1,3 @@
+Hello,
+
+Please note that this comment was generated automatically. If you feel that this output has mistakes, please contact me via email (ignatenkobrain@fedoraproject.org).

From 352cea0b775314c8c46779012adfbc387ec48ec2 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 2/30] ftbfs-fti: Do not close bugs if they block any other important tracker


Signed-off-by: Igor Raits <ignatenkobrain@fedoraproject.org>
Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 9ab262e..41ded53 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -13,6 +13,17 @@ import solv
 
 TEMPLATE_DIR = os.path.dirname(os.path.realpath(__file__))
 
+TRACKERS = {
+    "F30FailsToInstall": 1700323,
+    "F31FailsToInstall": 1700324,
+    "F32FailsToInstall": 1750909,
+    "F33FailsToInstall": 1803235,
+    "F30FTBFS": 1674516,
+    "F31FTBFS": 1700317,
+    "F32FTBFS": 1750908,
+    "F33FTBFS": 1803234,
+}
+
 
 def _bzdate_to_python(date):
     return datetime.datetime.strptime(str(date), "%Y%m%dT%H:%M:%S")
@@ -137,6 +148,7 @@ def follow_policy():
             "creation_time",
             "assigned_to",
             "flags",
+            "blocks",
         ],
     )
     query_fti["blocks"] = ftibug.id
@@ -168,13 +180,22 @@ def follow_policy():
 
     fixed_ftis = {src: b for src, b in current_ftis.items() if src not in fti_report}
     if fixed_ftis:
-        close_fti_update = bz.build_update(
-            comment=env.get_template("close-fti.j2").render(),
-            status="CLOSED",
-            resolution="RAWHIDE",
+        comment = env.get_template("close-fti.j2").render()
+        close = bz.build_update(comment=comment, status="CLOSED", resolution="RAWHIDE")
+        unblock = bz.build_update(comment=comment, blocks_remove=ftibug.id)
+        to_close = [
+            b.id
+            for b in fixed_ftis.values()
+            if not (set(b.blocks) - {ftibug.id}) & set(TRACKERS.values())
+        ]
+        to_unblock = [b.id for b in fixed_ftis.values() if b.id not in to_close]
+        print(f"Closing FTI bugs for fixed components: {to_close}", file=sys.stderr)
+        bz.update_bugs(to_close, close)
+        print(
+            f"Unblocking FTI tracker for fixed components: {to_unblock}",
+            file=sys.stderr,
         )
-        print(f"Closing FTI bugs for fixed components: {', '.join(fixed_ftis.keys())}")
-        bz.update_bugs([b.id for b in fixed_ftis.values()], close_fti_update)
+        bz.update_bugs(to_unblock, unblock)
         current_ftis = {
             src: b for src, b in current_ftis.items() if src not in fixed_ftis
         }

From 9f13caa4aefbfcec96b87b60c6b9c3bd6bc73f15 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 3/30] ftbfs-fti: Show more information about dates


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 41ded53..b7bf447 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -210,7 +210,10 @@ def follow_policy():
         creation_time = _bzdate_to_python(b.creation_time)
         diff = now - creation_time
         if diff < datetime.timedelta(weeks=1):
-            print("Skipping because week did not pass yet", file=sys.stderr)
+            print(
+                f"Skipping because week did not pass yet since creation time ({creation_time})",
+                file=sys.stderr,
+            )
             continue
 
         # Only reliable way to get whether needinfos were set is go through history
@@ -239,7 +242,10 @@ This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fes
                     if now - _bzdate_to_python(n["when"]) >= datetime.timedelta(weeks=1)
                 )
             except StopIteration:
-                print("No needinfo older than 1 week", file=sys.stderr)
+                print(
+                    f"No needinfo older than 1 week (oldest is from {_bzdate_to_python(needinfos[0]['when'])})",
+                    file=sys.stderr,
+                )
                 continue
             try:
                 needinfo_after_four_weeks = next(

From 250d542e2665f4d9265d576c7513f0b9515bcc33 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 4/30] ftbfs-fti: Move bugkeeping to the separate function


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index b7bf447..5e87781 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -13,6 +13,7 @@ import solv
 
 TEMPLATE_DIR = os.path.dirname(os.path.realpath(__file__))
 
+NOW = datetime.datetime.now()
 TRACKERS = {
     "F30FailsToInstall": 1700323,
     "F31FailsToInstall": 1700324,
@@ -29,6 +30,96 @@ def _bzdate_to_python(date):
     return datetime.datetime.strptime(str(date), "%Y%m%dT%H:%M:%S")
 
 
+def handle_orphaning(bug):
+    bz = bug.bugzilla
+    creation_time = _bzdate_to_python(bug.creation_time)
+    diff = NOW - creation_time
+    if diff < datetime.timedelta(weeks=1):
+        print(
+            f"Skipping because week did not pass yet since creation time ({creation_time})",
+            file=sys.stderr,
+        )
+        return
+
+    # Only reliable way to get whether needinfos were set is go through history
+    history = bug.get_history_raw()["bugs"][0]["history"]
+    needinfos = [
+        u
+        for u in history
+        for c in u["changes"]
+        if f"needinfo?({bug.assigned_to})" in c["added"]
+    ]
+    bzupdate = None
+    flag = {"name": "needinfo", "status": "?", "requestee": bug.assigned_to}
+    if not needinfos:
+        print("Asking for a first needinfo")
+        bzupdate = bz.build_update(
+            comment="""Hello,
+
+This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).""",
+            flags=[flag],
+        )
+    else:
+        try:
+            needinfo_after_week = next(
+                _bzdate_to_python(n["when"])
+                for n in needinfos
+                if NOW - _bzdate_to_python(n["when"]) >= datetime.timedelta(weeks=1)
+            )
+        except StopIteration:
+            print(
+                f"No needinfo older than 1 week (oldest is from {_bzdate_to_python(needinfos[0]['when'])})",
+                file=sys.stderr,
+            )
+            return
+        try:
+            needinfo_after_four_weeks = next(
+                _bzdate_to_python(n["when"])
+                for n in needinfos
+                if _bzdate_to_python(n["when"]) - needinfo_after_week
+                >= datetime.timedelta(weeks=3)
+            )
+        except StopIteration:
+            print(
+                f"No needinfo older than 3 weeks starting from first needinfo ({needinfo_after_week})",
+                file=sys.stderr,
+            )
+            if NOW - needinfo_after_week >= datetime.timedelta(weeks=3):
+                print("Asking for another needinfo")
+                # RHBZ can have multiple needinfo flags, but python-bugzilla does not like it much
+                # https://github.com/python-bugzilla/python-bugzilla/issues/118
+                flags = [f for f in bug.flags if f["name"] == "needinfo"]
+                if flags:
+                    # If there are any needinfos, we will drop all of them and then create a new one
+                    bz.update_bugs(
+                        [bug.id],
+                        bz.build_update(
+                            flags=[
+                                {"name": "needinfo", "id": f["id"], "status": "X"}
+                                for f in flags
+                            ]
+                        ),
+                    )
+                bzupdate = bz.build_update(
+                    comment="""Hello,
+
+This is the second reminder (step 4 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).""",
+                    flags=[flag],
+                )
+                # TODO: Implement
+        if NOW - needinfo_after_four_weeks >= datetime.timedelta(weeks=4):
+            print("Opening releng ticket")
+            # TODO: Implement
+        else:
+            print(
+                f"No needinfo older than 4 weeks starting from second needinfo ({needinfo_after_four_weeks})",
+                file=sys.stderr,
+            )
+
+    if bzupdate is not None:
+        bz.update_bugs([bug.id], bzupdate)
+
+
 def find_broken_packages(pool):
     solver = pool.Solver()
     solver.set_flag(solv.Solver.SOLVER_FLAG_IGNORE_RECOMMENDED, True)
@@ -204,95 +295,9 @@ def follow_policy():
 
     # Now we care only about bugs in NEW state
     current_ftis = {src: b for src, b in current_ftis.items() if b.status == "NEW"}
-    now = datetime.datetime.now()
     for src, b in current_ftis.items():
         print(f"Checking {b.id} ({src})…")
-        creation_time = _bzdate_to_python(b.creation_time)
-        diff = now - creation_time
-        if diff < datetime.timedelta(weeks=1):
-            print(
-                f"Skipping because week did not pass yet since creation time ({creation_time})",
-                file=sys.stderr,
-            )
-            continue
-
-        # Only reliable way to get whether needinfos were set is go through history
-        history = b.get_history_raw()["bugs"][0]["history"]
-        needinfos = [
-            u
-            for u in history
-            for c in u["changes"]
-            if f"needinfo?({b.assigned_to})" in c["added"]
-        ]
-        bzupdate = None
-        flag = {"name": "needinfo", "status": "?", "requestee": b.assigned_to}
-        if not needinfos:
-            print("Asking for a first needinfo")
-            bzupdate = bz.build_update(
-                comment="""Hello,
-
-This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).""",
-                flags=[flag],
-            )
-        else:
-            try:
-                needinfo_after_week = next(
-                    _bzdate_to_python(n["when"])
-                    for n in needinfos
-                    if now - _bzdate_to_python(n["when"]) >= datetime.timedelta(weeks=1)
-                )
-            except StopIteration:
-                print(
-                    f"No needinfo older than 1 week (oldest is from {_bzdate_to_python(needinfos[0]['when'])})",
-                    file=sys.stderr,
-                )
-                continue
-            try:
-                needinfo_after_four_weeks = next(
-                    _bzdate_to_python(n["when"])
-                    for n in needinfos
-                    if _bzdate_to_python(n["when"]) - needinfo_after_week
-                    >= datetime.timedelta(weeks=3)
-                )
-            except StopIteration:
-                print(
-                    f"No needinfo older than 3 weeks starting from first needinfo ({needinfo_after_week})",
-                    file=sys.stderr,
-                )
-                if now - needinfo_after_week >= datetime.timedelta(weeks=3):
-                    print("Asking for another needinfo")
-                    # RHBZ can have multiple needinfo flags, but python-bugzilla does not like it much
-                    # https://github.com/python-bugzilla/python-bugzilla/issues/118
-                    flags = [f for f in b.flags if f["name"] == "needinfo"]
-                    if flags:
-                        # If there are any needinfos, we will drop all of them and then create a new one
-                        bz.update_bugs(
-                            [b.id],
-                            bz.build_update(
-                                flags=[
-                                    {"name": "needinfo", "id": f["id"], "status": "X"}
-                                    for f in flags
-                                ]
-                            ),
-                        )
-                    bzupdate = bz.build_update(
-                        comment="""Hello,
-
-This is the second reminder (step 4 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).""",
-                        flags=[flag],
-                    )
-                    # TODO: Implement
-            if now - needinfo_after_four_weeks >= datetime.timedelta(weeks=4):
-                print("Opening releng ticket")
-                # TODO: Implement
-            else:
-                print(
-                    f"No needinfo older than 4 weeks starting from second needinfo ({needinfo_after_four_weeks})",
-                    file=sys.stderr,
-                )
-
-        if bzupdate is not None:
-            bz.update_bugs([b.id], bzupdate)
+        handle_orphaning(b)
 
 
 if __name__ == "__main__":

From c9eb2c7acbd8b7798cf4da3686f35589767d786f Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 5/30] ftbfs-fti: Improve wording for new bugs


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/create-footer.j2 b/scripts/ftbfs-fti/create-footer.j2
index d0bb5b9..6e278bd 100644
--- a/scripts/ftbfs-fti/create-footer.j2
+++ b/scripts/ftbfs-fti/create-footer.j2
@@ -1,4 +1,4 @@
-According to a policy for FTBFS/FTI bugs (https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/), your package may be orphaned in 8+ weeks if you won't reply to this bug.
+If you don't react accordingly to the policy for FTBFS/FTI bugs (https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/), your package may be orphaned in 8+ weeks.
 
 P.S. The data was generated solely from koji buildroot, so it might be newer than the latest rawhide compose.
 

From 036b6e6a822eeb7e30d09a2c48b276af31826783 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 6/30] ftbfs-fti: Improve messaging when closing bug to handle retired packages


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/close-fti.j2 b/scripts/ftbfs-fti/close-fti.j2
index 5b5ec2e..8539c80 100644
--- a/scripts/ftbfs-fti/close-fti.j2
+++ b/scripts/ftbfs-fti/close-fti.j2
@@ -1,5 +1,5 @@
 {% include "header.j2" %}
 
-All subpackages of a package agaisnt which this bug was filled are installable now in Fedora 33.
+All subpackages of a package agaisnt which this bug was filled are now installable or removed from Fedora 33.
 
 Thanks for taking care of it!

From defb0608935ae1a820d7a2855f5e1215881f71d8 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 7/30] ftbfs-fti: Fix crash after asking for second needinfo


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 5e87781..4ac00ab 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -106,7 +106,7 @@ This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fes
 This is the second reminder (step 4 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).""",
                     flags=[flag],
                 )
-                # TODO: Implement
+                return
         if NOW - needinfo_after_four_weeks >= datetime.timedelta(weeks=4):
             print("Opening releng ticket")
             # TODO: Implement

From e90ac824afe02240d78809d3fe1473abd8741f0d Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 8/30] ftbfs-fti: Truncate summary if needed


It can be only 255 characters long.

Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 4ac00ab..313f313 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -258,11 +258,14 @@ def follow_policy():
             continue
 
         description = fti_template.render(src=src, pkg_problems=pkgs)
+        summary = f"F{release}FailsToInstall: {', '.join(pkgs)}"
+        if len(summary) > 255:
+            summary = f"F{release}FailsToInstall: Multiple packages built from {src}"
         create_fti_info = bz.build_createbug(
             product="Fedora",
             version="rawhide",
             component=src,
-            summary=f"FTI: {src}: {', '.join(pkgs)}",
+            summary=summary,
             description=description,
             blocks=ftibug.id,
         )

From c06bd8e798cb3573aa7198a7d380e66c5ebe56b7 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 9/30] ftbfs-fti: Add support for opening bugs against non-rawhide


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/close-fti.j2 b/scripts/ftbfs-fti/close-fti.j2
index 8539c80..45bbf15 100644
--- a/scripts/ftbfs-fti/close-fti.j2
+++ b/scripts/ftbfs-fti/close-fti.j2
@@ -1,5 +1,5 @@
 {% include "header.j2" %}
 
-All subpackages of a package agaisnt which this bug was filled are now installable or removed from Fedora 33.
+All subpackages of a package agaisnt which this bug was filled are now installable or removed from Fedora {{ release }}.
 
 Thanks for taking care of it!
diff --git a/scripts/ftbfs-fti/create-footer.j2 b/scripts/ftbfs-fti/create-footer.j2
index 6e278bd..1ac2bbe 100644
--- a/scripts/ftbfs-fti/create-footer.j2
+++ b/scripts/ftbfs-fti/create-footer.j2
@@ -1,6 +1,6 @@
 If you don't react accordingly to the policy for FTBFS/FTI bugs (https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/), your package may be orphaned in 8+ weeks.
 
-P.S. The data was generated solely from koji buildroot, so it might be newer than the latest rawhide compose.
+P.S. The data was generated solely from koji buildroot, so it might be newer than the latest compose or the content on mirrors.
 
 P.P.S. If this bug has been reported in the middle of upgrading multiple dependent packages, please consider using side tags: https://docs.fedoraproject.org/en-US/rawhide-gating/multi-builds/
 
diff --git a/scripts/ftbfs-fti/create-fti.j2 b/scripts/ftbfs-fti/create-fti.j2
index 926bce1..5ca5559 100644
--- a/scripts/ftbfs-fti/create-fti.j2
+++ b/scripts/ftbfs-fti/create-fti.j2
@@ -1,6 +1,6 @@
 {% include "header.j2" %}
 
-Your package ({{ src }}) Fails To Install in Fedora 33:
+Your package ({{ src }}) Fails To Install in Fedora {{ release }}:
 
 {% for pkg, problems in pkg_problems.items() -%}
 can't install {{ pkg }}:
diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 313f313..98f7e71 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -185,7 +185,14 @@ def find_broken_packages(pool):
 
 
 @click.command()
-def follow_policy():
+@click.option(
+    "--release",
+    type=click.Choice(sorted(set(t[1:3] for t in TRACKERS.keys()))),
+    default="33",
+    show_default=True,
+    help="Fedora release",
+)
+def follow_policy(release):
     pool = solv.Pool()
     pool.setarch()
 
@@ -201,9 +208,9 @@ def follow_policy():
     ftbfs, fti = find_broken_packages(pool)
 
     bz = bugzilla.Bugzilla("https://bugzilla.redhat.com")
-    # ftbfsbug = bz.getbug("F33FTBFS")
+    # ftbfsbug = bz.getbug(f"F{release}FTBFS")
 
-    ftibug = bz.getbug("F33FailsToInstall")
+    ftibug = bz.getbug(f"F{release}FailsToInstall")
     fti_report = {}
     # TODO: report obsoleted packages
     for src, pkg_rules in fti.items():
@@ -229,7 +236,7 @@ def follow_policy():
         if problems:
             fti_report[src] = problems
 
-    ftibug = bz.getbug("F33FailsToInstall")
+    ftibug = bz.getbug(f"F{release}FailsToInstall")
     query_fti = bz.build_query(
         product="Fedora",
         include_fields=[
@@ -246,6 +253,7 @@ def follow_policy():
     current_ftis = {b.component: b for b in bz.query(query_fti) if b.status != "CLOSED"}
 
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_DIR))
+    env.globals["release"] = release
 
     fti_template = env.get_template("create-fti.j2")
 

From 02dcb8097bf43c7451e85a24e86997cdaf95ade7 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 10/30] ftbfs-fti: Ignore ON_QA bugs


Those are almost never set by users, but rather bodhi, so let's just
hope that we don't miss bugs for auto-closing.

Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 98f7e71..a62b6c0 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -250,7 +250,7 @@ def follow_policy(release):
         ],
     )
     query_fti["blocks"] = ftibug.id
-    current_ftis = {b.component: b for b in bz.query(query_fti) if b.status != "CLOSED"}
+    current_ftis = {b.component: b for b in bz.query(query_fti) if b.status not in {"CLOSED", "ON_QA"}}
 
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_DIR))
     env.globals["release"] = release

From 3fa7f44b35613227b6ce0761fd9c3b63c99fd699 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 11/30] ftbfs-fti: Ignore only MODIFIED/ON_QA bugs if they have bodhi update for appropriate release


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index a62b6c0..4083efc 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -250,7 +250,7 @@ def follow_policy(release):
         ],
     )
     query_fti["blocks"] = ftibug.id
-    current_ftis = {b.component: b for b in bz.query(query_fti) if b.status not in {"CLOSED", "ON_QA"}}
+    current_ftis = {b.component: b for b in bz.query(query_fti) if b.status != "CLOSED"}
 
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_DIR))
     env.globals["release"] = release
@@ -281,6 +281,26 @@ def follow_policy(release):
         bz.createbug(create_fti_info)
 
     fixed_ftis = {src: b for src, b in current_ftis.items() if src not in fti_report}
+    # Ignore bugs which have pending updates in Bodhi
+    for src, b in list(fixed_ftis.items()):
+        if b.status not in {"MODIFIED", "ON_QA"}:
+            continue
+        print(
+            f"Checking {b.id} if it was submitted as an update to appropriate release",
+            file=sys.stderr,
+        )
+        comments = b.getcomments()
+        try:
+            next(
+                c
+                for c in comments
+                if c["creator"] == "updates@fedoraproject.org"
+                and f"Fedora {release}" in c["text"]
+            )
+            print(f"Bug for {src} ({b.id}) has a pending update, ignoring")
+            del fixed_ftis[src]
+        except StopIteration:
+            pass
     if fixed_ftis:
         comment = env.get_template("close-fti.j2").render()
         close = bz.build_update(comment=comment, status="CLOSED", resolution="RAWHIDE")

From a4ba1208211bc2997adc53c5079affc271e6cd21 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 12/30] ftbfs-fti: Do not bother opening bugs for orphaned packages


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 4083efc..7648de1 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -9,6 +9,7 @@ import sys
 import bugzilla
 import click
 import jinja2
+import requests
 import solv
 
 TEMPLATE_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -236,6 +237,10 @@ def follow_policy(release):
         if problems:
             fti_report[src] = problems
 
+    pkg_owners = requests.get(
+        "https://src.fedoraproject.org/extras/pagure_poc.json"
+    ).json()
+
     ftibug = bz.getbug(f"F{release}FailsToInstall")
     query_fti = bz.build_query(
         product="Fedora",
@@ -250,7 +255,11 @@ def follow_policy(release):
         ],
     )
     query_fti["blocks"] = ftibug.id
-    current_ftis = {b.component: b for b in bz.query(query_fti) if b.status != "CLOSED"}
+    current_ftis = {
+        b.component: b
+        for b in bz.query(query_fti)
+        if b.status != "CLOSED"
+    }
 
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_DIR))
     env.globals["release"] = release
@@ -265,6 +274,10 @@ def follow_policy(release):
             )
             continue
 
+        if pkg_owners["rpms"][src]["fedora"] == "orphan":
+            # Skip reporting bugs for orphaned packages
+            continue
+
         description = fti_template.render(src=src, pkg_problems=pkgs)
         summary = f"F{release}FailsToInstall: {', '.join(pkgs)}"
         if len(summary) > 255:

From 8c614af093bdd0c3d55501feb7d15b61aa8cb763 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 13/30] ftbfs-fti: Close bugs with WORKSFORME instead of RAWHIDE


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 7648de1..0d64aac 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -316,7 +316,7 @@ def follow_policy(release):
             pass
     if fixed_ftis:
         comment = env.get_template("close-fti.j2").render()
-        close = bz.build_update(comment=comment, status="CLOSED", resolution="RAWHIDE")
+        close = bz.build_update(comment=comment, status="CLOSED", resolution="WORKSFORME")
         unblock = bz.build_update(comment=comment, blocks_remove=ftibug.id)
         to_close = [
             b.id

From e46269c41d6725fd801c01e5fa7049f7be33e1ff Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:51 +0000
Subject: [PATCH 14/30] ftbfs-fti: Create bugs for a specific release instead of rawhide


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 0d64aac..64d39db 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -25,6 +25,7 @@ TRACKERS = {
     "F32FTBFS": 1750908,
     "F33FTBFS": 1803234,
 }
+RAWHIDE = "33"
 
 
 def _bzdate_to_python(date):
@@ -282,9 +283,10 @@ def follow_policy(release):
         summary = f"F{release}FailsToInstall: {', '.join(pkgs)}"
         if len(summary) > 255:
             summary = f"F{release}FailsToInstall: Multiple packages built from {src}"
+        bz_version = release if release != RAWHIDE else "rawhide"
         create_fti_info = bz.build_createbug(
             product="Fedora",
-            version="rawhide",
+            version=bz_version,
             component=src,
             summary=summary,
             description=description,

From 7100097980a4ee419b92649eb19b0bf67eefb205 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 15/30] ftbfs-fti: Improve output a bit


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 64d39db..c27d387 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -256,11 +256,7 @@ def follow_policy(release):
         ],
     )
     query_fti["blocks"] = ftibug.id
-    current_ftis = {
-        b.component: b
-        for b in bz.query(query_fti)
-        if b.status != "CLOSED"
-    }
+    current_ftis = {b.component: b for b in bz.query(query_fti) if b.status != "CLOSED"}
 
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(TEMPLATE_DIR))
     env.globals["release"] = release
@@ -318,7 +314,9 @@ def follow_policy(release):
             pass
     if fixed_ftis:
         comment = env.get_template("close-fti.j2").render()
-        close = bz.build_update(comment=comment, status="CLOSED", resolution="WORKSFORME")
+        close = bz.build_update(
+            comment=comment, status="CLOSED", resolution="WORKSFORME"
+        )
         unblock = bz.build_update(comment=comment, blocks_remove=ftibug.id)
         to_close = [
             b.id
@@ -326,13 +324,15 @@ def follow_policy(release):
             if not (set(b.blocks) - {ftibug.id}) & set(TRACKERS.values())
         ]
         to_unblock = [b.id for b in fixed_ftis.values() if b.id not in to_close]
-        print(f"Closing FTI bugs for fixed components: {to_close}", file=sys.stderr)
-        bz.update_bugs(to_close, close)
-        print(
-            f"Unblocking FTI tracker for fixed components: {to_unblock}",
-            file=sys.stderr,
-        )
-        bz.update_bugs(to_unblock, unblock)
+        if to_close:
+            print(f"Closing FTI bugs for fixed components: {to_close}", file=sys.stderr)
+            bz.update_bugs(to_close, close)
+        if to_unblock:
+            print(
+                f"Unblocking FTI tracker for fixed components: {to_unblock}",
+                file=sys.stderr,
+            )
+            bz.update_bugs(to_unblock, unblock)
         current_ftis = {
             src: b for src, b in current_ftis.items() if src not in fixed_ftis
         }

From cdbb23465df24a4b620a24bccd67337c385bcbc3 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 16/30] ftbfs-fti: Drop F30 trackers, those are EOL anyway


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index c27d387..d33bbea 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -16,11 +16,9 @@ TEMPLATE_DIR = os.path.dirname(os.path.realpath(__file__))
 
 NOW = datetime.datetime.now()
 TRACKERS = {
-    "F30FailsToInstall": 1700323,
     "F31FailsToInstall": 1700324,
     "F32FailsToInstall": 1750909,
     "F33FailsToInstall": 1803235,
-    "F30FTBFS": 1674516,
     "F31FTBFS": 1700317,
     "F32FTBFS": 1750908,
     "F33FTBFS": 1803234,

From fdbe1dfed3b4730b1199c135f80162cbb2f8f7f8 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 17/30] Actually send second needinfo


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index d33bbea..820e6f9 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -79,6 +79,14 @@ This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fes
                 if _bzdate_to_python(n["when"]) - needinfo_after_week
                 >= datetime.timedelta(weeks=3)
             )
+            if NOW - needinfo_after_four_weeks >= datetime.timedelta(weeks=4):
+                print("Opening releng ticket")
+                # TODO: Implement
+            else:
+                print(
+                    f"No needinfo older than 4 weeks starting from second needinfo ({needinfo_after_four_weeks})",
+                    file=sys.stderr,
+                )
         except StopIteration:
             print(
                 f"No needinfo older than 3 weeks starting from first needinfo ({needinfo_after_week})",
@@ -106,15 +114,6 @@ This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fes
 This is the second reminder (step 4 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).""",
                     flags=[flag],
                 )
-                return
-        if NOW - needinfo_after_four_weeks >= datetime.timedelta(weeks=4):
-            print("Opening releng ticket")
-            # TODO: Implement
-        else:
-            print(
-                f"No needinfo older than 4 weeks starting from second needinfo ({needinfo_after_four_weeks})",
-                file=sys.stderr,
-            )
 
     if bzupdate is not None:
         bz.update_bugs([bug.id], bzupdate)

From 0ecf2a56a8b3bcaad4d5421aed44033c06ad0341 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 18/30] ftbfs-fti: Try to be smarter when dealing bugzilla wrt flag setting


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 820e6f9..9595d5d 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -94,20 +94,6 @@ This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fes
             )
             if NOW - needinfo_after_week >= datetime.timedelta(weeks=3):
                 print("Asking for another needinfo")
-                # RHBZ can have multiple needinfo flags, but python-bugzilla does not like it much
-                # https://github.com/python-bugzilla/python-bugzilla/issues/118
-                flags = [f for f in bug.flags if f["name"] == "needinfo"]
-                if flags:
-                    # If there are any needinfos, we will drop all of them and then create a new one
-                    bz.update_bugs(
-                        [bug.id],
-                        bz.build_update(
-                            flags=[
-                                {"name": "needinfo", "id": f["id"], "status": "X"}
-                                for f in flags
-                            ]
-                        ),
-                    )
                 bzupdate = bz.build_update(
                     comment="""Hello,
 
@@ -116,7 +102,33 @@ This is the second reminder (step 4 from https://docs.fedoraproject.org/en-US/fe
                 )
 
     if bzupdate is not None:
-        bz.update_bugs([bug.id], bzupdate)
+        result = bz.update_bugs([bug.id], bzupdate)
+        if "flags" in bzupdate and not result["bugs"][0]["changes"]:
+            # FIXME: Probably bug(s) in bugzilla and should be reported there
+            # 1. Accounts which change email do not force needinfo change
+            # 2. RHBZ can have multiple flags of the same type, but python-bugzilla does not like it much
+            #    https://github.com/python-bugzilla/python-bugzilla/issues/118
+            flags = bzupdate["flags"]
+            flags_to_unset = [
+                f for f in bug.flags if f["name"] in set(f["name"] for f in flags)
+            ]
+            flags = [f for f in bug.flags if f["name"] == "needinfo"]
+            if not flags_to_unset:
+                raise AssertionError(
+                    "Flags update did not happen, neither there are flags to remove"
+                )
+                # If there are any needinfos, we will drop all of them and then create a new one
+            bz.update_bugs(
+                [bug.id],
+                bz.build_update(
+                    flags=[
+                        {"name": "needinfo", "id": f["id"], "status": "X"}
+                        for f in flags
+                    ]
+                ),
+            )
+            # Retry setting a flag
+            bz.update_bugs([bug.id], bz.build_update(flags=bzupdate["flags"]))
 
 
 def find_broken_packages(pool):

From 6f1ea7edf1cb7eac6466f17d91d4cc7ad258919d Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 19/30] ftbfs-fti: Improve debuggning messages


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 9595d5d..83f0062 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -36,7 +36,7 @@ def handle_orphaning(bug):
     diff = NOW - creation_time
     if diff < datetime.timedelta(weeks=1):
         print(
-            f"Skipping because week did not pass yet since creation time ({creation_time})",
+            f"→ Week did not pass since creation time ({creation_time}), skipping…",
             file=sys.stderr,
         )
         return
@@ -52,7 +52,7 @@ def handle_orphaning(bug):
     bzupdate = None
     flag = {"name": "needinfo", "status": "?", "requestee": bug.assigned_to}
     if not needinfos:
-        print("Asking for a first needinfo")
+        print("Asking for the first needinfo")
         bzupdate = bz.build_update(
             comment="""Hello,
 
@@ -68,7 +68,7 @@ This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fes
             )
         except StopIteration:
             print(
-                f"No needinfo older than 1 week (oldest is from {_bzdate_to_python(needinfos[0]['when'])})",
+                f"→ Week did not pass since first needinfo ({_bzdate_to_python(needinfos[0]['when'])}), skipping…",
                 file=sys.stderr,
             )
             return
@@ -84,12 +84,12 @@ This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fes
                 # TODO: Implement
             else:
                 print(
-                    f"No needinfo older than 4 weeks starting from second needinfo ({needinfo_after_four_weeks})",
+                    f"→ 4 weeks did not pass since second needinfo ({needinfo_after_four_weeks}), skipping…",
                     file=sys.stderr,
                 )
         except StopIteration:
             print(
-                f"No needinfo older than 3 weeks starting from first needinfo ({needinfo_after_week})",
+                f"→ 3 weeks did not pass since first needinfo ({needinfo_after_week}), skipping…",
                 file=sys.stderr,
             )
             if NOW - needinfo_after_week >= datetime.timedelta(weeks=3):

From 023338914968e2b0a8931e58a4e0ba8760867611 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 20/30] ftbfs-fti: Print closing/unblocking message to the stdout


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 83f0062..3995782 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -334,13 +334,10 @@ def follow_policy(release):
         ]
         to_unblock = [b.id for b in fixed_ftis.values() if b.id not in to_close]
         if to_close:
-            print(f"Closing FTI bugs for fixed components: {to_close}", file=sys.stderr)
+            print(f"Closing FTI bugs for fixed components: {to_close}")
             bz.update_bugs(to_close, close)
         if to_unblock:
-            print(
-                f"Unblocking FTI tracker for fixed components: {to_unblock}",
-                file=sys.stderr,
-            )
+            print(f"Unblocking FTI tracker for fixed components: {to_unblock}")
             bz.update_bugs(to_unblock, unblock)
         current_ftis = {
             src: b for src, b in current_ftis.items() if src not in fixed_ftis

From bfc2f259fb5b844f0f68196a8cbc1ac5763b9d1e Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 21/30] ftbfs-fti: Count time in orphaning process from the last relevant blocked tracker


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 3995782..350dfb8 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -30,19 +30,29 @@ def _bzdate_to_python(date):
     return datetime.datetime.strptime(str(date), "%Y%m%dT%H:%M:%S")
 
 
-def handle_orphaning(bug):
+def handle_orphaning(bug, tracker):
     bz = bug.bugzilla
-    creation_time = _bzdate_to_python(bug.creation_time)
-    diff = NOW - creation_time
+
+    history = bug.get_history_raw()["bugs"][0]["history"]
+
+    start_time = _bzdate_to_python(
+        next(
+            u["when"]
+            for u in history
+            for c in u["changes"]
+            if c["field_name"] == "blocks"
+            and str(tracker.id) in {b.strip() for b in c["added"].split(",")}
+        )
+    )
+    diff = NOW - start_time
     if diff < datetime.timedelta(weeks=1):
         print(
-            f"→ Week did not pass since creation time ({creation_time}), skipping…",
+            f"→ Week did not pass since bug started to block tracker ({start_time}), skipping…",
             file=sys.stderr,
         )
         return
 
     # Only reliable way to get whether needinfos were set is go through history
-    history = bug.get_history_raw()["bugs"][0]["history"]
     needinfos = [
         u
         for u in history
@@ -240,7 +250,11 @@ def follow_policy(release):
                     # Only interested in missing providers of a package itself
                     continue
                 # Skip hacky multilib packages
-                if "(x86-32)" in str(info["dep"]):
+                if (
+                    "(x86-32)" in str(info["dep"])
+                    or "dssi-vst-wine" in str(info["dep"])
+                    or "lmms-vst" in str(info["dep"])
+                ):
                     continue
                 problems[pkg].append(info["str"])
 
@@ -254,15 +268,7 @@ def follow_policy(release):
     ftibug = bz.getbug(f"F{release}FailsToInstall")
     query_fti = bz.build_query(
         product="Fedora",
-        include_fields=[
-            "id",
-            "status",
-            "component",
-            "creation_time",
-            "assigned_to",
-            "flags",
-            "blocks",
-        ],
+        include_fields=["id", "status", "component", "assigned_to", "flags", "blocks"],
     )
     query_fti["blocks"] = ftibug.id
     current_ftis = {b.component: b for b in bz.query(query_fti) if b.status != "CLOSED"}
@@ -349,7 +355,7 @@ def follow_policy(release):
     current_ftis = {src: b for src, b in current_ftis.items() if b.status == "NEW"}
     for src, b in current_ftis.items():
         print(f"Checking {b.id} ({src})…")
-        handle_orphaning(b)
+        handle_orphaning(b, ftibug)
 
 
 if __name__ == "__main__":

From 49bf3f2af2d9a2cfeb0bc9e320030732ce62c931 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 22/30] ftbfs-fti: When closing bugs, ensure minor_update


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 350dfb8..9334895 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -333,6 +333,7 @@ def follow_policy(release):
             comment=comment, status="CLOSED", resolution="WORKSFORME"
         )
         unblock = bz.build_update(comment=comment, blocks_remove=ftibug.id)
+        unblock["minor_update"] = True
         to_close = [
             b.id
             for b in fixed_ftis.values()

From 08fb9291f63695aeaa4cd283247adec956557d6e Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 23/30] ftbfs-fti: If the FTI tracker was added at the creation time, use it


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 9334895..7121cfd 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -35,15 +35,18 @@ def handle_orphaning(bug, tracker):
 
     history = bug.get_history_raw()["bugs"][0]["history"]
 
-    start_time = _bzdate_to_python(
-        next(
-            u["when"]
-            for u in history
-            for c in u["changes"]
-            if c["field_name"] == "blocks"
-            and str(tracker.id) in {b.strip() for b in c["added"].split(",")}
+    try:
+        start_time = _bzdate_to_python(
+            next(
+                u["when"]
+                for u in history
+                for c in u["changes"]
+                if c["field_name"] == "blocks"
+                and str(tracker.id) in {b.strip() for b in c["added"].split(",")}
+            )
         )
-    )
+    except StopIteration:
+        start_time = _bzdate_to_python(bug.creation_time)
     diff = NOW - start_time
     if diff < datetime.timedelta(weeks=1):
         print(
@@ -268,7 +271,15 @@ def follow_policy(release):
     ftibug = bz.getbug(f"F{release}FailsToInstall")
     query_fti = bz.build_query(
         product="Fedora",
-        include_fields=["id", "status", "component", "assigned_to", "flags", "blocks"],
+        include_fields=[
+            "id",
+            "status",
+            "component",
+            "assigned_to",
+            "flags",
+            "blocks",
+            "creation_time",
+        ],
     )
     query_fti["blocks"] = ftibug.id
     current_ftis = {b.component: b for b in bz.query(query_fti) if b.status != "CLOSED"}

From f5dccce6753f2a742d6dfe031f68b0ab4abb8967 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 24/30] ftbfs-fti: Exit early when skipping bugs


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 7121cfd..b0a2281 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -100,11 +100,8 @@ This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fes
                     f"→ 4 weeks did not pass since second needinfo ({needinfo_after_four_weeks}), skipping…",
                     file=sys.stderr,
                 )
+                return
         except StopIteration:
-            print(
-                f"→ 3 weeks did not pass since first needinfo ({needinfo_after_week}), skipping…",
-                file=sys.stderr,
-            )
             if NOW - needinfo_after_week >= datetime.timedelta(weeks=3):
                 print("Asking for another needinfo")
                 bzupdate = bz.build_update(
@@ -113,6 +110,12 @@ This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fes
 This is the second reminder (step 4 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).""",
                     flags=[flag],
                 )
+            else:
+                print(
+                    f"→ 3 weeks did not pass since first needinfo ({needinfo_after_week}), skipping…",
+                    file=sys.stderr,
+                )
+                return
 
     if bzupdate is not None:
         result = bz.update_bugs([bug.id], bzupdate)

From 25a4b5047966fceca0c0b4dfd1c108c0e0583b33 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 25/30] ftbfs-fti: Improve notifications messages


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index b0a2281..d4b8e8c 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -69,7 +69,9 @@ def handle_orphaning(bug, tracker):
         bzupdate = bz.build_update(
             comment="""Hello,
 
-This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).""",
+This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).
+
+If you know about this problem and are planning on fixing it, please acknowledge so by setting the bug status to ASSIGNED. If you don't have time to maintain this package, consider orphaning it, so maintainers of dependent packages realize the problem.""",
             flags=[flag],
         )
     else:
@@ -107,7 +109,9 @@ This is the first reminder (step 3 from https://docs.fedoraproject.org/en-US/fes
                 bzupdate = bz.build_update(
                     comment="""Hello,
 
-This is the second reminder (step 4 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).""",
+This is the second reminder (step 4 from https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/#_package_removal_for_long_standing_ftbfs_and_fti_bugs).
+
+If you know about this problem and are planning on fixing it, please acknowledge so by setting the bug status to ASSIGNED. If you don't have time to maintain this package, consider orphaning it, so maintainers of dependent packages realize the problem.""",
                     flags=[flag],
                 )
             else:

From 3e000f6f6501772eae1df0e8537274325fda4a03 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 26/30] ftbfs-fti: All times should be in UTC


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index d4b8e8c..eca5f7f 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -14,7 +14,7 @@ import solv
 
 TEMPLATE_DIR = os.path.dirname(os.path.realpath(__file__))
 
-NOW = datetime.datetime.now()
+NOW = datetime.datetime.now(datetime.timezone.utc)
 TRACKERS = {
     "F31FailsToInstall": 1700324,
     "F32FailsToInstall": 1750909,
@@ -27,7 +27,7 @@ RAWHIDE = "33"
 
 
 def _bzdate_to_python(date):
-    return datetime.datetime.strptime(str(date), "%Y%m%dT%H:%M:%S")
+    return datetime.datetime.strptime(str(date), "%Y%m%dT%H:%M:%S").replace(tzinfo=datetime.timezone.utc)
 
 
 def handle_orphaning(bug, tracker):

From 8f4ffbc3176f5f9cd96d0562f98dd49319fab449 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 27/30] ftbfs-fti: Improve creation message


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/create-footer.j2 b/scripts/ftbfs-fti/create-footer.j2
index 1ac2bbe..c6ec8a1 100644
--- a/scripts/ftbfs-fti/create-footer.j2
+++ b/scripts/ftbfs-fti/create-footer.j2
@@ -1,3 +1,6 @@
+If you know about this problem and are planning on fixing it, please acknowledge so by setting the bug status to ASSIGNED. If you don't have time to maintain this package, consider orphaning it, so maintainers of dependent packages realize the problem.
+
+
 If you don't react accordingly to the policy for FTBFS/FTI bugs (https://docs.fedoraproject.org/en-US/fesco/Fails_to_build_from_source_Fails_to_install/), your package may be orphaned in 8+ weeks.
 
 P.S. The data was generated solely from koji buildroot, so it might be newer than the latest compose or the content on mirrors.

From 8ca0d95e90100ac3e8b75bb9cb7a52f91754e589 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 28/30] ftbfs-fti: Skip dummy test packages


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index eca5f7f..7e5fcce 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -249,6 +249,8 @@ def follow_policy(release):
         for pkg, rules in pkg_rules.items():
             if re.search(r"^rust-.*-devel$", pkg):
                 continue
+            if pkg.startswith("dummy-test-packagep"):
+                continue
 
             # All direct problems will be in the first rule
             for info in rules[0]:

From 777c5e954413180a230252259a465d21e50eb398 Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 29/30] ftbfs-fti: Update orphaned bugs with useful info and do not ask for needinfo


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/follow-policy.py b/scripts/ftbfs-fti/follow-policy.py
index 7e5fcce..58e2793 100755
--- a/scripts/ftbfs-fti/follow-policy.py
+++ b/scripts/ftbfs-fti/follow-policy.py
@@ -27,7 +27,9 @@ RAWHIDE = "33"
 
 
 def _bzdate_to_python(date):
-    return datetime.datetime.strptime(str(date), "%Y%m%dT%H:%M:%S").replace(tzinfo=datetime.timezone.utc)
+    return datetime.datetime.strptime(str(date), "%Y%m%dT%H:%M:%S").replace(
+        tzinfo=datetime.timezone.utc
+    )
 
 
 def handle_orphaning(bug, tracker):
@@ -372,8 +374,37 @@ def follow_policy(release):
     else:
         print("No FTI bugs to close, everything is still broken", file=sys.stderr)
 
+    # Update bugs for orphaned packages
+    orphaned = {
+        src: b
+        for src, b in current_ftis.items()
+        if b.assigned_to == "extras-orphan@fedoraproject.org"
+    }
+    for src, b in orphaned.items():
+        comments = b.getcomments()
+        update = False
+        try:
+            next(c for c in comments if "This package has been orphaned." in c["text"])
+            continue
+        except StopIteration:
+            pass
+
+        bz.update_bugs(
+            [b.id],
+            bz.build_update(
+                comment=f"""This package has been orphaned.
+
+You can pick it up at https://src.fedoraproject.org/rpms/{src} by clicking button "Take". If nobody picks it up, it will be retired and removed from a distribution.""",
+                status="NEW",
+            ),
+        )
+
     # Now we care only about bugs in NEW state
-    current_ftis = {src: b for src, b in current_ftis.items() if b.status == "NEW"}
+    current_ftis = {
+        src: b
+        for src, b in current_ftis.items()
+        if b.status == "NEW" and src not in orphaned
+    }
     for src, b in current_ftis.items():
         print(f"Checking {b.id} ({src})…")
         handle_orphaning(b, ftibug)

From 102831ba9c2d32bd6fdd1969efd223e3ffaaa31a Mon Sep 17 00:00:00 2001
From: Igor Raits <i.gnatenko.brain@gmail.com>
Date: Jul 21 2020 17:18:52 +0000
Subject: [PATCH 30/30] ftbfs-fti: Fix typo in closing message


Signed-off-by: Igor Raits <i.gnatenko.brain@gmail.com>

---

diff --git a/scripts/ftbfs-fti/close-fti.j2 b/scripts/ftbfs-fti/close-fti.j2
index 45bbf15..4a3bba5 100644
--- a/scripts/ftbfs-fti/close-fti.j2
+++ b/scripts/ftbfs-fti/close-fti.j2
@@ -1,5 +1,5 @@
 {% include "header.j2" %}
 
-All subpackages of a package agaisnt which this bug was filled are now installable or removed from Fedora {{ release }}.
+All subpackages of a package against which this bug was filled are now installable or removed from Fedora {{ release }}.
 
 Thanks for taking care of it!