From 7f6f7473a807d9ec4f6662cf7bccf1b9eafb676b Mon Sep 17 00:00:00 2001
From: Jan Kaluza <jkaluza@redhat.com>
Date: Oct 05 2016 08:52:29 +0000
Subject: modlint: Use asynchronous requests using asyncio to query Fedora dist-git when checking commit ids.


---

diff --git a/modlint b/modlint
index d77d9d9..70b15f6 100755
--- a/modlint
+++ b/modlint
@@ -1,20 +1,66 @@
 #!/usr/bin/python3
 
-
 import argparse
 import sys
-
 import requests
+import collections
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+
 from modulemd import ModuleMetadata
 
+class BacklogHTTPClient(object):
+    """
+    HTTP Client class sending HEAD requests asynchronously.
+    """
+    MAX_CONCURRENT_REQUESTS = 20
+
+    def __init__(self):
+        self.backlog = collections.deque()
+        self.concurrent_requests = 0
+        self.executor = ThreadPoolExecutor(
+            max_workers=self.MAX_CONCURRENT_REQUESTS)
+
+    def _schedule_requests(self):
+        loop = asyncio.get_event_loop()
+        while (self.backlog and
+               self.concurrent_requests < self.MAX_CONCURRENT_REQUESTS):
+            request, callback = self.backlog.popleft()
+            future = loop.run_in_executor(self.executor, requests.head,
+                                          request)
+            future.add_done_callback(callback)
+            self.concurrent_requests += 1
+        if len(self.backlog) == 0 and self.concurrent_requests == 0:
+            loop.stop()
+
+    def _get_callback(self, function, callback_data):
+        def wrapped(*args, **kwargs):
+            self.concurrent_requests -= 1
+            self._schedule_requests()
+            return function(data=callback_data, *args, **kwargs)
+        return wrapped
+
+    def head(self, url, callback=None, callback_data=None):
+        """
+        Schedules new HTTP HEAD request. Once the response is received,
+        callback function is called with response and callback data passed
+        to it.
+
+        :param string url: URL.
+        :param function callback: Callback function.
+        :param anytype callback_data: Data passed to callback function.
+        """
+        wrapped = self._get_callback(callback, callback_data)
+        self.backlog.append((url, wrapped))
+        self._schedule_requests()
 
 class RpmContentError(ValueError):
     """Metadata contains invalid/nonexistent srpm or commit."""
 
     #: Mapping of HTTP status codes to actual failure reasons.
     REASONS = {
-        400: 'Bad hash',
-        404: 'Nonexistent rpm',
+        400: "Bad hash",
+        404: "Nonexistent rpm",
     }
 
     def __init__(self, package, commit, status_code):
@@ -25,77 +71,122 @@ class RpmContentError(ValueError):
         self.status_code = status_code
 
     def __repr__(self):
-        return 'RpmContentError({pkg!r}, {commit!r}, {status!r})'.format(
+        return "RpmContentError({pkg!r}, {commit!r}, {status!r})".format(
             pkg=self.package,
             commit=self.commit,
             status=self.status_code
         )
 
     def __str__(self):
-        return '{pkg} [{commit}]: {reason}'.format(
+        return "{pkg} [{commit}]: {reason}".format(
             pkg=self.package,
             commit=self.commit,
-            reason=self.REASONS.get(self.status_code, 'Unknown error')
+            reason=self.REASONS.get(self.status_code, "Unknown error")
         )
 
+class VerificationError(ValueError):
+    """
+    ValueError subclass meaning that RPM verification has failed.
+    """
+    def __init__(self, desc):
+        super().__init__(self, desc)
+        self.desc = desc
 
-def verify_rpm_content(package, metadata, *, requests=requests):
-    """Verify the existence of dist-git repo for described package.
+    def __str__(self):
+        return self.desc
 
-    Keyword arguments:
-        package -- Name of the package to verify (key in
-            modulemd packages)
-        metadata -- Metadata associated with the package
-            (value in modulemd packages)
+class RPMVerifier(object):
+    def __init__(self):
+        self.http_client = BacklogHTTPClient()
+        self.packages_checked = 0
+        self.packages_to_check = 0
+        self.error = False
 
-        requests -- Dependency injection of networking library.
+    def _handle_git_response(self, response, data = None):
+        response = response.result()
+        package = data["package"]
+        commit = data["commit"]
 
-    Returns:
-        None if the associated repo exists.
+        self.packages_checked += 1
+        print("\r", end="")
 
-    Raises:
-        RpmContentError -- when the package (or commit) does not exist
-            in dist-git.
-    """
+        if response.status_code != 200:
+            raise RpmContentError(package, commit, response.status_code)
 
-    # TODO: Get URL from the metadata
-    CGIT_URL_TEMPLATE = 'http://pkgs.fedoraproject.org/cgit/rpms/{name}.git/commit'
+        print("Verifying RPM content %d%% (%d/%d)" % (
+            self.packages_checked / self.packages_to_check * 100,
+            self.packages_checked, self.packages_to_check), end = "",
+            flush = True)
 
-    commit = metadata.get('commit', 'HEAD')
+    def verify_rpm(self, package, metadata):
+        """
+        Verifies the existence of dist-git repo for described package.
 
-    response = requests.head(
-            CGIT_URL_TEMPLATE.format(name=package),
-            params={'id': commit}
-            )
+        :param string package: Name of the package to verify.
+        :param ModuleMetadata metadata: Metadata associated with the package.
+        """
 
-    if response.status_code != requests.codes.ok:
-        raise RpmContentError(package, commit, response.status_code)
+        # TODO: Get URL from the metadata
+        CGIT_URL_TEMPLATE = "http://pkgs.fedoraproject.org/cgit/rpms/{name}.git/commit"
 
+        commit = metadata.get("commit", "HEAD")
+        url = CGIT_URL_TEMPLATE.format(name=package) + "?id=" + commit
 
-def rpm_content(mmd):
-    """Generate (package, metadata) pairs from module.
+        data = {}
+        data["package"] = package
+        data["commit"] = commit
+        self.http_client.head(url, self._handle_git_response, data)
 
-    Keyword arguments:
-        mmd -- ModuleMetadata to be inspected.
+    def rpm_content(self, mmd):
+        """Generate (package, metadata) pairs from module.
 
-    Yields:
-        (package, metadata) from mmd, if any.
-    """
+        Positional arguments:
+            mmd -- ModuleMetadata to be inspected.
 
-    try:
-        packages = mmd.components.rpms.packages
-    except AttributeError:
-        # Module has no packages -- end generation with no output
-        return None
+        Yields:
+            (package, metadata) from mmd, if any.
+        """
 
-    yield from packages.items()
+        try:
+            packages = mmd.components.rpms.packages
+        except AttributeError:
+            # Module has no packages -- end generation with no output
+            return None
 
+        yield from packages.items()
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Validate module metadata.')
+    def _exception_handler(self, loop, context):
+        self.error = True
+        print(str(context["exception"]))
+
+    def verify_rpms(self, metadata):
+        """
+        Verifies the existence of dist-git repo for described package.
+
+        :param string package: Name of the package to verify.
+        :param ModuleMetadata metadata: Metadata associated with the package.
+        :raises VerificationError: If there is an error during verification.
+        """
+        loop = asyncio.get_event_loop()
+        loop.set_exception_handler(self._exception_handler)
+
+        self.packages_to_check = len(metadata.components.rpms.packages)
+        self.packages_checked = 0
+        for package in self.rpm_content(metadata):
+            self.verify_rpm(*package)
+
+        loop.run_forever()
+
+        print("\r", end="")
+
+        if self.error:
+            raise VerificationError("Some RPMs failed the verification")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Validate module metadata.")
 
     # Positional arguments
-    parser.add_argument('file', help='Input metadata file')
+    parser.add_argument("file", help="Input metadata file")
 
     args = parser.parse_args()
 
@@ -103,28 +194,34 @@ if __name__ == '__main__':
     metadata_errors = TypeError, ValueError
     all_went_well = True
 
+    # Load the ModuleMetadata file
     try:
         metadata.load(args.file)
     except metadata_errors as invalid_input_metadata:
-        message = 'ERROR: Invalid input: {!s}'.format(invalid_input_metadata)
+        message = "ERROR: Invalid input: {!s}".format(invalid_input_metadata)
         sys.exit(message)
 
+    # Try to validate it.
     try:
         metadata.validate()
     except metadata_errors as invalid_metadata_structure:
         all_went_well = False
-        print('ERROR: Invalid structure:', str(invalid_metadata_structure),
+        print("ERROR: Invalid structure:", str(invalid_metadata_structure),
               file=sys.stderr)
 
-    for package in rpm_content(metadata):
-        try:
-            verify_rpm_content(*package)
-        except RpmContentError as invalid_rpm:
-            all_went_well = False
-            print('RPM CONTENT ERROR:', str(invalid_rpm),
-                  file=sys.stderr)
+    # Try to verify RPM section of ModuleMetadata file.
+    rpm_verifier = RPMVerifier()
+    try:
+        rpm_verifier.verify_rpms(metadata)
+    except VerificationError as error:
+        all_went_well = False
+        print("ERROR:", str(error), file=sys.stderr)
+    except KeyboardInterrupt:
+        all_went_well = False
 
+    # Exit
     if all_went_well:
-        print('Everything OK')
+        print("")
+        print("Everything OK")
     else:
         sys.exit(1)

