From 7f6f7473a807d9ec4f6662cf7bccf1b9eafb676b Mon Sep 17 00:00:00 2001 From: Jan Kaluza Date: Oct 05 2016 08:52:29 +0000 Subject: modlint: Use asynchronous requests using asyncio to query Fedora dist-git when checking commit ids. --- diff --git a/modlint b/modlint index d77d9d9..70b15f6 100755 --- a/modlint +++ b/modlint @@ -1,20 +1,66 @@ #!/usr/bin/python3 - import argparse import sys - import requests +import collections +import asyncio +from concurrent.futures import ThreadPoolExecutor + from modulemd import ModuleMetadata +class BacklogHTTPClient(object): + """ + HTTP Client class sending HEAD requests asynchronously. + """ + MAX_CONCURRENT_REQUESTS = 20 + + def __init__(self): + self.backlog = collections.deque() + self.concurrent_requests = 0 + self.executor = ThreadPoolExecutor( + max_workers=self.MAX_CONCURRENT_REQUESTS) + + def _schedule_requests(self): + loop = asyncio.get_event_loop() + while (self.backlog and + self.concurrent_requests < self.MAX_CONCURRENT_REQUESTS): + request, callback = self.backlog.popleft() + future = loop.run_in_executor(self.executor, requests.head, + request) + future.add_done_callback(callback) + self.concurrent_requests += 1 + if len(self.backlog) == 0 and self.concurrent_requests == 0: + loop.stop() + + def _get_callback(self, function, callback_data): + def wrapped(*args, **kwargs): + self.concurrent_requests -= 1 + self._schedule_requests() + return function(data=callback_data, *args, **kwargs) + return wrapped + + def head(self, url, callback=None, callback_data=None): + """ + Schedules new HTTP HEAD request. Once the response is received, + callback function is called with response and callback data passed + to it. + + :param string url: URL. + :param function callback: Callback function. + :param anytype callback_data: Data passed to callback function. + """ + wrapped = self._get_callback(callback, callback_data) + self.backlog.append((url, wrapped)) + self._schedule_requests() class RpmContentError(ValueError): """Metadata contains invalid/nonexistent srpm or commit.""" #: Mapping of HTTP status codes to actual failure reasons. REASONS = { - 400: 'Bad hash', - 404: 'Nonexistent rpm', + 400: "Bad hash", + 404: "Nonexistent rpm", } def __init__(self, package, commit, status_code): @@ -25,77 +71,122 @@ class RpmContentError(ValueError): self.status_code = status_code def __repr__(self): - return 'RpmContentError({pkg!r}, {commit!r}, {status!r})'.format( + return "RpmContentError({pkg!r}, {commit!r}, {status!r})".format( pkg=self.package, commit=self.commit, status=self.status_code ) def __str__(self): - return '{pkg} [{commit}]: {reason}'.format( + return "{pkg} [{commit}]: {reason}".format( pkg=self.package, commit=self.commit, - reason=self.REASONS.get(self.status_code, 'Unknown error') + reason=self.REASONS.get(self.status_code, "Unknown error") ) +class VerificationError(ValueError): + """ + ValueError subclass meaning that RPM verification has failed. + """ + def __init__(self, desc): + super().__init__(self, desc) + self.desc = desc -def verify_rpm_content(package, metadata, *, requests=requests): - """Verify the existence of dist-git repo for described package. + def __str__(self): + return self.desc - Keyword arguments: - package -- Name of the package to verify (key in - modulemd packages) - metadata -- Metadata associated with the package - (value in modulemd packages) +class RPMVerifier(object): + def __init__(self): + self.http_client = BacklogHTTPClient() + self.packages_checked = 0 + self.packages_to_check = 0 + self.error = False - requests -- Dependency injection of networking library. + def _handle_git_response(self, response, data = None): + response = response.result() + package = data["package"] + commit = data["commit"] - Returns: - None if the associated repo exists. + self.packages_checked += 1 + print("\r", end="") - Raises: - RpmContentError -- when the package (or commit) does not exist - in dist-git. - """ + if response.status_code != 200: + raise RpmContentError(package, commit, response.status_code) - # TODO: Get URL from the metadata - CGIT_URL_TEMPLATE = 'http://pkgs.fedoraproject.org/cgit/rpms/{name}.git/commit' + print("Verifying RPM content %d%% (%d/%d)" % ( + self.packages_checked / self.packages_to_check * 100, + self.packages_checked, self.packages_to_check), end = "", + flush = True) - commit = metadata.get('commit', 'HEAD') + def verify_rpm(self, package, metadata): + """ + Verifies the existence of dist-git repo for described package. - response = requests.head( - CGIT_URL_TEMPLATE.format(name=package), - params={'id': commit} - ) + :param string package: Name of the package to verify. + :param ModuleMetadata metadata: Metadata associated with the package. + """ - if response.status_code != requests.codes.ok: - raise RpmContentError(package, commit, response.status_code) + # TODO: Get URL from the metadata + CGIT_URL_TEMPLATE = "http://pkgs.fedoraproject.org/cgit/rpms/{name}.git/commit" + commit = metadata.get("commit", "HEAD") + url = CGIT_URL_TEMPLATE.format(name=package) + "?id=" + commit -def rpm_content(mmd): - """Generate (package, metadata) pairs from module. + data = {} + data["package"] = package + data["commit"] = commit + self.http_client.head(url, self._handle_git_response, data) - Keyword arguments: - mmd -- ModuleMetadata to be inspected. + def rpm_content(self, mmd): + """Generate (package, metadata) pairs from module. - Yields: - (package, metadata) from mmd, if any. - """ + Positional arguments: + mmd -- ModuleMetadata to be inspected. - try: - packages = mmd.components.rpms.packages - except AttributeError: - # Module has no packages -- end generation with no output - return None + Yields: + (package, metadata) from mmd, if any. + """ - yield from packages.items() + try: + packages = mmd.components.rpms.packages + except AttributeError: + # Module has no packages -- end generation with no output + return None + yield from packages.items() -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Validate module metadata.') + def _exception_handler(self, loop, context): + self.error = True + print(str(context["exception"])) + + def verify_rpms(self, metadata): + """ + Verifies the existence of dist-git repo for described package. + + :param string package: Name of the package to verify. + :param ModuleMetadata metadata: Metadata associated with the package. + :raises VerificationError: If there is an error during verification. + """ + loop = asyncio.get_event_loop() + loop.set_exception_handler(self._exception_handler) + + self.packages_to_check = len(metadata.components.rpms.packages) + self.packages_checked = 0 + for package in self.rpm_content(metadata): + self.verify_rpm(*package) + + loop.run_forever() + + print("\r", end="") + + if self.error: + raise VerificationError("Some RPMs failed the verification") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Validate module metadata.") # Positional arguments - parser.add_argument('file', help='Input metadata file') + parser.add_argument("file", help="Input metadata file") args = parser.parse_args() @@ -103,28 +194,34 @@ if __name__ == '__main__': metadata_errors = TypeError, ValueError all_went_well = True + # Load the ModuleMetadata file try: metadata.load(args.file) except metadata_errors as invalid_input_metadata: - message = 'ERROR: Invalid input: {!s}'.format(invalid_input_metadata) + message = "ERROR: Invalid input: {!s}".format(invalid_input_metadata) sys.exit(message) + # Try to validate it. try: metadata.validate() except metadata_errors as invalid_metadata_structure: all_went_well = False - print('ERROR: Invalid structure:', str(invalid_metadata_structure), + print("ERROR: Invalid structure:", str(invalid_metadata_structure), file=sys.stderr) - for package in rpm_content(metadata): - try: - verify_rpm_content(*package) - except RpmContentError as invalid_rpm: - all_went_well = False - print('RPM CONTENT ERROR:', str(invalid_rpm), - file=sys.stderr) + # Try to verify RPM section of ModuleMetadata file. + rpm_verifier = RPMVerifier() + try: + rpm_verifier.verify_rpms(metadata) + except VerificationError as error: + all_went_well = False + print("ERROR:", str(error), file=sys.stderr) + except KeyboardInterrupt: + all_went_well = False + # Exit if all_went_well: - print('Everything OK') + print("") + print("Everything OK") else: sys.exit(1)