#25 Create a consumer to sign CoreOS artifacts
Merged 4 years ago by abompard. Opened 4 years ago by abompard.
abompard/robosignatory coreos  into  master

file modified
+5 -1
@@ -4,6 +4,7 @@ 

  config = {

      'robosignatory.enabled.tagsigner': True,

      'robosignatory.enabled.atomicsigner': True,

+     'robosignatory.enabled.coreossigner': True,

  

      'robosignatory.signing': {

          # This should be the name of an entrypoint plugin that provides
@@ -49,5 +50,8 @@ 

              'directory': '/mnt/koji/compose/atomic/25/',

              'key': 'fedora-25'

          }

-     }

+     },

+ 

+     'robosignatory.coreos.bucket': 'robosig-dev-fcos-builds',

+     'robosignatory.coreos.key': 'coreos',

  }

file modified
+21
@@ -1,8 +1,10 @@ 

+ import argparse

  import fedmsg.config

  import koji

  import sys

  

  from robosignatory.tagconsumer import TagSignerConsumer

+ from robosignatory.coreosconsumer import CoreOSSignerConsumer

  import robosignatory.utils as utils

  import robosignatory.work

  
@@ -57,3 +59,22 @@ 

      val = config['robosignatory.ostree_refs'][ref]

  

      robosignatory.work.process_atomic(signer, ref, commitid, doref=doref, **val)

+ 

+ 

+ def coreossigner():

+     parser = argparse.ArgumentParser()

+     parser.add_argument("file")

+     parser.add_argument("checksum")

+     args = parser.parse_args()

+ 

+     config = fedmsg.config.load_config([], None)

+     signing_config = config['robosignatory.signing']

+     signer = utils.get_signing_helper(**signing_config)

+ 

+     signer = CoreOSSignerConsumer(None)

+     contents = {

+         "artifacts": [

+             {"file": args.file, "checksum": args.checksum}

+         ]

+     }

+     signer.dowork(contents)

@@ -0,0 +1,68 @@ 

+ import shutil

+ import tempfile

+ 

+ import boto3

+ import fedmsg.consumers

+ import robosignatory.utils as utils

+ import robosignatory.work

+ 

+ import logging

+ log = logging.getLogger("robosignatory.coreosconsumer")

+ 

+ 

+ class CoreOSSignerConsumer(fedmsg.consumers.FedmsgConsumer):

+     config_key = 'robosignatory.enabled.coreossigner'

+ 

+     def __init__(self, hub):

+         if hub:

+             super(CoreOSSignerConsumer, self).__init__(hub)

+             self.config = self.hub.config

+         else:

+             # No hub, we are in ad-hoc mode

+             self.config = fedmsg.config.load_config()

+ 

+         prefix = self.config.get('topic_prefix')

+         env = self.config.get('environment')

+         self.topic = [

+             "%s.%s.coreos.build.request.artifacts-sign" % (prefix, env)

+         ]

+ 

+         aws_config = self.config['robosignatory.coreos.aws']

+         s3 = boto3.resource(

+             's3',

+             region_name=aws_config['region'],

+             aws_access_key_id=aws_config['access_key'],

+             aws_secret_access_key=aws_config['access_secret'],

+         )

+         bucket_name = self.config['robosignatory.coreos.bucket']

+         self.bucket = s3.Bucket(bucket_name)

+ 

+         signing_config = self.config['robosignatory.signing']

+         self.signer = utils.get_signing_helper(**signing_config)

+ 

+         log.info('CoreOSSignerConsumer ready for service')

+ 

+     def consume(self, msg):

+         topic = msg['topic']

+         if topic not in self.topic:

+             return

+         # Message structure: https://github.com/coreos/fedora-coreos-tracker/issues/198#issuecomment-513944390

+         contents = msg['body']['msg']

+         log.info(

+             'CoreOS wants to sign '

+             '%(build_id)s on %(stream)s for %(basearch)s' % contents

+         )

+         self.dowork(contents)

+ 

+     def dowork(self, contents):

+         # This is here and not in __init__ because we may want a stream or

+         # version-dependant key in the future.

+         key = self.config["robosignatory.coreos.key"]

+ 

+         tmpdir = tempfile.mkdtemp(prefix="/tmp/robosignatory-")

+         try:

+             for artifact in contents["artifacts"]:

+                 robosignatory.work.process_coreos(

+                     self.signer, key, self.bucket, tmpdir, artifact)

+         finally:

+             shutil.rmtree(tmpdir)

file modified
+14
@@ -71,6 +71,10 @@ 

      def build_atomic_cmdline(self, *args):

          pass

  

+     @abc.abstractmethod

+     def build_coreos_cmdline(self, *args):

+         pass

+ 

  

  class EchoHelper(BaseSigningHelper):

      """ A dummy "hello world" helper, used for debugging. """
@@ -92,6 +96,11 @@ 

          log.info(result)

          return result

  

+     def build_coreos_cmdline(self, *args, **kwargs):

+         result = ['echo', ' '.join(['build_coreos_cmdline:', str(args), str(kwargs)])]

+         log.info(result)

+         return result

+ 

  

  class SigulHelper(BaseSigningHelper):

      def __init__(self, user, passphrase_file, config_file=None):
@@ -130,3 +139,8 @@ 

          command = self.build_cmdline('sign-ostree', key, checksum, input_file,

                                       '--output', output_file)

          return command

+ 

+     def build_coreos_cmdline(self, key, input_file, output_file):

+         command = self.build_cmdline('sign-data', key, input_file,

+                                      '--output', output_file)

+         return command

file modified
+57
@@ -1,6 +1,8 @@ 

  import stat

  import os

+ from hashlib import sha256

  

+ from six.moves.urllib.parse import urlparse

  import robosignatory.utils as utils

  

  import logging
@@ -59,3 +61,58 @@ 

              f.write(commitid + '\n')

  

      log.info('Done')

+ 

+ 

+ def process_coreos(signer, key, bucket, tmpdir, artifact):

+     filepath = urlparse(artifact["file"]).path.lstrip("/")

+     local_filepath = os.path.join(tmpdir, os.path.basename(filepath))

+ 

+     log.info("Downloading %s", filepath)

+     bucket.download_file(filepath, local_filepath)

+ 

+     log.info("Checking %s", filepath)

+     hasher = sha256()

+     with open(local_filepath, "rb") as f:

+         while True:

+             content = f.read(1024)

+             if not content:

+                 break

+             hasher.update(content)

+     h = hasher.hexdigest()

+     if h != artifact["checksum"]:

+         log.error("Incorrect SHA256 for %s, not signing", filepath)

+         return

+ 

+     log.info("Signing %s", filepath)

+     sig_filepath = local_filepath + ".sig"

+     cmdline = signer.build_coreos_cmdline(

+         key, local_filepath, sig_filepath)

+     log.info('Signing command line: %s', cmdline)

+     ret, stdout, stderr = utils.run_command(cmdline)

+     if ret != 0:

+         log.error('Error signing! Signing output: %s, stdout: %s, '

+                   'stderr: %s', ret, stdout, stderr)

+         return

+     if not os.path.exists(sig_filepath):

+         log.error("Signer did not produce any signature file for %s", filepath)

+         return

+     log.debug('Fixing signature file permissions')

+     # Sigul writes it as 0600, which makes a lot of sense as a general file

+     # mode for it, but this is just a signature file that we want published

+     os.chmod(sig_filepath,

+              (stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH))

+ 

+     log.info("Uploading signature for %s", filepath)

+     bucket.upload_file(sig_filepath, filepath + ".sig")

+     # Check the uploaded file

+     uploaded = list(bucket.objects.filter(Prefix=filepath + ".sig"))

+     if len(uploaded) != 1:

+         log.warning("The signature for %s was not uploaded properly", filepath)

+     elif uploaded[0].size != os.stat(sig_filepath).st_size:

+         log.warning(

+             "The uploaded signature for %s does not have the right size",

+             filepath

+         )

+ 

+     os.remove(local_filepath)

+     os.remove(sig_filepath)

file modified
+4
@@ -13,6 +13,8 @@ 

          "koji",

          "psutil",

          "moksha.hub",

+         "boto3",

+         "six",

      ],

      tests_require=[

          "nose",
@@ -24,10 +26,12 @@ 

      [moksha.consumer]

      tagsignerconsumer = robosignatory.tagconsumer:TagSignerConsumer

      atomicsignerconsumer = robosignatory.atomicconsumer:AtomicSignerConsumer

+     coreossignerconsumer = robosignatory.coreosconsumer:CoreOSSignerConsumer

  

      [console_scripts]

      robosignatory-signtagbuild = robosignatory.cli:tagsigner

      robosignatory-signatomic = robosignatory.cli:atomicsigner

+     robosignatory-signcoreos = robosignatory.cli:coreossigner

  

      [robosignatory.signing.helpers]

      echo = robosignatory.utils:EchoHelper

This changeset adds a consumer to sign CoreOS artifacts.
I haven't tested it end-to-end yet because for now the CoreOS project does not produce the corresponding messages, but I think the code review can begin anyway.

Could we put the AWS credentials in the robosignatory configuration file as well?
That means you'll need to construct the boto3 client manually (rather than using the implicit default one), or manually configure the default one.
For reference, see setup_default_session in https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/boto3.html, together with https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html#boto3.session.Session.client for kwargs values.

I think that we'll want to avoid using f.read(), since f could be a file image of many gigabytes in size.
I think we'll want to construct the sha256 hasher, and then read f in blocks, and pass that into the hasher.

1 new commit added

  • Fix review comments
4 years ago

Looks good to me.
Let's merge this, and then see based on actual testing (when possible) what to change before making a release.

rebased onto 19879bc

4 years ago

Pull-Request has been merged by abompard

4 years ago