From 36309a674636e40d41c2e865950095ed230a5e49 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Oct 11 2017 17:52:53 +0000 Subject: Provide the sha256 and sha512 of the releases With this commit when a new release is uploading we create or update a `checksums` file in the release folder containing both the sha256 and the sha512 of the file(s) uploaded. Fixes https://pagure.io/pagure/issue/2622 Signed-off-by: Pierre-Yves Chibon --- diff --git a/pagure/lib/tasks.py b/pagure/lib/tasks.py index d9bf248..5fda99d 100644 --- a/pagure/lib/tasks.py +++ b/pagure/lib/tasks.py @@ -9,6 +9,7 @@ """ import gc +import hashlib import os import os.path import shutil @@ -682,3 +683,42 @@ def sync_pull_ref(name, namespace, user, requestid): session.remove() gc_clean() + + +@conn.task +def update_checksums_file(folder, filenames): + """ + """ + sha_file = os.path.join(folder, 'CHECKSUMS') + new_file = not os.path.exists(sha_file) + + if not new_file: + with open(sha_file) as stream: + row = stream.readline().strip() + if row != '# Generated and updated by pagure': + # This wasn't generated by pagure, don't touch it! + return + + algos = { + 'sha256': hashlib.sha256(), + 'sha512': hashlib.sha512(), + } + + for filename in filenames: + # for each files computes the different algorythm supported + with open(os.path.join(folder, filename), "rb") as stream: + while True: + buf = stream.read(2 * 2 ** 10) + if buf: + for hasher in algos.values(): + hasher.update(buf) + else: + break + + # Write them out to the output file + with open(sha_file, 'a') as stream: + if new_file: + stream.write('# Generated and updated by pagure\n') + for algo in algos: + stream.write('%s (%s) = %s\n' % ( + algo.upper(), filename, algos[algo].hexdigest())) diff --git a/pagure/templates/releases.html b/pagure/templates/releases.html index 8efb441..3cebcf5 100644 --- a/pagure/templates/releases.html +++ b/pagure/templates/releases.html @@ -31,6 +31,17 @@ release folder.

+{% if pagure_checksum %} +

+ The CHECKSUMS + file contains the checksums (sha256 and sha512) of the files uploaded to + pagure. +

+

+ To use this file, simply download it next to the tarball you downloaded and + run 

sha512sum -c CHECKSUMS
+

+{% endif %} {% endif %}
diff --git a/pagure/ui/repo.py b/pagure/ui/repo.py index 55bb5ba..d79c370 100644 --- a/pagure/ui/repo.py +++ b/pagure/ui/repo.py @@ -913,12 +913,18 @@ def view_tags(repo, username=None, namespace=None): repo = flask.g.repo tags = pagure.lib.git.get_git_tags_objects(repo) + pagure_checksum = os.path.exists(os.path.join( + APP.config['UPLOAD_FOLDER_PATH'], + repo.fullname, + 'CHECKSUMS')) + return flask.render_template( 'releases.html', select='tags', username=username, repo=repo, tags=tags, + pagure_checksum=pagure_checksum, ) @@ -950,8 +956,10 @@ def new_release(repo, username=None, namespace=None): form = pagure.forms.UploadFileForm() if form.validate_on_submit(): + filenames = [] for filestream in flask.request.files.getlist('filestream'): filename = werkzeug.secure_filename(filestream.filename) + filenames.append(filename) try: folder = os.path.join( APP.config['UPLOAD_FOLDER_PATH'], @@ -970,6 +978,13 @@ def new_release(repo, username=None, namespace=None): except Exception as err: # pragma: no cover _log.exception(err) flask.flash('Upload failed', 'error') + + task = pagure.lib.tasks.update_checksums_file.delay( + folder=folder, filenames=filenames) + _log.info( + 'Updating checksums for %s of project %s in task: %s' % ( + filenames, repo.fullname, task.id)) + return flask.redirect(flask.url_for( 'view_tags', repo=repo.name, username=username, namespace=repo.namespace)) diff --git a/tests/test_pagure_flask_ui_repo.py b/tests/test_pagure_flask_ui_repo.py index d8794e8..64a00d4 100644 --- a/tests/test_pagure_flask_ui_repo.py +++ b/tests/test_pagure_flask_ui_repo.py @@ -18,6 +18,7 @@ import re import shutil import sys import tempfile +import time import os import pygit2 @@ -3972,6 +3973,9 @@ index 0000000..fb7093d csrf_token = output.data.split( 'name="csrf_token" type="hidden" value="')[1].split('">')[0] + upload_dir = os.path.join(self.path, 'releases') + self.assertEqual(os.listdir(upload_dir), []) + # Upload successful with open(img, mode='rb') as stream: data = {'filestream': stream, 'csrf_token': csrf_token} @@ -3984,6 +3988,40 @@ index 0000000..fb7093d 'uploaded\n ', output.data) self.assertIn('This project has not been tagged.', output.data) + self.assertEqual(os.listdir(upload_dir), ['test']) + folder = os.path.join(upload_dir, 'test') + checksum_file = os.path.join(folder, 'CHECKSUMS') + + # Wait for the worker to create the checksums file + cnt = 0 + while not os.path.exists(checksum_file): + print os.listdir(os.path.join(upload_dir, 'test')) + cnt += 1 + if cnt == 40: + raise ValueError( + 'The worker did not create the checksums file ' + 'in a timely manner') + time.sleep(0.5) + + self.assertEqual(len(os.listdir(folder)), 2) + + self.assertTrue(os.path.exists(checksum_file)) + + # Check the content of the checksums file + with open(checksum_file) as stream: + data = stream.readlines() + self.assertEqual(len(data), 3) + self.assertEqual(data[0], '# Generated and updated by pagure\n') + self.assertTrue(data[1].startswith('SHA256 (')) + self.assertTrue(data[1].endswith( + 'tests_placebo.png) = 8a06845923010b27bfd8e7e75acff' + '7badc40d1021b4994e01f5e11ca40bc3abe\n')) + self.assertTrue(data[2].startswith('SHA512 (')) + self.assertTrue(data[2].endswith( + 'tests_placebo.png) = 65a4458df0acb29dc3c5ad4a3620e' + '98841d1fcf3f8df358f5348fdeddd1a86706491ac6e416768e' + '9f218aae8147d6ac524a59d3eb91fb925fdcb5c489e55ccbb\n')) + # Try uploading the same file -- fails with open(img, mode='rb') as stream: data = {'filestream': stream, 'csrf_token': csrf_token}