From 2e5b04dc45138192b34f6ea39cbed0023d716f9a Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Apr 26 2019 07:11:49 +0000 Subject: Use a safe NFS lock file mechanism for OpenShift deployment Signed-off-by: Clement Verna --- diff --git a/Dockerfile b/Dockerfile index 8fce400..7192825 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ LABEL maintainer "Clément Verna " EXPOSE 8080 -RUN dnf -y install python3-aiohttp python3-werkzeug python3-requests python3-sqlalchemy python3-fedora-messaging +RUN dnf -y install python3-aiohttp python3-werkzeug python3-requests python3-sqlalchemy python3-fedora-messaging python3-flufl-lock USER 1001 ENV MDAPI_CONFIG=/etc/mdapi/mdapi.cfg diff --git a/mdapi-get_repo_md b/mdapi-get_repo_md index d0f0414..a290ba7 100755 --- a/mdapi-get_repo_md +++ b/mdapi-get_repo_md @@ -52,9 +52,9 @@ import requests from sqlalchemy import text from fedora_messaging.api import Message, publish from fedora_messaging.exceptions import PublishReturned, ConnectionException +from flufl.lock import Lock import mdapi.lib as mdapilib -import mdapi.file_lock as file_lock KOJI_REPO = 'https://kojipkgs.fedoraproject.org/repos/' PKGDB2_URL = 'https://admin.fedoraproject.org/pkgdb/' @@ -329,7 +329,7 @@ def publish_changes(name, packages, repomd_url): def install_db(name, src, dest): print('%s Installing %s to %s.' % (name.ljust(padding), src, dest)) - with file_lock.FileFlock(dest + '.lock'): + with Lock(dest + '.lock'): shutil.move(src, dest) diff --git a/mdapi/__init__.py b/mdapi/__init__.py index df70827..fca3d29 100644 --- a/mdapi/__init__.py +++ b/mdapi/__init__.py @@ -34,9 +34,10 @@ import asyncio import werkzeug from aiohttp import web from multidict import MultiDict +from flufl.lock import Lock import mdapi.lib as mdapilib -import mdapi.file_lock as file_lock + CONFIG = dict() @@ -114,7 +115,7 @@ def _get_pkg(branch, name=None, action=None, srcname=None): wrongdb = False - with file_lock.FileFlock(dbfile + '.lock'): + with Lock(dbfile + '.lock'): session = yield from mdapilib.create_session( 'sqlite:///%s' % dbfile) if name: @@ -166,7 +167,7 @@ def _expand_pkg_info(pkgs, branch, repotype=None): dbfile = '%s/mdapi-%s%s-primary.sqlite' % ( CONFIG['DB_FOLDER'], branch, '-%s' % repotype if repotype else '') - with file_lock.FileFlock(dbfile + '.lock'): + with Lock(dbfile + '.lock'): session = yield from mdapilib.create_session( 'sqlite:///%s' % dbfile) # Fill in some extra info @@ -263,7 +264,7 @@ def get_pkg_files(request): if not os.path.exists(dbfile): raise web.HTTPBadRequest() - with file_lock.FileFlock(dbfile + '.lock'): + with Lock(dbfile + '.lock'): session2 = yield from mdapilib.create_session( 'sqlite:///%s' % dbfile) filelist = yield from mdapilib.get_files(session2, pkg.pkgId) @@ -296,7 +297,7 @@ def get_pkg_changelog(request): if not os.path.exists(dbfile): raise web.HTTPBadRequest() - with file_lock.FileFlock(dbfile + '.lock'): + with Lock(dbfile + '.lock'): session2 = yield from mdapilib.create_session( 'sqlite:///%s' % dbfile) changelogs = yield from mdapilib.get_changelog(session2, pkg.pkgId) diff --git a/mdapi/file_lock.py b/mdapi/file_lock.py deleted file mode 100644 index 9fdb867..0000000 --- a/mdapi/file_lock.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright © derpston. -# -# Source: https://github.com/derpston/python-simpleflock/ -# -# License: WTFPL -# - -''' -Simple file lock mechanism -''' - -import time -import os -import fcntl -import errno - - -class FileFlock: - """ Provides the simplest possible interface to flock-based file locking. - Intended for use with the `with` syntax. It will create/truncate/delete - the lock file as necessary. - - """ - - def __init__(self, path, timeout=None, wait_for=0.1): - self._path = path - self._timeout = timeout - self._fd = None - self.wait_for = wait_for - - def __enter__(self): - self._fd = os.open(self._path, os.O_CREAT) - start_lock_search = time.time() - while True: - try: - fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - # Lock acquired! - return - except IOError as ex: - # Keep going if resource temporarily unavailable - if ex.errno != errno.EAGAIN: - raise - # Exceeded the user-specified timeout. - elif self._timeout is not None \ - and time.time() > (start_lock_search + self._timeout): - raise - - # Let's not spin continuously - time.sleep(self.wait_for) - - def __exit__(self, *args): - fcntl.flock(self._fd, fcntl.LOCK_UN) - os.close(self._fd) - self._fd = None - - # Try to remove the lock file, but don't try too hard because it is - # unnecessary. This is mostly to help the user see whether a lock - # exists by examining the filesystem. - try: - os.unlink(self._path) - except: - pass diff --git a/requirements.txt b/requirements.txt index 2c713dd..c1f4c6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ multidict requests sqlalchemy werkzeug +flufl.lock