Package backend :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module backend.helpers

  1  from __future__ import print_function 
  2  from __future__ import unicode_literals 
  3  from __future__ import division 
  4  from __future__ import absolute_import 
  5  import fcntl 
  6   
  7  from operator import methodcaller 
  8  import optparse 
  9  import ConfigParser 
 10  import os 
 11  import sys 
 12  import time 
 13   
 14  from bunch import Bunch 
 15  import datetime 
 16   
 17  from copr.client import CoprClient 
 18   
 19  from backend.constants import DEF_BUILD_USER, DEF_BUILD_TIMEOUT, DEF_CONSECUTIVE_FAILURE_THRESHOLD, \ 
 20      CONSECUTIVE_FAILURE_REDIS_KEY 
 21  from backend.exceptions import CoprBackendError 
 22   
 23  from redis import StrictRedis 
 24   
 25  try: 
 26      import fedmsg 
 27  except ImportError: 
 28      # fedmsg is optional 
 29      fedmsg = None 
 30   
 31   
32 -class SortedOptParser(optparse.OptionParser):
33 """Optparser which sorts the options by opt before outputting --help""" 34
35 - def format_help(self, formatter=None):
36 self.option_list.sort(key=methodcaller("get_opt_string")) 37 return optparse.OptionParser.format_help(self)
38 39
40 -def _get_conf(cp, section, option, default, mode=None):
41 """ 42 To make returning items from config parser less irritating 43 44 :param mode: convert obtained value, possible modes: 45 - None (default): do nothing 46 - "bool" or "boolean" 47 - "int" 48 - "float" 49 """ 50 51 if cp.has_section(section) and cp.has_option(section, option): 52 if mode is None: 53 return cp.get(section, option) 54 elif mode in ["bool", "boolean"]: 55 return cp.getboolean(section, option) 56 elif mode == "int": 57 return cp.getint(section, option) 58 elif mode == "float": 59 return cp.getfloat(section, option) 60 elif mode == "path": 61 path = cp.get(section, option) 62 if path.startswith("~"): 63 path = os.path.expanduser(path) 64 path = os.path.abspath(path) 65 path = os.path.normpath(path) 66 67 return path 68 return default
69 70
71 -class BackendConfigReader(object):
72 - def __init__(self, config_file=None, ext_opts=None):
73 self.config_file = config_file or "/etc/copr/copr-be.conf" 74 self.ext_opts = ext_opts
75
76 - def read(self):
77 try: 78 opts = self._read_unsafe() 79 if self.ext_opts: 80 for key, value in self.ext_opts.items(): 81 setattr(opts, key, value) 82 83 if not opts.destdir: 84 raise CoprBackendError( 85 "Incomplete Config - must specify" 86 " destdir in configuration") 87 88 return opts 89 90 except ConfigParser.Error as e: 91 raise CoprBackendError( 92 "Error parsing config file: {0}: {1}".format( 93 self.config_file, e))
94
95 - def _read_unsafe(self):
96 cp = ConfigParser.ConfigParser() 97 cp.read(self.config_file) 98 99 opts = Bunch() 100 opts.results_baseurl = _get_conf( 101 cp, "backend", "results_baseurl", "http://copr") 102 103 # TODO: this should be built from frontend_base_url + '/backend' 104 opts.frontend_url = _get_conf( 105 cp, "backend", "frontend_url", "http://coprs/rest/api") 106 107 # We need this to access public api 108 opts.frontend_base_url = _get_conf( 109 cp, "backend", "frontend_base_url", "http://coprs/") 110 111 opts.frontend_auth = _get_conf( 112 cp, "backend", "frontend_auth", "PASSWORDHERE") 113 114 opts.do_sign = _get_conf( 115 cp, "backend", "do_sign", False, mode="bool") 116 117 opts.keygen_host = _get_conf( 118 cp, "backend", "keygen_host", "copr-keygen.cloud.fedoraproject.org") 119 120 opts.build_user = _get_conf( 121 cp, "backend", "build_user", DEF_BUILD_USER) 122 123 opts.build_groups_count = _get_conf( 124 cp, "backend", "build_groups", 1, mode="int") 125 126 opts.build_groups = [] 127 for group_id in range(int(opts.build_groups_count)): 128 archs = _get_conf(cp, "backend", 129 "group{0}_archs".format(group_id), 130 default="i386,x86_64").split(",") 131 group = { 132 "id": int(group_id), 133 "name": _get_conf(cp, "backend", "group{0}_name".format(group_id), "PC"), 134 "archs": archs, 135 "spawn_playbook": _get_conf( 136 cp, "backend", "group{0}_spawn_playbook".format(group_id), 137 default="/srv/copr-work/provision/builderpb-PC.yml"), 138 "terminate_playbook": _get_conf( 139 cp, "backend", "group{0}_terminate_playbook".format(group_id), 140 default="/srv/copr-work/provision/terminatepb-PC.yml"), 141 "max_workers": _get_conf( 142 cp, "backend", "group{0}_max_workers".format(group_id), 143 default=8, mode="int") 144 } 145 opts.build_groups.append(group) 146 147 opts.destdir = _get_conf(cp, "backend", "destdir", None, mode="path") 148 149 opts.exit_on_worker = _get_conf( 150 cp, "backend", "exit_on_worker", False, mode="bool") 151 opts.fedmsg_enabled = _get_conf( 152 cp, "backend", "fedmsg_enabled", False, mode="bool") 153 opts.sleeptime = _get_conf( 154 cp, "backend", "sleeptime", 10, mode="int") 155 opts.timeout = _get_conf( 156 cp, "builder", "timeout", DEF_BUILD_TIMEOUT, mode="int") 157 opts.consecutive_failure_threshold = _get_conf( 158 cp, "builder", "consecutive_failure_threshold", 159 DEF_CONSECUTIVE_FAILURE_THRESHOLD, mode="int") 160 opts.logfile = _get_conf( 161 cp, "backend", "logfile", "/var/log/copr/backend.log") 162 opts.error_logfile = _get_conf( 163 cp, "backend", "error_logfile", "/var/log/copr/backend_error.log") 164 opts.verbose = _get_conf( 165 cp, "backend", "verbose", False, mode="bool") 166 opts.worker_logdir = _get_conf( 167 cp, "backend", "worker_logdir", "/var/log/copr/workers/") 168 opts.terminate_vars = _get_conf(cp, "backend", "terminate_vars", "").split(",") 169 170 opts.prune_days = _get_conf(cp, "backend", "prune_days", None, mode="int") 171 opts.prune_script = _get_conf(cp, "backend", "prune_script", None, mode="path") 172 173 opts.spawn_in_advance = _get_conf( 174 cp, "backend", "spawn_in_advance", False, mode="bool") 175 176 # ssh options 177 opts.ssh = Bunch() 178 # TODO: ansible Runner show some magic bugs with transport "ssh", using paramiko 179 opts.ssh.transport = _get_conf( 180 cp, "ssh", "transport", "paramiko") 181 182 # thoughts for later 183 # ssh key for connecting to builders? 184 # cloud key stuff? 185 # 186 return opts
187 188
189 -def get_auto_createrepo_status(front_url, username, projectname):
190 client = CoprClient(copr_url=front_url) 191 result = client.get_project_details(projectname, username) 192 193 if "auto_createrepo" in result.data["detail"]: 194 return bool(result.data["detail"]["auto_createrepo"]) 195 else: 196 return True
197 198
199 -def log(lf, msg, quiet=None):
200 if lf: 201 now = datetime.datetime.utcnow().isoformat() 202 try: 203 with open(lf, "a") as lfh: 204 fcntl.flock(lfh, fcntl.LOCK_EX) 205 lfh.write(str(now) + ":" + msg + "\n") 206 fcntl.flock(lfh, fcntl.LOCK_UN) 207 except (IOError, OSError) as e: 208 sys.stderr.write( 209 "Could not write to logfile {0} - {1}\n".format(lf, str(e))) 210 if not quiet: 211 print(msg)
212 213
214 -def register_build_result(opts=None, failed=False):
215 """ 216 Remember fails to redis. 217 Successful build resets counter to zero. 218 219 :param opts: BackendConfig, when opts not provided default config location will be used 220 :param boolean failed: failure flag 221 :param str origin: name of component produced failure, default: `builder` 222 """ 223 if opts is None: 224 opts = BackendConfigReader().read() 225 226 # TODO: add config options to specify redis host, port 227 conn = StrictRedis() # connecting to default local redis instance 228 229 key = CONSECUTIVE_FAILURE_REDIS_KEY 230 if not failed: 231 conn.set(key, 0) 232 else: 233 conn.incr(key)
234