1 from __future__ import print_function
2 from __future__ import unicode_literals
3 from __future__ import division
4 from __future__ import absolute_import
5 import fcntl
6
7 from operator import methodcaller
8 import optparse
9 import ConfigParser
10 import os
11 import sys
12 import time
13
14 from bunch import Bunch
15 import datetime
16
17 from copr.client import CoprClient
18
19 from backend.constants import DEF_BUILD_USER, DEF_BUILD_TIMEOUT, DEF_CONSECUTIVE_FAILURE_THRESHOLD, \
20 CONSECUTIVE_FAILURE_REDIS_KEY
21 from backend.exceptions import CoprBackendError
22
23 from redis import StrictRedis
24
25 try:
26 import fedmsg
27 except ImportError:
28
29 fedmsg = None
30
31
33 """Optparser which sorts the options by opt before outputting --help"""
34
38
39
40 -def _get_conf(cp, section, option, default, mode=None):
41 """
42 To make returning items from config parser less irritating
43
44 :param mode: convert obtained value, possible modes:
45 - None (default): do nothing
46 - "bool" or "boolean"
47 - "int"
48 - "float"
49 """
50
51 if cp.has_section(section) and cp.has_option(section, option):
52 if mode is None:
53 return cp.get(section, option)
54 elif mode in ["bool", "boolean"]:
55 return cp.getboolean(section, option)
56 elif mode == "int":
57 return cp.getint(section, option)
58 elif mode == "float":
59 return cp.getfloat(section, option)
60 elif mode == "path":
61 path = cp.get(section, option)
62 if path.startswith("~"):
63 path = os.path.expanduser(path)
64 path = os.path.abspath(path)
65 path = os.path.normpath(path)
66
67 return path
68 return default
69
70
72 - def __init__(self, config_file=None, ext_opts=None):
73 self.config_file = config_file or "/etc/copr/copr-be.conf"
74 self.ext_opts = ext_opts
75
77 try:
78 opts = self._read_unsafe()
79 if self.ext_opts:
80 for key, value in self.ext_opts.items():
81 setattr(opts, key, value)
82
83 if not opts.destdir:
84 raise CoprBackendError(
85 "Incomplete Config - must specify"
86 " destdir in configuration")
87
88 return opts
89
90 except ConfigParser.Error as e:
91 raise CoprBackendError(
92 "Error parsing config file: {0}: {1}".format(
93 self.config_file, e))
94
96 cp = ConfigParser.ConfigParser()
97 cp.read(self.config_file)
98
99 opts = Bunch()
100 opts.results_baseurl = _get_conf(
101 cp, "backend", "results_baseurl", "http://copr")
102
103
104 opts.frontend_url = _get_conf(
105 cp, "backend", "frontend_url", "http://coprs/rest/api")
106
107
108 opts.frontend_base_url = _get_conf(
109 cp, "backend", "frontend_base_url", "http://coprs/")
110
111 opts.frontend_auth = _get_conf(
112 cp, "backend", "frontend_auth", "PASSWORDHERE")
113
114 opts.do_sign = _get_conf(
115 cp, "backend", "do_sign", False, mode="bool")
116
117 opts.keygen_host = _get_conf(
118 cp, "backend", "keygen_host", "copr-keygen.cloud.fedoraproject.org")
119
120 opts.build_user = _get_conf(
121 cp, "backend", "build_user", DEF_BUILD_USER)
122
123 opts.build_groups_count = _get_conf(
124 cp, "backend", "build_groups", 1, mode="int")
125
126 opts.build_groups = []
127 for group_id in range(int(opts.build_groups_count)):
128 archs = _get_conf(cp, "backend",
129 "group{0}_archs".format(group_id),
130 default="i386,x86_64").split(",")
131 group = {
132 "id": int(group_id),
133 "name": _get_conf(cp, "backend", "group{0}_name".format(group_id), "PC"),
134 "archs": archs,
135 "spawn_playbook": _get_conf(
136 cp, "backend", "group{0}_spawn_playbook".format(group_id),
137 default="/srv/copr-work/provision/builderpb-PC.yml"),
138 "terminate_playbook": _get_conf(
139 cp, "backend", "group{0}_terminate_playbook".format(group_id),
140 default="/srv/copr-work/provision/terminatepb-PC.yml"),
141 "max_workers": _get_conf(
142 cp, "backend", "group{0}_max_workers".format(group_id),
143 default=8, mode="int")
144 }
145 opts.build_groups.append(group)
146
147 opts.destdir = _get_conf(cp, "backend", "destdir", None, mode="path")
148
149 opts.exit_on_worker = _get_conf(
150 cp, "backend", "exit_on_worker", False, mode="bool")
151 opts.fedmsg_enabled = _get_conf(
152 cp, "backend", "fedmsg_enabled", False, mode="bool")
153 opts.sleeptime = _get_conf(
154 cp, "backend", "sleeptime", 10, mode="int")
155 opts.timeout = _get_conf(
156 cp, "builder", "timeout", DEF_BUILD_TIMEOUT, mode="int")
157 opts.consecutive_failure_threshold = _get_conf(
158 cp, "builder", "consecutive_failure_threshold",
159 DEF_CONSECUTIVE_FAILURE_THRESHOLD, mode="int")
160 opts.logfile = _get_conf(
161 cp, "backend", "logfile", "/var/log/copr/backend.log")
162 opts.error_logfile = _get_conf(
163 cp, "backend", "error_logfile", "/var/log/copr/backend_error.log")
164 opts.verbose = _get_conf(
165 cp, "backend", "verbose", False, mode="bool")
166 opts.worker_logdir = _get_conf(
167 cp, "backend", "worker_logdir", "/var/log/copr/workers/")
168 opts.terminate_vars = _get_conf(cp, "backend", "terminate_vars", "").split(",")
169
170 opts.prune_days = _get_conf(cp, "backend", "prune_days", None, mode="int")
171 opts.prune_script = _get_conf(cp, "backend", "prune_script", None, mode="path")
172
173 opts.spawn_in_advance = _get_conf(
174 cp, "backend", "spawn_in_advance", False, mode="bool")
175
176
177 opts.ssh = Bunch()
178
179 opts.ssh.transport = _get_conf(
180 cp, "ssh", "transport", "paramiko")
181
182
183
184
185
186 return opts
187
188
190 client = CoprClient(copr_url=front_url)
191 result = client.get_project_details(projectname, username)
192
193 if "auto_createrepo" in result.data["detail"]:
194 return bool(result.data["detail"]["auto_createrepo"])
195 else:
196 return True
197
198
199 -def log(lf, msg, quiet=None):
200 if lf:
201 now = datetime.datetime.utcnow().isoformat()
202 try:
203 with open(lf, "a") as lfh:
204 fcntl.flock(lfh, fcntl.LOCK_EX)
205 lfh.write(str(now) + ":" + msg + "\n")
206 fcntl.flock(lfh, fcntl.LOCK_UN)
207 except (IOError, OSError) as e:
208 sys.stderr.write(
209 "Could not write to logfile {0} - {1}\n".format(lf, str(e)))
210 if not quiet:
211 print(msg)
212
213
215 """
216 Remember fails to redis.
217 Successful build resets counter to zero.
218
219 :param opts: BackendConfig, when opts not provided default config location will be used
220 :param boolean failed: failure flag
221 :param str origin: name of component produced failure, default: `builder`
222 """
223 if opts is None:
224 opts = BackendConfigReader().read()
225
226
227 conn = StrictRedis()
228
229 key = CONSECUTIVE_FAILURE_REDIS_KEY
230 if not failed:
231 conn.set(key, 0)
232 else:
233 conn.incr(key)
234