1 import os
2 import pipes
3 import socket
4 from subprocess import Popen, PIPE
5 import time
6 from urlparse import urlparse
7
8 from ansible.runner import Runner
9
10 from ..exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError
11
12 from ..constants import mockchain, rsync
16
17 - def __init__(self, opts, hostname, username, job,
18 timeout, chroot, buildroot_pkgs,
19 callback,
20 remote_basedir, remote_tempdir=None,
21 macros=None, repos=None):
22
23
24 self.opts = opts
25 self.hostname = hostname
26 self.username = username
27 self.job = job
28 self.timeout = timeout
29 self.chroot = chroot
30 self.repos = repos or []
31 self.macros = macros or {}
32 self.callback = callback
33
34 self.buildroot_pkgs = buildroot_pkgs or ""
35
36 self._remote_tempdir = remote_tempdir
37 self._remote_basedir = remote_basedir
38
39 self.conn = self._create_ans_conn()
40 self.root_conn = self._create_ans_conn(username="root")
41
42
43
44
45
46
47 @property
50
51 @property
53 if self._remote_tempdir:
54 return self._remote_tempdir
55
56 create_tmpdir_cmd = "/bin/mktemp -d {0}/{1}-XXXXX".format(
57 self._remote_basedir, "mockremote")
58
59 results = self._run_ansible(create_tmpdir_cmd)
60
61 tempdir = None
62
63 for _, resdict in results["contacted"].items():
64 tempdir = resdict["stdout"]
65
66
67 if not tempdir:
68 raise BuilderError("Could not make tmpdir on {0}".format(
69 self.hostname))
70
71 self._run_ansible("/bin/chmod 755 {0}".format(tempdir))
72 self._remote_tempdir = tempdir
73
74 return self._remote_tempdir
75
76 @tempdir.setter
78 self._remote_tempdir = value
79
81 ans_conn = Runner(remote_user=username or self.username,
82 host_list=self.hostname + ",",
83 pattern=self.hostname,
84 forks=1,
85 transport=self.opts.ssh.transport,
86 timeout=self.timeout)
87 return ans_conn
88
89 - def run_ansible_with_check(self, cmd, module_name=None, as_root=False,
90 err_codes=None, success_codes=None):
91
92 results = self._run_ansible(cmd, module_name, as_root)
93
94 try:
95 check_for_ans_error(
96 results, self.hostname, err_codes, success_codes)
97 except AnsibleResponseError as response_error:
98 raise AnsibleCallError(
99 msg="Failed to execute ansible command",
100 cmd=cmd, module_name=module_name, as_root=as_root,
101 return_code=response_error.return_code,
102 stdout=response_error.stdout, stderr=response_error.stderr
103 )
104
105 return results
106
107 - def _run_ansible(self, cmd, module_name=None, as_root=False):
108 """
109 Executes single ansible module
110
111 :param str cmd: module command
112 :param str module_name: name of the invoked module
113 :param bool as_root:
114 :return: ansible command result
115 """
116 if as_root:
117 conn = self.root_conn
118 else:
119 conn = self.conn
120
121 conn.module_name = module_name or "shell"
122 conn.module_args = str(cmd)
123 return conn.run()
124
126
127
128 s_pkg = os.path.basename(pkg)
129 pdn = s_pkg.replace(".src.rpm", "")
130 remote_pkg_dir = os.path.normpath(
131 os.path.join(self.remote_build_dir, "results",
132 self.chroot, pdn))
133
134 return remote_pkg_dir
135
137 """
138 Modify mock config for current chroot.
139
140 Packages in buildroot_pkgs are added to minimal buildroot
141 """
142
143 if ("'{0} '".format(self.buildroot_pkgs) !=
144 pipes.quote(str(self.buildroot_pkgs) + ' ')):
145
146
147
148 raise BuilderError("Do not try this kind of attack on me")
149
150 self.callback.log("putting {0} into minimal buildroot of {1}"
151 .format(self.buildroot_pkgs, self.chroot))
152
153 kwargs = {
154 "chroot": self.chroot,
155 "pkgs": self.buildroot_pkgs
156 }
157 buildroot_cmd = (
158 "dest=/etc/mock/{chroot}.cfg"
159 " line=\"config_opts['chroot_setup_cmd'] = 'install @buildsys-build {pkgs}'\""
160 " regexp=\"^.*chroot_setup_cmd.*$\""
161 )
162
163 disable_networking_cmd = (
164 "dest=/etc/mock/{chroot}.cfg"
165 " line=\"config_opts['use_host_resolv'] = False\""
166 " regexp=\"^.*user_host_resolv.*$\""
167 )
168 try:
169 self.run_ansible_with_check(buildroot_cmd.format(**kwargs),
170 module_name="lineinfile", as_root=True)
171 if not self.job.enable_net:
172 self.run_ansible_with_check(disable_networking_cmd.format(**kwargs),
173 module_name="lineinfile", as_root=True)
174 except BuilderError as err:
175 self.callback.log(str(err))
176 raise
177
179 self.callback.log("Listing built binary packages")
180
181
182 results = self._run_ansible(
183 "cd {0} && "
184 "for f in `ls *.rpm |grep -v \"src.rpm$\"`; do"
185 " rpm -qp --qf \"%{{NAME}} %{{VERSION}}\n\" $f; "
186 "done".format(pipes.quote(self._get_remote_pkg_dir(pkg)))
187 )
188
189 build_details["built_packages"] = list(results["contacted"].values())[0][u"stdout"]
190 self.callback.log("Packages:\n{}".format(build_details["built_packages"]))
191
196
198 """
199 Local file will be sent into the build chroot,
200 if pkg is a url, it will be returned as is.
201
202 :param str pkg: path to the local file or URL
203 :return str: fixed pkg location
204 """
205 if os.path.exists(pkg):
206 dest = os.path.normpath(
207 os.path.join(self.tempdir, os.path.basename(pkg)))
208
209 self.callback.log(
210 "Sending {0} to {1} to build".format(
211 os.path.basename(pkg), self.hostname))
212
213
214 self._run_ansible("src={0} dest={1}".format(pkg, dest), module_name="copy")
215 else:
216 dest = pkg
217
218 return dest
219
221 self.callback.log("Getting package information: version")
222 results = self._run_ansible("rpm -qp --qf \"%{{EPOCH}}\$\$%{{VERSION}}\$\$%{{RELEASE}}\" {}".format(pkg))
223 if "contacted" in results:
224
225 raw = list(results["contacted"].values())[0][u"stdout"]
226 try:
227 epoch, version, release = raw.split("$$")
228
229 if epoch == "(none)" or epoch == "0":
230 epoch = None
231 if release == "(none)":
232 release = None
233
234 self.job.pkg_main_version = version
235 self.job.pkg_epoch = epoch
236 self.job.pkg_release = release
237 except ValueError:
238 pass
239
241 """
242 Expands variables and sanitize repo url to be used for mock config
243 """
244 try:
245 parsed_url = urlparse(repo_url)
246 if parsed_url.scheme == "copr":
247 user = parsed_url.netloc
248 prj = parsed_url.path.split("/")[1]
249 repo_url = "/".join([self.opts.results_baseurl, user, prj, self.chroot])
250
251 else:
252 if "rawhide" in self.chroot:
253 repo_url = repo_url.replace("$releasever", "rawhide")
254
255 repo_url = repo_url.replace("$chroot", self.chroot)
256 repo_url = repo_url.replace("$distname", self.chroot.split("-")[0])
257
258 return pipes.quote(repo_url)
259 except Exception as err:
260 self.callback.log("Failed not pre-process repo url: {}".format(err))
261 return None
262
264 buildcmd = "{0} -r {1} -l {2} ".format(
265 mockchain, pipes.quote(self.chroot),
266 pipes.quote(self.remote_build_dir))
267 for repo in self.repos:
268 repo = self.pre_process_repo_url(repo)
269 if repo is not None:
270 buildcmd += "-a {0} ".format(repo)
271
272 for k, v in self.macros.items():
273 mock_opt = "--define={0} {1}".format(k, v)
274 buildcmd += "-m {0} ".format(pipes.quote(mock_opt))
275 buildcmd += dest
276 return buildcmd
277
279 self.callback.log("executing: {0}".format(buildcmd))
280 self.conn.module_name = "shell"
281 self.conn.module_args = buildcmd
282 _, poller = self.conn.run_async(self.timeout)
283 waited = 0
284 results = None
285 while True:
286
287
288 results = poller.poll()
289
290 if results["contacted"] or results["dark"]:
291 break
292
293 if waited >= self.timeout:
294 msg = "Build timeout expired. Time limit: {}s, time spent: {}s".format(
295 self.timeout, waited)
296 self.callback.log(msg)
297 raise BuilderTimeOutError(msg)
298
299 time.sleep(10)
300 waited += 10
301 return results
302
332
334
335
336 rpd = self._get_remote_pkg_dir(pkg)
337
338 destdir = "'" + destdir.replace("'", "'\\''") + "'"
339
340
341 remote_src = "{0}@{1}:{2}".format(self.username, self.hostname, rpd)
342 ssh_opts = "'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'"
343
344 rsync_log_filepath = os.path.join(destdir, "build-{}.rsync.log".format(self.job.build_id))
345 command = "{} -avH -e {} {} {}/ &> {}".format(
346 rsync, ssh_opts, remote_src, destdir,
347 rsync_log_filepath)
348
349
350
351
352 try:
353 cmd = Popen(command, shell=True)
354 cmd.wait()
355 except Exception as error:
356 raise BuilderError(msg="Failed to download from builder due to rsync error, "
357 "see logs dir. Original error: {}".format(error))
358 if cmd.returncode != 0:
359 raise BuilderError(msg="Failed to download from builder due to rsync error, "
360 "see logs dir.", return_code=cmd.returncode)
361
390
393 if hostname in results["dark"]:
394 return results["dark"][hostname]
395 if hostname in results["contacted"]:
396 return results["contacted"][hostname]
397
398 return {}
399
402 """
403 dict includes 'msg'
404 may include 'rc', 'stderr', 'stdout' and any other requested result codes
405
406 :raises AnsibleResponseError:
407 """
408
409 if err_codes is None:
410 err_codes = []
411 if success_codes is None:
412 success_codes = [0]
413
414 if "dark" in results and hostname in results["dark"]:
415 raise AnsibleResponseError(
416 msg="Error: Could not contact/connect to {}.".format(hostname))
417
418 error = False
419 err_results = {}
420 if err_codes or success_codes:
421 if hostname in results["contacted"]:
422 if "rc" in results["contacted"][hostname]:
423 rc = int(results["contacted"][hostname]["rc"])
424 err_results["return_code"] = rc
425
426 if rc in err_codes:
427 error = True
428 err_results["msg"] = "rc {0} matched err_codes".format(rc)
429 elif rc not in success_codes:
430 error = True
431 err_results["msg"] = "rc {0} not in success_codes".format(rc)
432
433 elif ("failed" in results["contacted"][hostname] and
434 results["contacted"][hostname]["failed"]):
435
436 error = True
437 err_results["msg"] = "results included failed as true"
438
439 if error:
440 for item in ["stdout", "stderr"]:
441 if item in results["contacted"][hostname]:
442 err_results[item] = results["contacted"][hostname][item]
443
444 if error:
445 raise AnsibleResponseError(**err_results)
446