Package backend :: Package mockremote :: Module builder
[hide private]
[frames] | no frames]

Source Code for Module backend.mockremote.builder

  1  import os 
  2  import pipes 
  3  import socket 
  4  from subprocess import Popen, PIPE 
  5  import time 
  6  from urlparse import urlparse 
  7   
  8  from ansible.runner import Runner 
  9   
 10  from ..exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError 
 11   
 12  from ..constants import mockchain, rsync 
13 14 15 -class Builder(object):
16
17 - def __init__(self, opts, hostname, username, job, 18 timeout, chroot, buildroot_pkgs, 19 callback, 20 remote_basedir, remote_tempdir=None, 21 macros=None, repos=None):
22 23 # TODO: remove fields obtained from opts 24 self.opts = opts 25 self.hostname = hostname 26 self.username = username 27 self.job = job 28 self.timeout = timeout 29 self.chroot = chroot 30 self.repos = repos or [] 31 self.macros = macros or {} # rename macros to mock_ext_options 32 self.callback = callback 33 34 self.buildroot_pkgs = buildroot_pkgs or "" 35 36 self._remote_tempdir = remote_tempdir 37 self._remote_basedir = remote_basedir 38 # if we're at this point we've connected and done stuff on the host 39 self.conn = self._create_ans_conn() 40 self.root_conn = self._create_ans_conn(username="root")
41 42 # self.callback.log("Created builder: {}".format(self.__dict__)) 43 44 # Before use: check out the host - make sure it can build/be contacted/etc 45 # self.check() 46 47 @property
48 - def remote_build_dir(self):
49 return self.tempdir + "/build/"
50 51 @property
52 - def tempdir(self):
53 if self._remote_tempdir: 54 return self._remote_tempdir 55 56 create_tmpdir_cmd = "/bin/mktemp -d {0}/{1}-XXXXX".format( 57 self._remote_basedir, "mockremote") 58 59 results = self._run_ansible(create_tmpdir_cmd) 60 61 tempdir = None 62 # TODO: use check_for_ans_error 63 for _, resdict in results["contacted"].items(): 64 tempdir = resdict["stdout"] 65 66 # if still nothing then we"ve broken 67 if not tempdir: 68 raise BuilderError("Could not make tmpdir on {0}".format( 69 self.hostname)) 70 71 self._run_ansible("/bin/chmod 755 {0}".format(tempdir)) 72 self._remote_tempdir = tempdir 73 74 return self._remote_tempdir
75 76 @tempdir.setter
77 - def tempdir(self, value):
78 self._remote_tempdir = value
79
80 - def _create_ans_conn(self, username=None):
81 ans_conn = Runner(remote_user=username or self.username, 82 host_list=self.hostname + ",", 83 pattern=self.hostname, 84 forks=1, 85 transport=self.opts.ssh.transport, 86 timeout=self.timeout) 87 return ans_conn
88
89 - def run_ansible_with_check(self, cmd, module_name=None, as_root=False, 90 err_codes=None, success_codes=None):
91 92 results = self._run_ansible(cmd, module_name, as_root) 93 94 try: 95 check_for_ans_error( 96 results, self.hostname, err_codes, success_codes) 97 except AnsibleResponseError as response_error: 98 raise AnsibleCallError( 99 msg="Failed to execute ansible command", 100 cmd=cmd, module_name=module_name, as_root=as_root, 101 return_code=response_error.return_code, 102 stdout=response_error.stdout, stderr=response_error.stderr 103 ) 104 105 return results
106
107 - def _run_ansible(self, cmd, module_name=None, as_root=False):
108 """ 109 Executes single ansible module 110 111 :param str cmd: module command 112 :param str module_name: name of the invoked module 113 :param bool as_root: 114 :return: ansible command result 115 """ 116 if as_root: 117 conn = self.root_conn 118 else: 119 conn = self.conn 120 121 conn.module_name = module_name or "shell" 122 conn.module_args = str(cmd) 123 return conn.run()
124
125 - def _get_remote_pkg_dir(self, pkg):
126 # the pkg will build into a dir by mockchain named: 127 # $tempdir/build/results/$chroot/$packagename 128 s_pkg = os.path.basename(pkg) 129 pdn = s_pkg.replace(".src.rpm", "") 130 remote_pkg_dir = os.path.normpath( 131 os.path.join(self.remote_build_dir, "results", 132 self.chroot, pdn)) 133 134 return remote_pkg_dir
135
137 """ 138 Modify mock config for current chroot. 139 140 Packages in buildroot_pkgs are added to minimal buildroot 141 """ 142 143 if ("'{0} '".format(self.buildroot_pkgs) != 144 pipes.quote(str(self.buildroot_pkgs) + ' ')): 145 146 # just different test if it contains only alphanumeric characters 147 # allowed in packages name 148 raise BuilderError("Do not try this kind of attack on me") 149 150 self.callback.log("putting {0} into minimal buildroot of {1}" 151 .format(self.buildroot_pkgs, self.chroot)) 152 153 kwargs = { 154 "chroot": self.chroot, 155 "pkgs": self.buildroot_pkgs 156 } 157 buildroot_cmd = ( 158 "dest=/etc/mock/{chroot}.cfg" 159 " line=\"config_opts['chroot_setup_cmd'] = 'install @buildsys-build {pkgs}'\"" 160 " regexp=\"^.*chroot_setup_cmd.*$\"" 161 ) 162 163 disable_networking_cmd = ( 164 "dest=/etc/mock/{chroot}.cfg" 165 " line=\"config_opts['use_host_resolv'] = False\"" 166 " regexp=\"^.*user_host_resolv.*$\"" 167 ) 168 try: 169 self.run_ansible_with_check(buildroot_cmd.format(**kwargs), 170 module_name="lineinfile", as_root=True) 171 if not self.job.enable_net: 172 self.run_ansible_with_check(disable_networking_cmd.format(**kwargs), 173 module_name="lineinfile", as_root=True) 174 except BuilderError as err: 175 self.callback.log(str(err)) 176 raise
177
178 - def collect_built_packages(self, build_details, pkg):
179 self.callback.log("Listing built binary packages") 180 # self.conn.module_name = "shell" 181 182 results = self._run_ansible( 183 "cd {0} && " 184 "for f in `ls *.rpm |grep -v \"src.rpm$\"`; do" 185 " rpm -qp --qf \"%{{NAME}} %{{VERSION}}\n\" $f; " 186 "done".format(pipes.quote(self._get_remote_pkg_dir(pkg))) 187 ) 188 189 build_details["built_packages"] = list(results["contacted"].values())[0][u"stdout"] 190 self.callback.log("Packages:\n{}".format(build_details["built_packages"]))
191
192 - def check_build_success(self, pkg):
193 successfile = os.path.join(self._get_remote_pkg_dir(pkg), "success") 194 ansible_test_results = self._run_ansible("/usr/bin/test -f {0}".format(successfile)) 195 check_for_ans_error(ansible_test_results, self.hostname)
196
197 - def check_if_pkg_local_or_http(self, pkg):
198 """ 199 Local file will be sent into the build chroot, 200 if pkg is a url, it will be returned as is. 201 202 :param str pkg: path to the local file or URL 203 :return str: fixed pkg location 204 """ 205 if os.path.exists(pkg): 206 dest = os.path.normpath( 207 os.path.join(self.tempdir, os.path.basename(pkg))) 208 209 self.callback.log( 210 "Sending {0} to {1} to build".format( 211 os.path.basename(pkg), self.hostname)) 212 213 # FIXME should probably check this but <shrug> 214 self._run_ansible("src={0} dest={1}".format(pkg, dest), module_name="copy") 215 else: 216 dest = pkg 217 218 return dest
219
220 - def update_job_pkg_version(self, pkg):
221 self.callback.log("Getting package information: version") 222 results = self._run_ansible("rpm -qp --qf \"%{{EPOCH}}\$\$%{{VERSION}}\$\$%{{RELEASE}}\" {}".format(pkg)) 223 if "contacted" in results: 224 # TODO: do more sane 225 raw = list(results["contacted"].values())[0][u"stdout"] 226 try: 227 epoch, version, release = raw.split("$$") 228 229 if epoch == "(none)" or epoch == "0": 230 epoch = None 231 if release == "(none)": 232 release = None 233 234 self.job.pkg_main_version = version 235 self.job.pkg_epoch = epoch 236 self.job.pkg_release = release 237 except ValueError: 238 pass
239
240 - def pre_process_repo_url(self, repo_url):
241 """ 242 Expands variables and sanitize repo url to be used for mock config 243 """ 244 try: 245 parsed_url = urlparse(repo_url) 246 if parsed_url.scheme == "copr": 247 user = parsed_url.netloc 248 prj = parsed_url.path.split("/")[1] 249 repo_url = "/".join([self.opts.results_baseurl, user, prj, self.chroot]) 250 251 else: 252 if "rawhide" in self.chroot: 253 repo_url = repo_url.replace("$releasever", "rawhide") 254 # custom expand variables 255 repo_url = repo_url.replace("$chroot", self.chroot) 256 repo_url = repo_url.replace("$distname", self.chroot.split("-")[0]) 257 258 return pipes.quote(repo_url) 259 except Exception as err: 260 self.callback.log("Failed not pre-process repo url: {}".format(err)) 261 return None
262
263 - def gen_mockchain_command(self, dest):
264 buildcmd = "{0} -r {1} -l {2} ".format( 265 mockchain, pipes.quote(self.chroot), 266 pipes.quote(self.remote_build_dir)) 267 for repo in self.repos: 268 repo = self.pre_process_repo_url(repo) 269 if repo is not None: 270 buildcmd += "-a {0} ".format(repo) 271 272 for k, v in self.macros.items(): 273 mock_opt = "--define={0} {1}".format(k, v) 274 buildcmd += "-m {0} ".format(pipes.quote(mock_opt)) 275 buildcmd += dest 276 return buildcmd
277
278 - def run_command_and_wait(self, buildcmd):
279 self.callback.log("executing: {0}".format(buildcmd)) 280 self.conn.module_name = "shell" 281 self.conn.module_args = buildcmd 282 _, poller = self.conn.run_async(self.timeout) 283 waited = 0 284 results = None 285 while True: 286 # TODO: try replace with ``while waited < self.timeout`` 287 # extract method and return waited time, raise timeout error in `else` 288 results = poller.poll() 289 290 if results["contacted"] or results["dark"]: 291 break 292 293 if waited >= self.timeout: 294 msg = "Build timeout expired. Time limit: {}s, time spent: {}s".format( 295 self.timeout, waited) 296 self.callback.log(msg) 297 raise BuilderTimeOutError(msg) 298 299 time.sleep(10) 300 waited += 10 301 return results
302
303 - def build(self, pkg):
304 # build the pkg passed in 305 # add pkg to various lists 306 # check for success/failure of build 307 308 # build_details = {} 309 self.modify_mock_chroot_config() 310 311 # check if pkg is local or http 312 dest = self.check_if_pkg_local_or_http(pkg) 313 314 # srpm version 315 self.update_job_pkg_version(pkg) 316 317 # construct the mockchain command 318 buildcmd = self.gen_mockchain_command(dest) 319 320 # run the mockchain command async 321 ansible_build_results = self.run_command_and_wait(buildcmd) # now raises BuildTimeoutError 322 check_for_ans_error(ansible_build_results, self.hostname) # on error raises AnsibleResponseError 323 324 # we know the command ended successfully but not if the pkg built 325 # successfully 326 self.check_build_success(pkg) 327 build_out = get_ans_results(ansible_build_results, self.hostname).get("stdout", "") 328 329 build_details = {"pkg_version": self.job.pkg_version} 330 self.collect_built_packages(build_details, pkg) 331 return build_details, build_out
332
333 - def download(self, pkg, destdir):
334 # download the pkg to destdir using rsync + ssh 335 336 rpd = self._get_remote_pkg_dir(pkg) 337 # make spaces work w/our rsync command below :( 338 destdir = "'" + destdir.replace("'", "'\\''") + "'" 339 340 # build rsync command line from the above 341 remote_src = "{0}@{1}:{2}".format(self.username, self.hostname, rpd) 342 ssh_opts = "'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'" 343 344 rsync_log_filepath = os.path.join(destdir, "build-{}.rsync.log".format(self.job.build_id)) 345 command = "{} -avH -e {} {} {}/ &> {}".format( 346 rsync, ssh_opts, remote_src, destdir, 347 rsync_log_filepath) 348 349 # dirty magic with Popen due to IO buffering 350 # see http://thraxil.org/users/anders/posts/2008/03/13/Subprocess-Hanging-PIPE-is-your-enemy/ 351 # alternative: use tempfile.Tempfile as Popen stdout/stderr 352 try: 353 cmd = Popen(command, shell=True) 354 cmd.wait() 355 except Exception as error: 356 raise BuilderError(msg="Failed to download from builder due to rsync error, " 357 "see logs dir. Original error: {}".format(error)) 358 if cmd.returncode != 0: 359 raise BuilderError(msg="Failed to download from builder due to rsync error, " 360 "see logs dir.", return_code=cmd.returncode)
361
362 - def check(self):
363 # do check of host 364 try: 365 socket.gethostbyname(self.hostname) 366 except socket.gaierror: 367 raise BuilderError("{0} could not be resolved".format( 368 self.hostname)) 369 370 try: 371 # check_for_ans_error(res, self.hostname) 372 self.run_ansible_with_check("/bin/rpm -q mock rsync") 373 except AnsibleCallError: 374 raise BuilderError(msg="Build host `{0}` does not have mock or rsync installed" 375 .format(self.hostname)) 376 377 # test for path existence for mockchain and chroot config for this chroot 378 try: 379 self.run_ansible_with_check("/usr/bin/test -f {0}".format(mockchain)) 380 except AnsibleCallError: 381 raise BuilderError(msg="Build host `{}` missing mockchain binary `{}`" 382 .format(self.hostname, mockchain)) 383 384 try: 385 self.run_ansible_with_check("/usr/bin/test -f /etc/mock/{}.cfg" 386 .format(self.chroot)) 387 except AnsibleCallError: 388 raise BuilderError(msg="Build host `{}` missing mock config for chroot `{}`" 389 .format(self.hostname, self.chroot))
390
391 392 -def get_ans_results(results, hostname):
393 if hostname in results["dark"]: 394 return results["dark"][hostname] 395 if hostname in results["contacted"]: 396 return results["contacted"][hostname] 397 398 return {}
399
400 401 -def check_for_ans_error(results, hostname, err_codes=None, success_codes=None):
402 """ 403 dict includes 'msg' 404 may include 'rc', 'stderr', 'stdout' and any other requested result codes 405 406 :raises AnsibleResponseError: 407 """ 408 409 if err_codes is None: 410 err_codes = [] 411 if success_codes is None: 412 success_codes = [0] 413 414 if "dark" in results and hostname in results["dark"]: 415 raise AnsibleResponseError( 416 msg="Error: Could not contact/connect to {}.".format(hostname)) 417 418 error = False 419 err_results = {} 420 if err_codes or success_codes: 421 if hostname in results["contacted"]: 422 if "rc" in results["contacted"][hostname]: 423 rc = int(results["contacted"][hostname]["rc"]) 424 err_results["return_code"] = rc 425 # check for err codes first 426 if rc in err_codes: 427 error = True 428 err_results["msg"] = "rc {0} matched err_codes".format(rc) 429 elif rc not in success_codes: 430 error = True 431 err_results["msg"] = "rc {0} not in success_codes".format(rc) 432 433 elif ("failed" in results["contacted"][hostname] and 434 results["contacted"][hostname]["failed"]): 435 436 error = True 437 err_results["msg"] = "results included failed as true" 438 439 if error: 440 for item in ["stdout", "stderr"]: 441 if item in results["contacted"][hostname]: 442 err_results[item] = results["contacted"][hostname][item] 443 444 if error: 445 raise AnsibleResponseError(**err_results)
446