Package backend :: Package mockremote
[hide private]
[frames] | no frames]

Source Code for Package backend.mockremote

  1  #!/usr/bin/python -tt 
  2  # by skvidal 
  3  # This program is free software; you can redistribute it and/or modify 
  4  # it under the terms of the GNU General Public License as published by 
  5  # the Free Software Foundation; either version 2 of the License, or 
  6  # (at your option) any later version. 
  7  # 
  8  # This program is distributed in the hope that it will be useful, 
  9  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 10  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 11  # GNU Library General Public License for more details. 
 12  # 
 13  # You should have received a copy of the GNU General Public License 
 14  # along with this program; if not, write to the Free Software 
 15  # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA. 
 16  # copyright 2012 Red Hat, Inc. 
 17   
 18   
 19  # Original approach was: 
 20  # take list of pkgs 
 21  # take single hostname 
 22  # send 1 pkg at a time to host 
 23  # build in remote w/mockchain 
 24  # rsync results back 
 25  # repeat 
 26  # take args from mockchain (more or less) 
 27   
 28  # now we build only one package per MockRemote instance 
 29   
 30  from __future__ import print_function 
 31  from __future__ import unicode_literals 
 32  from __future__ import division 
 33  from __future__ import absolute_import 
 34   
 35  import fcntl 
 36  import urllib 
 37  import os 
 38  from bunch import Bunch 
 39   
 40  from ..constants import DEF_REMOTE_BASEDIR, DEF_BUILD_TIMEOUT, DEF_REPOS, \ 
 41      DEF_BUILD_USER, DEF_MACROS 
 42  from ..exceptions import MockRemoteError, BuilderError 
 43   
 44   
 45  # TODO: replace sign & createrepo with dependency injection 
 46  from ..sign import sign_rpms_in_dir, get_pubkey 
 47  from ..createrepo import createrepo 
 48   
 49  from .builder import Builder 
 50  from .callback import DefaultCallBack 
51 52 53 -def get_target_dir(chroot_dir, pkg_name):
54 source_basename = os.path.basename(pkg_name).replace(".src.rpm", "") 55 return os.path.normpath(os.path.join(chroot_dir, source_basename))
56
57 58 -class MockRemote(object):
59 # TODO: Refactor me! 60 # mock remote now do too much things 61 # idea: send events according to the build progress to handler 62
63 - def __init__(self, builder_host=None, job=None, 64 repos=None, 65 callback=None, 66 macros=None, 67 opts=None, 68 lock=None):
69 70 """ 71 :param builder_host: builder hostname or ip 72 73 :param backend.job.BuildJob job: Job object with the following attributes:: 74 :ivar timeout: ssh timeout 75 :ivar destdir: target directory to put built packages 76 :ivar chroot: chroot config name/base to use in the mock build 77 (e.g.: fedora20_i386 ) 78 :ivar buildroot_pkgs: whitespace separated string with additional 79 packages that should present during build 80 :ivar build_id: copr build.id 81 :ivar pkg: pkg to build 82 83 84 :param repos: additional repositories for mock 85 :param backend.mockremote.callback.DefaultCallBack callback: object with hooks for notifications 86 about build progress 87 88 :param macros: { "copr_username": ..., 89 "copr_projectname": ..., 90 "vendor": ...} 91 :param multiprocessing.Lock lock: instance of Lock shared between 92 Copr backend process 93 :param DefaultCallback callback: build progress handler 94 95 :param Bunch opts: builder options, used keys:: 96 :ivar build_user: user to run as/connect as on builder systems 97 :ivar do_sign: enable package signing, require configured 98 signer host and correct /etc/sign.conf 99 :ivar frontend_base_url: url to the copr frontend 100 :ivar results_baseurl: base url for the built results 101 :ivar remote_basedir: basedir on builder 102 :ivar remote_tempdir: tempdir on builder 103 104 # Removed: 105 # :param cont: if a pkg fails to build, continue to the next one-- 106 # :param bool recurse: if more than one pkg and it fails to build, 107 # try to build the rest and come back to it 108 """ 109 self.opts = Bunch( 110 do_sign=False, 111 frontend_base_url=None, 112 results_baseurl=u"", 113 build_user=DEF_BUILD_USER, 114 remote_basedir=DEF_REMOTE_BASEDIR, 115 remote_tempdir=None, 116 ) 117 if opts: 118 self.opts.update(opts) 119 120 self.max_retry_count = 2 # TODO: add config option 121 122 self.job = job 123 self.repos = repos or DEF_REPOS 124 125 # TODO: remove or re-implement 126 # self.cont = cont # unused since we build only one pkg at time 127 # self.recurse = recurse 128 129 self.callback = callback 130 self.macros = macros or DEF_MACROS 131 self.lock = lock 132 133 if not self.callback: 134 self.callback = DefaultCallBack() 135 136 self.callback.log("Setting up builder: {0}".format(builder_host)) 137 self.builder = Builder( 138 opts=self.opts, 139 hostname=builder_host, 140 username=self.opts.build_user, 141 job=self.job, 142 chroot=self.job.chroot, 143 timeout=self.job.timeout or DEF_BUILD_TIMEOUT, 144 buildroot_pkgs=self.job.buildroot_pkgs, 145 callback=self.callback, 146 remote_basedir=self.opts.remote_basedir, 147 remote_tempdir=self.opts.remote_tempdir, 148 macros=self.macros, repos=self.repos) 149 150 self.failed = [] 151 self.finished = []
152 153 # self.callback.log("MockRemote: {}".format(self.__dict__)) 154
155 - def check(self):
156 """ 157 Checks that MockRemote configuration and environment are correct. 158 159 :raises MockRemoteError: when configuration is wrong or 160 some expected resource is unavailable 161 """ 162 if not self.job.chroot: 163 raise MockRemoteError("No chroot specified!") 164 165 self.builder.check()
166 167 @property
168 - def chroot_dir(self):
169 return os.path.normpath(os.path.join(self.job.destdir, self.job.chroot))
170 171 @property
172 - def pkg(self):
173 return self.job.pkg
174
175 - def _get_pkg_destpath(self, pkg):
176 s_pkg = os.path.basename(pkg) 177 pdn = s_pkg.replace(".src.rpm", "") 178 return os.path.normpath( 179 "{0}/{1}/{2}".format(self.job.destdir, self.job.chroot, pdn))
180
181 - def add_pubkey(self):
182 """ 183 Adds pubkey.gpg with public key to ``chroot_dir`` 184 using `copr_username` and `copr_projectname` from self.job. 185 """ 186 self.callback.log("Retrieving pubkey ") 187 # TODO: sign repodata as well ? 188 user = self.job.project_owner 189 project = self.job.project_name 190 pubkey_path = os.path.join(self.job.destdir, "pubkey.gpg") 191 try: 192 # TODO: uncomment this when key revoke/change will be implemented 193 # if os.path.exists(pubkey_path): 194 # return 195 196 get_pubkey(user, project, pubkey_path) 197 self.callback.log( 198 "Added pubkey for user {} project {} into: {}". 199 format(user, project, pubkey_path)) 200 201 except Exception as e: 202 self.callback.error( 203 "failed to retrieve pubkey for user {} project {} due to: \n" 204 "{}".format(user, project, e))
205
206 - def sign_built_packages(self):
207 """ 208 Sign built rpms 209 using `copr_username` and `copr_projectname` from self.job 210 by means of obs-sign. If user builds doesn't have a key pair 211 at sign service, it would be created through ``copr-keygen`` 212 213 :param chroot_dir: Directory with rpms to be signed 214 :param pkg: path to the source package 215 216 """ 217 218 self.callback.log("Going to sign pkgs from source: {} in chroot: {}". 219 format(self.pkg, self.chroot_dir)) 220 221 try: 222 sign_rpms_in_dir(self.job.project_owner, 223 self.job.project_name, 224 get_target_dir(self.chroot_dir, self.pkg), 225 opts=self.opts, 226 callback=self.callback,) 227 except Exception as e: 228 self.callback.error( 229 "failed to sign packages " 230 "built from `{}` with error: \n" 231 "{}".format(self.pkg, e) 232 ) 233 if isinstance(e, MockRemoteError): 234 raise e 235 236 self.callback.log("Sign done")
237 238 @staticmethod
239 - def log_to_file_safe(filepath, to_out_list, to_err_list):
240 r_log = open(filepath, 'a') 241 fcntl.flock(r_log, fcntl.LOCK_EX) 242 for to_out in to_out_list: 243 r_log.write(to_out) 244 if to_err_list: 245 r_log.write("\nstderr\n") 246 for to_err in to_err_list: 247 r_log.write(str(to_err)) 248 fcntl.flock(r_log, fcntl.LOCK_UN) 249 r_log.close()
250
251 - def do_createrepo(self):
252 base_url = "/".join([self.opts.results_baseurl, self.job.project_owner, 253 self.job.project_name, self.job.chroot]) 254 self.callback.log("Createrepo:: owner: {}; project: {}; " 255 "front url: {}; path: {}; base_url: {}" 256 .format(self.job.project_owner, self.job.project_name, 257 self.opts.frontend_base_url, self.chroot_dir, base_url)) 258 259 _, _, err = createrepo( 260 path=self.chroot_dir, 261 front_url=self.opts.frontend_base_url, 262 base_url=base_url, 263 username=self.job.project_owner, 264 projectname=self.job.project_name, 265 lock=self.lock, 266 ) 267 if err.strip(): 268 self.callback.error( 269 "Error making local repo: {0}".format(self.chroot_dir)) 270 271 self.callback.error(str(err))
272 # FIXME - maybe clean up .repodata and .olddata 273 # here? 274
275 - def on_success_build(self):
276 self.callback.log("Success building {0}".format(os.path.basename(self.pkg))) 277 278 if self.opts.do_sign: 279 self.sign_built_packages() 280 281 # self.built_packages.append(self.pkg) 282 283 # createrepo with the new pkgs 284 self.do_createrepo()
285
286 - def prepare_build_dir(self):
287 p_path = self._get_pkg_destpath(self.pkg) 288 # if it's marked as fail, nuke the failure and try to rebuild it 289 if os.path.exists(os.path.join(p_path, "fail")): 290 os.unlink(os.path.join(p_path, "fail")) 291 292 # mkdir to download results 293 if not os.path.exists(p_path): 294 os.makedirs(p_path) 295 296 self.mark_dir_with_build_id()
297
299 self.prepare_build_dir() 300 301 # building 302 self.callback.start_build(self.pkg) 303 # b_status, b_out, b_err, build_details = self.builder.build(self.pkg) 304 build_stdout = "" 305 build_error = build_details = None 306 try: 307 build_details, build_stdout = self.builder.build(self.pkg) 308 self.callback.log("builder.build finished; details: {}\n stdout: {}".format(build_details, build_stdout)) 309 except BuilderError as error: 310 self.callback.log("builder.build error building pkg `{}`: {}".format(self.pkg, error)) 311 build_error = error 312 313 # TODO: try to use 314 # finally: 315 316 self.callback.end_build(self.pkg) 317 318 # downloading 319 self.callback.start_download(self.pkg) 320 self.builder.download(self.pkg, self.chroot_dir) 321 self.callback.end_download(self.pkg) 322 323 # add record to mockchain.log # TODO: 324 if build_error is not None: 325 stderr_to_log = build_error.stderr 326 else: 327 stderr_to_log = "" 328 self.log_to_file_safe(os.path.join(self.chroot_dir, "mockchain.log"), 329 ["\n\n{0}\n\n".format(self.pkg), build_stdout], [stderr_to_log]) 330 331 if build_error: 332 raise MockRemoteError("Error occurred during build {}: {}" 333 .format(os.path.basename(self.pkg), build_error)) 334 335 self.on_success_build() 336 return build_details
337
338 - def build_pkg(self):
339 """Build pkg defined in self.job 340 341 342 :return: build_details 343 """ 344 # before mockremote was able to build more src pkgs at once 345 # but now we expect only one src pkg at time 346 347 retry_count = self.max_retry_count 348 while retry_count > 0: 349 retry_count -= 1 350 try: 351 build_details = self.build_pkg_and_process_results() 352 break 353 except MockRemoteError as exception: 354 self.callback.error(exception.msg) 355 356 else: 357 # retry count reached 0 without successful results 358 msg = "Build pkg {} failed {} times".format(self.pkg, self.max_retry_count) 359 self.callback.log(msg) 360 raise MockRemoteError(msg) 361 362 return build_details
363
364 - def mark_dir_with_build_id(self):
365 """ 366 Places "build.info" which contains job build_id 367 into the directory with downloaded files. 368 369 """ 370 info_file_path = os.path.join(self._get_pkg_destpath(self.job.pkg), 371 "build.info") 372 self.callback.log("mark build dir with build_id, ") 373 try: 374 with open(info_file_path, 'w') as info_file: 375 info_file.writelines(["build_id={}".format(self.job.build_id),]) 376 377 except Exception as error: 378 msg = "Failed to mark build {} with build_id".format(error) 379 self.callback.log(msg)
380