copr / copr

Clone

#50 [backend] more standalone builder
Merged 7 years ago by clime. Opened 7 years ago by praiskup.
Unknown source standalone-builder  into  master

file modified
+6 
@@ -287,6 +287,12 @@
 

          opts.consecutive_failure_threshold = _get_conf(
 

              cp, "builder", "consecutive_failure_threshold",
 

              DEF_CONSECUTIVE_FAILURE_THRESHOLD, mode="int")
 

+         opts.standalone_builder = _get_conf(
 

+             cp, "builder", "standalone", False, mode="bool")
 

+         opts.standalone_builder_config = _get_conf(
 

+             cp, "builder", "config", "/etc/copr-builder/fedora-copr.conf")
 

+ 

+ 

          opts.log_dir = _get_conf(
 

              cp, "backend", "log_dir", "/var/log/copr-backend/")
 

          opts.log_level = _get_conf(

file modified
+74 -16 
@@ -37,6 +37,7 @@
 

          self.conn = SSHConnection(user=self.opts.build_user, host=self.hostname, config_file=self.opts.ssh.builder_config)
 

  

          self.module_dist_tag = self._load_module_dist_tag()
 

+         self._build_pid = None
 

  

      def _load_module_dist_tag(self):
 

          module_md_filepath = os.path.join(self.job.destdir, self.job.chroot, "module_md.yaml")
 
@@ -65,9 +66,12 @@
 

          if self._remote_tempdir:
 

              return self._remote_tempdir
 

  

-         tempdir_path = "{0}/{1}-{2}".format(
 

-             self._remote_basedir, "mockremote", self.job.task_id)
 

-         self._run_ssh_cmd("/bin/mkdir -m 755 -p {0}".format(tempdir_path))
 

+         if self.opts.standalone_builder:
 

+             tempdir_path = "/var/lib/copr-builder"
 

+         else:
 

+             tempdir_path = "{0}/{1}-{2}".format(
 

+                 self._remote_basedir, "mockremote", self.job.task_id)
 

+             self._run_ssh_cmd("/bin/mkdir -m 755 -p {0}".format(tempdir_path))
 

  

          self._remote_tempdir = tempdir_path
 

          return self._remote_tempdir
 
@@ -98,6 +102,9 @@
 

          return out, err
 

  

      def _get_remote_results_dir(self):
 

+         if self.opts.standalone_builder:
 

+             return os.path.join(self.tempdir, 'results')
 

+ 

          if any(x is None for x in [self.remote_build_dir,
 

                                     self.remote_pkg_name,
 

                                     self.job.chroot]):
 
@@ -232,9 +239,18 @@
 

  

      @property
 

      def livelog_name(self):
 

+         if self.opts.standalone_builder:
 

+             return "/var/lib/copr-builder/live-log"
 

          return pipes.quote('/tmp/{}.log'.format(self.job.task_id))
 

  

-     def run_mockchain_async(self):
 

+     def run_async_build(self):
 

+         if self.opts.standalone_builder:
 

+             cmd = self._copr_builder_cmd()
 

+             pid, _ = self._run_ssh_cmd(cmd)
 

+             self._build_pid =int(pid.strip())
 

+             return
 

+ 

+ 

          buildcmd = "timeout {} {} -r {} -l {} ".format(
 

              self.timeout, mockchain, pipes.quote(self.get_chroot_config_path(self.job.chroot)),
 

              pipes.quote(self.remote_build_dir))
 
@@ -260,10 +276,10 @@
 

          #   on std{out,err} which means that it's output are not line-buffered
 

          #   (which wouldn't be very useful live-log), so let's use `unbuffer`
 

          #   from expect.rpm to allocate _persistent_ server-side pseudo-terminal
 

-         buildcmd_async = 'trap "" SIGHUP; unbuffer {buildcmd} &>{livelog} &'.format(
 

+         buildcmd_async = 'trap "" SIGHUP; unbuffer {buildcmd} &>{livelog} & echo !$'.format(
 

              livelog=self.livelog_name, buildcmd=buildcmd)
 

- 

-         self._run_ssh_cmd(buildcmd_async)
 

+         pid, _ = self._run_ssh_cmd(buildcmd_async)
 

+         self._build_pid =int(pid.strip())
 

  

      def setup_pubsub_handler(self):
 

  
@@ -298,20 +314,56 @@
 

      #     buildcmd = self.gen_mockchain_command(dest)
 

      #
 

  

-     def attach_to_build(self):
 

+     @property
 

+     def build_pid(self):
 

+         if self._build_pid:
 

+             return self._build_pid
 

          try:
 

              pidof_cmd = "/usr/bin/pgrep -o -u {user} {command}".format(
 

                  user=self.opts.build_user, command="mockchain")
 

+ 

+             if self.opts.standalone_builder:
 

+                 pidof_cmd = "cat /var/lib/copr-builder/pid"
 

+ 

              out, _ = self._run_ssh_cmd(pidof_cmd)
 

-         except RemoteCmdError:
 

+         except:
 

+             return None
 

+ 

+         return int(out.strip())
 

+ 

+ 

+     def _copr_builder_cmd(self):
 

+         template = 'copr-builder --config {config} --copr {copr} ' \
 

+                  + '--package {package} --revision {revision} ' \
 

+                  + '--host-resolv {net} --timeout {timeout} ' \
 

+                  + '--chroot {chroot} --detached '
 

+ 

+         # Repo name like <user/group>/<copr>/<package>
 

+         git_repo_path = self.job.git_repo.split('/')
 

+ 

+         copr = '{0}/{1}'.format(git_repo_path[0], git_repo_path[1])
 

+         package = git_repo_path[2]
 

+ 

+         return template.format(
 

+             config=self.opts.standalone_builder_config,
 

+             copr=copr,
 

+             package=package,
 

+             revision=self.job.git_hash,
 

+             net="True" if self.job.enable_net else "False",
 

+             chroot=self.job.chroot,
 

+             timeout=self.timeout,
 

+         )
 

+ 

+     def attach_to_build(self):
 

+         if not self.build_pid:
 

              self.log.info("Build is not running. Continuing...")
 

              return
 

  

          ensure_dir_exists(self.job.results_dir, self.log)
 

          live_log = os.path.join(self.job.results_dir, 'mockchain-live.log')
 

  

-         live_cmd = '/usr/bin/tail -f --pid={pid} {log}'.format(
 

-             pid=out.strip(), log=self.livelog_name)
 

+         live_cmd = '/usr/bin/tail -n +0 -f --pid={pid} {log}'.format(
 

+             pid=self.build_pid, log=self.livelog_name)
 

  

          self.log.info("Attaching to live build log: " + live_cmd)
 

          with open(live_log, 'w') as logfile:
 
@@ -320,14 +372,15 @@
 

  

  

      def build(self):
 

-         # make mock config
 

-         self.setup_mock_chroot_config()
 

+         if not self.opts.standalone_builder:
 

+             # make mock config
 

+             self.setup_mock_chroot_config()
 

  

-         # download the package to the builder
 

-         self.download_job_pkg_to_builder()
 

+             # download the package to the builder
 

+             self.download_job_pkg_to_builder()
 

  

          # run the build
 

-         self.run_mockchain_async()
 

+         self.run_async_build()
 

  

          # attach to building output
 

          self.attach_to_build()
 
@@ -375,6 +428,11 @@
 

          self.rsync_call(self._get_remote_config_dir(), target_path)
 

  

      def check(self):
 

+         if self.opts.standalone_builder:
 

+             # We simply expect that 'copr-builder' package is installed on the
 

+             # builder machine.
 

+             return
 

+ 

          # do check of host
 

          try:
 

              # requires name resolve facility

When copr-be.conf contains this snippet ..

[builder]
standalone=true
config=/etc/copr-builder/<INSTANCE>.conf

.. then instead of controlling all the build peculiarities
manually from backend, the build is fully controlled builder-side,
while the build process logic is concentrated in one, easily
reproducible command:

copr-builder --config /tmp/copr.conf [opts] --detached

This extremely simplifies debugging when something goes wrong, but
it is also the first step towards "heterogeneous sets of builders"
feature https://bugzilla.redhat.com/show_bug.cgi?id=1334701

The builder VMs just need to have installed package 'copr-builder'.

.
Edited 7 years ago by praiskup

Also note that there are some prereqs:
https://pagure.io/copr/copr/pull-request/48
https://pagure.io/copr/copr/pull-request/44

This also helps the idea with "submit build" and "poll for results", because instead of running a lot of remote commands, this shrinks the workflow to (almost) one remote command.

Once we are on the "submit && poll" workflow, we either have to start some daemonized live-log transport.

For red-hatters only, unfortunately:
There's a running copr instance on https://dev-copr.devel.redhat.com/ where you can give it a try in devel environment (at least it "works" now, that is work-in progress machine and I'll probably revert the changes today or tomorrow anyway).

Some additional links:
Related bugreport: https://bugzilla.redhat.com/1357562
copr-builder TODO list before merge can happen: https://github.com/praiskup/copr-bulider/blob/master/TODO

If we were able to use this approach everywhere unconditionally, we could simplify the backend code base (builder.py) a lot. But for now, any comment is welcome. Are there any obvious issues?

rebased
7 years ago

rebased
7 years ago

I believe copr-bulder script is now ready 1:1 replacement for the actual backend-controlled workflow.

One valid point from @clime was that copr-builder doesn't use mock's new dist-git feature for generating srpm (it only does mock --srpmbuild). This isn't blocker IMO. Once we consider switch to mock's dist-git feature good enough, it might be done in copr-builder (that's very trivial script, ~200 lines of shell code).

As this is still proposed as optional feature, patching isn't intrusive so I'm pretty inclined to go with downstream patch for the time being ... (we have very slow builders, and the s/mockchain/mock/ switch together with live log is really tempting).

rebased
7 years ago

What about now? Still opt-in only feature in this proposal with hope that we
could discuss issues and potentially merge so I don't have to downstream patch.

This extremely simplifies debugging when something goes wrong (only one
command is executed remotely, which is trivial to reproduce without backend),
but it is also the first step towards "heterogeneous sets of builders" feature
https://bugzilla.redhat.com/show_bug.cgi?id=1334701

Note that it is not obvious from the first sight, but the simplification of
backend-side would be huge if we made it the only&default option.

2 new commits added

  • [backend] standalone builder
  • [backend] unless reattaching, don't pgrep for mockchain pid
7 years ago

2 new commits added

  • [backend] standalone builder
  • [backend] unless reattaching, don't pgrep for mockchain pid
7 years ago

I agree adopting copr-builder package probably fully could be beneficial as it provides better separation of concerns and it makes lots of tasks (e.g. running builds detached and reattaching) easier.

rebased
7 years ago

Thank you! Merging this now. We can continue working on this and make it an only option once it's ready and well tested.

Pull-Request has been merged by clime
7 years ago
Metadata
None
No Tags
Changes Summary 2
+6 -0
file changed
backend/backend/helpers.py
+74 -16
file changed
backend/backend/mockremote/builder.py
Powered by Pagure 5.13.3
Documentation • File an Issue • About • SSH Hostkey/Fingerprint
© Red Hat, Inc. and others.