| |
@@ -29,7 +29,10 @@
|
| |
import time
|
| |
from itertools import groupby
|
| |
from multiprocessing.dummy import Pool as ThreadPool
|
| |
+ from tempfile import NamedTemporaryFile
|
| |
from operator import itemgetter
|
| |
+ from munch import Munch
|
| |
+ from pipes import quote
|
| |
|
| |
import git
|
| |
import requests
|
| |
@@ -48,7 +51,7 @@
|
| |
from pyrpkg.sources import SourcesFile
|
| |
from pyrpkg.utils import (cached_property, extract_srpm, find_me,
|
| |
is_file_tracked, is_lookaside_eligible_file,
|
| |
- log_result)
|
| |
+ log_result, run, macro_helper_cmd)
|
| |
|
| |
from .gitignore import GitIgnore
|
| |
|
| |
@@ -92,7 +95,7 @@
|
| |
build_client, user=None,
|
| |
dist=None, target=None, quiet=False,
|
| |
distgit_namespaced=False, realms=None, lookaside_namespaced=False,
|
| |
- git_excludes=None):
|
| |
+ git_excludes=None, preprocess_spec=False, base_output_path=None):
|
| |
"""Init the object and some configuration details."""
|
| |
|
| |
# Path to operate on, most often pwd
|
| |
@@ -214,6 +217,19 @@
|
| |
self.git_excludes = git_excludes or []
|
| |
# Layout setup
|
| |
self.layout = layout.build(self.path)
|
| |
+ # preproc attributes
|
| |
+ self._preproc_preprocess_spec = preprocess_spec
|
| |
+ self._preproc_base_output_path = base_output_path
|
| |
+ self._preproc_produce_sources = False
|
| |
+ self._preproc_use_existing_outdir = False
|
| |
+ self._preproc_no_regenerate = False
|
| |
+ self._preproc_version_bump = False
|
| |
+ self._preproc_release_bump = False
|
| |
+ self._preproc_spec_content = None
|
| |
+ self._preproc_spec_path = None
|
| |
+ self._preproc_git_props = None
|
| |
+ self._preproc_outdir = None
|
| |
+ self._preproc_done = False
|
| |
|
| |
# Define properties here
|
| |
# Properties allow us to "lazy load" various attributes, which also means
|
| |
@@ -676,13 +692,20 @@
|
| |
def load_nameverrel(self):
|
| |
"""Set the release of a package."""
|
| |
|
| |
+ tmp_spec_created = False
|
| |
+ if self._preproc_preprocess_spec and not self._preproc_done:
|
| |
+ spec = self._final_tmp_spec()
|
| |
+ tmp_spec_created = True
|
| |
+ else:
|
| |
+ spec = self.spec
|
| |
+
|
| |
cmd = ['rpm']
|
| |
cmd.extend(self.rpmdefines)
|
| |
# We make sure there is a space at the end of our query so that
|
| |
# we can split it later. When there are subpackages, we get a
|
| |
# listing for each subpackage. We only care about the first.
|
| |
cmd.extend(['-q', '--qf', '"??%{NAME} %{EPOCH} %{VERSION} %{RELEASE}??"',
|
| |
- '--specfile', '"%s"' % os.path.join(self.path, self.spec)])
|
| |
+ '--specfile', '"%s"' % os.path.join(self.path, spec)])
|
| |
joined_cmd = ' '.join(cmd)
|
| |
try:
|
| |
proc = subprocess.Popen(joined_cmd, shell=True,
|
| |
@@ -697,13 +720,17 @@
|
| |
self.log.error(err)
|
| |
raise rpkgError('Could not query n-v-r of %s: %s'
|
| |
% (self.repo_name, e))
|
| |
+ finally:
|
| |
+ if tmp_spec_created:
|
| |
+ os.remove(spec)
|
| |
+
|
| |
if err:
|
| |
self.log.debug('Errors occoured while running following command to get N-V-R-E:')
|
| |
self.log.debug(joined_cmd)
|
| |
self.log.error(err)
|
| |
if proc.returncode > 0:
|
| |
raise rpkgError('Could not get n-v-r-e from %s'
|
| |
- % os.path.join(self.path, self.spec))
|
| |
+ % os.path.join(self.path, spec))
|
| |
|
| |
# Get just the output, then split it by ??, grab the first and split
|
| |
# again to get ver and rel
|
| |
@@ -785,6 +812,10 @@
|
| |
self.load_spec()
|
| |
return self._spec
|
| |
|
| |
+ @spec.setter
|
| |
+ def spec(self, value):
|
| |
+ self._spec = value
|
| |
+
|
| |
def load_spec(self):
|
| |
"""This sets the spec attribute"""
|
| |
|
| |
@@ -1361,6 +1392,315 @@
|
| |
}
|
| |
return giturl
|
| |
|
| |
+ #####################################################
|
| |
+ ### PREPROCESSING HELPERS START
|
| |
+ #####################################################
|
| |
+
|
| |
+ def _preprocess_spec(self, produce_sources=True,
|
| |
+ use_existing_outdir=False,
|
| |
+ no_regenerate=False):
|
| |
+ """Does nothing if preprocessing is disabled.
|
| |
+
|
| |
+ Otherwise, runs preprocessing on the spec file
|
| |
+ and updates the instance attributes accordingly.
|
| |
+
|
| |
+ The new spec file will be generated into a separate
|
| |
+ newly created directory and self.path and self.layout
|
| |
+ will be updated to point there.
|
| |
+
|
| |
+ :param bool produce_sources: whether source files should be generated into
|
| |
+ the outdir (needed for further building)
|
| |
+ :param bool use_existing_outdir: whether we should operate on a previously
|
| |
+ created outdir instead of creating new one
|
| |
+ :param bool no_regenerate: if the target outdir already contains a
|
| |
+ previously generated spec file, do nothing.
|
| |
+ In this mode, nothing will be produced even
|
| |
+ if produce_sources is set to True.
|
| |
+ """
|
| |
+ # check if preprocessing is enabled first
|
| |
+ if not self._preproc_preprocess_spec:
|
| |
+ return
|
| |
+
|
| |
+ # preload repo information for the current self.path
|
| |
+ # if not preloaded yet
|
| |
+ if not self._repo:
|
| |
+ self.load_repo()
|
| |
+
|
| |
+ # set the preprocessing flags
|
| |
+ self._preproc_produce_sources = produce_sources
|
| |
+ self._preproc_use_existing_outdir = use_existing_outdir
|
| |
+ self._preproc_no_regenerate = no_regenerate
|
| |
+
|
| |
+ # do the job
|
| |
+ self.spec = self._final_spec()
|
| |
+
|
| |
+ # update context
|
| |
+ self.layout = layout.build(self._outdir)
|
| |
+ self._path = self._outdir
|
| |
+
|
| |
+ # load rpmdefines now after path is updated
|
| |
+ self.load_rpmdefines()
|
| |
+
|
| |
+ # make note that spec was already preprocessed
|
| |
+ self._preproc_done = True
|
| |
+
|
| |
+ @property
|
| |
+ def _outdir(self):
|
| |
+ if self._preproc_outdir:
|
| |
+ return self._preproc_outdir
|
| |
+
|
| |
+ dirname_base = os.path.splitext(
|
| |
+ os.path.splitext(os.path.basename(self._spec_path))[0])[0]
|
| |
+
|
| |
+ cnt = len(glob.glob(os.path.join(self._preproc_base_output_path, dirname_base+'*')))
|
| |
+
|
| |
+ if not self._preproc_use_existing_outdir:
|
| |
+ prefix = dirname_base + '-' + str(cnt+1) + '-'
|
| |
+
|
| |
+ if not os.path.isdir(self._preproc_base_output_path):
|
| |
+ prev_umask = os.umask(0)
|
| |
+ os.makedirs(self._preproc_base_output_path, mode=0o1777)
|
| |
+ os.umask(prev_umask)
|
| |
+
|
| |
+ self._preproc_outdir = tempfile.mkdtemp(dir=self._preproc_base_output_path, prefix=prefix)
|
| |
+ return self._preproc_outdir
|
| |
+
|
| |
+ try:
|
| |
+ prefix = dirname_base + '-' + str(cnt) + '-'
|
| |
+ self._preproc_outdir = glob.glob(os.path.join(self._preproc_base_output_path, prefix+'*'))[0]
|
| |
+ return self._preproc_outdir
|
| |
+ except IndexError:
|
| |
+ raise rpkgError('Could not load the latest outdir with prefix "%s" at %s'
|
| |
+ % (prefix, self._preproc_base_output_path))
|
| |
+
|
| |
+ @property
|
| |
+ def _spec_content(self):
|
| |
+ if not self._preproc_spec_content:
|
| |
+ with open(self._spec_path, 'r') as f:
|
| |
+ self._preproc_spec_content = f.read()
|
| |
+ return self._preproc_spec_content
|
| |
+
|
| |
+ @property
|
| |
+ def _spec_path(self):
|
| |
+ if not self._preproc_spec_path:
|
| |
+ self._preproc_spec_path = self._locate_spec(self.path)
|
| |
+ return self._preproc_spec_path
|
| |
+
|
| |
+ @property
|
| |
+ def _git_props(self):
|
| |
+ if self._preproc_git_props:
|
| |
+ return self._preproc_git_props
|
| |
+
|
| |
+ git_props = Munch()
|
| |
+
|
| |
+ git_props.root = run(
|
| |
+ macro_helper_cmd('git_root'),
|
| |
+ capture_stderr=True, throw=False).out
|
| |
+
|
| |
+ git_props.branch = run(
|
| |
+ macro_helper_cmd('git_branch'),
|
| |
+ env={'GIT_ROOT': git_props.root},
|
| |
+ capture_stderr=True, throw=False).out
|
| |
+
|
| |
+ git_props.remote = run(
|
| |
+ macro_helper_cmd('git_remote'),
|
| |
+ env={'GIT_ROOT': git_props.root, 'GIT_BRANCH': git_props.branch},
|
| |
+ capture_stderr=True, throw=False).out
|
| |
+
|
| |
+ git_props.remote_url = run(
|
| |
+ macro_helper_cmd('git_remote_url'),
|
| |
+ env={'GIT_ROOT': git_props.root, 'GIT_REMOTE': git_props.remote},
|
| |
+ capture_stderr=True, throw=False).out
|
| |
+
|
| |
+ git_props.head = run(
|
| |
+ macro_helper_cmd('git_head'),
|
| |
+ env={'GIT_ROOT': git_props.root},
|
| |
+ capture_stderr=True, throw=False).out
|
| |
+
|
| |
+ git_props.head_short = run(
|
| |
+ macro_helper_cmd('git_head_short'),
|
| |
+ env={'GIT_ROOT': git_props.root, 'GIT_HEAD': git_props.head},
|
| |
+ capture_stderr=True, throw=False).out
|
| |
+
|
| |
+ git_props.status = run(
|
| |
+ macro_helper_cmd('git_status'),
|
| |
+ env={'GIT_ROOT': git_props.root},
|
| |
+ capture_stderr=True, throw=False).out
|
| |
+
|
| |
+ git_props.merged_tag_refs = run(
|
| |
+ macro_helper_cmd('git_merged_tag_refs'),
|
| |
+ env={'GIT_ROOT': git_props.root, 'GIT_HEAD': git_props.head},
|
| |
+ capture_stderr=True, throw=False).out
|
| |
+
|
| |
+ git_props.submodule_refs = run(
|
| |
+ macro_helper_cmd('git_submodule_refs'),
|
| |
+ env={'GIT_ROOT': git_props.root, 'GIT_HEAD': git_props.head},
|
| |
+ capture_stderr=True, throw=False).out
|
| |
+
|
| |
+ git_props.remote_netloc = urllib.parse.urlparse(git_props.remote_url).netloc
|
| |
+ git_props.dirty = 'yes' if (git_props.status or (
|
| |
+ git_props.root and not git_props.head)) else ''
|
| |
+
|
| |
+ self._preproc_git_props = git_props
|
| |
+ return self._preproc_git_props
|
| |
+
|
| |
+ def _preproc_source_params(self):
|
| |
+ return '-s /usr/lib/rpkg.macros.d/git.bash'
|
| |
+
|
| |
+ def _preproc_env_params(self, spec_templ):
|
| |
+ spec_templ_dir = os.path.dirname(spec_templ)
|
| |
+
|
| |
+ env_params = '-e INPUT_PATH={0} -e INPUT_DIR_PATH={1}'.format(
|
| |
+ quote(spec_templ), quote(spec_templ_dir))
|
| |
+
|
| |
+ if self._preproc_produce_sources:
|
| |
+ env_params += ' -e OUTDIR={0}'.format(quote(self._outdir))
|
| |
+
|
| |
+ if self._preproc_version_bump:
|
| |
+ env_params += ' -e VERSION_BUMP=1'
|
| |
+
|
| |
+ if self._preproc_release_bump:
|
| |
+ env_params += ' -e RELEASE_BUMP=1'
|
| |
+
|
| |
+ if self.verbose:
|
| |
+ env_params += ' -e VERBOSE=1'
|
| |
+
|
| |
+ return env_params
|
| |
+
|
| |
+ def _preproc_git_props_params(self, status_file,
|
| |
+ merged_tag_refs_file,
|
| |
+ submodule_refs_file):
|
| |
+ env_params = ''
|
| |
+
|
| |
+ merged_tag_refs_file.write(self._git_props.merged_tag_refs+'\n')
|
| |
+ submodule_refs_file.write(self._git_props.submodule_refs+'\n')
|
| |
+ status_file.write(self._git_props.status+'\n')
|
| |
+
|
| |
+ merged_tag_refs_file.close()
|
| |
+ submodule_refs_file.close()
|
| |
+ status_file.close()
|
| |
+
|
| |
+ for key, val in self._git_props.items():
|
| |
+ if key == 'merged_tag_refs':
|
| |
+ env_params += ' -e GIT_MERGED_TAG_REFS_FILE={0}'.format(
|
| |
+ quote(merged_tag_refs_file.name))
|
| |
+ elif key == 'submodule_refs':
|
| |
+ env_params += ' -e GIT_SUBMODULE_REFS_FILE={0}'.format(
|
| |
+ quote(submodule_refs_file.name))
|
| |
+ elif key == 'status':
|
| |
+ env_params += ' -e GIT_STATUS_FILE={0}'.format(
|
| |
+ quote(status_file.name))
|
| |
+ else:
|
| |
+ env_params += ' -e GIT_{0}={1}'.format(
|
| |
+ key.upper(), quote(val))
|
| |
+
|
| |
+ return env_params.strip()
|
| |
+
|
| |
+ def _locate_spec(cls, path):
|
| |
+ spec_path = None
|
| |
+
|
| |
+ for f in os.listdir(path):
|
| |
+ if (f.endswith('.spec') or f.endswith('.spec.rpkg')) and not f.startswith('.'):
|
| |
+ spec_path = os.path.abspath(os.path.join(path, f))
|
| |
+ break
|
| |
+
|
| |
+ if not spec_path:
|
| |
+ raise rpkgError('No spec file found at {0}.'
|
| |
+ .format(path))
|
| |
+ return spec_path
|
| |
+
|
| |
+ def _final_spec_text(self):
|
| |
+ if not self._preproc_preprocess_spec:
|
| |
+ return self._spec_content
|
| |
+
|
| |
+ source_params = self._preproc_source_params()
|
| |
+ env_params = self._preproc_env_params(self._spec_path)
|
| |
+
|
| |
+ tmp0_file = NamedTemporaryFile(
|
| |
+ 'w', prefix='tmp_', dir='/tmp/', delete=False)
|
| |
+ tmp1_file = NamedTemporaryFile(
|
| |
+ 'w', prefix='tmp_', dir='/tmp/', delete=False)
|
| |
+ tmp2_file = NamedTemporaryFile(
|
| |
+ 'w', prefix='tmp_', dir='/tmp/', delete=False)
|
| |
+ tmp3_file = NamedTemporaryFile(
|
| |
+ 'w', prefix='tmp_', dir='/tmp/', delete=False)
|
| |
+
|
| |
+ git_props_params = self._preproc_git_props_params(
|
| |
+ tmp1_file, tmp2_file, tmp3_file)
|
| |
+
|
| |
+ tmp0_file.write(self._spec_content)
|
| |
+ tmp0_file.close()
|
| |
+
|
| |
+ cmd = 'cat {0} | preproc -C {1} {2} {3} {4}'.format(
|
| |
+ quote(tmp0_file.name), quote(self.path), source_params,
|
| |
+ env_params, git_props_params)
|
| |
+
|
| |
+ try:
|
| |
+ final_spec_text = run(cmd, shell=True, capture=True).out
|
| |
+ finally:
|
| |
+ os.unlink(tmp0_file.name)
|
| |
+ os.unlink(tmp1_file.name)
|
| |
+ os.unlink(tmp2_file.name)
|
| |
+ os.unlink(tmp3_file.name)
|
| |
+
|
| |
+ return final_spec_text
|
| |
+
|
| |
+ def _final_spec_path(self):
|
| |
+ final_spec_basename = os.path.basename(
|
| |
+ self._spec_path[:-5] if self._spec_path.endswith('.rpkg') else self._spec_path)
|
| |
+ return os.path.join(self._outdir, final_spec_basename)
|
| |
+
|
| |
+ def _final_spec(self):
|
| |
+ if not self._preproc_preprocess_spec:
|
| |
+ return self.spec
|
| |
+
|
| |
+ final_spec_path = self._final_spec_path()
|
| |
+
|
| |
+ if os.path.exists(final_spec_path) and self._preproc_no_regenerate:
|
| |
+ return final_spec_path
|
| |
+
|
| |
+ with open(final_spec_path, 'w') as f:
|
| |
+ f.write(self._final_spec_text()+'\n')
|
| |
+
|
| |
+ if self._preproc_produce_sources:
|
| |
+ self._setup_source_symlinks(final_spec_path)
|
| |
+
|
| |
+ log.info('Wrote: {0}'.format(final_spec_path))
|
| |
+ return final_spec_path
|
| |
+
|
| |
+ def _final_tmp_spec(self):
|
| |
+ final_tmp_spec = NamedTemporaryFile(
|
| |
+ 'w', prefix='tmp_',
|
| |
+ dir='/tmp',
|
| |
+ delete=False)
|
| |
+
|
| |
+ self._preproc_produce_sources = False
|
| |
+ final_tmp_spec.write(self._final_spec_text())
|
| |
+ final_tmp_spec.close()
|
| |
+
|
| |
+ return final_tmp_spec.name
|
| |
+
|
| |
+ def _setup_source_symlinks(self, final_spec_path):
|
| |
+ ts = rpm.ts()
|
| |
+
|
| |
+ try:
|
| |
+ rpm_spec = ts.parseSpec(final_spec_path)
|
| |
+ except ValueError as e:
|
| |
+ raise rpkgError("Could not parse spec file with error: "+str(e))
|
| |
+
|
| |
+ for (filename, num, flags) in rpm_spec.sources:
|
| |
+ filename = os.path.basename(filename)
|
| |
+ symlink_source = os.path.join(self.path, filename)
|
| |
+ symlink_dest = os.path.join(self._outdir, filename)
|
| |
+
|
| |
+ if os.path.isfile(symlink_source) and not \
|
| |
+ os.path.isfile(symlink_dest):
|
| |
+ os.symlink(symlink_source, symlink_dest)
|
| |
+
|
| |
+ #####################################################
|
| |
+ ### PREPROCESSING HELPERS END
|
| |
+ #####################################################
|
| |
+
|
| |
def add_tag(self, tagname, force=False, message=None, file=None):
|
| |
"""Add a git tag to the repository
|
| |
|
| |
@@ -2341,6 +2681,12 @@
|
| |
def clog(self, raw=False):
|
| |
"""Write the latest spec changelog entry to a clog file"""
|
| |
|
| |
+ # remember orig path to write the "clog" file there
|
| |
+ # becase preprocessing changes it if enabled
|
| |
+ orig_path = self.path
|
| |
+
|
| |
+ self._preprocess_spec(produce_sources=False)
|
| |
+
|
| |
spec_file = os.path.join(self.path, self.spec)
|
| |
# TODO: remove when fixed
|
| |
# Command contains workaround (undefines _changelog_trimtime) described at:
|
| |
@@ -2374,7 +2720,7 @@
|
| |
buf.close()
|
| |
|
| |
# Now open the clog file and write out the lines
|
| |
- with open(os.path.join(self.path, 'clog'), 'w') as clog:
|
| |
+ with open(os.path.join(orig_path, 'clog'), 'w') as clog:
|
| |
clog.writelines(clog_lines)
|
| |
|
| |
def compile(self, arch=None, short=False, builddir=None, nocheck=False,
|
| |
@@ -2387,6 +2733,8 @@
|
| |
Logs the output and returns nothing
|
| |
"""
|
| |
|
| |
+ self._preprocess_spec(produce_sources=True)
|
| |
+
|
| |
# setup the rpm command
|
| |
cmd = ['rpmbuild']
|
| |
cmd.extend(self.rpmdefines)
|
| |
@@ -2463,6 +2811,8 @@
|
| |
Parameter buildrootdir.
|
| |
"""
|
| |
|
| |
+ self._preprocess_spec(produce_sources=True)
|
| |
+
|
| |
# setup the rpm command
|
| |
cmd = ['rpmbuild']
|
| |
cmd.extend(self.rpmdefines)
|
| |
@@ -2500,6 +2850,10 @@
|
| |
specified by the command line argument.
|
| |
"""
|
| |
|
| |
+ self._preprocess_spec(produce_sources=False,
|
| |
+ use_existing_outdir=True,
|
| |
+ no_regenerate=True)
|
| |
+
|
| |
# Check for srpm
|
| |
srpm = "%s-%s-%s.src.rpm" % (self.repo_name, self.ver, self.rel)
|
| |
if not os.path.exists(os.path.join(self.path, srpm)):
|
| |
@@ -2563,6 +2917,8 @@
|
| |
Parameter buildrootdir.
|
| |
"""
|
| |
|
| |
+ self._preprocess_spec(produce_sources=True)
|
| |
+
|
| |
# This could really use a list of arches to build for and loop over
|
| |
# Get the sources
|
| |
# build up the rpm command
|
| |
@@ -2872,6 +3228,8 @@
|
| |
Parameter buildrootdir.
|
| |
"""
|
| |
|
| |
+ self._preprocess_spec(produce_sources=True)
|
| |
+
|
| |
# setup the rpm command
|
| |
cmd = ['rpmbuild']
|
| |
cmd.extend(self.rpmdefines)
|
| |
@@ -2914,6 +3272,8 @@
|
| |
the rpmbuild command.
|
| |
"""
|
| |
|
| |
+ self._preprocess_spec(produce_sources=True)
|
| |
+
|
| |
self.srpmname = os.path.join(self.path,
|
| |
"%s-%s-%s.src.rpm"
|
| |
% (self.repo_name, self.ver, self.rel))
|
| |
@@ -3025,6 +3385,10 @@
|
| |
Parameter buildrootdir.
|
| |
"""
|
| |
|
| |
+ self._preprocess_spec(produce_sources=False,
|
| |
+ use_existing_outdir=True,
|
| |
+ no_regenerate=True)
|
| |
+
|
| |
# setup the rpm command
|
| |
cmd = ['rpmbuild']
|
| |
cmd.extend(self.rpmdefines)
|
| |
Hello,
this pull request enables spec preprocessing functionality that originated in rpkg-util. I tested this PR successfully with fedpkg tool. I haven't yet tested other tools like
rfpkg
but I assume they could work if they work in similar fashion to fedpkg. This complements work that was done in mock previously: https://github.com/rpm-software-management/mock/commit/bda814a743ef74e6dfbee261a4db6e2f219382d1If this is accepted, I plan to continue the integration by extending the
tag
subcommand to be able to populate tag messages automatically from git commits from the latest reachable annotated tag and also add some more options (like--name
,--version
, or--release
and possibly others to make the integration smooth).Basically, what this commit does is that it enables usage of rpkg-macros in spec files (if
preprocess_spec
configuration option is set to True). This is a possible solution for automatic release and changelog generation problem but it offers more than that.I am open to any required code changes and improvements in this PR or in future.
NOTES from the commit:
be put into an auto-generated directory under /tmp/<name>
(e.g. /tmp/fedpkg for the fedpkg tool)
srpm, local, prep, install, compile, lint, verify-files, clog
mockbuild, koji scratch build, copr build
spec
is introduced just to renderthe spec file without further building srpm etc.
utils.changelog_entry or utils.edit). The functions are
intended to be used in a future commit that extends the
tag
subcommand to automatically pregenerate tag messages.also python-munch
Signed-off-by: clime clime@fedoraproject.org