#54 Allow spec file generation directly from local crates and toml files
Merged 5 years ago by ignatenkobrain. Opened 5 years ago by zbyszek.
fedora-rust/ zbyszek/rust2rpm local-crates  into  master

file modified
+90 -35
@@ -1,5 +1,6 @@ 

  import argparse

  import configparser

+ import contextlib

  from datetime import datetime, timezone

  import difflib

  import itertools
@@ -81,36 +82,31 @@ 

      t = datetime.fromtimestamp(os.stat(path).st_mtime, timezone.utc)

      return t.astimezone().isoformat()

  

- def main():

-     parser = argparse.ArgumentParser()

-     parser.add_argument("-", "--stdout", action="store_true",

-                         help="Print spec and patches into stdout")

-     parser.add_argument("-t", "--target", action="store",

-                         choices=("plain", "fedora", "mageia", "opensuse"), default=get_default_target(),

-                         help="Distribution target")

-     parser.add_argument("-p", "--patch", action="store_true",

-                         help="Do initial patching of Cargo.toml")

-     parser.add_argument("crate", help="crates.io name")

-     parser.add_argument("version", nargs="?", help="crates.io version")

-     args = parser.parse_args()

+ def local_toml(toml, version):

+     if os.path.isdir(toml):

+         toml = os.path.join(toml, 'Cargo.toml')

  

-     if args.patch:

-         editor = detect_editor()

+     return toml, None, version

  

-     if args.version is None:

+ def local_crate(crate, version):

+     cratename, version = os.path.basename(crate)[:-6].rsplit('-', 1)

+     return crate, cratename, version

+ 

+ def download(crate, version):

+     if version is None:

          # Now we need to get latest version

-         url = requests.compat.urljoin(API_URL, "crates/{}/versions".format(args.crate))

+         url = requests.compat.urljoin(API_URL, "crates/{}/versions".format(crate))

          req = requests.get(url)

          req.raise_for_status()

          versions = req.json()["versions"]

-         args.version = next(version["num"] for version in versions if not version["yanked"])

+         version = next(version["num"] for version in versions if not version["yanked"])

  

      if not os.path.isdir(CACHEDIR):

          os.mkdir(CACHEDIR)

-     cratef_base = "{}-{}.crate".format(args.crate, args.version)

+     cratef_base = "{}-{}.crate".format(crate, version)

      cratef = os.path.join(CACHEDIR, cratef_base)

      if not os.path.isfile(cratef):

-         url = requests.compat.urljoin(API_URL, "crates/{}/{}/download#".format(args.crate, args.version))

+         url = requests.compat.urljoin(API_URL, "crates/{}/{}/download#".format(crate, version))

          req = requests.get(url, stream=True)

          req.raise_for_status()

          total = int(req.headers["Content-Length"])
@@ -118,7 +114,10 @@ 

              for chunk in tqdm.tqdm(req.iter_content(), "Downloading {}".format(cratef_base),

                                     total=total, unit="B", unit_scale=True):

                  f.write(chunk)

+     return cratef, crate, version

  

+ @contextlib.contextmanager

+ def toml_from_crate(cratef, crate, version):

      with tempfile.TemporaryDirectory() as tmpdir:

          target_dir = "{}/".format(tmpdir)

          with tarfile.open(cratef, "r") as archive:
@@ -126,28 +125,84 @@ 

                  if not os.path.abspath(os.path.join(target_dir, n)).startswith(target_dir):

                      raise Exception("Unsafe filenames!")

              archive.extractall(target_dir)

-         toml_relpath = "{}-{}/Cargo.toml".format(args.crate, args.version)

+         toml_relpath = "{}-{}/Cargo.toml".format(crate, version)

          toml = "{}/{}".format(tmpdir, toml_relpath)

-         assert os.path.isfile(toml)

- 

-         if args.patch:

-             mtime_before = file_mtime(toml)

-             with open(toml, "r") as fobj:

-                 toml_before = fobj.readlines()

-             subprocess.check_call([editor, toml])

-             mtime_after = file_mtime(toml)

-             with open(toml, "r") as fobj:

-                 toml_after = fobj.readlines()

-             diff = list(difflib.unified_diff(toml_before, toml_after,

-                                              fromfile=toml_relpath, tofile=toml_relpath,

-                                              fromfiledate=mtime_before, tofiledate=mtime_after))

+         if not os.path.isfile(toml):

+             raise IOError('crate does not contain Cargo.toml file')

+         yield toml

+ 

+ def make_patch(toml, enabled=True, tmpfile=False):

+     if not enabled:

+         return []

+ 

+     editor = detect_editor()

+ 

+     mtime_before = file_mtime(toml)

+     toml_before = open(toml).readlines()

+ 

+     # When we are editing a git checkout, we should not modify the real file.

+     # When we are editing an unpacked crate, we are free to edit anything.

+     # Let's keep the file name as close as possible to make editing easier.

+     if tmpfile:

+         tmpfile = tempfile.NamedTemporaryFile('w+t', dir=os.path.dirname(toml),

+                                               prefix='Cargo.', suffix='.toml')

+         tmpfile.writelines(toml_before)

+         tmpfile.flush()

+         fname = tmpfile.name

+     else:

+         fname = toml

+     subprocess.check_call([editor, fname])

+     mtime_after = file_mtime(toml)

+     toml_after = open(fname).readlines()

+     toml_relpath = '/'.join(toml.split('/')[-2:])

+     diff = list(difflib.unified_diff(toml_before, toml_after,

+                                      fromfile=toml_relpath, tofile=toml_relpath,

+                                      fromfiledate=mtime_before, tofiledate=mtime_after))

+     return diff

  

+ def _is_path(path):

+     return '/' in path or path in {'.', '..'}

+ 

+ def make_diff_metadata(crate, version, patch=False):

+     if _is_path(crate):

+         # Only things that look like a paths are considered local arguments

+         if crate.endswith('.crate'):

+             cratef, crate, version = local_crate(crate, version)

+         else:

+             toml, crate, version = local_toml(crate, version)

+             diff = make_patch(toml, enabled=patch, tmpfile=True)

+             metadata = Metadata.from_file(toml)

+             return metadata.name, diff, metadata

+     else:

+         cratef, crate, version = download(crate, version)

+ 

+     with toml_from_crate(cratef, crate, version) as toml:

+         diff = make_patch(toml, enabled=patch)

          metadata = Metadata.from_file(toml)

+     return crate, diff, metadata

+ 

+ def main():

+     parser = argparse.ArgumentParser('rust2rpm',

+                                      formatter_class=argparse.RawTextHelpFormatter)

+     parser.add_argument("-", "--stdout", action="store_true",

+                         help="Print spec and patches into stdout")

+     parser.add_argument("-t", "--target", action="store",

+                         choices=("plain", "fedora", "mageia", "opensuse"), default=get_default_target(),

+                         help="Distribution target")

+     parser.add_argument("-p", "--patch", action="store_true",

+                         help="Do initial patching of Cargo.toml")

+     parser.add_argument("crate", help="crates.io name\n"

+                                       "path/to/local.crate\n"

+                                       "path/to/project/")

+     parser.add_argument("version", nargs="?", help="crates.io version")

+     args = parser.parse_args()

+ 

+     crate, diff, metadata = make_diff_metadata(args.crate, args.version, patch=args.patch)

  

      template = JINJA_ENV.get_template("main.spec")

  

      if args.patch and len(diff) > 0:

-         patch_file = "{}-{}-fix-metadata.diff".format(args.crate, args.version)

+         patch_file = "{}-fix-metadata.diff".format(crate)

Why don't we put version there anymore?

      else:

          patch_file = None

  
@@ -193,7 +248,7 @@ 

          kwargs["date"] = time.strftime("%a %b %d %Y")

      kwargs["packager"] = detect_packager()

  

-     spec_file = "rust-{}.spec".format(args.crate)

+     spec_file = "rust-{}.spec".format(crate)

      spec_contents = template.render(md=metadata, patch_file=patch_file, **kwargs)

      if args.stdout:

          print("# {}".format(spec_file))

This PR includes #51, updated with the style issues that were pointed out in review.
On top of that, it adds the possibility to generate the spec file from a rust project checkout, by specifying either the toml file or just the directory name. This is nice when developing local projects and tweaking the metadata. The expected way to use this is 'rust2rpm .../project/', look at the generated spec file, do fixes to source, regenerate spec file, etc.

2 new commits added

  • Allow generating the spec file from rust project checkout
  • First shot at handling local crates
5 years ago

Been testing this for a while:

  • It should not try to use basename if local path is given (in any case)
⋊> ~/P/u/rust2rpm on master ↑ python3 -m rust2rpm - ../stratisd                                                                     14:37:32
Traceback (most recent call last):
  File "/usr/lib64/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/lib64/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/brain/Projects/upstream/rust2rpm/rust2rpm/__main__.py", line 264, in <module>
    main()
  File "/home/brain/Projects/upstream/rust2rpm/rust2rpm/__main__.py", line 196, in main
    crate, diff, metadata = make_diff_metadata(args.crate, args.version, patch=args.patch)
  File "/home/brain/Projects/upstream/rust2rpm/rust2rpm/__main__.py", line 176, in make_diff_metadata
    cratef, crate, version = download(crate, version)
  File "/home/brain/Projects/upstream/rust2rpm/rust2rpm/__main__.py", line 109, in download
    req.raise_for_status()
  File "/usr/lib/python3.7/site-packages/requests/models.py", line 939, in raise_for_status
    raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 404 Client Error: Not Found for url: https://crates.io/api/v1/stratisd/versions
⋊> ~/P/u/rust2rpm on master ↑ ls ../stratisd                                                                                        14:38:33
ls: cannot access '../stratisd': No such file or directory
  • rust2rpm -p PATH should not modify local copy of Cargo.toml

Once this is fixed, I will be happy to merge.

3 new commits added

  • Do not clobber real files when creating a patch
  • Do not attempt to try to download things that look like paths
  • Make nested function non-nested
5 years ago

Updated.

I'm not 100% convinced that this "automagic" guessing whether the arg is a local thing or not is good. Maybe it should be more explicit, like "if '/' is in the argument, treat it as local path. Otherwise, download." ? I would prefer to merge this, and then change the interface later. I hope a release is not planned to soon.

"if '/' is in the argument, treat it as local path. Otherwise, download."

Works for me. Should we merge this and then you will change it or opposite?

Let's wait then. I'll rebase the series.

rebased onto b5647ab

5 years ago

4 new commits added

  • Do not clobber real files when creating a patch
  • Allow generating the spec file from rust project checkout
  • Make nested function non-nested
  • First shot at handling local crates
5 years ago

Updated.

It turns out that this new semantics is easier to implement. It also generated better error messages because we know that something is a path and we can print the error we get without second-guessing anything.

I also fixed an error in the patch generation for local checkouts and updated help output.

Why don't we put version there anymore?

I don't think the version is useful. The patch was created for version X, but later on it'll be rebased to version Y, and it might be exactly the same. I don't think there's any need to put the version in the filename. Git already can tell us how the patch looked when it was used with version X, and it can also tell us how it looked rebased to version Y, so putting the version in the filename only adds noise.

Pull-Request has been merged by ignatenkobrain

5 years ago
Metadata