#197 Guess crate from dir name, make %doc|%license lists multiline, skip non-release versions
Merged 2 years ago by zbyszek. Opened 2 years ago by zbyszek.
fedora-rust/ zbyszek/rust2rpm three-featurelets  into  main

file modified
+68 -17
@@ -6,6 +6,7 @@ 

  import difflib

  import functools

  import itertools

+ import glob

  import os

  import pathlib

  import re
@@ -135,6 +136,10 @@ 

          os.unlink(path)

          raise

  

+ def package_name_suffixed(name, suffix):

+     joiner = '_' if suffix and name[-1].isdigit() else ''

+     return 'rust-' + name + joiner + (suffix or '')

+ 

  def local_toml(toml, version):

      if os.path.isdir(toml):

          doc_files = get_doc_files(toml)
@@ -150,14 +155,27 @@ 

      cratename, version = os.path.basename(crate)[:-6].rsplit("-", 1)

      return crate, cratename, version

  

+ def query_newest_version(crate):

+     url = requests.compat.urljoin(API_URL, f"crates/{crate}/versions")

+     req = requests.get(url, headers={"User-Agent": "rust2rpm"})

+     req.raise_for_status()

+     versions = req.json()["versions"]

+     for struct in versions:

+         version = struct["num"]

+         if struct["yanked"]:

+             print(f'Ignoring yanked version {version}')

+         elif re.search('alpha|beta|rc|pre', version):

+             print(f'Ignoring pre-release version {version}')

+         else:

+             print(f'Found version {version}')

+             return version

+ 

+     raise ValueError("Couldn't find any release versions. Specify a version explicitly.")

+ 

  def download(crate, version):

      if version is None:

          # Now we need to get latest version

-         url = requests.compat.urljoin(API_URL, f"crates/{crate}/versions")

-         req = requests.get(url, headers={"User-Agent": "rust2rpm"})

-         req.raise_for_status()

-         versions = req.json()["versions"]

-         version = next(version["num"] for version in versions if not version["yanked"])

+         version = query_newest_version(crate)

  

      os.makedirs(CACHEDIR, exist_ok=True)

      cratef_base = f"{crate}-{version}.crate"
@@ -343,6 +361,44 @@ 

      return any(re.match(autochangelog_re, line) for line in text.splitlines())

  

  

+ def guess_crate_name():

+     """Guess crate name from directory name and/or spec file name

+ 

+     If a spec file is present, we use the %crate variable. This is the best

+     option. But if we're in a new directory, and don't have a spec file yet,

+     let's assume that this is a dist-git directory and extract the crate name.

+     For compat packages, the directory name will contain a version suffix,

+     but a compat package would almost always be created from an existing

+     dist-git directory, hence there'd be a spec file, so we can ignore this.

+     """

+     specs = glob.glob('*.spec')

+     if len(specs) > 1:

+         print('Multiple spec files found, cannot guess crate name')

+         return None

+     if len(specs) == 1:

+         crate = None

+         for line in open(specs[0]):

+             if m := re.match(r'^%(?:global|define)\s+crate\s+(\S+)\s+', line):

+                 if crate:

+                     print(f'{specs[0]}: Found duplicated %crate define, cannot guess crate name')

+                     return None

+                 crate = m.group(1)

+                 if '%' in crate:

+                     print(f'{specs[0]}: Crate name appears to use a macro, and evaluation is not implemented')

+                     return None

+         if crate:

+             print(f'{specs[0]}: Found crate name {crate!r}')

+         else:

+             print(f'{specs[0]}: %crate define not found, cannot guess crate name')

+         return crate

+ 

+     dirname = os.path.basename(os.getcwd())

+     if m := re.match('^rust-([a-z+0-9_-]+)$', dirname):

+         print(f'Using crate name {m.group(1)!r} based on the directory name')

+         return m.group(1)

+ 

+     return None

+ 

  @contextlib.contextmanager

  def exit_on_common_errors():

      """Suppress tracebacks on common "expected" exceptions"""
@@ -419,7 +475,9 @@ 

          return

  

      if args.crate is None:

-         parser.error("required crate/path argument missing")

+         args.crate = guess_crate_name()

+         if args.crate is None:

+             parser.error("crate/path argument missing and autodetection failed")

  

      crate, diff, metadata, doc_files, license_files = make_diff_metadata(args, args.crate, args.version)

  
@@ -450,21 +508,14 @@ 

          raise ValueError("No bins and no libs")

      kwargs["include_devel"] = is_lib

  

-     if args.suffix is not None:

-         if metadata.name[-1].isdigit():

-             suffix = f'_{args.suffix}'

-         else:

-             suffix = args.suffix

-     else:

-         suffix = ""

-     kwargs["pkg_suffix"] = suffix

-     spec_file = pathlib.Path(f"rust-{metadata.name}{suffix}.spec")

+     kwargs["pkg_name"] = pkg_name = package_name_suffixed(metadata.name, args.suffix)

+     spec_file = pathlib.Path(f"{pkg_name}.spec")

  

      if args.target in {"fedora"} and args.existence_check and not os.path.exists(spec_file):

          # No specfile, so this is probably a new package

-         package_info = get_package_info(f"rust-{metadata.name}{suffix}")

+         package_info = get_package_info(pkg_name)

          if package_info:

-             print(f"Crate {metadata.name}{suffix} is already packaged in Fedora ({package_info['full_url']}).")

+             print(f"Crate {pkg_name} is already packaged in Fedora ({package_info['full_url']}).")

Note that this is now broken, it prints something like:

Crate git2_0.13 is already packaged in Fedora (https://src.fedoraproject.org/rpms/rust-git2_0.13).

This is not the crate name, this is crate name + suffix. But at this point, shouldn't pkg_name be rust-git2_0.13?

              print("Re-run with --no-existence-check if you still want to convert it.")

              sys.exit(1)

  

file modified
+13 -11
@@ -13,7 +13,7 @@ 

  %global real_crate {{ crate }}

  {% endif %}

  

- Name:           rust-%{crate}{{ pkg_suffix }}

+ Name:           {{ pkg_name }}

This causes a small problem - it generates rust-foo instead of rust-%{crate}.

I think we should stick to the latter to avoid duplication and the possibility of mistakes.

  Version:        {{ md.version }}

  Release:        {{ pkg_release }}

  {% if md.description is none %}
@@ -100,13 +100,15 @@ 

  

  %files       -n %{crate}

    {% if license_files|length > 0 %}

- %license {{ license_files|join(' ') }}

+     {% for file in license_files %}

+ %license {{ file }}

+     {% endfor %}

    {% else %}

  # FIXME: no license files detected

    {% endif %}

-   {% if doc_files|length > 0 %}

- %doc {{ doc_files|join(' ') }}

-   {% endif %}

+   {% for file in doc_files %}

+ %doc {{ file }}

+   {% endfor %}

    {% for bin in bins %}

  %{_bindir}/{{ bin.name }}

    {% endfor %}
@@ -160,13 +162,13 @@ 

  %files       {{ pkg }}

      {% if feature is none %}

        {% if license_files|length > 0 %}

-         {% if relative_license_paths %}

- %license {{ license_files|join(' ') }}

-         {% else %}

-           {% for file in license_files %}

+         {% for file in license_files %}

+           {% if relative_license_paths %}

+ %license {{ file }}

+           {% else %}

  %license %{crate_instdir}/{{ file }}

-           {% endfor %}

-         {% endif %}

+           {% endif %}

+         {% endfor %}

        {% else %}

  # FIXME: no license files detected

        {% endif %}

file was moved with no change to the file
no initial comment

"draft" sounds weird here. Shouldn't this be "pre-release" or "development release"?

rebased onto 53d715d

2 years ago

Changed to "pre-release'.

This (EDIT: meaning falling back to "CWD == rust-{crate}") will also be completely broken with compat packages, so I don't think using CWD as a data source is a good idea.

But if you really want to do this, the message should say something like "Falling back to \" {args.crate}\" because the crate name was not supplied on the command line." because I think this should be graceful fallback in case somebody forgets to add that argument, and not the default case.


Meh, I submitted this comment while you pushed your rebase, so the link is broken.

rebased onto c185145

2 years ago

I adjusted the heuristic to take into account compat package names.

I think this should be graceful fallback in case somebody forgets to add that argument, and not the default case.

The heuristic should work fine in 99% of cases. Essentially, the only case where you'll need to supply a name explicitly is when not using normal dist-git repo, but a renamed directory for some reason. And I don't think we should ask people to provide the name if we can almost always figure it out ourselves.

No, I just don't agree with using heuristics here. There's starting to be a lot of compat packages for post-1.0 crates (i.e. rust-dirs3, etc.) and there's just no way to distinguish those from packages for crates that just happen to be named like socket2 or memmap2 (this is a gripe that I have with the compat package naming guidelines in general, this is not specific to Rust crates).

Rather than guess the crate name from the name of directory (or the name of the .spec file, for that matter, which would already be an improvement, since that should always be %{name}.spec), it would be much safer to read the value of the %crate macro, like here, with no need to heuristics that take inconsistently-applied (i.e. from before you fixed rust2rpm to use underscores) compat package naming guidelines into account:

https://src.fedoraproject.org/rpms/rust-socket2/blob/rawhide/f/rust-socket2.spec#_5

5 new commits added

  • Simplify handling of package suffix
  • Guess crate name based on %crate or directory name
  • Skip pre-release versions when looking for the lastest
  • Use separate lines for each file in %doc/%license
  • Move old test file under src/tests/
2 years ago

Updated. I changed the detection to just grep for %global crate. This should DTRT in most cases.

5 new commits added

  • Simplify handling of package suffix
  • Guess crate name based on %crate or directory name
  • Skip pre-release versions when looking for the lastest
  • Use separate lines for each file in %doc/%license
  • Move old test file under src/tests/
2 years ago

Thanks, looks good to me now. I'll try to use rust2rpm with the patches from this PR applied for a few updates, though, since verifying Python code to work correctly just by reading it is kind of hard :)

This causes a small problem - it generates rust-foo instead of rust-%{crate}.

I think we should stick to the latter to avoid duplication and the possibility of mistakes.

The "grep" for the value of %crate also doesn't seem to work , I suspect that the regex isn't correct (I tested it with the rust-cxx-build package).

5 new commits added

  • Simplify handling of package suffix
  • Guess crate name based on %crate or directory name
  • Skip pre-release versions when looking for the lastest
  • Use separate lines for each file in %doc/%license
  • Move old test file under src/tests/
2 years ago

This causes a small problem - it generates rust-foo instead of rust-%{crate}.

I think we should stick to the latter to avoid duplication and the possibility of mistakes.

It just moves the substitution a bit earlier: instead of jinja2 creating an spec pattern to use a macro, we do the substitution directly in jinja2. So if anything, this is simpler and has a lesser chance of a mistake. In practice, I don't think it matters too much, since we create the whole spec file for each rust package from scratch and the two methods are equivalent.

The motivating case for the change when the package name has a suffix: we used Name: rust-%create{{pkg_suffix}}, i.e. part would be substituted by jinja2 and part would be substituted by rpm. This is overly complex for no good reason.

The "grep" for the value of %crate also doesn't seem to work , I suspect that the regex isn't correct (I tested it with the rust-cxx-build package).

Arrgh, \w, doesn't include -. I changed the pattern to any non-whitespace now.

Using Name: rust-%{crate} would have the benefit to match exactly the Packaging Guidelines wrt. the rules for naming packages for Rust crates.

I don't think this is a useful argument: the result will be the same both ways. The guidelines certainly don't say that a macro must be used, but only how the end result looks.

Sure, the result will be the same ... meh.
I don't like the duplication of putting the crate name into two separate places, but I don't care about this strongly enough.

I'll go ahead and merge this. I want to restructure the code to add tests, and I don't want to deal with conflicts later.

Pull-Request has been merged by zbyszek

2 years ago

Note that this is now broken, it prints something like:

Crate git2_0.13 is already packaged in Fedora (https://src.fedoraproject.org/rpms/rust-git2_0.13).

This is not the crate name, this is crate name + suffix. But at this point, shouldn't pkg_name be rust-git2_0.13?

Let me amend that (pagure isn't letting me edit the last comment): This is already broken, and it's still broken after this PR. :(