#216 [DRAFT] use tomlkit for stripping dependencies for "foreign" targets
Opened 3 months ago by decathorpe. Modified a month ago
fedora-rust/ decathorpe/rust2rpm main  into  main

file modified
+1
@@ -2,4 +2,5 @@ 

  pyparsing

  requests

  termcolor

+ tomlkit>=0.11.2

  tqdm

file modified
+67 -64
@@ -14,8 +14,10 @@ 

  import tarfile

  import tempfile

  import subprocess

+ from typing import Optional

  

  import requests

+ import tomlkit

  import tqdm

  

  from . import cfg, licensing, generator, log, util
@@ -37,21 +39,6 @@ 

      re.VERBOSE,

  )

  

- # [target.'cfg(not(any(target_os="windows", target_os="macos")))'.dependencies]

- # [target."cfg(windows)".dependencies.winapi]

- # [target."cfg(target_arch = \"wasm32\")".dev-dependencies.wasm-bindgen-test]

- 

- TARGET_DEPENDENCY_LINE = re.compile(

-     r"""

-     ^ \[ target\.(?P<cfg>(?P<quote>['"])cfg\(.*\)(?P=quote))

-     \.

-     (?P<type> dependencies|build-dependencies|dev-dependencies)

-     (?:\. (?P<feature>[a-zA-Z0-9_-]+) )?

-     \] \s* $

-     """,

-     re.VERBOSE,

- )

- 

  

  def sortify(func):

      """Return a sorted list from a generator"""
@@ -202,56 +189,68 @@ 

      )

  

  

- def drop_foreign_dependencies(lines):

-     dropped_lines = 0

-     dropped_features = set()

-     good_lines = []

- 

-     value = True

-     for line in lines:

-         if m := TARGET_DEPENDENCY_LINE.match(line):

-             expr = m.group("cfg")

-             expr = ast.literal_eval(expr)

-             try:

-                 value = cfg.parse_and_evaluate(expr)

-             except (ValueError, cfg.ParseException):

-                 log.warn(f"Could not evaluate {expr!r}, treating as true.")

-                 value = True

- 

-             if not value:

-                 feature = m.group("feature")

-                 log.info(f"Dropping target-specific dependency on {feature!r}.")

-                 dropped_features.add(feature)

- 

-         elif line.startswith("["):

-             # previous section ended, let's keep printing lines again

-             value = True

- 

-         if value:

-             good_lines += [line]

-         else:

-             dropped_lines += 1

+ def drop_foreign_dependencies(toml_str: str) -> Optional[str]:

+     toml_data = tomlkit.parse(toml_str)

+ 

+     target_table = toml_data.get("target")

+     feature_table = toml_data.get("features")

  

-     if not dropped_lines:

-         # nothing to do, let's bail out

+     # there is no [target] table: nothing to do

+     if target_table is None:

          return None

  

-     good_lines2 = []

-     in_features = False

-     filt = filter_out_features_re(dropped_features)

-     for line in good_lines:

-         if line.rstrip() == "[features]":

-             in_features = True

-         elif line.startswith("["):

-             in_features = False

-         elif in_features:

-             line = re.sub(filt, "", line)

-             if not line:

-                 continue

+     remove_targets = []

+     remove_features = []

+ 

+     # iterate over entries in the [target] table

+     for key in target_table.keys():

+         if not cfg.parse_and_evaluate(key):

+             remove_targets.append(key)

  

-         good_lines2 += [line]

+             cfg_table = target_table[key]

+ 

+             # remember which optional {dev,build,}dependencies are being removed

+             for dep_type in ["dependencies", "dev-dependencies", "build-dependencies"]:

+                 if cfg_deps := cfg_table.get(dep_type):

+                     for dep_name in cfg_deps.keys():

+                         if cfg_deps[dep_name].get("optional") is True:

+                             remove_features.append(dep_name)

+ 

+     # no dependencies were removed

+     if not remove_targets:

+         return None

  

-     return good_lines2

+     # remove entries from [target] table

+     for key in remove_targets:

+         del target_table[key]

+ 

+     if feature_table is None:

+         # there is no [features] list: we are done

+         toml_out = tomlkit.dumps(toml_data)

+         return toml_out

+ 

+     # update feature dependency lists for dropped optional dependencies

+     for feature_name in feature_table.keys():

+         feature_deps = feature_table[feature_name]

+ 

+         remove_deps = []

+ 

+         for feature_dep in feature_deps:

+             # TODO: support for Rust 1.60+ feature dependency syntax

+ 

+             if "/" in feature_dep:

+                 dep_name = feature_dep.split("/")[0]

+             else:

+                 dep_name = feature_dep

+ 

+             if dep_name in remove_features:

+                 remove_deps.append(feature_dep)

+ 

+         for remove_dep in remove_deps:

+             feature_deps.remove(remove_dep)

+ 

+     toml_out = tomlkit.dumps(toml_data)

+     return toml_out

  

  

  def make_diff(path, lines1, mtime1, lines2, mtime2):
@@ -267,7 +266,9 @@ 

      made after the editor returns.

      """

      mtime_before = file_mtime(toml)

-     toml_before = open(toml).readlines()

+ 

+     with open(toml) as toml_file:

+         toml_before = toml_file.read()

  

      diff1 = diff2 = None

      toml_path = f"{args.crate}-{version}/Cargo.toml"
@@ -292,7 +293,7 @@ 

          with open(toml, "w") as file:

              file.writelines(toml_after)

  

-         diff1 = make_diff(toml_path, toml_before, mtime_before, toml_after, mtime_before)

+         diff1 = make_diff(toml_path, toml_before.splitlines(), mtime_before, toml_after.splitlines(), mtime_before)

      else:

          toml_after = toml_before

  
@@ -302,9 +303,11 @@ 

          subprocess.check_call([editor, toml])

  

          mtime_after2 = file_mtime(toml)

-         toml_after2 = open(toml).readlines()

  

-         diff2 = make_diff(toml_path, toml_after, mtime_before, toml_after2, mtime_after2)

+         with open(toml) as toml_file:

+             toml_after2 = toml_file.read()

+ 

+         diff2 = make_diff(toml_path, toml_after.splitlines(), mtime_before, toml_after2.splitlines(), mtime_after2)

  

      return diff1, diff2

  

@@ -0,0 +1,180 @@ 

+ # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO

+ #

+ # When uploading crates to the registry Cargo will automatically

+ # "normalize" Cargo.toml files for maximal compatibility

+ # with all versions of Cargo and also rewrite `path` dependencies

+ # to registry (e.g., crates.io) dependencies.

+ #

+ # If you are reading this file be aware that the original Cargo.toml

+ # will likely look very different (and much more reasonable).

+ # See Cargo.toml.orig for the original contents.

+ 

+ [package]

+ edition = "2018"

+ rust-version = "1.46"

+ name = "nix"

+ version = "0.24.1"

+ authors = ["The nix-rust Project Developers"]

+ include = [

+     "src/**/*",

+     "test/**/*",

+     "LICENSE",

+     "README.md",

+     "CHANGELOG.md",

+ ]

+ description = "Rust friendly bindings to *nix APIs"

+ readme = "README.md"

+ categories = ["os::unix-apis"]

+ license = "MIT"

+ repository = "https://github.com/nix-rust/nix"

+ 

+ [package.metadata.docs.rs]

+ rustdoc-args = [

+     "--cfg",

+     "docsrs",

+ ]

+ targets = [

+     "x86_64-unknown-linux-gnu",

+     "aarch64-linux-android",

+     "x86_64-apple-darwin",

+     "aarch64-apple-ios",

+     "x86_64-unknown-freebsd",

+     "x86_64-unknown-openbsd",

+     "x86_64-unknown-netbsd",

+     "x86_64-unknown-dragonfly",

+     "x86_64-fuchsia",

+     "x86_64-unknown-redox",

+     "x86_64-unknown-illumos",

+ ]

+ 

+ [[test]]

+ name = "test"

+ path = "test/test.rs"

+ 

+ [[test]]

+ name = "test-aio-drop"

+ path = "test/sys/test_aio_drop.rs"

+ 

+ [[test]]

+ name = "test-clearenv"

+ path = "test/test_clearenv.rs"

+ 

+ [[test]]

+ name = "test-lio-listio-resubmit"

+ path = "test/sys/test_lio_listio_resubmit.rs"

+ 

+ [[test]]

+ name = "test-mount"

+ path = "test/test_mount.rs"

+ harness = false

+ 

+ [[test]]

+ name = "test-ptymaster-drop"

+ path = "test/test_ptymaster_drop.rs"

+ 

+ [dependencies.bitflags]

+ version = "1.1"

+ 

+ [dependencies.cfg-if]

+ version = "1.0"

+ 

+ [dependencies.libc]

+ version = "0.2.121"

+ features = ["extra_traits"]

+ 

+ [dev-dependencies.assert-impl]

+ version = "0.1"

+ 

+ [dev-dependencies.lazy_static]

+ version = "1.2"

+ 

+ [dev-dependencies.parking_lot]

+ version = "0.11.2"

+ 

+ [dev-dependencies.rand]

+ version = "0.8"

+ 

+ [dev-dependencies.semver]

+ version = "1.0.0"

+ 

+ [dev-dependencies.tempfile]

+ version = "3.2.0"

+ 

+ [features]

+ acct = []

+ aio = []

+ default = [

+     "acct",

+     "aio",

+     "dir",

+     "env",

+     "event",

+     "feature",

+     "fs",

+     "hostname",

+     "inotify",

+     "ioctl",

+     "kmod",

+     "mman",

+     "mount",

+     "mqueue",

+     "net",

+     "personality",

+     "poll",

+     "process",

+     "pthread",

+     "ptrace",

+     "quota",

+     "reboot",

+     "resource",

+     "sched",

+     "signal",

+     "socket",

+     "term",

+     "time",

+     "ucontext",

+     "uio",

+     "user",

+     "zerocopy",

+ ]

+ dir = ["fs"]

+ env = []

+ event = []

+ feature = []

+ fs = []

+ hostname = []

+ inotify = []

+ ioctl = []

+ kmod = []

+ mman = []

+ mount = ["uio"]

+ mqueue = ["fs"]

+ net = ["socket"]

+ personality = []

+ poll = []

+ process = []

+ pthread = []

+ ptrace = ["process"]

+ quota = []

+ reboot = []

+ resource = []

+ sched = ["process"]

+ signal = ["process"]

+ socket = ["memoffset"]

+ term = []

+ time = []

+ ucontext = ["signal"]

+ uio = []

+ user = ["feature"]

+ zerocopy = [

+     "fs",

+     "uio",

+ ]

+ 

+ [target."cfg(any(target_os = \"android\", target_os = \"linux\"))".dev-dependencies.caps]

+ version = "0.5.1"

+ 

+ [target."cfg(not(target_os = \"redox\"))".dependencies.memoffset]

+ version = "0.6.3"

+ optional = true

+ 

@@ -7,7 +7,7 @@ 

  Name:           rust-tokio-1.19.2

  Version:        1.19.2

  Release:        %autorelease

- Summary:        Event-driven, non-blocking I/O platform for writing asynchronous I/O

+ Summary:        Event-driven, non-blocking I/O platform for writing asynchronous I/O backed applications

  

  License:        MIT

  URL:            https://crates.io/crates/tokio-1.19.2
@@ -38,8 +38,8 @@ 

  %endif

  

  %global _description %{expand:

- Event-driven, non-blocking I/O platform for writing asynchronous I/O

- backed applications.}

+ Event-driven, non-blocking I/O platform for writing asynchronous I/O backed

+ applications.}

  

  %description %{_description}

  

@@ -7,7 +7,7 @@ 

  Name:           rust-tokio-1.19.2

  Version:        1.19.2

  Release:        %mkrel 1

- Summary:        Event-driven, non-blocking I/O platform for writing asynchronous I/O

+ Summary:        Event-driven, non-blocking I/O platform for writing asynchronous I/O backed applications

  Group:          Development/Rust

  

  License:        MIT
@@ -39,8 +39,8 @@ 

  %endif

  

  %global _description %{expand:

- Event-driven, non-blocking I/O platform for writing asynchronous I/O

- backed applications.}

+ Event-driven, non-blocking I/O platform for writing asynchronous I/O backed

+ applications.}

  

  %description %{_description}

  

@@ -24,7 +24,7 @@ 

  Name:           rust-tokio-1.19.2

  Version:        1.19.2

  Release:        0

- Summary:        Event-driven, non-blocking I/O platform for writing asynchronous I/O

+ Summary:        Event-driven, non-blocking I/O platform for writing asynchronous I/O backed applications

  Group:          Development/Libraries/Rust

  

  License:        MIT
@@ -56,8 +56,8 @@ 

  %endif

  

  %global _description %{expand:

- Event-driven, non-blocking I/O platform for writing asynchronous I/O

- backed applications.}

+ Event-driven, non-blocking I/O platform for writing asynchronous I/O backed

+ applications.}

  

  %description %{_description}

  

@@ -7,7 +7,7 @@ 

  Name:           rust-tokio-1.19.2

  Version:        1.19.2

  Release:        1%{?dist}

- Summary:        Event-driven, non-blocking I/O platform for writing asynchronous I/O

+ Summary:        Event-driven, non-blocking I/O platform for writing asynchronous I/O backed applications

  

  License:        MIT

  URL:            https://crates.io/crates/tokio-1.19.2
@@ -38,8 +38,8 @@ 

  %endif

  

  %global _description %{expand:

- Event-driven, non-blocking I/O platform for writing asynchronous I/O

- backed applications.}

+ Event-driven, non-blocking I/O platform for writing asynchronous I/O backed

+ applications.}

  

  %description %{_description}

  

@@ -0,0 +1,199 @@ 

+ # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO

+ #

+ # When uploading crates to the registry Cargo will automatically

+ # "normalize" Cargo.toml files for maximal compatibility

+ # with all versions of Cargo and also rewrite `path` dependencies

+ # to registry (e.g., crates.io) dependencies.

+ #

+ # If you are reading this file be aware that the original Cargo.toml

+ # will likely look very different (and much more reasonable).

+ # See Cargo.toml.orig for the original contents.

+ 

+ [package]

+ edition = "2018"

+ rust-version = "1.49"

+ name = "tokio"

+ version = "1.19.2"

+ authors = ["Tokio Contributors <team@tokio.rs>"]

+ description = """

+ An event-driven, non-blocking I/O platform for writing asynchronous I/O

+ backed applications.

+ """

+ homepage = "https://tokio.rs"

+ readme = "README.md"

+ keywords = [

+     "io",

+     "async",

+     "non-blocking",

+     "futures",

+ ]

+ categories = [

+     "asynchronous",

+     "network-programming",

+ ]

+ license = "MIT"

+ repository = "https://github.com/tokio-rs/tokio"

+ 

+ [package.metadata.docs.rs]

+ all-features = true

+ rustdoc-args = [

+     "--cfg",

+     "docsrs",

+     "--cfg",

+     "tokio_unstable",

+ ]

+ rustc-args = [

+     "--cfg",

+     "tokio_unstable",

+ ]

+ 

+ [package.metadata.playground]

+ features = [

+     "full",

+     "test-util",

+ ]

+ 

+ [dependencies.bytes]

+ version = "1.0.0"

+ optional = true

+ 

+ [dependencies.memchr]

+ version = "2.2"

+ optional = true

+ 

+ [dependencies.mio]

+ version = "0.8.1"

+ optional = true

+ 

+ [dependencies.num_cpus]

+ version = "1.8.0"

+ optional = true

+ 

+ [dependencies.once_cell]

+ version = "1.5.2"

+ optional = true

+ 

+ [dependencies.parking_lot]

+ version = "0.12.0"

+ optional = true

+ 

+ [dependencies.pin-project-lite]

+ version = "0.2.0"

+ 

+ [dependencies.socket2]

+ version = "0.4.4"

+ features = ["all"]

+ optional = true

+ 

+ [dependencies.tokio-macros]

+ version = "1.7.0"

+ optional = true

+ 

+ [dev-dependencies.async-stream]

+ version = "0.3"

+ 

+ [dev-dependencies.futures]

+ version = "0.3.0"

+ features = ["async-await"]

+ 

+ [dev-dependencies.mockall]

+ version = "0.11.1"

+ 

+ [dev-dependencies.tempfile]

+ version = "3.1.0"

+ 

+ [dev-dependencies.tokio-stream]

+ version = "0.1"

+ 

+ [dev-dependencies.tokio-test]

+ version = "0.4.0"

+ 

+ [features]

+ default = []

+ fs = []

+ full = [

+     "fs",

+     "io-util",

+     "io-std",

+     "macros",

+     "net",

+     "parking_lot",

+     "process",

+     "rt",

+     "rt-multi-thread",

+     "signal",

+     "sync",

+     "time",

+ ]

+ io-std = []

+ io-util = [

+     "memchr",

+     "bytes",

+ ]

+ macros = ["tokio-macros"]

+ net = [

+     "libc",

+     "mio/os-poll",

+     "mio/os-ext",

+     "mio/net",

+     "socket2",

+ ]

+ process = [

+     "bytes",

+     "once_cell",

+     "libc",

+     "mio/os-poll",

+     "mio/os-ext",

+     "mio/net",

+     "signal-hook-registry",

+ ]

+ rt = ["once_cell"]

+ rt-multi-thread = [

+     "num_cpus",

+     "rt",

+ ]

+ signal = [

+     "once_cell",

+     "libc",

+     "mio/os-poll",

+     "mio/net",

+     "mio/os-ext",

+     "signal-hook-registry",

+ ]

+ stats = []

+ sync = []

+ test-util = [

+     "rt",

+     "sync",

+     "time",

+ ]

+ time = []

+ 

+ [target."cfg(not(target_arch = \"wasm32\"))".dev-dependencies.proptest]

+ version = "1"

+ 

+ [target."cfg(not(target_arch = \"wasm32\"))".dev-dependencies.rand]

+ version = "0.8.0"

+ 

+ [target."cfg(not(target_arch = \"wasm32\"))".dev-dependencies.socket2]

+ version = "0.4"

+ 

+ [target."cfg(unix)".dependencies.libc]

+ version = "0.2.42"

+ optional = true

+ 

+ [target."cfg(unix)".dependencies.signal-hook-registry]

+ version = "1.1.1"

+ optional = true

+ 

+ [target."cfg(unix)".dev-dependencies.libc]

+ version = "0.2.42"

+ 

+ [target."cfg(unix)".dev-dependencies.nix]

+ version = "0.24"

+ features = [

+     "fs",

+     "socket",

+ ]

+ default-features = false

+ 

@@ -40,11 +40,13 @@ 

      args = get_parser().parse_args(["foobar", f"--target={target}", "-a" if target == "fedora" else "--no-rpmautospec"])

      pkg_name = package_name_suffixed(crate, args.suffix)

  

-     toml_before = open(tomlfile).readlines()

+     with open(tomlfile) as file:

+         toml_before = file.read()

+ 

      toml_after = drop_foreign_dependencies(toml_before) or toml_before

  

      fake_toml = tmpdir / "Cargo.toml"

-     fake_toml.write_text("\n".join(toml_after))

+     fake_toml.write_text(toml_after)

  

      (metadata,) = Metadata.from_file(fake_toml)

  

@@ -0,0 +1,20 @@ 

+ import glob

+ import os

+ 

+ import pytest

+ 

+ from rust2rpm.__main__ import drop_foreign_dependencies

+ 

+ toml_files = glob.glob(os.path.join(os.path.dirname(__file__), "samples", "*.toml"))

+ 

+ 

+ @pytest.mark.parametrize("toml_path", toml_files)

+ def test_strip(toml_path):

+     with open(toml_path) as file:

+         orig = file.read()

+ 

+     if stripped := drop_foreign_dependencies(orig):

+         with open(toml_path + ".stripped") as file:

+             expected = file.read()

+ 

+         assert stripped == expected

file modified
+1
@@ -28,6 +28,7 @@ 

      pyparsing

      requests

      termcolor

+     tomlkit>=0.11.1

      tqdm

  

  [options.package_data]

This removes the custom, regex-based and error-prone parsing of Cargo.toml files and replaces it with logic based on an actual, style-preserving TOML library (tomlkit).

To accommidate this change, the drop_foreign_dependencies function no longer operates on individual lines from Cargo.toml, but the entire contents.

Using an actual TOML parser also fixes an edge case where multi-line string literals were truncated (see the tokio spec file generator tests, where the "Summary" was missing the last two words).

I also added tests for drop_foreign_dependencies function, which was previously not covered by tests. If it returns None, then no modifications were made. Otherwise, the stripped Cargo.toml is compared to the expected contents.

Since tomlkit is not without flaws, the drop_foreign_dependencies function also contains two small workarounds for formatting inconsistencies.


Additionally, the version of tomlkit in Fedora is very old and very buggy, I only tested with 0.11.1 (the latest upstream release). Not sure what to do about that ...

EDIT: The tomlkit package in Fedora 37+ is now recent enough to have the bugfixes needed.

This change would also fix #220.

rebased onto 82f6284

a month ago