#288 Create test suite to test DNF5.
Closed 5 months ago by lruzicka. Opened 2 years ago by lruzicka.
dnf5  into  main

file added
+211
@@ -0,0 +1,211 @@ 

+ package dnf;

+ 

+ use strict;

+ 

+ use base 'Exporter';

+ use Exporter;

+ use lockapi;

+ use testapi;

+ use utils;

+ 

+ our @EXPORT = qw(install check remove reinstall parse_list_output parse_history_info latest_item parse_info parse_package_name confirm_in_lines confirm_in_output);

+ 

+ 

+ # This subroutine install a $package using DNF5.

+ # It dies if the command returns a non-zero exit

+ # code or if it times out.

+ sub install {

+     my $package = shift;

+     assert_script_run("dnf5 install -y $package", timeout => 120);

+ }

+ 

+ # This subroutine checks if the $package is installed

+ # using the rpm command. It returns True if it is

+ # installed.

+ sub check {

+     my $package = shift;

+     if (script_run("rpm -q $package", timeout => 60)) {

+         return 0;

+     }

+     else {

+         return 1;

+     }

+ }

+ 

+ # This subroutine removes the package using DNF5.

+ sub remove {

+     my $package = shift;

+     assert_script_run("dnf5 remove -y $package", timeout => 120);

+ }

+ 

+ # This subroutine reinstalls the package using DNF5.

+ sub reinstall {

+     my $package = shift;

+     assert_script_run("dnf5 reinstall -y $package", timeout => 120);

+ }

+ 

+ # This subroutine will parse the DNF5 cli output

+ # and return a hash with activities.

+ # Parsing the output is quite fragile, because

+ # the info is only divided by whitespaces and some

+ # columns are empty, so we retain column information

+ # in this case. Therefore, we will only consider

+ # the following structure, that will do for simple

+ # transactions:

+ # number : dnf5 : command : value : date

+ sub parse_list_output {

+     my $dnf = shift;

+     # Let us split the whole ouput into single lines

+     my @lines = split("\n", $dnf);

+     # The final hash with output information.

+     my $parsed_output = {};

+     # Let's iterate over the single lines and split

+     # them into single columns and store in the

+     # parsed output.

+     my $values = [];

+     for my $line (@lines) {

+         # Split columns on a white space.

+         my @columns = split(" ", $line);

+         # Take the columns

+         $values->[0] = $columns[1];

+         $values->[1] = $columns[2];

+         # When date comes on the third place, it means

+         # that the command did not have any arguments.

+         # We need to remember this and we will store

+         # an empty string instead.

+         if ($columns[3] =~ /\d\d\d\d-\d\d-\d\d/) {

+             $values->[2] = "";

+             $values->[3] = $columns[3];

+         }

+         # All DNF commands have the "-y" argument

+         # which we also need to think about.

+         elsif ($columns[3] eq "-y") {

+             $values->[2] = $columns[4];

+             $values->[3] = $columns[5];

+         }

+         else {

+             $values->[2] = $columns[3];

+             $values->[3] = $columns[4];

+         }

+         # The output also contains a description line that

+         # we do not want to store. That line leaves the "ID"

+         # string in the first place, so it is easy to find.

+         unless ($columns[0] eq "ID") {

+             $parsed_output->{$columns[0]} = $values;

+         }

+     }

+     return $parsed_output;

+ }

+ 

+ # This subroutine parses the DNF5 history info output. We will

+ # only consider several fields in order not to make things overly

+ # complicated, i.e. ID, user, status, releasever, and description.

+ sub parse_history_info {

+     my $dnf = shift;

+     # First, split into single lines.

+     my @lines = split("\n", $dnf);

+     # The final hash with output info

+     my $parsed_output = {};

+     # These are columns we are interested in.

+     my $selected = ["Transaction ID", "User", "Status", "Releasever", "Description"];

+     # Let us iterate over the lines and only choose info we want

+     for my $line (@lines) {

+         my @columns = split(" : ", $line);

+         # Trim the whitespaces from the column name

+         my $name = $columns[0];

+         $name =~ s/^\s+|\s+$//g;

+         # Only remember lines where $name matches

+         if ($name ~~ @$selected) {

+             $parsed_output->{$name} = $columns[1];

+         }

+     }

+     return $parsed_output;

+ }

+ 

+ # This subroutine returns the id number of the latest history item,

+ # based on the parsed history output, see parse_list_output.

+ sub latest_item {

+     # It takes the parsed data already.

+     my $dnfdata = shift;

+     # Take the keys list of ID numbers.

+     my @ids = keys %$dnfdata;

+     # Sort the keys as keys might not be sorted.

+     @ids = sort { $b <=> $a } @ids;

+     # The latest item, with highest number will now be at index 0.

+     return $ids[0];

+ }

+ 

+ # This subroutine parses info from the 'info' output

+ # and returns a hash where the output comprises

+ # of keys and values.

+ sub parse_info {

+     my $output = shift;

+     my $info = {};

+ 

+     my @output = split("\n", $output);

+     foreach (@output) {

+         my ($key, $value) = split(" : ", $_);

+         $key =~ s/^\s+|\s+$//g;

+         $value =~ s/^\s+|\s+$//g;

+         $info->{$key} = $value;

+     }

+     return $info;

+ }

+ 

+ # This routine takes the package name

+ # and splits it into various parts that are

+ # returned as a hash where the values are

+ # easily accessible.

+ sub parse_package_name {

+     my $rpm = shift;

+     my @rpm = split("-", $rpm);

+     my ($name, $version, $suffix) = split("-", $rpm);

+     my ($release, $os, $arch);

+     ($release, $os, $arch, $suffix) = split(/\./, $suffix);

+     my $package = {};

+     $package->{name} = $name;

+     $package->{version} = $version;

+     $package->{release} = "$release.$os";

+     $package->{arch} = $arch;

+     return $package;

+ }

+ 

+ 

+ # This subroutine checks all the lines of $output

+ # for the $keyword and records all lines where

+ # the $keyword is not found and then it dies.

+ sub confirm_in_lines {

+     my ($output, $keyword) = @_;

+     my @unmatched;

+     my @lines = split("\n", $output);

+     # Iterate over the lines and record any line with a discrepancy

+     # into the unmatched array.

+     foreach (@lines) {

+         unless ($_ =~ /Updating and loading/ or $_ =~ /Repositories loaded/) {

+             unless ($_ =~ /$keyword/) {

+                 push(@unmatched, $_);

+             }

+         }

+     }

+     # If there are errors in unmatched, log them and die.

+     if (scalar(@unmatched) > 0) {

+         diag("DNF5 Repoquerry errors:");

+         foreach (@unmatched) {

+             diag($_);

+         }

+         die("dnf5 repoquery returned lines that did not match the chosen pattern.");

+     }

+ }

+ 

+ # This subroutine checks that the $keyword

+ # exists in the $output and dies if it does not.

+ sub confirm_in_output {

+     my ($output, $keywords) = @_;

+     my @lines = split("\n", $output);

+     foreach (@$keywords) {

+         unless (grep(/$_/, @lines)) {

+             die("The $_ was not found in the output.");

+         }

+     }

+ }

+ 

file modified
+16
@@ -2363,6 +2363,22 @@ 

                  "TEST_TARGET": "NONE"

              }

          },

+         "dnf5": {

+             "profiles": {

+                 "fedora-Server-dvd-iso-aarch64-*-aarch64": 30,

+                 "fedora-Server-dvd-iso-ppc64le-*-ppc64le": 30,

+                 "fedora-Server-dvd-iso-x86_64-*-64bit": 30

+             },

+             "settings": {

+                 "BOOTFROM": "c",

+                 "HDD_1": "disk_%FLAVOR%_%MACHINE%.qcow2",

+                 "LOGIN_SNAPSHOT": "1",

+                 "POSTINSTALL_PATH": "tests/dnf5",

+                 "ROOT_PASSWORD": "weakpassword",

+                 "START_AFTER_TEST": "%DEPLOY_UPLOAD_TEST%",

+                 "USER_LOGIN": "false"

+             }

+         },

          "podman": {

              "profiles": {

                  "fedora-CoreOS-colive-iso-x86_64-*-64bit": 20,

@@ -0,0 +1,50 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ 

+ # The setup script for the DNF5 test suite.

+ #

+ # This script will test if DNF5 is installed on the system.

+ # Currently, this is not the case, but later in the process,

+ # so we will softfail if it is not and we will install it.

+ # Later, when DNF5 is part of Fedora, we will fail instead.

+ 

+ sub run {

+     my $self = shift;

+     # Switch to console

+     $self->root_console(tty => 3);

+ 

+     # Check that the package is installed. If it is not,

+     # the following command will fail.

+     if (script_run("rpm -q dnf5", timeout => 20)) {

+         # FIXME: When DNF5 is part of Fedora, change this to die instead.

+         record_soft_failure("The dnf5 package is not installed. I am installing it now.");

+         # Install the package, so we can move forward.

+         assert_script_run("dnf install -y dnf5 dnf5-plugins", timeout => 180);

+     }

+     else {

+         # This is here as notice. When we are seeing this, DNF5 has become

+         # the part of Fedora.

+         record_info("DNF5 is already installed on the system.");

+     }

+ 

+     # Currently, DNF5 is being invoked by calling "dnf5". We assume,

+     # that later it will be changed into "dnf" when DNF4 will be gone.

+     my $dnfversion = script_output("dnf --version");

+     if ("dnf5" =~ $dnfversion) {

+         record_info("DNF version", "The 'dnf' command uses version 5.");

+     }

+     else {

+         record_info("DNF version", "The 'dnf' command uses version 4.");

+     }

+ 

+ }

+ 

+ sub test_flags {

+     return {fatal => 1, milestone => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

@@ -0,0 +1,103 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ use dnf;

+ 

+ # This script follows the basic_sanity test as described

+ # https://fedoraproject.org/wiki/QA:Testcase_DNF_basic_sanity

+ 

+ 

+ # We select some packages and we'll do the installation tests.

+ sub run {

+     my $self = shift;

+     my @packages = ("fortune-mod", "mc", "cowsay", "python3-ipython");

+ 

+     # Install packages.

+     for my $package (@packages) {

+         install($package);

+         # Check if installed

+         die("$package not installed") unless (check($package));

+     }

+ 

+     # Remove packages.

+     for my $package (@packages) {

+         remove($package);

+         # Check if removed

+         die("$package not removed") if (check($package));

+     }

+ 

+     # Reinstall package.

+     #

+     # We will install elinks again and check that

+     # its executable is present on the system.

+     # Then we will delete the executable and check

+     # that it is not on the system any more.

+     # Then we will reinstall the package and check

+     # that everything is ok again.

+     install("elinks");

+     # Check that it is there

+     assert_script_run("which elinks");

+     # Remove the elinks executable

+     assert_script_run("rm -f /usr/bin/elinks");

+     # Check again for non-existence

+     assert_script_run("! which elinks");

+     # Reinstall

+     reinstall("elinks");

+     # Check once more for existence

+     assert_script_run("which elinks");

+ 

+     # Download package

+     # Create a download directory and cd into it.

+     assert_script_run("mkdir /download && cd /download");

+     assert_script_run("rm -f *");

+     # Download the package

+     assert_script_run("dnf5 download -y mc");

+     # Check that the rpm file has been downloaded

+     assert_script_run("ls mc*.rpm");

+     # Remove everything

+     assert_script_run("rm -f *");

+ 

+     # Download package and its dependencies

+     # Download the package

+     if (script_run("dnf5 download -y --resolve mc")) {

+         # FIXME: Currently, this command will result in an error, because the --resolve command

+         # does not work as expected. Bug has been reported.

+         record_soft_failure("Bug already reported: https://bugzilla.redhat.com/show_bug.cgi?id=2187981");

+     }

+     else {

+         # Check that the rpm files have been downloaded

+         assert_script_run("ls mc*.rpm");

+         assert_script_run("ls slang*.rpm");

+     }

+     # Remove the downloaded packages.

+     assert_script_run("rm -f *");

+ 

+     # Download package and all dependencies. There should

+     # be more than 200 files downloaded.

+     # Download the packages

+     if (script_run("dnf5 download -y --resolve --alldeps mc")) {

+         # See above.

+         record_soft_failure("Bug already reported: https://bugzilla.redhat.com/show_bug.cgi?id=2187981");

+     }

+     else {

+         # Check that the rpm files have been downloaded, but we will

+         # not be doing an exact comparison of all the dependencies,

+         # because they might vary through time. Instead, we will only

+         # check that a reasonably big number of rpms have been downloaded.

+         my $package_count = script_output("ls | wc -l");

+         if ($package_count < 200) {

+             die("The count of all dependencies is less than expected.");

+         }

+     }

+     # Remove everything

+     assert_script_run("rm -f *");

+ }

+ 

+ sub test_flags {

+     return {always_rollback => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

file added
+46
@@ -0,0 +1,46 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ use dnf;

+ 

+ # This script will make sure that DNF5 is able to

+ # clean cache and recreate it according to

+ # https://fedoraproject.org/wiki/QA:Testcase_DNF_makecache_and_clean.

+ 

+ 

+ sub run {

+     my $self = shift;

+ 

+     # The cache should be clean at this moment, so we will

+     # check that there are no records in libdnf5 directory.

+     #

+     # We will create the cache and check that the data is

+     # there.

+     assert_script_run("dnf5 -y makecache");

+     # We will see how many lines there are in the

+     # directory listings when metadata is present.

+     my $metadata = script_output("ls /var/cache/libdnf5/* | wc -l");

+     #

+     # We will clean the metadata.

+     assert_script_run("dnf5 -y clean metadata");

+     # And we will check again. Now, the metadata

+     # count should be much less, because the

+     # directories should be empty.

+     my $postdelete = script_output("ls /var/cache/libdnf5/* | wc -l");

+     if ($postdelete >= $metadata) {

+         die("The created metadata had $metadata lines and after deletion they have $postdelete.");

+     }

+ 

+     # We will clean everything again.

+     assert_script_run("dnf5 -y clean all");

+     assert_script_run("! ls /var/cache/libdnf5/*");

+ }

+ 

+ sub test_flags {

+     return {always_rollback => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

@@ -0,0 +1,67 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ use dnf;

+ 

+ # This script will make sure that DNF5 is able to

+ # distro sync the system according to

+ # https://fedoraproject.org/wiki/QA:Testcase_DNF_distro-sync.

+ 

+ 

+ sub run {

+     my $self = shift;

+     my $version = get_var("VERSION");

+     $version = get_var("RAWREL") if ($version eq "Rawhide");

+     # Calculate the version numbers around the current release.

+     my $higher = $version + 1;

+     my $lower = $version - 1;

+ 

+     # Install the packages that we will work with.

+     assert_script_run("dnf5 install -y mc elinks");

+ 

+     # Downgrade one of the packages.

+     assert_script_run("dnf5 downgrade -y mc --releasever=$lower");

+ 

+     # Upgrade the other package using the rawhide repositories.

+     if ($version ne "Rawhide") {

+         assert_script_run("dnf5 install -y fedora-repos-rawhide");

+     }

+     assert_script_run("dnf5 upgrade -y elinks --enablerepo=rawhide");

+ 

+     # Check the versions.

+     my $mcver = script_output("rpm -qa mc");

+     my $elinksver = script_output("rpm -qa elinks");

+ 

+     my $lmark = "fc$lower";

+     my $hmark = "fc$higher";

+     unless (index($mcver, $lmark) != -1) {

+         die("The mc package could not be downgraded to $lower version.");

+     }

+     unless (index($elinksver, $hmark) != -1) {

+         die("The elinks package could not be upgraded to $higher version.");

+     }

+ 

+     # Distro-sync the versions

+     assert_script_run("dnf5 distro-sync -y", timeout => 240);

+ 

+     # Check the versions.

+     $mcver = script_output("rpm -qa mc");

+     $elinksver = script_output("rpm -qa elinks");

+ 

+     my $mark = "fc$version";

+     unless (index($mcver, $mark) != -1) {

+         die("The mc package was not synced to the current version.");

+     }

+     unless (index($elinksver, $mark) != -1) {

+         die("The elinks package was not synced to the current version.");

+     }

+ }

+ 

+ sub test_flags {

+     return {always_rollback => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

file added
+104
@@ -0,0 +1,104 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ use dnf;

+ 

+ # This script will make sure that DNF5 is able to

+ # record the history of transactions and that it

+ # can perform various activities based on that.

+ # https://fedoraproject.org/wiki/QA:Testcase_DNF_history

+ 

+ sub run {

+     my $self = shift;

+ 

+     # Install one package

+     assert_script_run("dnf5 install -y mc");

+ 

+     # Check that the transaction appears in DNF history.

+     my $output = script_output("dnf5 history list");

+     my $history = parse_list_output($output);

+     my $last = latest_item($history);

+     # Read the last transaction

+     my $transaction = $history->{$last};

+     # Read transaction information

+     my $command = $transaction->[1];

+     my $package = $transaction->[2];

+     # Check for errors.

+     my $errors = [];

+     if ($command ne "install") {

+         push(@$errors, "command should be install but is $command.");

+     }

+     if ($package ne "mc") {

+         push(@$errors, "package should be mc but is $package.");

+     }

+     # Check if errors were recorded.

+     my $ecount = scalar @$errors;

+     if ($ecount > 0) {

+         diag("DNF5 history list errors:");

+         foreach (@$errors) {

+             diag($_);

+         }

+         die("The last transaction in the history differs from what we expected.");

+     }

+ 

+     # Check that info can be listed about the last transaction.

+     $errors = [];

+     $output = script_output("dnf5 history info $last");

+     my $info = parse_history_info($output);

+     if ($info->{User} != 0) {

+         push(@$errors, "user");

+     }

+     if ($info->{Status} ne "Ok") {

+         push(@$errors, "status");

+     }

+     if ($info->{Description} ne "dnf5 install -y mc") {

+         push(@$errors, "description");

+     }

+     if ($info->{"Transaction ID"} ne $last) {

+         push(@$errors, "id");

+     }

+     $ecount = scalar @$errors;

+     if ($ecount > 0) {

+         diag("DNF5 history info errors:");

+         foreach (@$errors) {

+             diag($_);

+         }

+         die("The last transaction info differs from what we expected");

+     }

+ 

+     # Check that the transaction can be undone and redone.

+     $errors = [];

+ 

+     # FIXME WARNING:

+     # The subcommands redo, undo, rollback, etc. are not implemented

+     # yet, so we cannot use them. However, we will try to use them

+     # anyway and let the test softfail if they cannot be used.

+     # Once it stops softfailing, we'll know that they have been

+     # implemented and we'll start checking more strictly.

+     #

+     # See, if subcommands work.

+     my @subcommands = qw(undo redo rollback);

+     foreach (@subcommands) {

+         my $result = script_run("dnf5 history $_ -y $last");

+         # Error code 2 means that the command is not supported

+         # yet and that usage info was returned instead.

+         if ($result == 2) {

+             diag("DNF5 history: $_ is probably not implemented yet.");

+             push(@$errors, $_);

+         }

+     }

+     # See, if we have errors and softfail if we do.

+     $ecount = scalar @$errors;

+     if ($ecount > 0) {

+         record_soft_failure("The DNF5 subcommands could not be run properly. Check the logs for more info.");

+     }

+ }

+ 

+ sub test_flags {

+     return {always_rollback => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

file added
+62
@@ -0,0 +1,62 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ use dnf;

+ 

+ 

+ # This script tests if information can be display about a package.

+ # https://fedoraproject.org/wiki/QA:Testcase_DNF_info

+ # For this test we will parse a package NEVRA

+ # and then find out whether the package info matches

+ # the package.

+ 

+ sub run {

+     my $self = shift;

+ 

+     # Install the 'mc' package

+     assert_script_run("dnf5 install -y mc");

+ 

+     # Check that info can be listed about the last transaction.

+     my $errors = [];    # This will be used to collect possible errors.

+                         # Get the info output.

+     my $output = script_output("dnf5 info mc");

+     # Get the package name from the system.

+     my $pname = script_output("rpm -q mc");

+ 

+ 

+     my $info = parse_info($output);

+     my $package = parse_package_name($pname);

+ 

+     if ($info->{Name} ne $package->{name}) {

+         push(@$errors, "Package name differs.");

+     }

+     if ($info->{Version} ne $package->{version}) {

+         push(@$errors, "Package version differs.");

+     }

+     if ($info->{Release} ne $package->{release}) {

+         push(@$errors, "Package release differs.");

+     }

+     if ($info->{"Architecture"} ne $package->{arch}) {

+         push(@$errors, "Package architecture differs.");

+     }

+ 

+     # Now, go over the recorded errors and die if you

+     # find any.

+     my $ecount = scalar @$errors;

+     if ($ecount > 0) {

+         diag("DNF5 history info errors:");

+         foreach (@$errors) {

+             diag($_);

+         }

+         die("The info differs from what we expected");

+     }

+ }

+ 

+ sub test_flags {

+     return {always_rollback => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

file added
+119
@@ -0,0 +1,119 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ use dnf;

+ 

+ # This script will make sure that DNF5 is able to

+ # list the packages via dnf list.

+ # http://fedoraproject.org/wiki/QA:Testcase_DNF_list

+ 

+ sub run {

+     my $self = shift;

+ 

+     # First, we will look for available packages. This search should result

+     # in a list of all available packages which is almost 70000 in standard

+     # Fedora repositories.

+     # We will list all those packages and count them via wc, then we will

+     # decide what to do with them.

+     my $count = script_output("dnf5 list --available | wc -l", timeout => 120);

+     # Now, if there are more than 60000 packages, we assume that it is correct.

+     if ($count > 60000) {

+         record_info("Packages counted", "It seems that the number of available packages is realistic ($count)");

+     }

+     elsif ($count > 20000) {

+         record_info("Warning", "The number of available packages seems a little to low. Check that repositories are correctly enabled.");

+     }

+     elsif ($count > 1000) {

+         record_soft_failure("The number of available packages is critically low, check the repositories.");

+     }

+     else {

+         die("The number of available packages does not meet any expectations, either there are no repositories or the tested command does not work correctly.");

+     }

+ 

+     # Also list available packages and look for a certain keyword to see that real packages are listed

+     assert_script_run("dnf5 list --available | grep firefox");

+     assert_script_run("dnf5 list --available | grep vim");

+     assert_script_run("dnf5 list --available | grep bash");

+     assert_script_run("dnf5 list --available | grep python3");

+     assert_script_run("dnf5 list --available | grep gimp");

+ 

+     # Second, we will list installed packages. The Fedora custom installation has slightly above 400

+     # packages, if the count matches, we can assume the command worked correctly.

+     $count = script_output("dnf5 list --installed | wc -l", timeout => 60);

+     if ($count > 400) {

+         record_info("Packages counted", "It seems that the number of listed packages is realistic ($count).");

+     }

+     else {

+         die("The number of installed packages is too low, the tested system might be corrupted.");

+     }

+ 

+ 

+     # Third, we will try to list a single package. If grep finds the name, we can assume that the

+     # package was listed.

+     assert_script_run("dnf5 list inkscape | grep inkscape");

+ 

+     # The --showduplicates switch shows all available packages. The output is divided into Installed

+     # and Available packages. With the kernel package, we know that it is installed for sure.

+     # There will be one installed package and one or more available packages. If more, we know

+     # the command works as expected.

+     my $output = script_output("dnf5 list kernel --showduplicates");

+     my @olist = split("\n", $output);

+     my ($installed, $available) = (0, 0);

+     my $a = 0;

+     # Let's go over the output

+     foreach (@olist) {

+         # Switch to installed if the label found.

+         if ($_ =~ /Installed/) {

+             $a = 0;

+         }

+         # Switch to available if the label found.

+         elsif ($_ =~ /Available/) {

+             $a = 1;

+         }

+         # When package line found, count it based on

+         # which mode we are in.

+         else {

+             if ($_ =~ /kernel/) {

+                 $installed += 1 if ($a == 0);

+                 $available += 1 if ($a == 1);

+             }

+         }

+     }

+     # Now, die under certain conditions

+     if ($installed != 1) {

+         die("Either no kernel or more kernels are installed which indicates an error in the command.");

+     }

+     if ($available == 1) {

+         record_soft_failure("There is only one available kernel package which is not an error, but we could not test if --showduplicates really work either.");

+     }

+     elsif ($available > 1) {

+         record_info("Argument OK", "The --showduplicates argument works normally.");

+     }

+     else {

+         die("There are problems with the --showduplicates argument.");

+     }

+ 

+     # Now, we test that we can list packages with --recent, we will grep for the word updates

+     # because that is the repository where new packages are expected.

+     assert_script_run("dnf5 list --recent | grep updates");

+ 

+     # And test that we can list new updates with --updates, however we will not know if

+     # there will be updates, so just running the command successfully will do.

+     assert_script_run("dnf5 list --updates | grep 'Repositories loaded'");

+ }

+ 

+ sub test_flags {

+     return {always_rollback => 1};

+ }

+ 

+ 

+ my $output = `dnf5 search python3-koza`;

+ print($output);

+ if ($output =~ /No matches found/) {

+     print("Package not found.\n");

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

@@ -0,0 +1,54 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ use dnf;

+ 

+ # This script will make sure that DNF5 is able to

+ # perform repolist and repoquery --info transactions.

+ # https://fedoraproject.org/wiki/QA:Testcase_DNF_repoquery

+ # We will not rely on exact repositories in this test,

+ # but rather if there is some output and how big the

+ # output is. The repositories might change with various

+ # releases and this would break the tests.

+ 

+ 

+ sub run {

+     my $self = shift;

+ 

+     # Let's take the total number of available repositories.

+     my $total = script_output("dnf5 repolist --all | wc -l");

+     # The output has one description line in the beginning that

+     # we should not calculate into the total count of repositories,

+     # therefore:

+     $total -= 1;

+ 

+     # Now let's the number of enabled repositories

+     my $enabled = script_output("dnf5 repolist | wc -l");

+     $enabled -= 1;    # See above

+ 

+     # And finally the number of disabled repositories

+     my $disabled = script_output("dnf5 repolist --disabled | wc -l");

+     $disabled -= 1;    # See above

+ 

+     # It seems logical that enable plus disabled should make all.

+     my $added = $enabled + $disabled;

+     diag("There are $enabled enabled repos and $disabled repos and $total all repos.");

+     if ($added != $total) {

+         die("The addition of enabled and disabled repositories does not match the count of all repos.");

+     }

+ 

+     # Let's also check that at least 'fedora' and 'updates'

+     # are part of enabled repositories.

+     my $output = script_output("dnf5 repolist");

+     my @repos = qw(fedora updates);

+     confirm_in_output($output, \@repos);

+ }

+ 

+ sub test_flags {

+     return {always_rollback => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

@@ -0,0 +1,57 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ use dnf;

+ 

+ # This script will make sure that DNF5 is able to

+ # perform several repoquery searches.

+ # https://fedoraproject.org/wiki/QA:Testcase_DNF_repoquery

+ # As of now, the test is based on the examples of

+ # the repoquery command stated in the DNF documentation

+ # at https://dnf.readthedocs.io/en/latest/command_ref.html#repoquery-command

+ # With DNF5 many options are still not implemented. We can add more

+ # querries and commands as needed later.

+ 

+ sub run {

+     my $self = shift;

+ 

+     # Let's take the total number of available packages in the repos.

+     my $total = script_output("dnf5 repoquery | wc -l", timeout => 60);

+     # The number of packages in default repositories is above 70000,

+     # so if the $total lies there, we could assume that the command

+     # returned something reasonable and thus works correctly:

+     if ($total < 10000) {

+         die("The number of packages is unrealistically low.");

+     }

+     elsif ($total < 70000) {

+         record_soft_failure("The number of packages is lower than we expected (70000).");

+     }

+ 

+     # Let's check the repoquery can be limited with a search pattern.

+     my $output = script_output("dnf5 repoquery 'light*'", timeout => 60);

+     confirm_in_lines($output, "light");

+     $output = script_output("dnf5 repoquery '*.noarch'", timeout => 60);

+     confirm_in_lines($output, "noarch");

+ 

+     # Let's check the repoquery can show what a meta package provides.

+     $output = script_output("dnf repoquery --whatprovides webserver", timeout => 60);

+     my @keywords = qw(caddy httpd lighttpd nginx);

+     # Check that correct packages are returned.

+     confirm_in_output($output, \@keywords);

+ 

+     # Let's check the repoquery can show what a package requires.

+     $output = script_output("dnf repoquery --requires lighttpd", timeout => 60);

+     @keywords = qw(libcrypt libnettle libuuid lighttpd-filesystems systemd);

+     # Check that correct packages are returned.

+     confirm_in_output($output, \@keywords);

+ 

+ }

+ 

+ sub test_flags {

+     return {always_rollback => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

file added
+29
@@ -0,0 +1,29 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ use dnf;

+ 

+ # This script will make sure that DNF5 is able to

+ # search repositories and get package info.

+ # https://fedoraproject.org/wiki/QA:Testcase_DNF_search

+ 

+ sub run {

+     my $self = shift;

+ 

+     # We will search for the existing package, python3-tkinter

+     # might be a safe option as python is always ready on Fedora.

+     #

+     assert_script_run("dnf5 search python3-tkinter | grep tkinter", timeout => 60);

+     # The third check checks that "No matches found" string is shown

+     # when we attempt to look for a non-existing package.

+     assert_script_run("dnf5 search jezibaba | grep 'No matches found'", timeout => 60);

+ }

+ 

+ sub test_flags {

+     return {always_rollback => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

file added
+45
@@ -0,0 +1,45 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ use dnf;

+ 

+ # This script will make sure that DNF5 is able to

+ # swap (remove and install) two packages that are

+ # conflicting with each other, so we can install

+ # and remove them in one single command.

+ 

+ sub run {

+     my $self = shift;

+ 

+     # Install one package

+     assert_script_run("dnf5 install -y i3", timeout => 180);

+ 

+     # Check that the package is installed

+     assert_script_run("rpm -q i3");

+ 

+     # The I3 package requires the `i3-config` or `i3-config-fedora`

+     # to be installed. By default the first one should be installed.

+     # Let's check

+     assert_script_run("rpm -q i3-config");

+ 

+     # Try to install the `i3-config-fedora` package ->

+     # this time the operation should fail because the

+     # packages conflict with each other.

+     assert_script_run("! dnf5 install -y i3-config-fedora");

+ 

+     # Swap packages. The operation should run successfully.

+     assert_script_run("dnf5 swap -y i3-config i3-config-fedora", timeout => 180);

+ 

+     # Check that the first package was removed and the second installed.

+     assert_script_run("! rpm -q i3-config");

+     assert_script_run("rpm -q i3-config-fedora");

+ }

+ 

+ sub test_flags {

+     return {always_rollback => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

@@ -0,0 +1,35 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ use packagetest;

+ use dnf;

+ 

+ # This script will make sure that DNF5 is able to

+ # upgrade the system when it has some older packages.

+ # https://fedoraproject.org/wiki/QA:Testcase_DNF_basic_update_packages

+ #

+ # This is an adaptation from base_update_cli.

+ 

+ 

+ sub run {

+     my $self = shift;

+ 

+     # Install and verify an outdated package.

+     prepare_test_packages;

+     verify_installed_packages;

+ 

+     # Upgrade system

+     assert_script_run("dnf5 upgrade -y pandoc-common", timeout => 120);

+ 

+     # Check the versions.

+     verify_updated_packages;

+ }

+ 

+ sub test_flags {

+     return {always_rollback => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

DNF5 is a major upcoming change for the future Fedora. It is wise if we started to run tests as soon as possible, so that we could see if anything breaks during the transition process and in the early adoption days.

This test suite is based on the test cases of the DNF5 test day and performs the same (or similar steps) to test the DNF5. Some of the tests from that test day, however, have not been implemented yet, because the commands are not supported by the current version of DNF5.

Only the commands, that are at least partly supported, became part of this suite.

rebased onto d607e29d9aeee0d95de5ee974a38afea7f5c8adb

2 years ago

rebased onto e41ee8750ebb989e3c47e61c8ba7c64de6f0d4db

2 years ago

this seems overly complex. you can just do if (script_run("rpm -q dnf5")) here; if that is true, it means dnf5 is not installed (because rpm -q returns 1 when the package is not installed), if it's false it means the package is installed (rpm -q returns 0 when the package is installed).

we may want to give this a longer timeout; dnf operations occasionally hit a slow server for metadata download or whatever, which can make them take a while.

see above note, just use the return code of rpm -q $package instead.

again may need a longer timeout.

and again, use return codes.

Uh, doesn't this result in $lower being -1 if version is Rawhide?

this feels like it'd be prone to failing or causing pretty extensive package changes if dependencies were too far out of whack across releases...maybe we should use dummy packages here, like we already do for various other package tests?

this seems like a stray line?

in fact, uh, this entire function appears to be unused?

this timeout feels maybe low?

well, the test is called repoquery.pm, and it has this comment, but it does not actually test repoquery at all?

A more realistic test would be to swap two alternate providers of a provide that is required by the system. That's what dnf swap is really for - making that operation possible without using rpm -e --nodeps or a dnf shell.

This test seems to basically duplicate base_update_cli.pm, only using a real package not a dummy one; using a dummy package is more reliable. It would seem to make more sense to just use base_update_cli here, and tweak it slightly so it can run dnf5 instead of dnf when desired.

Note that test specifies the package to update, rather than just using dnf upgrade -y. This is intentional, because we don't know what other upgrades may be available, there may be dozens, any one of which might be broken, which would cause the test to fail when it shouldn't (as dnf isn't broken).

Also note the wiki test case itself is basically a duplicate of https://fedoraproject.org/wiki/QA:Testcase_base_update_cli ...

1 new commit added

  • Try giving aarch64 VMs more RAM to see if it helps
2 years ago

rebased onto ddc5131f42418d3e01f68dd46ec4c6ede1892338

2 years ago

1 new commit added

  • Fix according to the review. Part 1.
2 years ago

2 new commits added

  • Fix according to the review. Part 1.
  • Create test suite to test DNF5.
2 years ago

1 new commit added

  • Update upgrade.pm after review.
2 years ago

1 new commit added

  • Fix typo
2 years ago

1 new commit added

  • Increase timeout
2 years ago

1 new commit added

  • Begin work on info.pm
2 years ago

1 new commit added

  • Fix timeout keywords.
2 years ago

1 new commit added

  • Update info.pm
2 years ago

1 new commit added

  • Fix info.pm!
2 years ago

rebased onto 8e3c1fd03d968d3f6840b50df6b3b319bdd1f6df

2 years ago

1 new commit added

  • Add repoquery and change to previous script to repolist.
2 years ago

1 new commit added

  • Update to use library.
2 years ago

1 new commit added

  • Add library
2 years ago

1 new commit added

  • Add repolist.
2 years ago

1 new commit added

  • Tidy scripts.
2 years ago

rebased onto 2dad3cdb1a526b7b5726bb715080f1285c95aa11

2 years ago

I tried to update the suite according to the information in review:

  • I prolonged the time outs.
  • I changed the if statements to use the script_run directly, where indicated.
  • I updated the upgrade.pm to mimic the basic_update_cli
  • I moved some of the code into the lib/dnf.pm library
  • I renamed repoquery.pm into repolist.pm because it is what it did and created a new repoquery,pm to test the basic queries.
  • I fixed the info.pm which somehow remained unfinished

rebased onto e8be61b9545e2732de0c6030f4dd086e21be4803

2 years ago

rebased onto 9ece3b8d4cb3e427815e453ee88ea6030992f4ca

2 years ago

rebased onto 0c4cb69c747a08966b96a2b53beaad279485c2ac

2 years ago

rebased onto ea0e2c0199d8a47f6d03662eb11f32cb21829ede

2 years ago

So this failed on Rawhide today: https://openqa.stg.fedoraproject.org/tests/2873527#step/distro_sync/14 . I think the immediate problem is that Rawhide does not have the repos for non-Rawhide releases enabled at all (so you can't easily distro-sync).

The repoquery test also softfailed because it had just under 70k packages.

So this failed on Rawhide today: https://openqa.stg.fedoraproject.org/tests/2873527#step/distro_sync/14 . I think the immediate problem is that Rawhide does not have the repos for non-Rawhide releases enabled at all (so you can't easily distro-sync).

Yeah, the distro-sync will be a little tough, because I could easily obtain fake packages from the test repo, how do I obtain a newer fake package? Do you think a super new package could be made so that it "always" remains newer than the normal repo packages?

The repoquery test also softfailed because it had just under 70k packages.

Well, I actually assumed that this will not be a problem that comes often, because I did not expect that the number of repo packages will vary, unless there is a package build problem or something like that ... and in that case a softfail warning could come handy. When I wrote the numbers, I tested with "current" repos and there were about 78000 packages, so the 70000 is in reality far less to expect. However, we can lower the number to make it softfail when even less packages are there in the repository.

Or do you have any other idea how we could test that the repoquery command actually works on everyday repositories? Or shall we move more packages into the testing repository and only do repoquery on that one?

The repoquery test also softfailed because it had just under 70k packages.

Well, I actually assumed that this will not be a problem that comes often, because I did not expect that the number of repo packages will vary, unless there is a package build problem or something like that ... and in that case a softfail warning could come handy. When I wrote the numbers, I tested with "current" repos and there were about 78000 packages, so the 70000 is in reality far less to expect. However, we can lower the number to make it softfail when even less packages are there in the repository.

Or do you have any other idea how we could test that the repoquery command actually works on everyday repositories? Or shall we move more packages into the testing repository and only do repoquery on that one?

So this failed on Rawhide today: https://openqa.stg.fedoraproject.org/tests/2873527#step/distro_sync/14 . I think the immediate problem is that Rawhide does not have the repos for non-Rawhide releases enabled at all (so you can't easily distro-sync).

Yeah, the distro-sync will be a little tough, because I could easily obtain fake packages from the test repo, how do I obtain a newer fake package? Do you think a super new package could be made so that it "always" remains newer than the normal repo packages?

Well, you could just give it a very high epoch (99999, say). It is going to be a bit more work to implement distro-sync with dummy packages/repos, though, I guess, as we'd need to set up all the appropriately-versioned repos...

The repoquery test also softfailed because it had just under 70k packages.

Well, I actually assumed that this will not be a problem that comes often, because I did not expect that the number of repo packages will vary, unless there is a package build problem or something like that ... and in that case a softfail warning could come handy. When I wrote the numbers, I tested with "current" repos and there were about 78000 packages, so the 70000 is in reality far less to expect. However, we can lower the number to make it softfail when even less packages are there in the repository.

Or do you have any other idea how we could test that the repoquery command actually works on everyday repositories? Or shall we move more packages into the testing repository and only do repoquery on that one?

Did you test with Rawhide? Maybe Rawhide returns a lower number for some reason (no updates-testing repo)?

rebased onto ca807fa7b9ddf93729adabe0d75d5e6df02f840e

2 years ago

rebased onto e26af0b6901d4debe67c8127b7406be1694e70f7

2 years ago

Build failed. More information on how to proceed and troubleshoot errors available at https://fedoraproject.org/wiki/Zuul-based-ci
https://fedora.softwarefactory-project.io/zuul/buildset/abb74ac7e14e45c6991c2f31172c828d

rebased onto 72ed01ffe008b44021caf87077daf218ac7f36f5

2 years ago

another failure on Rawhide today: https://openqa.stg.fedoraproject.org/tests/2896120#step/list/25

command 'dnf5 list --recent | grep updates' failed at /usr/lib/os-autoinst/autotest.pm line 387.

rebased onto 7afe926acf03d94890a2f276fcf60ab4f3ac7a4a

2 years ago

rebased onto 75ee73d645186b12a5263143ed35ec7fef2c383f

2 years ago

rebased onto 46f5300d262f2180ebf1e547924ef92141b7a614

2 years ago

rebased onto 957fd23c1f2a4a01f9d229f97ccecef0a8a65f74

2 years ago

rebased onto 33dc3c5c962f84c8c71078b10b851f9742accc9e

2 years ago

rebased onto 67e81e7ccc1c592d1a71efae4c6c53ca20a9abc5

2 years ago

rebased onto b006786794695b4375e65f72abea952646335f28

2 years ago

rebased onto e8bd9e5786c7343175dd96fe4bd2239e01e59c40

2 years ago

rebased onto 42594517843b83356a658473d9aa71bece3e05f6

2 years ago

rebased onto 1da4aa7a65185432b90651ed36829473e84135ef

2 years ago

rebased onto 395c994ee4890bc6e95603eb4c8fe512c2b5ca10

2 years ago

rebased onto 3db849437becd2c8d675b5ace46b21f77557302d

2 years ago

rebased onto 148d2cfb2793a17e753367652ecd32e484511ba4

2 years ago

rebased onto 4ac92404ea1138e1db838589b361d2c42d8f2da9

2 years ago

rebased onto 8c7cd8413c65ad217bbe50b454d8f31aada51b2d

2 years ago

rebased onto 3565bee9b4db13fccf0c32fb304e070b65328099

2 years ago

rebased onto 4d57a62bc06927786c890f64d1a3bcf2b0bced3d

2 years ago

rebased onto 21d0c18ffb444949b06889d60895fe78336c426b

2 years ago

rebased onto 26befc3

2 years ago

rebased onto 725c9b6

5 months ago

Pull-Request has been closed by lruzicka

5 months ago