#305 Create a test for basic Flatpak operations.
Merged a year ago by adamwill. Opened a year ago by lruzicka.

file modified
+22 -5
@@ -307,6 +307,14 @@ 

              assert_script_run "echo SCAN ${user}-finger-1 | socat STDIN UNIX-CONNECT:/run/fprintd-virt";

              send_key "ctrl-alt-f${tty}";

          }

+         elsif (match_has_tag "auth_required_locked") {

+             # When console operation takes a long time, the screen locks

+             # and typing password fails. If that happens, unlock

+             # the screen first and then type password.

+             send_key("ret");

+             wait_still_screen(2);

+             type_very_safely "weakpassword\n";

+         }

          else {

              # bit sloppy but in all cases where this is used, this is the

              # correct password
@@ -1540,6 +1548,9 @@ 

  # The data repository is located at https://pagure.io/fedora-qa/openqa_testdata.

  

  sub download_testdata {

+     # We can select which Data to copy over.

+     my $data = shift;

+     $data = 'structure' unless ($data);

      # Navigate to the user's home directory

      my $user = get_var("USER_LOGIN") // "test";

      assert_script_run("cd /home/$user/");
@@ -1551,11 +1562,17 @@ 

      # Untar it.

      assert_script_run("tar -zxvf repository.tar.gz");

      # Copy out the files into the VMs directory structure.

-     assert_script_run("cp music/* /home/$user/Music");

-     assert_script_run("cp documents/* /home/$user/Documents");

-     assert_script_run("cp pictures/* /home/$user/Pictures");

-     assert_script_run("cp video/* /home/$user/Videos");

-     assert_script_run("cp reference/* /home/$user/");

+     if ($data eq "structure") {

+         assert_script_run("cp music/* /home/$user/Music");

+         assert_script_run("cp documents/* /home/$user/Documents");

+         assert_script_run("cp pictures/* /home/$user/Pictures");

+         assert_script_run("cp video/* /home/$user/Videos");

+         assert_script_run("cp reference/* /home/$user/");

+     }

+     else {

+         assert_script_run("mkdir /home/$user/$data");

+         assert_script_run("cp $data/* /home/$user/$data/");

+     }

      # Delete the temporary directory and the downloaded file.

      assert_script_run("cd");

      assert_script_run("rm -rf /home/$user/temp");

@@ -0,0 +1,15 @@ 

+ {

+   "area": [

+     {

+       "xpos": 814,

+       "ypos": 43,

+       "width": 79,

+       "height": 99,

+       "type": "match"

+     }

+   ],

+   "properties": [],

+   "tags": [

+     "apps_run_focuswriter"

+   ]

+ } 

\ No newline at end of file

empty or binary file added
@@ -0,0 +1,15 @@ 

+ {

+   "area": [

+     {

+       "xpos": 446,

+       "ypos": 119,

+       "width": 129,

+       "height": 19,

+       "type": "match"

+     }

+   ],

+   "properties": [],

+   "tags": [

+     "apps_run_gvim"

+   ]

+ } 

\ No newline at end of file

empty or binary file added
@@ -0,0 +1,16 @@ 

+ {

+   "area": [

+     {

+       "type": "match",

+       "xpos": 415,

+       "ypos": 491,

+       "width": 194,

+       "height": 34

+     }

+   ],

+   "properties": [],

+   "tags": [

+     "auth_required_locked",

+     "auth_required"

+   ]

+ }

empty or binary file added
file modified
+12
@@ -1154,6 +1154,18 @@ 

                  "START_AFTER_TEST": "%DEPLOY_UPLOAD_TEST%"

              }

          },

+         "flatpak": {

+             "profiles": {

+                 "fedora-Silverblue-dvd_ostree-iso-ppc64le-*-ppc64le": 50,

+                 "fedora-Silverblue-dvd_ostree-iso-x86_64-*-64bit": 50

+             },

+             "settings": {

+                 "BOOTFROM": "c",

+                 "HDD_1": "disk_%FLAVOR%_%MACHINE%.qcow2",

+                 "POSTINSTALL": "flatpak_fedora flatpak_flathub flatpak_update",

+                 "START_AFTER_TEST": "%DEPLOY_UPLOAD_TEST%"

+             }

+         },

          "gnome-panel": {

              "profiles": {

                  "fedora-Silverblue-dvd_ostree-iso-ppc64le-*-ppc64le": 50,

@@ -0,0 +1,55 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ 

+ # This script tests that the flatpak technology is correctly set up

+ # and that it can be used without tweaking any settings or installing

+ # any packages on the system.

+ 

+ sub run {

+ 

+     my $self = shift;

+     $self->root_console(tty => 3);

+ 

+     # Check that that Flatpak is installed on the System.

+     # If the following command succeeds, we can assume that Flatpak is installed.

+     assert_script_run("flatpak --version");

+ 

+     # Check that at least Fedora remote repository is properly configured

+     validate_script_output("flatpak remotes", sub { m/fedora/ });

+ 

+     # Check that an application exists in the repository

+     validate_script_output("flatpak search gvim", sub { m/org.vim.Vim/ });

+ 

+     # Check that the application can be installed.

+     assert_script_run("flatpak -y install org.vim.Vim", timeout => 720);

+ 

+     # Check that it is listed as installed

+     assert_script_run("flatpak list | grep GVim");

+ 

+     # Now, we will switch into the Desktop and we will try to run the application

+     desktop_vt();

+     wait_still_screen(3);

+     menu_launch_type("gvim");

+     assert_screen("apps_run_gvim");

+     # Switch off the application

+     type_very_safely(":qa\n");

+ 

+     # We will switch to the CLI again

+     $self->root_console(tty => 3);

+ 

+     ## Now, we will remove the application again.

+     assert_script_run("flatpak -y remove org.vim.Vim", timeout => 240);

+ 

+     # Check that it the application is not listed among installed any more.

+     assert_script_run("! flatpak list | grep GVim");

+ }

+ 

+ sub test_flags {

+     return {fatal => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

@@ -0,0 +1,55 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ 

+ # This script tests that the Flathub repository can be added and that applications

+ # from that repository can be installed.

+ 

+ sub run {

+ 

+     my $self = shift;

+     $self->root_console(tty => 3);

+ 

+     # On Silverblue, Flathub is not set as a Flatpak remote by default, only when Third Party Repos

+     # are enabled. To make sure, we have it enabled, we will use the following command to

+     # add the Flathub repository.

+     assert_script_run("sudo flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo");

+ 

+     # Check that the Flathub repository has been added into the repositories.

+     validate_script_output("flatpak remotes", sub { m/flathub/ });

+ 

+     # Now, we can search for an application that only exists in Flathub.

+     validate_script_output("flatpak search focuswriter", sub { m/org.gottcode.FocusWriter/ });

+ 

+     # And we can install it

+     assert_script_run("flatpak install -y org.gottcode.FocusWriter", timeout => 600);

+ 

+     # Check that now the application is listed in the installed flatpaks.

+     assert_script_run("flatpak list | grep org.gottcode.FocusWriter");

+ 

+ 

+     # Switch to desktop and try to run the application.

+     desktop_vt();

+     wait_still_screen(3);

+     menu_launch_type("focuswriter");

+     # Check that it started

+     assert_screen("apps_run_focuswriter");

+     # Stop the application

+     send_key("alt-f4");

+ 

+     # Switch to console again.

+     $self->root_console(tty => 3);

+ 

+     # Now, remove the package and test that it is not listed.

+     assert_script_run("flatpak remove -y org.gottcode.FocusWriter");

+     assert_script_run("! flatpak list | grep org.gottcode.FocusWriter");

+ }

+ 

+ sub test_flags {

+     return {fatal => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

@@ -0,0 +1,53 @@ 

+ use base "installedtest";

+ use strict;

+ use testapi;

+ use utils;

+ 

+ # This script tests that we can update (and downgrade) flatpaks. We will

+ # use a test repository with a Dummy application.

+ 

+ sub run {

+ 

+     my $self = shift;

+     $self->root_console(tty => 3);

+     # We will need Flathub to pull dependencies.

+     # Flathub is not set as a Flatpak remote by default, only when Third Party Repos

+     # are enabled. To make sure, we have it enabled, we will use the following command to

+     # add the Flathub repository.

+     assert_script_run("sudo flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo", timeout => 120);

I think as things stand, we don't need this? Each module is set as 'fatal', we don't use rollbacks, and this module runs after flatpak_flathub. So we shouldn't be able to reach here with the remote not present, right?

+     # We will also add the test repository.

+     assert_script_run("flatpak remote-add --if-not-exists flatpaktest https://lruzicka.fedorapeople.org/flatpaktest/flatpaktest.flatpakrepo", timeout => 120);

+ 

+     # Install the Dummy application.

+     assert_script_run("flatpak install -y org.flatpak.Dummy", timeout => 600);

+     # Check that the application has been installed

+     assert_script_run("flatpak list | grep org.flatpak.Dummy");

+ 

+     # The application is installed in version 2. Let's check the output.

+     validate_script_output("flatpak run org.flatpak.Dummy", sub { m/Dummy flatpak: version 2/ });

+ 

+     # Now, we will attempt to downgrade the application to force the previous commit

+     assert_script_run("flatpak update -y --commit=37be70fa26aa652379f968a7aaf7b63fa515483b9381756cd371c8174ae68626 org.flatpak.Dummy");

+ 

+     # If that was successful, the output of the application will show version 1.

+     validate_script_output("flatpak run org.flatpak.Dummy", sub { m/Dummy flatpak: version 1/ });

+ 

+     # Now we can update the application again, using the standard command, which will update to the

+     # newest version (version 2) of the application again.

+     assert_script_run("flatpak update -y org.flatpak.Dummy");

+     # Let's check the application now provides the correct output.

+     validate_script_output("flatpak run org.flatpak.Dummy", sub { m/Dummy flatpak: version 2/ });

+ 

+     # Now, remove the package and test that it is not listed and that it cannot be run.

+     assert_script_run("flatpak remove -y org.flatpak.Dummy");

+     validate_script_output("flatpak list", sub { $_ !~ m/org\.flatpak\.Dummy/ });

+     assert_script_run("! flatpak run org.flatpak.Dummy", sub { $_ !~ m/Dummy flatpak: version 2/ });

+ }

+ 

+ sub test_flags {

+     return {fatal => 1};

+ }

+ 

+ 1;

+ 

+ # vim: set sw=4 et:

This PR tries to respond to issue #294.
On Silverblue, this will try:

  • flatpak install
  • flatpak remote-add
  • flatpak list
  • flatpak remotes
  • flatpak remove
  • flatpak update

and also it tests that a flatpak can be built.

rebased onto e97d0d7b97f5c5969613537f0fc2097257d2fa53

a year ago

thanks for this! I'm, uh, not sure about the approach. My instinct would have been to have the flatpaks and remotes as 'external assets' - external relative to the test, that is - not built in-line. Fedora Flatpaks aren't built this way, so there isn't a lot of value to Fedora in us testing that building Flatpaks this way works (unlike e.g. how we test building lives and netinsts in a very similar way to how the 'real' ones are built). So I was figuring we'd do something like https://pagure.io/fedora-qa/qa-misc/blob/master/f/testpackages - the script that builds the test packages and repo that's used for the package tests - and keep the remotes and flatpaks permanently on fedorapeople, like we do the dummy package repos...

I'm also...not sure if it might be better to just test installing some known-existent-but-not-installed-by-default flatpaks from the official remotes...that would have the drawback of not letting us test updates reliably, but the advantage of using the official configuration and making sure the official remotes are actually usable...hmm...

rebased onto 695ea65eb9e06299357fded6230c49f0bfef3676

a year ago

Hmm, and I was proud of this elegant solution that does not require any external sources. However, I suggest the following:

  • create one test script, that will test if flatpaks can be installed from the Fedora repository and then removed.
  • create another which will set up Flathub and install a flatpak from there, then remove it.
  • leave the existing script to know that a flatpak can be built and updated - I do not think that we need to create an external repository when we will know that two external repositories work already. It will be easier to maintain.

1 new commit added

  • Change test structure.
a year ago

1 new commit added

  • Change name
a year ago

1 new commit added

  • Tidy the script
a year ago

1 new commit added

  • Update the fedora test and add the flathub test.
a year ago

1 new commit added

  • Add needles.
a year ago

1 new commit added

  • Add needles for the second test.
a year ago

Oh no, I agree it's nicely done! I'm just considering the benefits/drawbacks of doing it this way.

The two potential drawbacks I see to building the flatpaks inline every time are:

  1. Time taken. How long does it take for all the 'let's build a flatpak' steps to happen? If we do those inline, we're taking that time every time this test is run. If we keep them externally we really only have to do it once (we only have to redo it if the flatpak format changes, or something), and it doesn't take up time and resources in the test system.
  2. What if it fails? We don't have a "the system must be able to build flatpaks using flatpak-builder from flathub" requirement for Fedora. That's not a critical function. It's not like the live image build test or the netinst build test or the ostree installer image build test, where we imitate how those operations work in releng very closely, and we absolutely do need to know if an update breaks those workflows. If building flatpaks using flatpak-builder is broken in Fedora, that doesn't really endanger any of our key Fedora workflows. So...say it gets broken. That means the rest of the test can't run - we don't know if installing and updating flatpaks works on Fedora, which we do care about. But we can't easily push anyone to hurry up unblocking that, because the thing it's blocked on is not critical.

With the 'build the flatpaks' step inline in the tests, if it ever breaks, the more important part of the test is immediately blocked. With that work done outside of the tests, the tests aren't blocked if building the flatpaks stops working - it'd only be a problem if we needed to rebuild the flatpaks for some reason. And even if we did, we'd have more flexibility to work around the problem somehow (build the flatpaks on some other release or distro, for instance) if they are outside of the test flow.

There are benefits to doing it this way too, of course. It involves fewer moving parts - we don't have to remember "oh, yeah, there's a script that builds those, then we have to remember where to scp them to afterwards..." if we ever do need to fiddle with the flatpaks, like we do with the RPM repos. (I actually did forget about the RPM script and reinvent it once...) And it does let us know if building flatpaks in this way is broken - the value of that isn't zero, it's just probably less than the value of knowing whether installing and updating them works...

Oh, I guess there's another possible concern. Most of the data transfer that happens in openQA tests is within Fedora infra. But this design involves pulling the platform and SDK runtimes from flathub. Are those big? Would flathub maybe get mad if we were pulling them down constantly?

I guess if we add the 'install a real flatpak from flathub' test we'll still be pulling the platform runtime for that test, but at least not the SDK.

Either way, maybe we need to figure out a way to cache the runtime(s) somehow...

1 new commit added

  • Remove flathub verification
a year ago

I see your point. I updated the structure now according to my last comment. I am using the official repositories to install and remove a package and I am also using the original script to build a flatpak, install it, rebuild and update. My observation is:

  • The Fedora flatpak test installs GVim and to be able to do it, it needs to download approximately 900 MB of data from fedora repos.
  • The Flathub test installs FocusWriter and downloads approximately 400 MB of data from Flathub.
  • The building and updating test downloads about 400 MB of data from Flathub.

I am quite surprised, how much that is ... almost 1.8 GB on each test run. However, if we want to test on real data, I do not think we can do much about it.

I was thinking about the possibility to create a dummy repository to hold the dummy flatpaks, but in order to be able to update a package, at first it needs to have one version in the repository and then it needs to have a newer version in that repository which cannot be static as it requires that the version are changed as the test runs. An alternative, I do not know how much valid, would be to have two repositories. The one with the older package and another with a newer one and before attempting to update, the second repository would be added to the system.

How do you put things on fedorapeople? Can't we have the dummy repos on Pagure or something similar?

Transferring data from the Fedora repos shouldn't be an issue - we already have tests doing that alllll the time. I guess we might want to talk to flathub about whether they'd have issues with us doing this kind of transfer load for testing. With the PR as-is at least we'd only be doing it for composes, but it probably would make sense to run this on updates too and then we'd be hitting them dozens of times a day...

An alternative, I do not know how much valid, would be to have two repositories. The one with the older package and another with a newer one and before attempting to update, the second repository would be added to the system.

yeah, this was the approach I was thinking of. I admit I haven't checked if it's possible, though...I guess flatpak might tie flatpaks to repos somehow?

1 new commit added

  • Rewrite the update test using external repository.
a year ago

rebased onto de28ccca66c8d2fabf4c5e7d1c3287e65532c5c1

a year ago

1 new commit added

  • Prolong timeout.
a year ago

1 new commit added

  • Unlock locked screen.
a year ago

1 new commit added

  • Finish the update script.
a year ago

rebased onto 78c9e60ad5e885596508c68d06d898450379c5f0

a year ago

rebased onto 101d99d95aeefe083c81e25ea3fc93aa03461d0d

a year ago

The two repositories suggestion to provide two different version of one application from which one could be able to update to another has proven impossible. It seems that it is only possible to update the application when newer version appears in the same repository from which the application was installed in the scope of one tree.

If the newer version of the application is contained in a different repository, Flatpak is unable to update to it, as it cannot confirm that the version is newer. The only way to update to that version is to remove the original application and to install that anew from the update repository.

When reading through the Flatpak documentation, I realized that one could also downgrade the package using the flatpak update --commit=xxxxx command and I decided to use this in the new version of the update test:

I have created an external repository with two versions of the application and the test:

  • adds the repository
  • installs Version 2 of the application
  • downgrades to Version 1 using the --commit switch.
  • updates the application to Version 2 using flatpak update again.

rebased onto 618479f788ea32653394f6565b124ba06ef0e794

a year ago

rebased onto 56f1616609a4b60063765d3e374f40805ab0f09e

a year ago

I think as things stand, we don't need this? Each module is set as 'fatal', we don't use rollbacks, and this module runs after flatpak_flathub. So we shouldn't be able to reach here with the remote not present, right?

on the whole I think this looks pretty good. However, I think it would be best to try and contact flathub somehow and explain what we're planning to do, and ask if the load is likely to be an issue at all. Then we can put a copy of the response here for the record.

thanks!

rebased onto 3eb298c764ab0f4b7f6992394984d911c1334178

a year ago

rebased onto 5517d3d2a73beb0353a4a755930113fd55fec952

a year ago

rebased onto eba06573fba34623f10ec34a14b506cec4e28f94

a year ago

rebased onto 99aa41478ee3beaf1812c6c642e561b64eda316c

a year ago

rebased onto 41a0fe1c93131e077ac6760a8b9b57fcb06f7042

a year ago

rebased onto 2bee3a3500784fa8826f0ee852992bbded83bcaf

a year ago

rebased onto 318d321f1e5ee8d30fcf54a0cdc894032d9784d1

a year ago

Thanks, I will update this and also notify flathub and will ask them, if they are fine with this running a couple of times a day and will post the reply here as requested.

rebased onto 153845b93deacb962bd02969fb3d33618405784b

a year ago

rebased onto 3a5bca43bce3d609cbbf3ca1edc678c4c763fc51

a year ago

rebased onto 03649bd025514e2bd5b8ed7dbcee54ced38447c5

a year ago

rebased onto d50885e

a year ago

So, today I have received the reply from the Flathub guys saying:

... I think it's fine. Our infrastructure will manage the additional load.
Some considerations below, but don't consider these a blocker.

The considerations they provided is:

  • make the runtimes part of the image installations which basically is not the solution to anything because the data would have to be downloaded anyway, just during another step.
  • Set up a reverse caching proxy on the Fedora Infra according to this page https://dataswamp.org/~solene/2023-04-05-lan-cache-flatpak.html

I believe we have a confirmation to spin off and if needed, we can set up the proxy later as I do not have any idea, where to set it.

OK, so that's fine for running at a compose level. Ideally I'd really want to run this test at an update level but then we'd be running it a lot more than "a couple of times a day", so the reverse proxy would be a very good idea at that point. I'll file an infra ticket for it, the response might end up being "well, you're sysadmin-main, do it yourself" though. :D

Pull-Request has been merged by adamwill

a year ago