#4205 Test the dev-data.py utility script in the unit-tests
Merged 5 years ago by pingou. Opened 5 years ago by pingou.

file modified
+66 -59
@@ -34,34 +34,16 @@ 

  _config = pagure.config.reload_config()

  

  

- def init_database():

-     DB_URL = _config['DB_URL']

- 

-     # create the table if it doesnt exist

-     pagure.lib.model.create_tables(

-         DB_URL,

-         _config.get('PATH_ALEMBIC_INI', None),

-         acls=_config.get('ACLS', {}),

-         debug=True)

- 

-     engine = create_engine('%s' % DB_URL, echo=True)

- 

-     metadata = MetaData(engine)

-     metadata.reflect(bind=engine)

-     return engine, metadata

- 

- 

- def empty_dev_db(metadata, engine):

+ def empty_dev_db(session):

      print('')

-     print('')

-     print('WARNING: Deleting all data from ', _config['DB_URL'])

-     # Dangerous: this will wipe the data from the table but keep the schema

-     print('')

-     response = six.moves.input('Do you want to continue? (yes/no)    ')

-     if 'yes'.startswith(response.lower()):

-         for tbl in reversed(metadata.sorted_tables):

-             if tbl.fullname != 'acls':

-                 engine.execute(tbl.delete())

+     print('WARNING: Deleting all data from', _config['DB_URL'])

+     response = os.environ.get("FORCE_DELETE")

+     if not response:

+         response = six.moves.input('Do you want to continue? (yes/no)    ')

+     if response.lower().startswith('y'):

+         tables = reversed(pagure.lib.model_base.BASE.metadata.sorted_tables)

+         for tbl in tables:

+             session.execute(tbl.delete())

      else:

          exit("Aborting.")

  
@@ -70,10 +52,6 @@ 

      _config['EMAIL_SEND'] = False

      _config['TESTING'] = True

  

-     # Populate with default statuses

-     create_default_status(session)

-     print('Default statuses populated')

- 

      ######################################

      # tags

      item = pagure.lib.model.Tag(
@@ -430,7 +408,7 @@ 

      session.commit()

  

  

- #####################################

+     #####################################

      # tokens

      tests.create_tokens(session, user_id=pingou.id, project_id=project1.id)

  
@@ -668,6 +646,46 @@ 

      shutil.rmtree(newfolder)

  

  

+ def _get_username():

+     invalid_option = ['pingou', 'foo']

+     user_name = os.environ.get("USER_NAME")

+     if not user_name:

+         print("")

+         user_name = six.moves.input(

+             "Enter your username so we can add you into the test data:  ")

+     cnt = 0

+     while not user_name.strip() or user_name in invalid_option:

+         print("Reserved names: " + str(invalid_option))

+         user_name = six.moves.input(

+             "Enter your username so we can add you into the "

+             "test data:  ")

+         cnt += 1

+         if cnt == 4:

+             print("We asked too many times, bailing")

+             sys.exit(1)

+ 

+     return user_name

+ 

+ 

+ def _get_user_email():

+     invalid_option = ['bar@pingou.com', 'foo@bar.com']

+     user_email = os.environ.get("USER_EMAIL")

+     if not user_email:

+         print("")

+         user_email = six.moves.input("Enter your user email:  ")

+ 

+     cnt = 0

+     while not user_email.strip() or user_email in invalid_option:

+         print("Reserved names: " + str(invalid_option))

+         user_email = six.moves.input("Enter your user email:  ")

+         cnt += 1

+         if cnt == 4:

+             print("We asked too many times, bailing")

+             sys.exit(1)

+ 

+     return user_email

+ 

+ 

  if __name__ == "__main__":

      desc = "Run the dev database initialization/insertion/deletion " \

             "script for db located  " + str(_config['DB_URL'])
@@ -679,7 +697,7 @@ 

      parser.add_argument('-d', '--delete', action="store_true",

                          help="Wipe the dev db")

      parser.add_argument('-a', '--all', action="store_true",

-                         help="Create, Wipe, Populate the dev db")

+                         help="Create, Populate then Wipe the dev db")

  

      args = parser.parse_args()

  
@@ -687,34 +705,23 @@ 

      if not any(vars(args).values()):

          parser.error('No arguments provided.')

  

-     if args.init or args.delete or args.all:

-         eng, meta = init_database()

+     session = None

  

-     if args.delete or args.all:

-         empty_dev_db(meta, eng)

+     if args.init or args.all:

+         session = pagure.lib.model.create_tables(

+             db_url=_config["DB_URL"],

+             alembic_ini=None,

+             acls=_config["ACLS"],

+             debug=False)

+         print("Database created")

  

      if args.populate or args.all:

-         session = pagure.lib.model_base.create_session(_config['DB_URL'])

-         invalid_option = ['pingou', 'bar@pingou.com', 'foo', 'foo@bar.com']

-         print("")

-         user_name = six.moves.input(

-             "Enter your username so we can add you into the test data:  ")

-         while user_name in invalid_option:

-             print("Reserved names: " + str(invalid_option))

-             user_name = six.moves.input(

-                 "Enter your username so we can add you into the test data:  ")

- 

-         if not user_name.replace(" ", ""):

-             user_name = 'pythagoras'

- 

-         print("")

-         user_email = six.moves.input("Enter your user email:  ")

- 

-         while user_email in invalid_option:

-             print("Reserved names: " + str(invalid_option))

-             user_email = six.moves.input("Enter your user email:  ")

- 

-         if not user_email.replace(" ", ""):

-             user_email = 'pythagoras@math.com'

+         if not session:

+             session = pagure.lib.query.create_session(_config['DB_URL'])

  

+         user_name = _get_username()

+         user_email = _get_user_email()

          insert_data(session, user_name, user_email)

+ 

+     if args.delete or args.all:

+         empty_dev_db(session)

file modified
+4 -2
@@ -373,7 +373,8 @@ 

  

          # Write a config file

          config_values = {

-             'path': self.path, 'dburl': self.dbpath,

+             'path': self.path,

+             'dburl': self.dbpath,

              'enable_docs': True,

              'docs_folder': '%s/repos/docs' % self.path,

              'enable_tickets': True,
@@ -389,10 +390,11 @@ 

              'repospanner_admin_migration': 'False',

          }

          config_values.update(self.config_values)

+         self.config_values = config_values

          config_path = os.path.join(self.path, 'config')

          if not os.path.exists(config_path):

              with open(config_path, 'w') as f:

-                 f.write(CONFIG_TEMPLATE % config_values)

+                 f.write(CONFIG_TEMPLATE % self.config_values)

          os.environ["PAGURE_CONFIG"] = config_path

          pagure_config.update(reload_config())

  

file added
+144
@@ -0,0 +1,144 @@ 

+ # -*- coding: utf-8 -*-

+ 

+ """

+  (c) 2019 - Copyright Red Hat Inc

+ 

+  Authors:

+    Pierre-Yves Chibon <pingou@pingoured.fr>

+ 

+ """

+ 

+ from __future__ import unicode_literals, absolute_import

+ 

+ import os

+ import subprocess

+ import sys

+ import unittest

+ 

+ import six

+ 

+ 

+ REPO_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))

+ 

+ import tests  # noqa

+ 

+ 

+ class TestDevData(tests.Modeltests):

+     """This test class contains tests pertaining to the dev-data utility

+     script."""

+ 

+     maxDiff = None

+ 

+     def test_dev_data_all(self):

+         """Check how dev-data --all performs

+ 

+         """

+ 

+         config_path = os.path.join(self.path, "config")

+         with open(config_path, "w") as f:

+             f.write("DB_URL = 'sqlite:///%s/db_dev_data.sqlite'\n" % self.path)

+             f.write("GIT_FOLDER = '%s/repos'\n" % self.path)

+             f.write(

+                 "BROKER_URL = 'redis+socket://%(global_path)s/broker'\n" % \

+                     self.config_values)

+             f.write("CELERY_CONFIG = {'task_always_eager': True}\n")

+ 

+         env = {

+             "USER_NAME": "testuser",

+             "USER_EMAIL": "testuser@example.com",

+             "FORCE_DELETE": "yes",

+             "PAGURE_CONFIG": config_path,

+         }

+         proc1 = subprocess.Popen(

+             [sys.executable, "dev-data.py", "--all"],

+             cwd=REPO_PATH,

+             stdout=subprocess.PIPE,

+             env=env,

+         )

+         stdout, stderr = proc1.communicate()

+         if isinstance(stdout, six.binary_type):

+             stdout = stdout.decode("utf-8")

+         output = (

+             """Database created

+ User created: pingou <bar@pingou.com>, testing123

+ User created: foo <foo@bar.com>, testing123

+ User created: testuser <testuser@example.com>, testing123

+ Created "admin" group. Pingou is a member.

+ Created "group" group. Pingou is a member.

+ Created "rel-eng" group. Pingou is a member.

+ git folder already deleted

+ docs folder already deleted

+ tickets folder already deleted

+ requests folder already deleted

+ 

+ WARNING: Deleting all data from sqlite:///%s/db_dev_data.sqlite

+ """

+             % self.path

+         )

+ 

+         self.assertEqual(len(stdout.split("\n")), 14)

+         self.assertEqual(stdout, output)

+ 

+     def test_dev_data_delete(self):

+         """Check how dev-data --init --delete performs

+ 

+         """

+ 

+         config_path = os.path.join(self.path, "config")

+ 

+         env = {

+             "USER_NAME": "testuser",

+             "USER_EMAIL": "testuser@example.com",

+             "FORCE_DELETE": "yes",

+             "PAGURE_CONFIG": config_path,

+         }

+         proc1 = subprocess.Popen(

+             [sys.executable, "dev-data.py", "--init", "--delete"],

+             cwd=REPO_PATH,

+             stdout=subprocess.PIPE,

+             env=env,

+         )

+         stdout, stderr = proc1.communicate()

+         if isinstance(stdout, six.binary_type):

+             stdout = stdout.decode("utf-8")

+         output = (

+             """Database created

+ 

+ WARNING: Deleting all data from %s

+ """

+             % self.dbpath

+         )

+ 

+         self.assertEqual(len(stdout.split("\n")), 4)

+         self.assertEqual(stdout.split("\n"), output.split("\n"))

+ 

+     def test_dev_data_init(self):

+         """Check how dev-data --init performs

+ 

+         """

+ 

+         config_path = os.path.join(self.path, "config")

+ 

+         env = {

+             "USER_NAME": "testuser",

+             "USER_EMAIL": "testuser@example.com",

+             "FORCE_DELETE": "yes",

+             "PAGURE_CONFIG": config_path,

+         }

+         proc1 = subprocess.Popen(

+             [sys.executable, "dev-data.py", "--init"],

+             cwd=REPO_PATH,

+             stdout=subprocess.PIPE,

+             env=env,

+         )

+         stdout, stderr = proc1.communicate()

+         if isinstance(stdout, six.binary_type):

+             stdout = stdout.decode("utf-8")

+         output = "Database created\n"

+ 

+         self.assertEqual(len(stdout.split("\n")), 2)

+         self.assertEqual(stdout.split("\n"), output.split("\n"))

+ 

+ 

+ if __name__ == "__main__":

+     unittest.main(verbosity=2)

no initial comment

pretty please pagure-ci rebuild

5 years ago

1 new commit added

  • Black fix
5 years ago

Jenkins is being most annoying here

@bstinson if you have a minute, if seem jenkins isn't exporting the actual output files from the run. If you check the failed tests in the shell output the failed test is py-test_dev_data but looking at the file exported in the artifact: https://ci.centos.org/job/pagure-pr/2094/artifact/pagure/results-py3/newfailed/*view*/ https://ci.centos.org/job/pagure-pr/2094/artifact/pagure/results-py2/newfailed/*view*/ neither of them show that this test failed and in both cases the exported files do not contain the output from this test.

rebased onto 45c4a3215fd63001f8530e614b9a73689411a433

5 years ago

rebased onto d0470f93162aaf7897017e9d5a4db3b491bc7147

5 years ago

pretty please pagure-ci rebuild

5 years ago

rebased onto 48344e8b22628a56c3cc214ecaff426d8495fa3e

5 years ago

1 new commit added

  • Small black fixes
5 years ago

rebased onto 0678534

5 years ago

2 new commits added

  • Fix getting the dev-data tests to run
  • Fix keeping in memory the config values to be accessible for all tests
5 years ago

5 new commits added

  • Fix getting the dev-data tests to run
  • Fix keeping in memory the config values to be accessible for all tests
  • Small code formatting change
  • Add unit-tests checking the behavior of the dev-data.py utility script
  • Small improvements to the dev-data.py script
5 years ago

The failing test is the one that fails everywhere, this PR is finally ready to go :)

Thanks for the review :)

Pull-Request has been merged by pingou

5 years ago