Commit a5d5f1b paginate /api/0/projects by default

3 files Authored by karsten 7 days ago , Committed by pingou 7 days ago ,
paginate /api/0/projects by default

On pagure instances with a huge number of projects querying /api/0/projects
could lead to out of memory errors. This change forces paginating the
output.

Signed-off-by: Karsten Hopp <karsten@redhat.com>

    
  1 @@ -344,11 +344,9 @@
  2       |               |          |               |   entrie project JSON    |
  3       |               |          |               |   or just a sub-set      |
  4       +---------------+----------+---------------+--------------------------+
  5 -     | ``page``      | int      | Optional      | | Specifies that         |
  6 -     |               |          |               |   pagination should be   |
  7 -     |               |          |               |   turned on and that     |
  8 -     |               |          |               |   this specific page     |
  9 -     |               |          |               |   should be displayed    |
 10 +     | ``page``      | int      | Optional      | | Specifies which        |
 11 +     |               |          |               |   page to return         |
 12 +     |               |          |               |   (defaults to: 1)       |
 13       +---------------+----------+---------------+--------------------------+
 14       | ``per_page``  | int      | Optional      | | The number of projects |
 15       |               |          |               |   to return per page.    |
 16 @@ -361,93 +359,6 @@
 17       ::
 18   
 19           {
 20 -           "total_projects": 2,
 21 -           "projects": [
 22 -             {
 23 -               "access_groups": {
 24 -                 "admin": [],
 25 -                 "commit": [],
 26 -                 "ticket": []
 27 -               },
 28 -               "access_users": {
 29 -                 "admin": [],
 30 -                 "commit": [
 31 -                   "some_user"
 32 -                 ],
 33 -                 "owner": [
 34 -                   "pingou"
 35 -                 ],
 36 -                 "ticket": []
 37 -               },
 38 -               "close_status": [],
 39 -               "custom_keys": [],
 40 -               "date_created": "1427441537",
 41 -               "date_modified": "1427441537",
 42 -               "description": "A web-based calendar for Fedora",
 43 -               "milestones": {},
 44 -               "namespace": null,
 45 -               "id": 7,
 46 -               "name": "fedocal",
 47 -               "fullname": "fedocal",
 48 -               "parent": null,
 49 -               "priorities": {},
 50 -               "tags": [],
 51 -               "user": {
 52 -                 "fullname": "Pierre-Yves C",
 53 -                 "name": "pingou"
 54 -               }
 55 -             },
 56 -             {
 57 -               "access_groups": {
 58 -                 "admin": [],
 59 -                 "commit": [],
 60 -                 "ticket": []
 61 -               },
 62 -               "access_users": {
 63 -                 "admin": [],
 64 -                 "commit": [],
 65 -                 "owner": [
 66 -                   "pingou"
 67 -                 ],
 68 -                 "ticket": []
 69 -               },
 70 -               "close_status": [],
 71 -               "custom_keys": [],
 72 -               "date_created": "1431666007",
 73 -               "description": "An awesome messaging servicefor everyone",
 74 -               "id": 12,
 75 -               "milestones": {},
 76 -               "name": "fedmsg",
 77 -               "namespace": null,
 78 -               "fullname": "forks/pingou/fedmsg",
 79 -               "parent": {
 80 -                 "date_created": "1433423298",
 81 -                 "description": "An awesome messaging servicefor everyone",
 82 -                 "id": 11,
 83 -                 "name": "fedmsg",
 84 -                 "fullname": "fedmsg",
 85 -                 "parent": null,
 86 -                 "user": {
 87 -                   "fullname": "Ralph B",
 88 -                   "name": "ralph"
 89 -                 }
 90 -               },
 91 -               "priorities": {},
 92 -               "tags": [],
 93 -               "user": {
 94 -                 "fullname": "Pierre-Yves C",
 95 -                 "name": "pingou"
 96 -               }
 97 -             }
 98 -           ]
 99 -         }
100 - 
101 -     Sample Response With Pagination
102 -     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
103 - 
104 -     ::
105 - 
106 -         {
107             "args": {
108               "fork": null,
109               "namespace": null,
110 @@ -562,7 +473,9 @@
111       pagination_metadata = None
112       query_start = None
113       query_limit = None
114 -     if page:
115 +     if not page:
116 +         page = 1
117 +     else:
118           try:
119               page = int(page)
120           except (TypeError, ValueError):
121 @@ -573,23 +486,23 @@
122               raise pagure.exceptions.APIError(
123                   400, error_code=APIERROR.EINVALIDREQ)
124   
125 -         if per_page:
126 -             try:
127 -                 per_page = int(per_page)
128 -             except (TypeError, ValueError):
129 -                 raise pagure.exceptions.APIError(
130 -                     400, error_code=APIERROR.EINVALIDREQ)
131 +     if per_page:
132 +         try:
133 +             per_page = int(per_page)
134 +         except (TypeError, ValueError):
135 +             raise pagure.exceptions.APIError(
136 +                 400, error_code=APIERROR.EINVALIDREQ)
137   
138 -             if per_page < 1 or per_page > 100:
139 -                 raise pagure.exceptions.APIError(
140 -                     400, error_code=APIERROR.EINVALIDPERPAGEVALUE)
141 -         else:
142 -             per_page = 20
143 +         if per_page < 1 or per_page > 100:
144 +             raise pagure.exceptions.APIError(
145 +                 400, error_code=APIERROR.EINVALIDPERPAGEVALUE)
146 +     else:
147 +         per_page = 20
148   
149 -         pagination_metadata = pagure.lib.get_pagination_metadata(
150 -             flask.request, page, per_page, project_count)
151 -         query_start = (page - 1) * per_page
152 -         query_limit = per_page
153 +     pagination_metadata = pagure.lib.get_pagination_metadata(
154 +         flask.request, page, per_page, project_count)
155 +     query_start = (page - 1) * per_page
156 +     query_limit = per_page
157   
158       projects = pagure.lib.search_projects(
159           flask.g.session, username=username, fork=fork, tags=tags,
  1 @@ -263,12 +263,15 @@
  2           data = json.loads(output.get_data(as_text=True))
  3           data['projects'][0]['date_created'] = "1436527638"
  4           data['projects'][0]['date_modified'] = "1436527638"
  5 +         del data['pagination']
  6           expected_data = {
  7             "args": {
  8               "fork": None,
  9               "namespace": None,
 10               "owner": None,
 11 +             "page": 1,
 12               "pattern": "test",
 13 +             "per_page": 20,
 14               "short": False,
 15               "tags": [],
 16               "username": None
 17 @@ -324,12 +327,15 @@
 18           output = self.app.get('/api/0/projects?pattern=te*&short=1')
 19           self.assertEqual(output.status_code, 200)
 20           data = json.loads(output.get_data(as_text=True))
 21 +         del data['pagination']
 22           expected_data = {
 23             "args": {
 24               "fork": None,
 25               "namespace": None,
 26               "owner": None,
 27 +             "page": 1,
 28               "pattern": "te*",
 29 +             "per_page": 20,
 30               "short": True,
 31               "tags": [],
 32               "username": None
 33 @@ -356,6 +362,7 @@
 34             ],
 35             "total_projects": 3
 36           }
 37 +         self.maxDiff = None
 38           self.assertDictEqual(data, expected_data)
 39   
 40       def test_api_projects(self):
 41 @@ -381,6 +388,8 @@
 42           output = self.app.get('/api/0/projects?tags=inf')
 43           self.assertEqual(output.status_code, 200)
 44           data = json.loads(output.get_data(as_text=True))
 45 +         null = None
 46 +         del data['pagination']
 47           self.assertDictEqual(
 48               data,
 49               {
 50 @@ -390,11 +399,13 @@
 51                       "fork": None,
 52                       "namespace": None,
 53                       "owner": None,
 54 +                     "page": 1,
 55                       "pattern": None,
 56 +                     "per_page": 20,
 57                       "short": False,
 58                       "tags": ["inf"],
 59                       "username": None
 60 -                 }
 61 +                 },
 62               }
 63           )
 64           output = self.app.get('/api/0/projects?tags=infra')
 65 @@ -402,12 +413,15 @@
 66           data = json.loads(output.get_data(as_text=True))
 67           data['projects'][0]['date_created'] = "1436527638"
 68           data['projects'][0]['date_modified'] = "1436527638"
 69 +         del data['pagination']
 70           expected_data = {
 71               "args": {
 72                   "fork": None,
 73                   "namespace": None,
 74                   "owner": None,
 75 +                 "page": 1,
 76                   "pattern": None,
 77 +                 "per_page": 20,
 78                   "short": False,
 79                   "tags": ["infra"],
 80                   "username": None
 81 @@ -459,12 +473,15 @@
 82           data['projects'][1]['date_modified'] = "1436527638"
 83           data['projects'][2]['date_created'] = "1436527638"
 84           data['projects'][2]['date_modified'] = "1436527638"
 85 +         del data['pagination']
 86           expected_data = {
 87               "args": {
 88                   "fork": None,
 89                   "namespace": None,
 90                   "owner": "pingou",
 91 +                 "page": 1,
 92                   "pattern": None,
 93 +                 "per_page": 20,
 94                   "short": False,
 95                   "tags": [],
 96                   "username": None
 97 @@ -592,12 +609,15 @@
 98           data['projects'][1]['date_modified'] = "1436527638"
 99           data['projects'][2]['date_created'] = "1436527638"
100           data['projects'][2]['date_modified'] = "1436527638"
101 +         del data['pagination']
102           expected_data = {
103               "args": {
104                   "fork": None,
105                   "namespace": None,
106                   "owner": None,
107 +                 "page": 1,
108                   "pattern": None,
109 +                 "per_page": 20,
110                   "short": False,
111                   "tags": [],
112                   "username": "pingou"
113 @@ -718,12 +738,15 @@
114           data = json.loads(output.get_data(as_text=True))
115           data['projects'][0]['date_created'] = "1436527638"
116           data['projects'][0]['date_modified'] = "1436527638"
117 +         del data['pagination']
118           expected_data = {
119               "args": {
120                   "fork": None,
121                   "namespace": None,
122                   "owner": None,
123 +                 "page": 1,
124                   "pattern": None,
125 +                 "per_page": 20,
126                   "short": False,
127                   "tags": ["infra"],
128                   "username": "pingou",
129 @@ -771,11 +794,14 @@
130           data = json.loads(output.get_data(as_text=True))
131           data['projects'][0]['date_created'] = "1436527638"
132           data['projects'][0]['date_modified'] = "1436527638"
133 +         del data['pagination']
134           expected_data = {
135               "args": {
136                   "fork": None,
137                   "owner": None,
138 +                 "page": 1,
139                   "namespace": "somenamespace",
140 +                 "per_page": 20,
141                   "pattern": None,
142                   "short": False,
143                   "tags": [],
144 @@ -3545,7 +3571,7 @@
145           """ Test the api_modify_acls method of the flask api when no ACL
146           are specified, so the user tries to remove their own access but the
147           user is the project owner. """
148 -          # Add the user `foo` to the project
149 +         # Add the user `foo` to the project
150           self.test_api_modify_acls_user()
151   
152           # Ensure `foo` was properly added:
  1 @@ -1097,6 +1097,7 @@
  2           output = self.app.get('/api/0/projects?tags=inf')
  3           self.assertEqual(output.status_code, 200)
  4           data = json.loads(output.get_data(as_text=True))
  5 +         del data['pagination']
  6           self.assertDictEqual(
  7               data,
  8               {
  9 @@ -1104,7 +1105,9 @@
 10                       'fork': None,
 11                       'namespace': None,
 12                       'owner': None,
 13 +                     'page': 1,
 14                       'pattern': None,
 15 +                     'per_page': 20,
 16                       'short': False,
 17                       'tags': ['inf'],
 18                       'username': None
 19 @@ -1118,6 +1121,7 @@
 20           output = self.app.get('/api/0/projects?tags=infra')
 21           self.assertEqual(output.status_code, 200)
 22           data = json.loads(output.get_data(as_text=True))
 23 +         del data['pagination']
 24           self.assertDictEqual(
 25               data,
 26               {
 27 @@ -1125,7 +1129,9 @@
 28                       'fork': None,
 29                       'namespace': None,
 30                       'owner': None,
 31 +                     'page': 1,
 32                       'pattern': None,
 33 +                     'per_page': 20,
 34                       'short': False,
 35                       'tags': ['infra'],
 36                       'username': None
 37 @@ -1141,6 +1147,7 @@
 38               output = self.app.get('/api/0/projects?tags=infra')
 39               self.assertEqual(output.status_code, 200)
 40               data = json.loads(output.get_data(as_text=True))
 41 +             del data['pagination']
 42               self.assertDictEqual(
 43                   data,
 44                   {
 45 @@ -1148,7 +1155,9 @@
 46                           'fork': None,
 47                           'namespace': None,
 48                           'owner': None,
 49 +                         'page': 1,
 50                           'pattern': None,
 51 +                         'per_page': 20,
 52                           'short': False,
 53                           'tags': ['infra'],
 54                           'username': None
 55 @@ -1164,6 +1173,7 @@
 56               output = self.app.get('/api/0/projects?tags=infra')
 57               self.assertEqual(output.status_code, 200)
 58               data = json.loads(output.get_data(as_text=True))
 59 +             del data['pagination']
 60               self.assertDictEqual(
 61                   data,
 62                   {
 63 @@ -1171,7 +1181,9 @@
 64                           'fork': None,
 65                           'namespace': None,
 66                           'owner': None,
 67 +                         'page': 1,
 68                           'pattern': None,
 69 +                         'per_page': 20,
 70                           'short': False,
 71                           'tags': ['infra'],
 72                           'username': None
 73 @@ -1186,6 +1198,7 @@
 74               data = json.loads(output.get_data(as_text=True))
 75               data['projects'][0]['date_created'] = "1436527638"
 76               data['projects'][0]['date_modified'] = "1436527638"
 77 +             del data['pagination']
 78               self.assertDictEqual(
 79                   data,
 80                   {
 81 @@ -1193,12 +1206,13 @@
 82                           "fork": None,
 83                           "namespace": None,
 84                           "owner": None,
 85 +                         "page": 1,
 86                           "pattern": None,
 87 +                         "per_page": 20,
 88                           "short": False,
 89                           "tags": [],
 90                           "username": "pingou"
 91                       },
 92 - 
 93                       "total_projects": 1,
 94                       "projects": [
 95                           {
 96 @@ -1243,6 +1257,7 @@
 97               data = json.loads(output.get_data(as_text=True))
 98               data['projects'][0]['date_created'] = "1436527638"
 99               data['projects'][0]['date_modified'] = "1436527638"
100 +             del data['pagination']
101               self.assertDictEqual(
102                   data,
103                   {
104 @@ -1250,7 +1265,9 @@
105                           "fork": None,
106                           "namespace": None,
107                           "owner": None,
108 +                         "page": 1,
109                           "pattern": None,
110 +                         "per_page": 20,
111                           "short": False,
112                           "tags": ["infra"],
113                           "username": "pingou"