From d1106768c4e0cf2baa4634eb63b0c9550c915ee9 Mon Sep 17 00:00:00 2001
From: Jakub Kadlcik <frostyx@email.cz>
Date: May 02 2022 10:12:15 +0000
Subject: backend: use pagination for S3 objects


By default only first 1000 results are returned, so we need to use
pagination. See

https://boto3.amazonaws.com/v1/documentation/api/latest/guide/paginators.html#creating-paginators

---

diff --git a/backend/run/copr-aws-s3-hitcounter b/backend/run/copr-aws-s3-hitcounter
index 2b7520d..e521558 100755
--- a/backend/run/copr-aws-s3-hitcounter
+++ b/backend/run/copr-aws-s3-hitcounter
@@ -52,10 +52,16 @@ class S3Bucket:
         """
         List all files within our AWS s3 bucket
         """
-        objects = self.s3.list_objects(
+        paginator = self.s3.get_paginator("list_objects")
+        page_iterator = paginator.paginate(
             Bucket=self.bucket,
             Prefix=self.directory)
-        return [x["Key"] for x in objects["Contents"]]
+
+        result = []
+        for page in page_iterator:
+            for obj in page["Contents"]:
+                result.append(obj["Key"])
+        return result
 
     def download_file(self, s3file, dstdir):
         """