From 791d41ec9c16626b01694d2187cbdaf90a1dbb9a Mon Sep 17 00:00:00 2001
From: Adam Saleh <asaleh@redhat.com>
Date: Oct 05 2021 15:10:39 +0000
Subject: Add test that generates the loglines to be parsed


---

diff --git a/countme/parse.py b/countme/parse.py
index b30e697..1fcf5f2 100644
--- a/countme/parse.py
+++ b/countme/parse.py
@@ -24,11 +24,11 @@ def pre_process(filepath: Union[str, Path]) -> Iterator[str]:
         yield tmpfile.name
 
 
-def parse(args=None):
+def parse_from_iterator(args, lines):
     if args.header or args.sqlite:
         args.writer.write_header()
 
-    for logf in ReadProgress(args.logs, display=args.progress, pre_process=pre_process):
+    for logf in lines:
         # Make an iterator object for the matching log lines
         match_iter = iter(args.matcher(logf))
 
@@ -53,3 +53,9 @@ def parse(args=None):
 
     if args.index:
         args.writer.write_index()
+
+
+def parse(args=None):
+    parse_from_iterator(
+        args, ReadProgress(args.logs, display=args.progress, pre_process=pre_process)
+    )
diff --git a/test_requirements.txt b/test_requirements.txt
index 15ff12a..d0c7072 100644
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -3,3 +3,4 @@ black
 mypy
 pytest
 pytest-cov
+hypothesis==5.43.9
\ No newline at end of file
diff --git a/tests/test_parse.py b/tests/test_parse.py
index fc1c86e..563f34d 100644
--- a/tests/test_parse.py
+++ b/tests/test_parse.py
@@ -1,12 +1,16 @@
+import datetime
 import tarfile
+import tempfile
 import sqlite3
+from hypothesis import given, strategies as st
+from hypothesis import settings, HealthCheck
 from pathlib import Path
 from typing import Any, List, NamedTuple
 
 import pytest
 
 from countme import CountmeMatcher, make_writer
-from countme.parse import parse
+from countme.parse import parse, parse_from_iterator
 
 
 HERE = Path(__file__).parent
@@ -83,3 +87,46 @@ def test_read_file(tmp_path_cwd, log_tar, db_tar):
     assert (
         missing is None and extra is None
     ), f"When comparing db's\n {missing} was missing and\n {extra} was extra"
+
+
+def create_logline(ip, date, repo):
+    dstr = date.strftime("%d/%b/%Y:%H:%M:%S +0000")
+    url = "/metalink?repo=updates-released-f33&arch=x86_64&countme=1"
+    agent = "libdnf (Fedora 33; workstation; Linux.x86_64)"
+    return f'{ip} - - [{dstr}] "GET {url} HTTP/1.1" 200 32015 "-" "{agent}"'
+
+
+@st.composite
+def log_data(draw):
+    ip_sample = st.lists(st.ip_addresses(), 10, unique=True)
+    repo = st.sampled_from(["Fedora", "epel-7", "centos8"])
+    ips = draw(ip_sample)
+    # datetime.fromisoformat('2020-12-04')
+    dates = st.lists(st.datetimes(datetime.datetime(2021, 8, 8, 0)), 2, unique=True)
+
+    return list(
+        sorted(((date, ip, draw(repo)) for ip in ips for date in draw(dates)), key=lambda x: x[0])
+    )
+
+
+@settings(suppress_health_check=(HealthCheck.too_slow,))
+@given(log_data())
+def test_log(loglines):
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        matcher = CountmeMatcher
+        args = Args(
+            writer=make_writer("sqlite", str(tmp_dir + "/test.db"), matcher.itemtuple),
+            matcher=matcher,
+            dupcheck=True,
+            index=True,
+            header=True,
+            progress=False,
+            matchmode="countme",
+            format="csv",
+            logs=[],
+            sqlite=str(tmp_dir + "/test.db"),
+        )
+        parse_from_iterator(args, [(create_logline(ip, date, repo) for date, ip, repo in loglines)])
+        db = sqlite3.connect(args.sqlite)
+        rows_no = db.execute("select count(*) from countme_raw;").fetchone()[0]
+        assert rows_no == len(loglines)