#!/usr/bin/python3

  import argparse

  import datetime

  from collections import Counter

  # Feb 11 2020 was the date that we branched F32 from Rawhide, so we've decided

  # to use that as the starting week for countme data.

  COUNTME_EPOCH = 345600  # =00:00:00 Mon Jan 5 00:00:00 1970 (UTC)

  # And here's how you convert a weeknum to a human-readable date

  def weekdate(weeknum, weekday=0):

      if weekday < 0 or weekday > 6:

          raise ValueError("weekday must be between 0 (Mon) and 6 (Sun)")

      return datetime.date.fromordinal(ordinal)

  def daterange(weeknum):

  # ===========================================================================

  # ====== Count Buckets & Items ==============================================

  # ===========================================================================

  class CountBucket(NamedTuple):

      os_name: str

      os_version: str

      os_variant: str

      os_arch: str

      repo_tag: str

      repo_arch: str

      def from_item(cls, item):

          return cls._make((weeknum(item.timestamp),) + item[2:])

  BucketSelect = CountBucket(

  )

  class CSVCountItem(NamedTuple):

      Represents one row in a countme_totals.csv file.

      In the interest of human-readability, we replace 'weeknum' with the

      start and end dates of that week.

      week_start: str

      week_end: str

      hits: int

      @classmethod

      def from_totalitem(cls, item):

          hits, weeknum, *rest = item

          week_start, week_end = daterange(weeknum)

          return cls._make([week_start, week_end, hits] + rest)

  # ===========================================================================

  # ====== SQL + Progress helpers =============================================

  # ===========================================================================

  class RawDB(SQLiteReader):

      def __init__(self, fp, **kwargs):

      def _minmax(self, column):

          cur = self._con.execute(f"SELECT min({column}),max({column}) FROM {self._tablename}")

          return cur.fetchone()

      def complete_weeks(self):

          weeknums contained in this database. The database may contain some

          data for `provweek`, but since it's provisional/incomplete it's

          # startweek can't be earlier than the first week of data

          startweek = max(weeknum(self.mintime()), COUNTME_START_WEEKNUM)

          # A week is provisional until the LOG_JITTER_WINDOW expires, so once

          return range(startweek, provweek)

      def week_count(self, weeknum):

          end_ts = start_ts + WEEK_LEN

          cur = self._con.execute(

              f"SELECT COUNT(*)"

              f" FROM {self._tablename}"

          return cur.fetchone()[0]

          if isinstance(select, (tuple, list)):

          elif isinstance(select, str):

              item_select = select

          else:

              raise ValueError(f"select should be a string or tuple, not {select.__class__.__name__}")

          end_ts = start_ts + WEEK_LEN

          return self._con.execute(

              f"SELECT {item_select}"

              f" FROM {self._tablename}"

  try:

      from tqdm import tqdm as Progress

      from countme.progress import diyprog as Progress

  # ===========================================================================

  # ====== CLI parser & main() ================================================

  # ===========================================================================

  def parse_args(argv=None):

      p = argparse.ArgumentParser(

      )

      args = p.parse_args(argv)

      args = parse_args()

      # Initialize the writer (better to fail early..)

      totals.write_header()

      # Are we doing an update?

              mon, sun = daterange(week)

              desc = f"week {week} ({mon} -- {sun})"

              total = rawdb.week_count(week)

              hitcount = Counter()

              # Select raw items into their buckets and count 'em up

                  prog.update()

              # Write the resulting totals into countme_totals

              prog.close()

          # Oh and make sure we index them by time.

          totals.write_index()

      # Was a CSV dump requested?

      if args.csv_dump:

          writer.write_header()

          for item in totalreader:

              writer.write_item(CSVCountItem.from_totalitem(item))

      try:

          main()

      except KeyboardInterrupt:

  # TODO: this should probably get cleaned up?

  __all__ = (

  )

  from urllib.parse import parse_qsl

  from .regex import COUNTME_LOG_RE, MIRRORS_LOG_RE

  # ===========================================================================

  # ====== Output item definitions and helpers ================================

  # ===========================================================================

  COUNTME_EPOCH = 345600  # =00:00:00 Mon Jan 5 00:00:00 1970 (UTC)

  MONTHIDX = {

  }

  def weeknum(timestamp):

      return (int(timestamp) - COUNTME_EPOCH) // WEEK_LEN

  def strptime_logtime(logtime):

      return datetime.strptime(logtime, "%d/%b/%Y:%H:%M:%S %z")

  def logtime_to_isoformat(logtime):

      # logtime: '29/Mar/2020:16:04:28 +0000'

      # ISO8601: '2020-03-29T16:04:28+00:00'

      offm = logtime[24:26]

      return f"{y}-{m:02}-{d}T{time}{offh}:{offm}"

  def offset_to_timezone(offset):

          offmin = -offmin

      return timezone(timedelta(minutes=offmin))

  def parse_logtime(logtime):

      # Equivalent to - but faster than - strptime_logtime.

      # It's like ~1.5usec vs 11usec, which might seem trivial but in my tests

      # (btw, slicing logtime by hand and using re.split are both marginally

      # slower. datetime.fromisoformat is slightly faster but not available

      # in Python 3.6 or earlier.)

  def parse_querydict(querystr):

      return dict(parse_qsl(querystr))

  class LogItem(NamedTuple):

      Generic access.log data holder.

      host: str

      identity: str

      time: str

      def querydict(self):

          return parse_querydict(self.query)

  # TODO: would be kinda nice if there was a clear subclass / translation

  # between item classes... or if compile_log_regex made the class for you?

  # Or something? It feels like these things should be more closely bound.

  class MirrorItem(NamedTuple):

      A basic mirrorlist/metalink metadata item.

      Each item has a timestamp, IP, and the requested repo= and arch= values.

      timestamp: int

      host: str

      repo_tag: Optional[str]

      repo_arch: Optional[str]

  class CountmeItem(NamedTuple):

      A "countme" match item.

      Includes the countme value and libdnf User-Agent fields.

      timestamp: int

      host: str

      os_name: str

  class LogMatcher:

      regex = NotImplemented

      def __init__(self, fileobj):

          self.fileobj = fileobj

      def iteritems(self):

          # TODO: at this point we're single-threaded and CPU-bound;

          # multithreading would speed things up here.

              match = self.regex.match(line)

              if match:

                  yield self.make_item(match)

      __iter__ = iteritems

      @classmethod

      def make_item(cls, match):

          raise NotImplementedError

  class MirrorMatcher(LogMatcher):

      regex = MIRRORS_LOG_RE

      itemtuple = MirrorItem

      @classmethod

      def make_item(cls, match):

  class CountmeMatcher(LogMatcher):

      regex = COUNTME_LOG_RE

      itemtuple = CountmeItem

      @classmethod

      def make_item(cls, match):

  # ===========================================================================

  # ====== ItemWriters - output formatting classes ============================

  # ===========================================================================

  class ItemWriter:

          self._fp = fp

          self._itemtuple = itemtuple

          self._fields = itemtuple._fields

          assert timefield in self._fields, f"{itemtuple.__name__!r} has no time field {timefield!r}"

          self._timefield = timefield

          self._get_writer(**kwargs)

      def _get_writer(self, **kwargs):

          raise NotImplementedError

      def write_item(self, item):

          raise NotImplementedError

      def write_items(self, items):

          for item in items:

              self.write_item(item)

      def write_header(self):

          pass

      def write_index(self):

          pass

  class JSONWriter(ItemWriter):

      def _get_writer(self, **kwargs):

          import json

          self._dump = json.dump

      def write_item(self, item):

          self._dump(item._asdict(), self._fp)

  class CSVWriter(ItemWriter):

      def _get_writer(self, **kwargs):

          import csv

          self._writer = csv.writer(self._fp)

      def write_header(self):

          self._writer.writerow(self._fields)

      def write_item(self, item):

          self._writer.writerow(item)

  class AWKWriter(ItemWriter):

          self._fieldsep = field_separator

      def _write_row(self, vals):

      def write_header(self):

          self._write_row(self._fields)

      def write_item(self, item):

          self._write_row(item)

  class SQLiteWriter(ItemWriter):

      # We have to get a little fancier with types here since SQL tables expect

      # typed values. Good thing Python has types now, eh?

      SQL_TYPE = {

          Optional[float]: "REAL",

          Optional[bytes]: "BLOB",

      }

      def _sqltype(self, fieldname):

          typehint = self._itemtuple.__annotations__[fieldname]

          return self.SQL_TYPE.get(typehint, "TEXT")

          import sqlite3

          if hasattr(self._fp, "name"):

              filename = self._fp.name

          else:

          # Generate SQL commands so we can use them later.

          # self._create_table creates the table, with column names and types

          # matching the names and types of the fields in self._itemtuple.

          )

          # self._insert_item is an "INSERT" command with '?' placeholders.

          )

          # self._create_time_index creates an index on 'timestamp' or whatever

          # the time-series field is.

          self._create_time_index = (

              "CREATE INDEX IF NOT EXISTS {timefield}_idx on {table} ({timefield})".format(

              )

          )

      def write_header(self):

          self._cur.execute(self._create_table)

      def write_item(self, item):

          self._cur.execute(self._insert_item, item)

      def write_items(self, items):

          with self._con:

              self._con.executemany(self._insert_item, items)

      def write_index(self):

          self._cur.execute(self._create_time_index)

          self._con.commit()

      def has_item(self, item):

          condition = " AND ".join(f"{field}=?" for field in self._fields)

          return bool(cur.fetchone()[0])

      def mintime(self):

          cur = self._cur.execute(f"SELECT MIN({self._timefield}) FROM {self._tablename}")

          return cur.fetchone()[0]

      def maxtime(self):

          cur = self._cur.execute(f"SELECT MAX({self._timefield}) FROM {self._tablename}")

          return cur.fetchone()[0]

  def make_writer(name, *args, **kwargs):

      if name == "csv":

          writer = CSVWriter

      elif name == "json":

          raise ValueError(f"Unknown writer '{name}'")

      return writer(*args, **kwargs)

  # ===========================================================================

  # ====== ItemReaders - counterpart to ItemWriter ============================

  # ===========================================================================

  class ReaderError(RuntimeError):

      pass

  class ItemReader:

      def __init__(self, fp, itemtuple, **kwargs):

          self._fp = fp

              raise ReaderError("no field names found")

          if filefields != self._itemfields:

              raise ReaderError(f"field mismatch: expected {self._itemfields}, got {filefields}")

      @property

      def fields(self):

          return self._itemfields

      def _get_reader(self):

          raise NotImplementedError

      def _get_fields(self):

          raise NotImplementedError

      def _iter_rows(self):

          raise NotImplementedError

      def _find_item(self, item):

          raise NotImplementedError

      def __iter__(self):

          for item in self._iter_rows():

              yield self._itemfactory(item)

      def __contains__(self, item):

          return self._find_item(item)

  class CSVReader(ItemReader):

      def _get_reader(self, **kwargs):

          import csv

          self._reader = csv.reader(self._fp)

      def _get_fields(self):

          filefields = tuple(next(self._reader))

          # Sanity check: if any fieldname is a number... this isn't a header

          if any(name.isnumeric() for name in filefields):

              raise ReaderError(f"header bad/missing: expected {self._itemfields}, got {header!r}")

          return filefields

      def _iter_rows(self):

          return self._reader

      def _dup(self):

          # This is pretty gross, but then, so's CSV

      def _find_item(self, item):

          stritem = self._itemfactory(str(v) for v in item)

  class AWKReader(CSVReader):

          self._reader = (line.split(field_separator) for line in self._fp)

  class JSONReader(CSVReader):

      def _get_reader(self, **kwargs):

          import json

          self._reader = (json.loads(line) for line in self._fp)

  class SQLiteReader(ItemReader):

          import sqlite3

          if hasattr(self._fp, "name"):

              filename = self._fp.name

          else:

              filename = self._fp

          self._con = sqlite3.connect(filename)

          self._cur = self._con.cursor()

          self._tablename = tablename

          self._timefield = timefield

          self._filename = filename

      def _get_fields(self):

          fields_sql = f"PRAGMA table_info('{self._tablename}')"

          filefields = tuple(r[1] for r in self._cur.execute(fields_sql))

          return filefields

      def _find_item(self, item):

          condition = " AND ".join(f"{field}=?" for field in self.fields)

          return bool(self._cur.fetchone()[0])

      def _iter_rows(self):

          fields = ",".join(self._itemfields)