From 94f059cbf8f1a688d5e8d920ab40b33d215586f4 Mon Sep 17 00:00:00 2001 From: Josseline Perdomo Date: Aug 07 2021 05:46:50 +0000 Subject: [PATCH 1/3] Refactor config to object approach Signed-off-by: Josseline Perdomo --- diff --git a/contributor_trends/app.py b/contributor_trends/app.py index f1a18e0..cc1b5b5 100644 --- a/contributor_trends/app.py +++ b/contributor_trends/app.py @@ -5,7 +5,10 @@ from contributor_trends.dashboard import create_dashboard def create_app() -> Flask: app = Flask(__name__, instance_relative_config=False) - app.config.from_pyfile("config.py") + if app.env == "development": + app.config.from_object("config.DevelopmentConfig") + else: + app.config.from_object("config.ProductionConfig") with app.app_context(): from contributor_trends.extensions import cache diff --git a/contributor_trends/config.py b/contributor_trends/config.py index 243b30c..b155218 100644 --- a/contributor_trends/config.py +++ b/contributor_trends/config.py @@ -1,10 +1,20 @@ import os -BASE_DIR = os.path.dirname(__file__) -APP_TITLE = "Fedora Contributor Trends" -APP_DATASOURCE_URL = os.environ.get("APP_DATASOURCE_URL") +class Config: + BASE_DIR = os.path.dirname(__file__) -CACHE_CONFIG = { - "CACHE_TYPE": "SimpleCache", -} + APP_TITLE = "Fedora Contributor Trends" + APP_DATASOURCE_URL = os.environ.get("APP_DATASOURCE_URL") + + CACHE_CONFIG = { + "CACHE_TYPE": "SimpleCache", + } + + +class DevelopmentConfig(Config): + APP_UPDATE_TIME = 30000 # 30 seconds in milliseconds + + +class ProductionConfig(Config): + APP_UPDATE_TIME = 86400000 # 24 hours in milliseconds diff --git a/contributor_trends/dashboard/figures.py b/contributor_trends/dashboard/figures.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/contributor_trends/dashboard/figures.py From 33aadc8c09b86f69f783bc97d01f35a876832000 Mon Sep 17 00:00:00 2001 From: Josseline Perdomo Date: Aug 07 2021 05:47:41 +0000 Subject: [PATCH 2/3] Updated .env example Signed-off-by: Josseline Perdomo --- diff --git a/.env.example b/.env.example index 8d0e118..9de48d8 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,6 @@ FLASK_ENV=development FLASK_APP=manage.py +FLASK_DEBUG=True +FLASK_RUN_PORT=42 APP_DATASOURCE_URL=http://example.com \ No newline at end of file From 595d9f54fd8bf507fce081c0fb4b432579659640 Mon Sep 17 00:00:00 2001 From: Josseline Perdomo Date: Aug 07 2021 05:48:44 +0000 Subject: [PATCH 3/3] Added a callback to allow live update of a chart component Signed-off-by: Josseline Perdomo --- diff --git a/contributor_trends/dashboard/callbacks.py b/contributor_trends/dashboard/callbacks.py index 9760fc2..3a80cf8 100644 --- a/contributor_trends/dashboard/callbacks.py +++ b/contributor_trends/dashboard/callbacks.py @@ -2,9 +2,10 @@ import dash import dash_html_components as html from dash.dependencies import Input, Output +from contributor_trends.dashboard import figures from contributor_trends.dashboard.datasets import active_contributors from contributor_trends.dashboard.layouts import ( - weekly_activity_layout, + chart_layout, not_found_layout, ) @@ -12,10 +13,27 @@ from contributor_trends.dashboard.layouts import ( def register_callbacks(dash_app: dash.Dash): @dash_app.callback( Output(component_id="page-content", component_property="children"), - [Input(component_id="url", component_property="pathname")], + Input(component_id="url", component_property="pathname"), ) def page_content(pathname: str) -> html.Div: if pathname == "/": - return weekly_activity_layout(active_contributors) + # TODO: quit directly the chart_layout call, is better add a container + return chart_layout(active_contributors) return not_found_layout(pathname) + @dash_app.callback( + Output( + component_id=f"{active_contributors.id}-chart", + component_property="children", + ), + Output( + component_id=f"{active_contributors.id}-chart-meta", + component_property="children", + ), + Input( + component_id=f"{active_contributors.id}-interval", + component_property="n_intervals", + ), + ) + def update_active_contributors(i: int): + return figures.update(active_contributors, figures.area_chart) diff --git a/contributor_trends/dashboard/datasets.py b/contributor_trends/dashboard/datasets.py index aa468cc..09a6b00 100644 --- a/contributor_trends/dashboard/datasets.py +++ b/contributor_trends/dashboard/datasets.py @@ -1,8 +1,8 @@ import abc -import datetime +from datetime import datetime from dataclasses import dataclass from enum import Enum -from typing import Tuple +from typing import Tuple, Union from urllib.parse import urljoin import pandas as pd @@ -13,12 +13,12 @@ from contributor_trends.utils.exceptions import InvalidDatasetFormat @dataclass class DatasetMeta: - filepath: str = "" - updated_at: datetime.datetime = None + filepath: str + updated_at: datetime def __post_init__(self): - if self.updated_at is not None: - self.updated_at = datetime.datetime.fromisoformat(self.updated_at) + if type(self.updated_at) == str: + self.updated_at = datetime.fromisoformat(self.updated_at) class ReportFrequency(Enum): @@ -30,33 +30,53 @@ class ReportFrequency(Enum): class Dataset(abc.ABC): def __init__(self) -> None: - self.source = "" - self._meta = DatasetMeta() - self.dataframe = pd.DataFrame() + self._source = "" + self._meta = DatasetMeta(filepath="", updated_at=datetime.min) + self._data = pd.DataFrame() - def load(self, source: str) -> None: - self.source = source - meta_updated, meta = self._get_meta() + def __str__(self) -> str: + return f"Last updated: {self.last_updated}" - if not meta_updated: - raise ConnectionError("Unable to identify remote datasource.") + @property + def id(self) -> str: + raise NotImplementedError - self._meta = meta - self.dataframe = self._get_dataframe() + @property + def title(self) -> str: + raise NotImplementedError + + @property + def data(self) -> pd.DataFrame: + return self._data @property - def last_updated(self) -> datetime.datetime: + def x(self) -> Union[pd.Series, str]: + raise NotImplementedError + + @property + def y(self) -> Union[pd.Series, str]: + raise NotImplementedError + + @property + def last_updated(self) -> datetime: return self._meta.updated_at @abc.abstractmethod - def _prepare_dataframe(self, dataframe: pd.DataFrame) -> pd.DataFrame: + def _prepare_data(self, dataframe: pd.DataFrame) -> pd.DataFrame: # It should be implemented by the inheritance classes pass + def _get_data(self) -> pd.DataFrame: + if self._meta.filepath.endswith(".csv"): + dataframe = pd.read_csv(self._meta.filepath) + return self._prepare_data(dataframe) + + raise InvalidDatasetFormat() + def _get_meta(self) -> Tuple[bool, DatasetMeta]: updated_status, meta = False, self._meta try: - response = requests.get(self.source) + response = requests.get(self._source) # TODO: logging the response if response.status_code == 201: updated_status, meta = True, DatasetMeta(**response.json()) @@ -66,18 +86,22 @@ class Dataset(abc.ABC): return updated_status, meta - def _get_dataframe(self) -> pd.DataFrame: - if self._meta.filepath.endswith(".csv"): - dataframe = pd.read_csv(self._meta.filepath) - return self._prepare_dataframe(dataframe) + def load(self, source: str) -> None: + self._source = source + meta_updated, meta = self._get_meta() - raise InvalidDatasetFormat() + if not meta_updated: + raise ConnectionError("Unable to identify remote datasource.") + + self._meta = meta + self._data = self._get_data() def update(self): + # TODO: Add logging to check when there is an update request meta_updated, meta = self._get_meta() if meta_updated: self._meta = meta - self.dataframe = self._get_dataframe() + self._data = self._get_data() class ActiveContributors(Dataset): @@ -85,11 +109,27 @@ class ActiveContributors(Dataset): super().__init__() self.report_frequency = ReportFrequency.WEEKLY.value + @property + def id(self) -> str: + return "active-contributors" + + @property + def title(self) -> str: + return "Active Contributors" + + @property + def x(self) -> Union[pd.Index, pd.Series, str]: + return self._data.index + + @property + def y(self) -> Union[pd.Series, str]: + return "ActiveUsers" + def load(self, base_url: str) -> None: endpoint = urljoin(base_url, "reports/contributors") super(ActiveContributors, self).load(endpoint) - def _prepare_dataframe(self, dataframe: pd.DataFrame) -> pd.DataFrame: + def _prepare_data(self, dataframe: pd.DataFrame) -> pd.DataFrame: dataframe.Date = pd.to_datetime(dataframe.Date, format="%m/%d/%y") return dataframe.resample(self.report_frequency, on="Date").sum() diff --git a/contributor_trends/dashboard/figures.py b/contributor_trends/dashboard/figures.py index e69de29..2b3fc30 100644 --- a/contributor_trends/dashboard/figures.py +++ b/contributor_trends/dashboard/figures.py @@ -0,0 +1,17 @@ +from typing import Type, Callable, Tuple + +import plotly.express as px +from plotly.graph_objs import Figure + +from contributor_trends.dashboard.datasets import Dataset + + +def area_chart(dataset: Type[Dataset]) -> Figure: + return px.area(dataset.data, x=dataset.x, y=dataset.y) + + +def update( + dataset: Type[Dataset], figure: Callable[[Type[Dataset]], Figure] +) -> Tuple[Figure, str]: + dataset.update() + return figure(dataset), str(dataset) diff --git a/contributor_trends/dashboard/layouts.py b/contributor_trends/dashboard/layouts.py index 6aa65ed..4d2584f 100644 --- a/contributor_trends/dashboard/layouts.py +++ b/contributor_trends/dashboard/layouts.py @@ -1,36 +1,48 @@ +from typing import Type + import dash_core_components as dcc import dash_html_components as html -import plotly.express as px +from dash.development.base_component import Component +from flask import current_app -from contributor_trends.dashboard.datasets import ActiveContributors +from contributor_trends.dashboard import figures +from contributor_trends.dashboard.datasets import Dataset -def weekly_activity_layout(dataset: ActiveContributors) -> html.Div: +def chart_layout(dataset: Type[Dataset]) -> Component: return html.Div( [ - html.H1("Weekly Active Contributors"), - html.Hr(), - dcc.Graph( - id="weekly-active-contributors", - figure=px.area( - dataset.dataframe, x=dataset.dataframe.index, y="ActiveUsers" - ), + html.H2(dataset.title, className="chart-title"), + html.Div( + [ + dcc.Graph( + id=f"{dataset.id}-chart", figure=figures.area_chart(dataset) + ), + html.P(str(dataset), id=f"{dataset.id}-chart-meta"), + ], + className="chart-content", ), - ] + dcc.Interval( + id=f"{dataset.id}-interval", + interval=current_app.config["APP_UPDATE_TIME"], # in milliseconds + ), + ], + id=f"{dataset.id}-container", + className="chart-container", ) -def index_layout() -> html.Div: +def index_layout() -> Component: return html.Div( [dcc.Location(id="url", refresh=False), html.Div(id="page-content")] ) -def not_found_layout(url) -> html.Div: +def not_found_layout(url) -> Component: return html.Div( [ html.H1("404: Not found"), html.Hr(), - html.P(f"The pathname {url} was not recognised..."), + html.P(f"The pathname {url} was not recognised."), ] )