PR#1: Generalized plot function and adding callback to live updates - contributor-trends-dashboard

contributor-trends-dashboard

#1 Generalized plot function and adding callback to live updates

Merged 2 years ago by josseline. Opened 2 years ago by josseline.

live-update into master

Added a callback to allow live update of a chart component

Josseline Perdomo • 2 years ago

595d9f5

Updated .env example

Josseline Perdomo • 2 years ago

33aadc8

Refactor config to object approach

Josseline Perdomo • 2 years ago

94f059c

.env.example

file modified

		`@@ -1,4 +1,6 @@`
		`FLASK_ENV=development`
		`FLASK_APP=manage.py`
		`+ FLASK_DEBUG=True`
		`+ FLASK_RUN_PORT=42`

		`APP_DATASOURCE_URL=http://example.com`
		`\ No newline at end of file`

contributor_trends/app.py

file modified

+4 -1

		`@@ -5,7 +5,10 @@`

		`def create_app() -> Flask:`
		`app = Flask(__name__, instance_relative_config=False)`
		`- app.config.from_pyfile("config.py")`
		`+ if app.env == "development":`
		`+ app.config.from_object("config.DevelopmentConfig")`
		`+ else:`
		`+ app.config.from_object("config.ProductionConfig")`

		`with app.app_context():`
		`from contributor_trends.extensions import cache`

contributor_trends/config.py

file modified

+16 -6

		`@@ -1,10 +1,20 @@`
		`import os`

		`- BASE_DIR = os.path.dirname(__file__)`

		`- APP_TITLE = "Fedora Contributor Trends"`
		`- APP_DATASOURCE_URL = os.environ.get("APP_DATASOURCE_URL")`
		`+ class Config:`
		`+ BASE_DIR = os.path.dirname(__file__)`

		`- CACHE_CONFIG = {`
		`- "CACHE_TYPE": "SimpleCache",`
		`- }`
		`+ APP_TITLE = "Fedora Contributor Trends"`
		`+ APP_DATASOURCE_URL = os.environ.get("APP_DATASOURCE_URL")`
		`+`
		`+ CACHE_CONFIG = {`
		`+ "CACHE_TYPE": "SimpleCache",`
		`+ }`
		`+`
		`+`
		`+ class DevelopmentConfig(Config):`
		`+ APP_UPDATE_TIME = 30000 # 30 seconds in milliseconds`
		`+`
		`+`
		`+ class ProductionConfig(Config):`
		`+ APP_UPDATE_TIME = 86400000 # 24 hours in milliseconds`

contributor_trends/dashboard/callbacks.py

file modified

+21 -3

		`@@ -2,9 +2,10 @@`
		`import dash_html_components as html`
		`from dash.dependencies import Input, Output`

		`+ from contributor_trends.dashboard import figures`
		`from contributor_trends.dashboard.datasets import active_contributors`
		`from contributor_trends.dashboard.layouts import (`
		`- weekly_activity_layout,`
		`+ chart_layout,`
		`not_found_layout,`
		`)`

		`@@ -12,10 +13,27 @@`
		`def register_callbacks(dash_app: dash.Dash):`
		`@dash_app.callback(`
		`Output(component_id="page-content", component_property="children"),`
		`- [Input(component_id="url", component_property="pathname")],`
		`+ Input(component_id="url", component_property="pathname"),`
		`)`
		`def page_content(pathname: str) -> html.Div:`
		`if pathname == "/":`
		`- return weekly_activity_layout(active_contributors)`
		`+ # TODO: quit directly the chart_layout call, is better add a container`
		`+ return chart_layout(active_contributors)`
		`return not_found_layout(pathname)`

		`+ @dash_app.callback(`
		`+ Output(`
		`+ component_id=f"{active_contributors.id}-chart",`
		`+ component_property="children",`
		`+ ),`
		`+ Output(`
		`+ component_id=f"{active_contributors.id}-chart-meta",`
		`+ component_property="children",`
		`+ ),`
		`+ Input(`
		`+ component_id=f"{active_contributors.id}-interval",`
		`+ component_property="n_intervals",`
		`+ ),`
		`+ )`
		`+ def update_active_contributors(i: int):`
		`+ return figures.update(active_contributors, figures.area_chart)`

contributor_trends/dashboard/datasets.py

file modified

+66 -26

		`@@ -1,8 +1,8 @@`
		`import abc`
		`- import datetime`
		`+ from datetime import datetime`
		`from dataclasses import dataclass`
		`from enum import Enum`
		`- from typing import Tuple`
		`+ from typing import Tuple, Union`
		`from urllib.parse import urljoin`

		`import pandas as pd`
		`@@ -13,12 +13,12 @@`

		`@dataclass`
		`class DatasetMeta:`
		`- filepath: str = ""`
		`- updated_at: datetime.datetime = None`
		`+ filepath: str`
		`+ updated_at: datetime`

		`def __post_init__(self):`
		`- if self.updated_at is not None:`
		`- self.updated_at = datetime.datetime.fromisoformat(self.updated_at)`
		`+ if type(self.updated_at) == str:`
		`+ self.updated_at = datetime.fromisoformat(self.updated_at)`


		`class ReportFrequency(Enum):`
		`@@ -30,33 +30,53 @@`

		`class Dataset(abc.ABC):`
		`def __init__(self) -> None:`
		`- self.source = ""`
		`- self._meta = DatasetMeta()`
		`- self.dataframe = pd.DataFrame()`
		`+ self._source = ""`
		`+ self._meta = DatasetMeta(filepath="", updated_at=datetime.min)`
		`+ self._data = pd.DataFrame()`

		`- def load(self, source: str) -> None:`
		`- self.source = source`
		`- meta_updated, meta = self._get_meta()`
		`+ def __str__(self) -> str:`
		`+ return f"Last updated: {self.last_updated}"`

		`- if not meta_updated:`
		`- raise ConnectionError("Unable to identify remote datasource.")`
		`+ @property`
		`+ def id(self) -> str:`
		`+ raise NotImplementedError`

		`- self._meta = meta`
		`- self.dataframe = self._get_dataframe()`
		`+ @property`
		`+ def title(self) -> str:`
		`+ raise NotImplementedError`
		`+`
		`+ @property`
		`+ def data(self) -> pd.DataFrame:`
		`+ return self._data`

		`@property`
		`- def last_updated(self) -> datetime.datetime:`
		`+ def x(self) -> Union[pd.Series, str]:`
		`+ raise NotImplementedError`
		`+`
		`+ @property`
		`+ def y(self) -> Union[pd.Series, str]:`
		`+ raise NotImplementedError`
		`+`
		`+ @property`
		`+ def last_updated(self) -> datetime:`
		`return self._meta.updated_at`

		`@abc.abstractmethod`
		`- def _prepare_dataframe(self, dataframe: pd.DataFrame) -> pd.DataFrame:`
		`+ def _prepare_data(self, dataframe: pd.DataFrame) -> pd.DataFrame:`
		`# It should be implemented by the inheritance classes`
		`pass`

		`+ def _get_data(self) -> pd.DataFrame:`
		`+ if self._meta.filepath.endswith(".csv"):`
		`+ dataframe = pd.read_csv(self._meta.filepath)`
		`+ return self._prepare_data(dataframe)`
		`+`
		`+ raise InvalidDatasetFormat()`
		`+`
		`def _get_meta(self) -> Tuple[bool, DatasetMeta]:`
		`updated_status, meta = False, self._meta`
		`try:`
		`- response = requests.get(self.source)`
		`+ response = requests.get(self._source)`
		`# TODO: logging the response`
		`if response.status_code == 201:`
		`updated_status, meta = True, DatasetMeta(**response.json())`
		`@@ -66,18 +86,22 @@`

		`return updated_status, meta`

		`- def _get_dataframe(self) -> pd.DataFrame:`
		`- if self._meta.filepath.endswith(".csv"):`
		`- dataframe = pd.read_csv(self._meta.filepath)`
		`- return self._prepare_dataframe(dataframe)`
		`+ def load(self, source: str) -> None:`
		`+ self._source = source`
		`+ meta_updated, meta = self._get_meta()`

		`- raise InvalidDatasetFormat()`
		`+ if not meta_updated:`
		`+ raise ConnectionError("Unable to identify remote datasource.")`
		`+`
		`+ self._meta = meta`
		`+ self._data = self._get_data()`

		`def update(self):`
		`+ # TODO: Add logging to check when there is an update request`
		`meta_updated, meta = self._get_meta()`
		`if meta_updated:`
		`self._meta = meta`
		`- self.dataframe = self._get_dataframe()`
		`+ self._data = self._get_data()`


		`class ActiveContributors(Dataset):`
		`@@ -85,11 +109,27 @@`
		`super().__init__()`
		`self.report_frequency = ReportFrequency.WEEKLY.value`

		`+ @property`
		`+ def id(self) -> str:`
		`+ return "active-contributors"`
		`+`
		`+ @property`
		`+ def title(self) -> str:`
		`+ return "Active Contributors"`
		`+`
		`+ @property`
		`+ def x(self) -> Union[pd.Index, pd.Series, str]:`
		`+ return self._data.index`
		`+`
		`+ @property`
		`+ def y(self) -> Union[pd.Series, str]:`
		`+ return "ActiveUsers"`
		`+`
		`def load(self, base_url: str) -> None:`
		`endpoint = urljoin(base_url, "reports/contributors")`
		`super(ActiveContributors, self).load(endpoint)`

		`- def _prepare_dataframe(self, dataframe: pd.DataFrame) -> pd.DataFrame:`
		`+ def _prepare_data(self, dataframe: pd.DataFrame) -> pd.DataFrame:`
		`dataframe.Date = pd.to_datetime(dataframe.Date, format="%m/%d/%y")`
		`return dataframe.resample(self.report_frequency, on="Date").sum()`

contributor_trends/dashboard/figures.py

file added

+17

		`@@ -0,0 +1,17 @@`
		`+ from typing import Type, Callable, Tuple`
		`+`
		`+ import plotly.express as px`
		`+ from plotly.graph_objs import Figure`
		`+`
		`+ from contributor_trends.dashboard.datasets import Dataset`
		`+`
		`+`
		`+ def area_chart(dataset: Type[Dataset]) -> Figure:`
		`+ return px.area(dataset.data, x=dataset.x, y=dataset.y)`
		`+`
		`+`
		`+ def update(`
		`+ dataset: Type[Dataset], figure: Callable[[Type[Dataset]], Figure]`
		`+ ) -> Tuple[Figure, str]:`
		`+ dataset.update()`
		`+ return figure(dataset), str(dataset)`

contributor_trends/dashboard/layouts.py

file modified

+26 -14

		`@@ -1,36 +1,48 @@`
		`+ from typing import Type`
		`+`
		`import dash_core_components as dcc`
		`import dash_html_components as html`
		`- import plotly.express as px`
		`+ from dash.development.base_component import Component`
		`+ from flask import current_app`

		`- from contributor_trends.dashboard.datasets import ActiveContributors`
		`+ from contributor_trends.dashboard import figures`
		`+ from contributor_trends.dashboard.datasets import Dataset`


		`- def weekly_activity_layout(dataset: ActiveContributors) -> html.Div:`
		`+ def chart_layout(dataset: Type[Dataset]) -> Component:`
		`return html.Div(`
		`[`
		`- html.H1("Weekly Active Contributors"),`
		`- html.Hr(),`
		`- dcc.Graph(`
		`- id="weekly-active-contributors",`
		`- figure=px.area(`
		`- dataset.dataframe, x=dataset.dataframe.index, y="ActiveUsers"`
		`- ),`
		`+ html.H2(dataset.title, className="chart-title"),`
		`+ html.Div(`
		`+ [`
		`+ dcc.Graph(`
		`+ id=f"{dataset.id}-chart", figure=figures.area_chart(dataset)`
		`+ ),`
		`+ html.P(str(dataset), id=f"{dataset.id}-chart-meta"),`
		`+ ],`
		`+ className="chart-content",`
		`),`
		`- ]`
		`+ dcc.Interval(`
		`+ id=f"{dataset.id}-interval",`
		`+ interval=current_app.config["APP_UPDATE_TIME"], # in milliseconds`
		`+ ),`
		`+ ],`
		`+ id=f"{dataset.id}-container",`
		`+ className="chart-container",`
		`)`


		`- def index_layout() -> html.Div:`
		`+ def index_layout() -> Component:`
		`return html.Div(`
		`[dcc.Location(id="url", refresh=False), html.Div(id="page-content")]`
		`)`


		`- def not_found_layout(url) -> html.Div:`
		`+ def not_found_layout(url) -> Component:`
		`return html.Div(`
		`[`
		`html.H1("404: Not found"),`
		`html.Hr(),`
		`- html.P(f"The pathname {url} was not recognised..."),`
		`+ html.P(f"The pathname {url} was not recognised."),`
		`]`
		`)`

josseline commented 2 years ago

With this function is available get a callback, it is in charge of making a request and update of the graph every day (in production) and every minute in the dev environment.