From 8ac31676a0dab006f8e3882350b10778cf570b71 Mon Sep 17 00:00:00 2001 From: mykaah Date: May 01 2021 23:24:51 +0000 Subject: [PATCH 1/2] refactored some modules --- diff --git a/create-all-time-userstats-csv.sh b/create-all-time-userstats-csv.sh index d616ede..a6e034a 100755 --- a/create-all-time-userstats-csv.sh +++ b/create-all-time-userstats-csv.sh @@ -12,6 +12,6 @@ ENDWEEK=$(( ( $(date +'%s') - $(date -ud '2012-01-01 0:0:0' +'%s') )/60/60/24/7 python new-and-old-users-report.py --csvh $(( $STARTWEEK - 1 )) | tee data/contributor-count.csv for week in $(seq $STARTWEEK $ENDWEEK ); do - python new-and-old-users-report.py --csv $week | tee -a data/contributor-count.csv + python3 new-and-old-users-report.py --csv $week | tee -a data/contributor-count.csv done diff --git a/generate-activity-charts.py b/generate-activity-charts.py index 83575e8..e8cdd8b 100755 --- a/generate-activity-charts.py +++ b/generate-activity-charts.py @@ -1,14 +1,12 @@ #!/usr/bin/python3 import os - import matplotlib as m -import matplotlib.pyplot as plt -import pandas +import pandas as pd +from utils import rename, get_pct, plot, save m.use("Agg") m.rcParams["font.size"] = 12 -m.rcParams["font.family"] = "Overpass" m.rcParams["legend.frameon"] = False try: @@ -16,419 +14,132 @@ try: except OSError: pass -datagit = pandas.read_csv( +# =============================datagit============================== +new_names = { + "users1": "Top 1%", + "users9": "Next 9%", + "users40": "Next 40%", + "userrest": "Remaining 50%", +} +color = ["#579d1c", "#ffd320", "#ff420e", "#004586"] + +datagit = pd.read_csv( "data/org.fedoraproject.prod.git.receive.bucketed-activity.csv", parse_dates=[0] ) datagit.set_index("weekstart", inplace=True) -graph = ( - datagit[["users1", "users9", "users40", "userrest"]] - .rename( - columns={ - "users1": "Top 1%", - "users9": "Next 9%", - "users40": "Next 40%", - "userrest": "Remaining 50%", - } - ) - .plot.area( - figsize=(16, 9), - color=["#579d1c", "#ffd320", "#ff420e", "#004586"], - grid=True, - yticks=range(0, 301, 25), - ) -) -# graph.legend(ncol=4) -# totally abusing this. -plt.suptitle("Number of Contributors Making Changes to Packages Each Week", fontsize=24) -graph.set_title("Grouped by Quarterly Activity Level of Each Contributor", fontsize=16) -graph.set_xlabel("") -fig = graph.get_figure() -fig.savefig("images/git.user.count.svg", dpi=300) +columns = ["users1", "users9", "users40", "userrest"] +df = rename(datagit, columns, new_names) -############################################# +yticks = range(0, 301, 25) +suptitle = "Number of Contributors Making Changes to Packages Each Week" +title = "Grouped by Quarterly Activity Level of Each Contributor" +path = "images/git.user.count.svg" -datagit["msgstotal"] = datagit[["msgs1", "msgs9", "msgs40", "msgsrest"]].sum(1) -datagit["msgs1%"] = 100 * datagit["msgs1"] / datagit["msgstotal"] -datagit["msgs9%"] = 100 * datagit["msgs9"] / datagit["msgstotal"] -datagit["msgs40%"] = 100 * datagit["msgs40"] / datagit["msgstotal"] -datagit["msgsrest%"] = 100 * datagit["msgsrest"] / datagit["msgstotal"] +graph = plot(df, suptitle, title, color, yticks=yticks) +save(graph, path) +get_pct(datagit) m.rcParams["legend.frameon"] = True -graph = ( - datagit[["msgs1%", "msgs9%", "msgs40%", "msgsrest%"]] - .rename( - columns={ - "msgs1%": "Top 1%", - "msgs9%": "Next 9%", - "msgs40%": "Next 40%", - "msgsrest%": "Remaining 50%", - } - ) - .plot.area( - figsize=(16, 9), - color=["#579d1c", "#ffd320", "#ff420e", "#004586"], - grid=True, - ylim=(0, 100), - ) -) -plt.suptitle( - "Percent of Package Changes Each Week From Each Activity Level Group", fontsize=24 -) -graph.set_title("", fontsize=16) -graph.set_xlabel("") - -fig = graph.get_figure() -fig.savefig("images/git.activity.share.svg", dpi=300) - -############################################### - -# graph=datagit[['newusercount']].rename(columns={"newusercount": "New Users"}).plot.area(figsize=(16, 9), -# color='#579d1c', -# grid=True,legend=False) -# plt.suptitle("New Packaging Contributor Count Per Week",fontsize=24) -# graph.set_title('') -# graph.set_xlabel('') -# fig=graph.get_figure() -# fig.savefig('images/git.newusers.svg',dpi=300) - -############################################# - -# datagit['newuseractions%']=100*datagit['newuseractions']/datagit['msgstotal'] -# datagit['monthuseractions%']=100*datagit['monthuseractions']/datagit['msgstotal'] -# datagit['yearuseractions%']=100*datagit['yearuseractions']/datagit['msgstotal'] -# datagit['olderuseractions%']=100*datagit['olderuseractions']/datagit['msgstotal'] - - -# m.rcParams['legend.frameon'] = True -# graph=datagit[['newuseractions%','monthuseractions%','yearuseractions%','olderuseractions%']][42:] -# .rename(columns={"newuseractions%": "New This Week","monthuseractions%":"New This Month", -# "yearuseractions%":"New This Year","olderuseractions%":"Old School"}) -# .plot.area(figsize=(16, 9), -# color=['#579d1c','#ffd320', '#ff420e', '#004586' ], -# grid=True,ylim=(0,100)) -# plt.suptitle("Percent of Package Changes Each Week By Time Since Packager's First Action",fontsize=24) -# graph.set_title("",fontsize=16) -# graph.set_xlabel('') -# -# fig=graph.get_figure() -# fig.savefig('images/git.activity.length.svg',dpi=300) -# -################################################################################################################ -################################################################################################################ - -databodhi = pandas.read_csv( + +columns = ["msgs1%", "msgs9%", "msgs40%", "msgsrest%"] +df = rename(datagit, columns, new_names) +ylim = (0, 100) +suptitle = "Percent of Package Changes Each Week From Each Activity Level Group" +title = "" +path = "images/git.activity.share.svg" +graph = plot(df, suptitle, title, color, ylim=ylim) +save(graph, path) + +# =============================databodhi============================== +databodhi = pd.read_csv( "data/org.fedoraproject.prod.bodhi.update.comment.bucketed-activity.csv", parse_dates=[0], ) databodhi.set_index("weekstart", inplace=True) -graph = ( - databodhi[["users1", "users9", "users40", "userrest"]] - .rename( - columns={ - "users1": "Top 1%", - "users9": "Next 9%", - "users40": "Next 40%", - "userrest": "Remaining 50%", - } - ) - .plot.area( - figsize=(16, 9), - color=["#579d1c", "#ffd320", "#ff420e", "#004586"], - grid=True, - yticks=range(0, 301, 25), - ) -) -# graph.legend(ncol=4) -# totally abusing this. -plt.suptitle( - "Number of Contributors Providing Feedback on Package Updates Each Week", - fontsize=24, -) -graph.set_title("Grouped by Quarterly Activity Level of Each Contributor", fontsize=16) -graph.set_xlabel("") -fig = graph.get_figure() -fig.savefig("images/bodhi.user.count.svg", dpi=300) +columns = ["users1", "users9", "users40", "userrest"] +df = rename(databodhi, columns, new_names) -############################################# +yticks = range(0, 301, 25) +suptitle = "Number of Contributors Providing Feedback on Package Updates Each Week" +title = "Grouped by Quarterly Activity Level of Each Contributor" +path = "images/bodhi.user.count.svg" +graph = plot(df, suptitle, title, color, yticks=yticks) +save(graph, path) -databodhi["msgstotal"] = databodhi[["msgs1", "msgs9", "msgs40", "msgsrest"]].sum(1) -databodhi["msgs1%"] = 100 * databodhi["msgs1"] / databodhi["msgstotal"] -databodhi["msgs9%"] = 100 * databodhi["msgs9"] / databodhi["msgstotal"] -databodhi["msgs40%"] = 100 * databodhi["msgs40"] / databodhi["msgstotal"] -databodhi["msgsrest%"] = 100 * databodhi["msgsrest"] / databodhi["msgstotal"] +get_pct(databodhi) m.rcParams["legend.frameon"] = True -graph = ( - databodhi[["msgs1%", "msgs9%", "msgs40%", "msgsrest%"]] - .rename( - columns={ - "msgs1%": "Top 1%", - "msgs9%": "Next 9%", - "msgs40%": "Next 40%", - "msgsrest%": "Remaining 50%", - } - ) - .plot.area( - figsize=(16, 9), - color=["#579d1c", "#ffd320", "#ff420e", "#004586"], - grid=True, - ylim=(0, 100), - ) -) -plt.suptitle( - "Percent of Update Feedback Each Week From Each Activity Level Group", fontsize=24 -) -graph.set_title("", fontsize=16) -graph.set_xlabel("") - -fig = graph.get_figure() -fig.savefig("images/bodhi.activity.share.svg", dpi=300) - -############################################### - -# graph=databodhi[['newusercount']].rename(columns={"newusercount": "New Users"}).plot.area(figsize=(16, 9), -# color='#579d1c', -# grid=True,legend=False) -# plt.suptitle("New Update Testing Contributor Count Per Week",fontsize=24) -# graph.set_title('') -# graph.set_xlabel('') -# fig=graph.get_figure() -# fig.savefig('images/bodhi.newusers.svg',dpi=300) - -############################################# - -# databodhi['newuseractions%']=100*databodhi['newuseractions']/databodhi['msgstotal'] -# databodhi['monthuseractions%']=100*databodhi['monthuseractions']/databodhi['msgstotal'] -# databodhi['yearuseractions%']=100*databodhi['yearuseractions']/databodhi['msgstotal'] -# databodhi['olderuseractions%']=100*databodhi['olderuseractions']/databodhi['msgstotal'] - - -# m.rcParams['legend.frameon'] = True -# graph=databodhi[['newuseractions%','monthuseractions%','yearuseractions%','olderuseractions%']][42:] -# .rename(columns={"newuseractions%": "New This Week","monthuseractions%":"New This Month", -# "yearuseractions%":"New This Year","olderuseractions%":"Old School"}).plot.area(figsize=(16, 9), -# color=['#579d1c','#ffd320', '#ff420e', '#004586' ], -# grid=True,ylim=(0,100)) -# plt.suptitle("Percent of Update Feedback Each Week By Time Since Packager's First Action",fontsize=24) -# graph.set_title("",fontsize=16) -# graph.set_xlabel('') -# -# fig=graph.get_figure() -# fig.savefig('images/bodhi.activity.length.svg',dpi=300) - - -################################################################################################################ -################################################################################################################ - -datawiki = pandas.read_csv( +columns = ["msgs1%", "msgs9%", "msgs40%", "msgsrest%"] +df = rename(databodhi, columns, new_names) + +ylim = (0, 100) +suptitle = "Percent of Update Feedback Each Week From Each Activity Level Group" +title = "" +path = "bodhi.activity.share.svg" +graph = plot(df, suptitle, title, color, ylim=ylim) +save(graph, path) + +# =============================datawiki============================== +datawiki = pd.read_csv( "data/org.fedoraproject.prod.wiki.article.edit.bucketed-activity.csv", parse_dates=[0], ) datawiki.set_index("weekstart", inplace=True) -graph = ( - datawiki[["users1", "users9", "users40", "userrest"]] - .rename( - columns={ - "users1": "Top 1%", - "users9": "Next 9%", - "users40": "Next 40%", - "userrest": "Remaining 50%", - } - ) - .plot.area( - figsize=(16, 9), - color=["#579d1c", "#ffd320", "#ff420e", "#004586"], - grid=True, - yticks=range(0, 301, 25), - ) -) -# graph.legend(ncol=4) -# totally abusing this. -plt.suptitle("Number of Wiki Editors Each Week", fontsize=24) -graph.set_title("Grouped by Quarterly Activity Level of Each Contributor", fontsize=16) -graph.set_xlabel("") -fig = graph.get_figure() -fig.savefig("images/wiki.user.count.svg", dpi=300) +columns = ["users1", "users9", "users40", "userrest"] +df = rename(datawiki, columns, new_names) -############################################# +yticks = range(0, 301, 25) +suptitle = "Number of Wiki Editors Each Week" +title = "Grouped by Quarterly Activity Level of Each Contributor" +path = "images/wiki.user.count.svg" +graph = plot(df, suptitle, title, color, yticks=yticks) +save(graph, path) -datawiki["msgstotal"] = datawiki[["msgs1", "msgs9", "msgs40", "msgsrest"]].sum(1) -datawiki["msgs1%"] = 100 * datawiki["msgs1"] / datawiki["msgstotal"] -datawiki["msgs9%"] = 100 * datawiki["msgs9"] / datawiki["msgstotal"] -datawiki["msgs40%"] = 100 * datawiki["msgs40"] / datawiki["msgstotal"] -datawiki["msgsrest%"] = 100 * datawiki["msgsrest"] / datawiki["msgstotal"] +get_pct(datawiki) m.rcParams["legend.frameon"] = True -graph = ( - datawiki[["msgs1%", "msgs9%", "msgs40%", "msgsrest%"]] - .rename( - columns={ - "msgs1%": "Top 1%", - "msgs9%": "Next 9%", - "msgs40%": "Next 40%", - "msgsrest%": "Remaining 50%", - } - ) - .plot.area( - figsize=(16, 9), - color=["#579d1c", "#ffd320", "#ff420e", "#004586"], - grid=True, - ylim=(0, 100), - ) -) -plt.suptitle( - "Percent of Wiki Edits Each Week From Each Activity Level Group", fontsize=24 -) -graph.set_title("", fontsize=16) -graph.set_xlabel("") - -fig = graph.get_figure() -fig.savefig("images/wiki.activity.share.svg", dpi=300) - -############################################### - -# graph=datawiki[['newusercount']].rename(columns={"newusercount": "New Users"}).plot.area(figsize=(16, 9), -# color='#579d1c', -# grid=True,legend=False) -# plt.suptitle("New Wiki Contributor Count Per Week",fontsize=24) -# graph.set_title('') -# graph.set_xlabel('') -# fig=graph.get_figure() -# fig.savefig('images/wiki.newusers.svg',dpi=300) -############################################### - -# graph=datawiki[['newusercount']].rename(columns={"newusercount": "New Users"}).plot.area(figsize=(16, 9), -# color='#579d1c', -# grid=True,legend=False) -# plt.suptitle("New Wiki Contributor Count Per Week",fontsize=24) -# graph.set_title('') -# graph.set_xlabel('') -# fig=graph.get_figure() -# fig.savefig('images/wiki.newusers.svg',dpi=300) - -############################################# - -# datawiki['newuseractions%']=100*datawiki['newuseractions']/datawiki['msgstotal'] -# datawiki['monthuseractions%']=100*datawiki['monthuseractions']/datawiki['msgstotal'] -# datawiki['yearuseractions%']=100*datawiki['yearuseractions']/datawiki['msgstotal'] -# datawiki['olderuseractions%']=100*datawiki['olderuseractions']/datawiki['msgstotal'] - - -# m.rcParams['legend.frameon'] = True -# graph=datawiki[['newuseractions%','monthuseractions%','yearuseractions%','olderuseractions%']][42:] -# .rename(columns={"newuseractions%": "New This Week","monthuseractions%":"New This Month", -# "yearuseractions%":"New This Year","olderuseractions%":"Old School"}).plot.area(figsize=(16, 9), -# color=['#579d1c','#ffd320', '#ff420e', '#004586' ], -# grid=True,ylim=(0,100)) -# plt.suptitle("Percent of Wiki Edits Each Week By Time Since Editor's First Edit",fontsize=24) -# graph.set_title("",fontsize=16) -# graph.set_xlabel('') -# -# fig=graph.get_figure() -# fig.savefig('images/wiki.activity.length.svg',dpi=300) - -############################################### -############################################### -datapagure = pandas.read_csv( +columns = ["msgs1%", "msgs9%", "msgs40%", "msgsrest%"] +df = rename(datawiki, columns, new_names) + +ylim = (0, 100) +suptitle = "Percent of Wiki Edits Each Week From Each Activity Level Group" +title = "" +path = "wiki.user.count.svg" +graph = plot(df, suptitle, title, color, ylim=ylim) +save(graph, path) + + +# =============================datapagure============================== +datapagure = pd.read_csv( "data/io.pagure.prod.pagure.git.receive.bucketed-activity.csv", parse_dates=[0] ) datapagure.set_index("weekstart", inplace=True) +columns = ["users1", "users9", "users40", "userrest"] +df = rename(datapagure, columns, new_names) -graph = ( - datapagure[["users1", "users9", "users40", "userrest"]] - .rename( - columns={ - "users1": "Top 1%", - "users9": "Next 9%", - "users40": "Next 40%", - "userrest": "Remaining 50%", - } - ) - .plot.area( - figsize=(16, 9), - color=["#579d1c", "#ffd320", "#ff420e", "#004586"], - grid=True, - yticks=range(0, 25, 5), - ) -) -# graph.legend(ncol=4) -# totally abusing this. -plt.suptitle("Number of Contributors Making Commits to Pagure Each Week", fontsize=24) -graph.set_title("Grouped by Quarterly Activity Level of Each Contributor", fontsize=16) -graph.set_xlabel("") -fig = graph.get_figure() -fig.savefig("images/pagure.user.count.svg", dpi=300) +yticks = range(0, 25, 5) +suptitle = "Number of Contributors Making Commits to Pagure Each Week" +title = "Grouped by Quarterly Activity Level of Each Contributor" +path = "images/pagure.user.count.svg" +graph = plot(df, suptitle, title, color, yticks=yticks) +save(graph, path) -############################################# - -datapagure["msgstotal"] = datapagure[["msgs1", "msgs9", "msgs40", "msgsrest"]].sum(1) -datapagure["msgs1%"] = 100 * datapagure["msgs1"] / datapagure["msgstotal"] -datapagure["msgs9%"] = 100 * datapagure["msgs9"] / datapagure["msgstotal"] -datapagure["msgs40%"] = 100 * datapagure["msgs40"] / datapagure["msgstotal"] -datapagure["msgsrest%"] = 100 * datapagure["msgsrest"] / datapagure["msgstotal"] +get_pct(datapagure) m.rcParams["legend.frameon"] = True -graph = ( - datapagure[["msgs1%", "msgs9%", "msgs40%", "msgsrest%"]] - .rename( - columns={ - "msgs1%": "Top 1%", - "msgs9%": "Next 9%", - "msgs40%": "Next 40%", - "msgsrest%": "Remaining 50%", - } - ) - .plot.area( - figsize=(16, 9), - color=["#579d1c", "#ffd320", "#ff420e", "#004586"], - grid=True, - ylim=(0, 100), - ) -) -plt.suptitle( - "Percent of Pagure Commits Each Week From Each Activity Level Group", fontsize=24 -) -graph.set_title("", fontsize=16) -graph.set_xlabel("") - -fig = graph.get_figure() -fig.savefig("images/pagure.activity.share.svg", dpi=300) - -############################################### - -# graph=datapagure[['newusercount']].rename(columns={"newusercount": "New Users"}).plot.area(figsize=(16, 9), -# color='#579d1c', -# grid=True,legend=False) -# plt.suptitle("New Pagure Contributor Count Per Week",fontsize=24) -# graph.set_title('') -# graph.set_xlabel('') -# fig=graph.get_figure() -# fig.savefig('images/pagure.newusers.svg',dpi=300) - -############################################# - -# datapagure['newuseractions%']=100*datapagure['newuseractions']/datapagure['msgstotal'] -# datapagure['monthuseractions%']=100*datapagure['monthuseractions']/datapagure['msgstotal'] -# datapagure['yearuseractions%']=100*datapagure['yearuseractions']/datapagure['msgstotal'] -# datapagure['olderuseractions%']=100*datapagure['olderuseractions']/datapagure['msgstotal'] - - -# m.rcParams['legend.frameon'] = True -# graph=datapagure[['newuseractions%','monthuseractions%','yearuseractions%','olderuseractions%']][42:] -# .rename(columns={"newuseractions%": "New This Week","monthuseractions%":"New This Month", -# "yearuseractions%":"New This Year","olderuseractions%":"Old School"}).plot.area(figsize=(16, 9), -# color=['#579d1c','#ffd320', '#ff420e', '#004586' ], -# grid=True,ylim=(0,100)) -# plt.suptitle("Percent of Pagure Commits Each Week By Time Since Packager's First Action",fontsize=24) -# graph.set_title("",fontsize=16) -# graph.set_xlabel('') -# -# fig=graph.get_figure() -# fig.savefig('images/pagure.activity.length.svg',dpi=300) +columns = ["msgs1%", "msgs9%", "msgs40%", "msgsrest%"] +df = rename(datapagure, columns, new_names) + +ylim = (0, 100) +suptitle = "Percent of Pagure Commits Each Week From Each Activity Level Group" +title = "" +path = "images/pagure.activity.share.svg" +graph = plot(df, suptitle, title, color, ylim=ylim) +save(graph, path) diff --git a/generate-contributor-charts.py b/generate-contributor-charts.py index 50c3adb..6f1b27e 100755 --- a/generate-contributor-charts.py +++ b/generate-contributor-charts.py @@ -2,13 +2,13 @@ import os import matplotlib as m -import matplotlib.pyplot as plt -import pandas +import pandas as pd + +from utils import rename, save, plot m.use("Agg") m.rcParams["font.size"] = 12 -m.rcParams["font.family"] = "Overpass" m.rcParams["legend.frameon"] = True try: @@ -16,75 +16,56 @@ try: except OSError: pass -data = pandas.read_csv("data/contributor-count.csv", parse_dates=[0]) -data.set_index("weekstart", inplace=True) - +new_names = { + "oldactive": "Old School", + "midactive": "Intermediate", + "newactive": "New Contributors", +} +color = ["#579d1c", "#ffd320", "#ff420e", "#004586"] -graph = ( - data[["oldactive", "midactive", "newactive"]] - .rename( - columns={ - "oldactive": "Old School", - "midactive": "Intermediate", - "newactive": "New Contributors", - } - ) - .plot.area( - figsize=(16, 9), - color=["#ff420e", "#ffd320", "#579d1c"], # '#004586' - grid=True, - stacked=True, - yticks=range(0, 451, 25), - ) -) -data[["rawcount"]].rename( - columns={"rawcount": "All Contributors\nincluding less active"} -).plot(figsize=(16, 9), ax=graph, yticks=range(0, 426, 25)) +data = pd.read_csv("data/contributor-count.csv", parse_dates=[0]) +data.set_index("weekstart", inplace=True) -graph.xaxis.grid(True, which="minor", linestyle="-", linewidth=0.25) -graph.yaxis.grid(True, which="major", linestyle="-", linewidth=0.25) +columns = ["oldactive", "midactive", "newactive"] +df = rename(data, columns) -plt.suptitle("Fedora Contributors by Week", fontsize=24) -graph.set_title( - "Stacked graph of contributors with measured activity each week — and at least 13 weeks total in the last year.\n" +yticks = range(0, 451, 25) +suptitle = "Fedora Contributors by Week" +title = ( + "Stacked graph of contributors with measured activity each week — and at least 13 weeks total in the \ + last year.\n" "“Old school” contributors have been active for longer than two years; new contributors, less than one.\n" - "Blue line shows all contributors active this week regardless of amount of other activity.", - fontsize=12, + "Blue line shows all contributors active this week regardless of amount of other activity." ) -graph.set_xlabel("") - -fig = graph.get_figure() -fig.savefig("images/active-contributors-by-week.svg", dpi=300) +path = "images/active-contributors-by-week.svg" - -graph = ( - data[["oldcore", "midcore", "newcore"]] - .rename( - columns={ - "oldcore": "Old School", - "midcore": "Intermediate", - "newcore": "New Contributors", - } - ) - .plot.area( - figsize=(16, 9), - color=["#ff420e", "#ffd320", "#579d1c"], # '#004586' - grid=True, - stacked=True, - yticks=range(0, 101, 25), - ) +color = ["#ff420e", "#ffd320", "#579d1c"] +new_names = {"rawcount": "All Contributors\nincluding less active"} +graph = plot(df, suptitle, title, yticks=yticks, color=color, stacked=True) +rename(df, columns=["rowcount"], new_names=new_names).plot( + figsize=(16, 9), ax=graph, yticks=range(0, 426, 25) ) +save(graph, path) -graph.xaxis.grid(True, which="minor", linestyle="-", linewidth=0.25) -graph.yaxis.grid(True, which="major", linestyle="-", linewidth=0.25) +columns = ["oldcore", "midcore", "newcore"] +new_names = new_names = { + "oldcore": "Old School", + "midcore": "Intermediate", + "newcore": "New Contributors", +} +df = rename(data, columns, new_names=new_names) -plt.suptitle("Core Fedora Contributors by Week", fontsize=24) -graph.set_title( - "Stacked graph of contributors with measured activity this week — and at least 13 weeks total in the last year.\n" +yticks = range(0, 101, 25) +suptitle = "Core Fedora Contributors by Week" +title = ( + "Stacked graph of contributors with measured activity this week — and at least 13 weeks total in the \ + last year.\n" "Old school contributors have been active for longer than two years; new contributors, less than one.\n" - "“Core” means part of the set doing about ⅔s of all actions over the past year.", - fontsize=12, + "“Core” means part of the set doing about ⅔s of all actions over the past year." ) -graph.set_xlabel("") -fig = graph.get_figure() -fig.savefig("images/active-core-contributors-by-week.svg", dpi=300) +path = "images/active-core-contributors-by-week.svg" + +color = ["#ff420e", "#ffd320", "#579d1c"] +new_names = {"rawcount": "All Contributors\nincluding less active"} +graph = plot(df, suptitle, title, yticks=yticks, color=color, stacked=True) +save(graph, path) diff --git a/get-weekly-user-stats.sh b/get-weekly-user-stats.sh index 5efee40..0b52936 100755 --- a/get-weekly-user-stats.sh +++ b/get-weekly-user-stats.sh @@ -1,11 +1,11 @@ #!/bin/sh -until python weekly-user-activity.py org.fedoraproject.prod.irc.karma; do sleep 5; done -until python weekly-user-activity.py org.fedoraproject.prod.git.receive; do sleep 5; done -until python weekly-user-activity.py org.fedoraproject.prod.bodhi.update.comment; do sleep 5; done -until python weekly-user-activity.py org.fedoraproject.prod.wiki.article.edit; do sleep 5; done -until python weekly-user-activity.py org.fedoraproject.prod.infragit.receive; do sleep 5; done -until python weekly-user-activity.py org.fedoraproject.prod.fedoratagger.rating.update; do sleep 5; done -until python weekly-user-activity.py io.pagure.prod.pagure.git.receive; do sleep 5; done -until python weekly-user-activity.py io.pagure.prod.pagure.issue.new; do sleep 5; done -until python weekly-user-activity.py org.fedoraproject.prod.fedbadges.badge.award; do sleep 5; done +until python3 weekly-user-activity.py org.fedoraproject.prod.irc.karma; do sleep 5; done +until python3 weekly-user-activity.py org.fedoraproject.prod.git.receive; do sleep 5; done +until python3 weekly-user-activity.py org.fedoraproject.prod.bodhi.update.comment; do sleep 5; done +until python3 weekly-user-activity.py org.fedoraproject.prod.wiki.article.edit; do sleep 5; done +until python3 weekly-user-activity.py org.fedoraproject.prod.infragit.receive; do sleep 5; done +until python3 weekly-user-activity.py org.fedoraproject.prod.fedoratagger.rating.update; do sleep 5; done +until python3 weekly-user-activity.py io.pagure.prod.pagure.git.receive; do sleep 5; done +until python3 weekly-user-activity.py io.pagure.prod.pagure.issue.new; do sleep 5; done +until python3 weekly-user-activity.py org.fedoraproject.prod.fedbadges.badge.award; do sleep 5; done diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8b6bc20..0000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -fedmsg==1.1.2 -matplotlib==3.4.1 -numpy==1.20.2 -pandas==1.2.3 -requests==2.25.1 -pytz==2021.1 -pytest==6.2.2 \ No newline at end of file diff --git a/utils.py b/utils.py index b3aa96c..367e2d1 100644 --- a/utils.py +++ b/utils.py @@ -1,4 +1,7 @@ import requests +import pandas as pd +import matplotlib.pyplot as plt +from typing import Dict, List url = "https://apps.fedoraproject.org/datagrepper/raw" @@ -38,3 +41,38 @@ def grep(tries=0, **kwargs): raise ValueError("Ran out of retries") for message in data.get("raw_messages", []): yield message + + +def rename(df: pd.DataFrame, columns: List, new_names: Dict) -> pd.DataFrame: + df.rename(columns=new_names) + return df[columns] + + +def get_pct(df: pd.DataFrame, columns: List = ["msgs1", "msgs9", "msgs40", "msgsrest"]): + df["msgstotal"] = df[columns].sum(1) + total = df["msgstotal"] + data_list = columns + for i in data_list: + df[i + "%"] = 100 * df[i] / total + + +def plot(df, suptitle: str, title: str, color, **kwargs) -> None: + graph = df.plot.area( + figsize=(16, 9), + color=color, + grid=False, + yticks=kwargs.get("yticks", None), + stacked=kwargs.get("stacked", True), + ylim=kwargs.get("ylim", None), + ) + plt.suptitle(suptitle, fontsize=24) + graph.set_title(title, fontsize=16) + graph.set_xlabel("") + graph.xaxis.grid(True, which="minor", linestyle="-", linewidth=0.25) + graph.yaxis.grid(True, which="major", linestyle="-", linewidth=0.25) + return graph + + +def save(graph, path: str) -> None: + fig = graph.get_figure() + fig.savefig(path, dpi=300) diff --git a/weekly-user-activity.py b/weekly-user-activity.py index 1e1d1a0..6c99700 100755 --- a/weekly-user-activity.py +++ b/weekly-user-activity.py @@ -112,7 +112,7 @@ with open("data/%s.bucketed-activity.csv" % (discriminant), "w") as bucketcsv: bucketcsv.flush() while starttime < datetime.datetime.now() + datetime.timedelta( - 42 + 42 ): # weeks in the future because see below endtime = starttime + datetime.timedelta(7) weekinfo = WeekActions( @@ -129,7 +129,7 @@ with open("data/%s.bucketed-activity.csv" % (discriminant), "w") as bucketcsv: ) msgcachefile = ( - "cache/" + discriminant + "." + starttime.strftime("%Y-%m-%d") + ".pickle" + "cache/" + discriminant + "." + starttime.strftime("%Y-%m-%d") + ".pickle" ) if os.path.exists(msgcachefile): @@ -300,7 +300,7 @@ with open("data/%s.bucketed-activity.csv" % (discriminant), "w") as bucketcsv: ) if any( - (bucketscores[1], bucketscores[2], bucketscores[3], bucketscores[4]) + (bucketscores[1], bucketscores[2], bucketscores[3], bucketscores[4]) ): bucketcsv.write( "%s,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d\n" @@ -330,7 +330,7 @@ with open("data/%s.bucketed-activity.csv" % (discriminant), "w") as bucketcsv: # unless we're in the future, save the weekly userdata csv if starttime < datetime.datetime.now(): with open( - "data/weekly/%s.userdata.%05d.csv" % (discriminant, weeknum), "w" + "data/weekly/%s.userdata.%05d.csv" % (discriminant, weeknum), "w" ) as weekcsv: weekcsv.write( "%s,%s,%s,%s\n" % ("user", "actions", "firstseen", "lastseen") From c6b571ea7f0de95e7c074ed4414c37a4360cc321 Mon Sep 17 00:00:00 2001 From: mykaah Date: May 06 2021 04:52:07 +0000 Subject: [PATCH 2/2] addedd requirements --- diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8b6bc20 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fedmsg==1.1.2 +matplotlib==3.4.1 +numpy==1.20.2 +pandas==1.2.3 +requests==2.25.1 +pytz==2021.1 +pytest==6.2.2 \ No newline at end of file