PR#128: Populate both groups and users from FAS. - fedora-hubs

fedora-hubs

#128 Populate both groups and users from FAS.

Merged 8 years ago by ralph. Opened 8 years ago by ralph.

feature/populate-enhancements into develop

Populate both groups and users from FAS.

Ralph Bean • 8 years ago

c9709c8

hubs/defaults.py

file modified

+47

		`@@ -80,3 +80,50 @@`
		`}))`
		`hub.widgets.append(widget)`
		`return hub`
		`+`
		`+`
		`+ def add_group_widgets(session, hub, name, summary,`
		`+ # These are all things that come from FAS`
		`+ apply_rules=None,`
		`+ irc_channel=None,`
		`+ irc_network=None,`
		`+ join_message=None,`
		`+ mailing_list=None,`
		`+ mailing_list_url=None):`
		`+ """ Some defaults for an automatically created group hub. """`
		`+`
		`+ if apply_rules:`
		`+ widget = hubs.models.Widget(`
		`+ plugin='sticky', index=0, left=True,`
		`+ _config=json.dumps({`
		`+ "text": apply_rules,`
		`+ }))`
		`+ hub.widgets.append(widget)`
		`+`
		`+ if join_message:`
		`+ widget = hubs.models.Widget(`
		`+ plugin='about', index=0,`
		`+ _config=json.dumps({`
		`+ "text": join_message,`
		`+ }))`
		`+ hub.widgets.append(widget)`
		`+`
		`+ widget = hubs.models.Widget(`
		`+ plugin='rules', index=1,`
		`+ _config=json.dumps({`
		`+ # TODO -- can we guess their urls?`
		`+ 'link': None,`
		`+ 'schedule_text': None,`
		`+ 'schedule_link': None,`
		`+ 'minutes_link': None,`
		`+ })`
		`+ )`
		`+ hub.widgets.append(widget)`
		`+`
		`+ # TODO - set up irc widget with the right irc channel`
		`+ # TODO - set up mailing list with the right mailing list.`
		`+`
		`+ # TODO - set up feed with aggregate of users stuff unless we have some`
		`+ # other preset.`
		`+`
		`+ return hub`

hubs/models.py

file modified

+21

		`@@ -160,6 +160,15 @@`
		`if idle > limit:`
		`return name`

		`+ @classmethod`
		`+ def get_or_create(cls, session, name, summary, **extra):`
		`+ self = cls.by_name(session, name)`
		`+ if not self:`
		`+ self = cls.create_group_hub(session, name, summary, **extra)`
		`+ session.add(self)`
		`+ session.commit()`
		`+ return self`
		`+`
		`@property`
		`def owners(self):`
		`return [assoc.user for assoc in self.associations`
		`@@ -227,6 +236,18 @@`

		`user = User.by_username(session, username)`
		`hub.subscribe(session, user, role='owner')`
		`+ return hub`
		`+`
		`+ @classmethod`
		`+ def create_group_hub(cls, session, name, summary, **extra):`
		`+ hub = cls(name=name, summary=summary,`
		`+ # TODO -- do something else, smarter for group avatars`
		`+ avatar=username2avatar(name),`
		`+ user_hub=False)`
		`+ session.add(hub)`
		`+`
		`+ hubs.defaults.add_group_widgets(session, hub, name, summary, **extra)`
		`+ return hub`

		`@property`
		`def right_width(self):`

hubs/templates/groups.html

file modified

+4 -2

		`@@ -82,7 +82,8 @@`
		`<div class="row">`
		`<div class="col-md-12">`
		`{% for group in promoted %}`
		`- {{ group.name }} is a promoted group, and it has {{ group.subscribers \| length }} subscribers and {{ group.members \| length }} members, and {{group.owners \| length }} owners`
		`+ <a href="{{ url_for('hub', name=group.name)}}">{{group.name}}</a>`
		`+ is a promoted group, and it has {{ group.subscribers \| length }} subscribers and {{ group.members \| length }} members, and {{group.owners \| length }} owners`
		`{% endfor %}`
		`</div>`
		`</div>`
		`@@ -108,7 +109,8 @@`
		`<div class="row">`
		`<div class="col-md-12">`
		`{% for group in secondary %}`
		`- {{ group.name }} is not a promoted group`
		`+ <a href="{{ url_for('hub', name=group.name)}}">{{group.name}}</a>`
		`+ is not a promoted group`
		`but it has {{ group.subscribers \| length }} subscribers and {{ group.members \| length }} members, and {{group.owners \| length }} owners`
		`<br/>`
		`{% endfor %}`

populate-from-fas.py ~~populate-all-users.py~~

file renamed

+61 -1

		`@@ -18,6 +18,22 @@`
		`timeout = socket.getdefaulttimeout()`
		`socket.setdefaulttimeout(None)`

		`+`
		`+ prefix_blacklist = [`
		`+ u'git',`
		`+ u'svn',`
		`+ u'bzr',`
		`+ u'cvs',`
		`+ u'hg',`
		`+ u'cla_',`
		`+ u'sysadmin-',`
		`+ ]`
		`+ suffix_blacklist = [`
		`+ u'commits',`
		`+ u'git',`
		`+ ]`
		`+`
		`+`
		`for letter in reversed(sorted(list(set(string.letters.lower())))):`
		`session = hubs.models.init(fedmsg_config['hubs.sqlalchemy.uri'], True, True)`
		`print "Querying FAS for the %r users.. hang on." % letter`
		`@@ -32,7 +48,51 @@`
		`fullname = user['human_name']`
		`openid = '%s.id.fedoraproject.org' % username`
		`print "Creating account for %r" % openid`
		`- hubs.models.User.get_or_create(`
		`+ hubs_user = hubs.models.User.get_or_create(`
		`session, openid=openid, fullname=fullname)`

		`session.commit()`
		`+`
		`+ # Go back now and set up the groups...`
		`+ session = hubs.models.init(fedmsg_config['hubs.sqlalchemy.uri'], True, True)`
		`+ for user in users:`
		`+ hubs_user = hubs.models.User.by_username(session, user['username'])`
		`+`
		`+ # Build a lookup by group id`
		`+ roles = dict([(r.group_id, r) for r in user['roles']])`
		`+`
		`+ for membership in user['memberships']:`
		`+ role = roles[membership['id']]`
		`+ if role['role_status'] != u'approved':`
		`+ continue`
		`+`
		`+ name = membership['name']`
		`+ summary = membership['display_name']`
		`+`
		`+ if any([name.startswith(prefix) for prefix in prefix_blacklist]):`
		`+ continue`
		`+ if any([name.endswith(suffix) for suffix in suffix_blacklist]):`
		`+ continue`
		`+`
		`+ hub = hubs.models.Hub.get_or_create(`
		`+ session,`
		`+ name=name,`
		`+ summary=summary,`
		`+ apply_rules=membership['apply_rules'],`
		`+ irc_channel=membership['irc_channel'],`
		`+ irc_network=membership['irc_network'],`
		`+ join_message=membership['joinmsg'],`
		`+ mailing_list=membership['mailing_list'],`
		`+ mailing_list_url=membership['mailing_list_url'],`
		`+ )`
		`+`
		`+`
		`+ if not hubs_user in hub.subscribers:`
		`+ hub.subscribe(session, hubs_user, role='subscriber')`
		`+ if not hubs_user in hub.members:`
		`+ hub.subscribe(session, hubs_user, role='member')`
		`+ if not hubs_user in hub.owners:`
		`+ if role['role_type'] in [u'administrator', u'sponsor']:`
		`+ hub.subscribe(session, hubs_user, role='owner')`
		`+`
		`+ session.commit()`

no initial comment

ralph commented 8 years ago

We can probably do better with some default widgets for automatically created group hubs.. but this will do for a start.

pingou commented 8 years ago

Looks fine to me, does this take long? Should slow down the initial start quite a bit no?

ralph commented 8 years ago

Yeah, it takes a good 10 minutes. It's a run-once script though to populate the demo with people and groups.

pingou commented 8 years ago

+1 to merge.

We should document where we cache which info so that when we want to invalidate some cache we're smarter than me and don't destroy everything, just what's needed :]

ralph commented 8 years ago

Yeah! For now, it's in:

/var/tmp/fedora-hubs-cache.db (that's the main cache that caches the JSON responses from the widgets).
/var/tmp/fmn-cache.db (this is an internal cache that caches some fmn responses for the feed widget)
/var/tmp/hubs.db (this is the sqlite db defining all the users and their widgets and their groups, etc. we'll move this to postgres in prod)