From 8a94f66ad3e27eb5713332616a3ac7f14bed6b31 Mon Sep 17 00:00:00 2001 From: Eric Barbour Date: Jun 24 2016 15:17:41 +0000 Subject: Implements feed widget in Reactjs Includes SSE functionality Fixes JS that was causing errors New React directory structure --- diff --git a/.gitignore b/.gitignore index a8c634a..7787cd3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ fedmsg.d/credentials.py node* build/ scratch/ + +npm-debug.log diff --git a/README.rst b/README.rst index 5ec6862..24d78eb 100644 --- a/README.rst +++ b/README.rst @@ -125,7 +125,11 @@ flexibility. To get this working, you're going to set up: Start with some required packages:: - $ sudo dnf install postgresql-server python-datanommer-consumer datanommer-commands fedmsg-hub + $ sudo dnf install postgresql-server python-datanommer-consumer datanommer-commands fedmsg-hub npm + +The Feed Widget runs on React so install those dependencies too:: + + $ cd hubs/static/client && npm install And there are some support libraries you'll also need:: @@ -184,6 +188,10 @@ will just grow and grow over time:: [2015-07-01 14:33:21][ fedmsg INFO] copr has 6 entries [2015-07-01 14:33:21][ fedmsg INFO] askbot has 2 entries +Build JavaScript assets. Add ``--w`` for live reloading:: + + $ cd hubs/static/client && node_modules/.bin/webpack + **Lastly**, (fingers crossed) start up the fedora-hubs webapp and load your profile page. Once there are some messages that get into your local database that *should* show up on your feed.. they should appear there. (At very least, diff --git a/ev-server/hubs-stream-server.py b/ev-server/hubs-stream-server.py new file mode 100644 index 0000000..56d5b50 --- /dev/null +++ b/ev-server/hubs-stream-server.py @@ -0,0 +1,134 @@ +# encoding=utf8 +''' +This file is for development purposes only. + +The end goal is for the streaming server to be implemented on the FMN side +''' +import json +import logging +import sys +import trollius +import trollius_redis +import urllib2 +from concurrent.futures import TimeoutError +from trollius import From + +reload(sys) +sys.setdefaultencoding('utf8') + +log = logging.getLogger(__name__) +SERVER = None +REDIS_HOST = '0.0.0.0' +REDIS_PORT = 6379 +REDIS_DB = 0 +EVENTSOURCE_PORT = 9090 +''' +you need to + +pip install trollius trollius_redis + +dnf install redis httpie +systemctl start redis + + +usage: http get 0.0.0.0:9090/ + +''' + + +def get_recent_posts(): + delta = "delta=86400" # one day worth of data + rows_per_page = "rows_per_page=50" + url = "https://apps.fedoraproject.org/datagrepper/raw" + "?" + delta + "&" + rows_per_page + request = urllib2.Request(url) + contents = urllib2.urlopen(request).read() + json_response = json.loads(contents) + return json_response['raw_messages'] + + +@trollius.coroutine +def handle_client(client_reader, client_writer): + origin = '*' + if origin.endswith('/'): + origin = origin[:-1] + + client_writer.write(("HTTP/1.0 200 OK\n" + "Content-Type: text/event-stream\n" + "Cache: nocache\n" + "Connection: keep-alive\n" + "Access-Control-Allow-Origin: %s\n\n" % origin + ).encode()) + + connection = yield trollius.From(trollius_redis.Connection.create( + host=REDIS_HOST, port=REDIS_PORT, + db=REDIS_DB)) + + try: + posts = get_recent_posts() + + # send 50 latest posts + num_post_sent = 0 + while num_post_sent < 50: + reply = posts[num_post_sent] + reply = json.dumps(reply) + log.info(reply) + log.info("Sending post %s %s", str(num_post_sent), reply) + client_writer.write(('data: %s\n\n' % reply).encode()) + yield trollius.From(client_writer.drain()) + num_post_sent += 1 + yield From(trollius.sleep(2)) + + except trollius.ConnectionResetError: + log.exception("ERROR: ConnectionResetError in handle_client") + except Exception: + log.exception("ERROR: Exception in handle_client") + finally: + # Wathever happens, close the connection. + connection.close() + client_writer.close() + + +def main(): + global SERVER + + try: + loop = trollius.get_event_loop() + coro = trollius.start_server(handle_client, + host=None, + port=EVENTSOURCE_PORT, + loop=loop) + SERVER = loop.run_until_complete(coro) + log.info('Serving server at {}'.format(SERVER.sockets[0].getsockname( + ))) + loop.run_forever() + except KeyboardInterrupt: + pass + except trollius.ConnectionResetError as err: + log.exception("ERROR: ConnectionResetError in main") + except Exception as err: + log.exception("ERROR: Exception in main") + finally: + # Close the server + SERVER.close() + log.info("End Connection") + loop.run_until_complete(SERVER.wait_closed()) + loop.close() + log.info("End") + + +if __name__ == '__main__': + log = logging.getLogger("") + formatter = logging.Formatter( + "%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s") + + # setup console logging + log.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + aslog = logging.getLogger("asyncio") + aslog.setLevel(logging.DEBUG) + + ch.setFormatter(formatter) + log.addHandler(ch) + main() diff --git a/hubs/app.py b/hubs/app.py index 64e6b91..1f30e15 100755 --- a/hubs/app.py +++ b/hubs/app.py @@ -14,6 +14,7 @@ import six from flask.ext.openid import OpenID +import fmn.lib import hubs.models import hubs.widgets @@ -665,17 +666,27 @@ def markup_fedmsg(): plugin = flask.request.args['plugin'] except KeyError: return flask.abort(400) + widget = hubs.models.Widget.by_plugin(session, plugin) + if not widget: return flask.abort(400) + context = widget.config.get('fmn_context') + messages = [] message = json.loads(data) + + if 'topic' not in message: + return flask.abort(400) + try: nickname = flask.g.auth.nickname - except AttributeError: # Not logged in + except AttributeError: # Not logged in return flask.abort(403) + preference = get_remote_preference(nickname, context) + if preference: try: preference = rehydrate_preference(preference) @@ -686,6 +697,7 @@ def markup_fedmsg(): if recipients: messages.append(message) matches = fedmsg.meta.conglomerate(messages, lexers=True, **fedmsg_config) + for match in matches: match['markup'] = apply_markup(match) for _, constituent in match['msg_ids'].items(): @@ -700,6 +712,8 @@ def markup_fedmsg(): else: markup = u"
{long_form}
".format(**constituent) constituent['long_form'] = markup + # And tack on a unique identifier for each top level entry. match['dom_id'] = six.text_type(uuid.uuid4()) + return flask.jsonify(matches) diff --git a/hubs/static/client/.babelrc b/hubs/static/client/.babelrc new file mode 100644 index 0000000..9926dbe --- /dev/null +++ b/hubs/static/client/.babelrc @@ -0,0 +1,5 @@ +{ + "presets" : ["es2015", "react"], + "plugins": ["transform-class-properties"] +} + diff --git a/hubs/static/client/app/components/Constituent.jsx b/hubs/static/client/app/components/Constituent.jsx new file mode 100644 index 0000000..17896a1 --- /dev/null +++ b/hubs/static/client/app/components/Constituent.jsx @@ -0,0 +1,52 @@ +import React from 'react'; + +export default class Constituent extends React.Component { + render() { + const match = this.props.match; + const hasConstituent = (match['msg_ids'].length === 1 && Object.keys(match['msg_ids'])[0]['long_form'] !== Object.keys(match['msg_ids'])[0]['subtitle']); + if ( hasConstituent ) { + const constituentKey = Object.keys(match['msg_ids'])[0]; + const constituent = match['msg_ids'][constituentKey]; + const differentIcon = constituent['icon'] != constituent['__icon__']; + const longLink = ( constituent['link'] && constituent['long_form'].indexOf(constituent['link']) === -1 ); + + let constituentIcon; + if (differentIcon) + constituentIcon = ( +
+ + {constituent['subtitle']}/ + +
+ + ) + + let readMore; + if(longLink) + readMore = ( +
+ {constituent['long_form']} + + Read more + +
+ ) + + return ( +
+ + {constituentIcon} + {readMore} +
+ ) + } else if (match['msg_ids'].length > 1) { +
+ + +
+ } + else { + return null + } + } +} diff --git a/hubs/static/client/app/components/ExpandCollapse.jsx b/hubs/static/client/app/components/ExpandCollapse.jsx new file mode 100644 index 0000000..80ab303 --- /dev/null +++ b/hubs/static/client/app/components/ExpandCollapse.jsx @@ -0,0 +1,18 @@ +import React from 'react'; + +export default class ExpandCollapse extends React.Component { + render() { + return ( +
+ + + expand + + + + collapse + +
+ ) + } +} diff --git a/hubs/static/client/app/components/Feed.jsx b/hubs/static/client/app/components/Feed.jsx new file mode 100644 index 0000000..9e6cccb --- /dev/null +++ b/hubs/static/client/app/components/Feed.jsx @@ -0,0 +1,67 @@ +import React from 'react'; +import {render} from 'react-dom'; + + +import Constituent from './Constituent.jsx'; +import ExpandCollapse from './ExpandCollapse.jsx'; +import Icon from './Icon.jsx'; +import Markup from './Markup.jsx'; +import {Message, MessageList} from './Messages.jsx'; + + +class Panel extends React.Component { + render() { + return ( +
+
+ + + +
+
+ ); + } +} + +export default class Feed extends React.Component { + constructor (props) { + super(props); + this.state = { + matches: this.props.matches, + sse: true, + } + this.source = (!!window.EventSource) ? new EventSource('http://localhost:9090') : {}; + this.source.addEventListener('error', () => { + this.state.sse = false; + }, false); + window.onbeforeunload = () => { + this.source.close(); + } + this.source.addEventListener('message', resp => { + $.get('/api/fedmsg/markup', { + message: resp.data, + plugin: 'feed', + }).done(data => { + if(!$.isEmptyObject(data)) { + this.state.matches.push.apply(this.state.matches, data); + this.setState({matches: this.state.matches}); + } + }); + }, false); + } + render() { + const feedNodes = this.state.matches.map(match => { + return + }); + return ( +
+ {feedNodes} +
+ ) + } +} + + +window.Feed = Feed; +window.React = React; +window.reactRender = render; diff --git a/hubs/static/client/app/components/Icon.jsx b/hubs/static/client/app/components/Icon.jsx new file mode 100644 index 0000000..577050e --- /dev/null +++ b/hubs/static/client/app/components/Icon.jsx @@ -0,0 +1,15 @@ +import React from 'react'; + +export default class Icon extends React.Component { + render() { + return ( +
+
+ + + +
+
+ ) + } +} diff --git a/hubs/static/client/app/components/Markup.jsx b/hubs/static/client/app/components/Markup.jsx new file mode 100644 index 0000000..45ccbe3 --- /dev/null +++ b/hubs/static/client/app/components/Markup.jsx @@ -0,0 +1,17 @@ +import React from 'react'; + +export default class Markup extends React.Component { + createMarkup() { + return {__html: this.props.match['markup']}; + } + render() { + return ( +
+

+ {this.props.match['human_time']} +
+ ) + } +} diff --git a/hubs/static/client/app/components/Messages.jsx b/hubs/static/client/app/components/Messages.jsx new file mode 100644 index 0000000..f034a87 --- /dev/null +++ b/hubs/static/client/app/components/Messages.jsx @@ -0,0 +1,36 @@ +import React from 'react'; + +class Message extends React.Component { + render() { + if(this.props.constituent['link']) { + return ( +
  • + {this.props.constituent['markup']} + + + +
    • + ) + } else { + return (
  • { this.props.constituent['markup'] }
    • ) + } + } +} + +class MessageList extends React.Component { + render() { + const messageNodes = this.props.items.map(function(item) { + return ; + }); + return ( +
      + {messageNodes} +
    + ) + } +} + +module.exports = { + Message: Message, + MessageList: MessageList +} diff --git a/hubs/static/client/app/index.jsx b/hubs/static/client/app/index.jsx new file mode 100644 index 0000000..d169745 --- /dev/null +++ b/hubs/static/client/app/index.jsx @@ -0,0 +1,10 @@ +import React from 'react'; +import { render } from 'react-dom'; + +import Feed from './components/Feed.jsx' + + +render( + , + document.getElementById('feed') +); diff --git a/hubs/static/client/package.json b/hubs/static/client/package.json new file mode 100644 index 0000000..abb023a --- /dev/null +++ b/hubs/static/client/package.json @@ -0,0 +1,33 @@ +{ + "name": "fedora-hubs", + "version": "0.0.0", + "description": "Fedora Hubs will provide a communication and collaboration center for Fedora contributors of all types. The idea is that contributors will be able to visit Hubs to check on their involvements across Fedora, discover new places that they can contribute, and more.", + "main": "webpack.config.js", + "directories": { + "doc": "docs" + }, + "dependencies": { + "babel-core": "~6.9.1", + "babel-loader": "~6.2.4", + "babel-preset-es2015": "~6.9.0", + "babel-preset-react": "~6.5.0", + "react": "~15.1.0", + "react-dom": "~15.1.0", + "reactify": "~1.1.1", + "webpack": "~1.13.1" + }, + "devDependencies": { + "eslint-config-airbnb": "~9.0.1", + "eslint-plugin-react": "~5.2.1", + "eslint-plugin-jsx-a11y": "~1.5.3", + "eslint-plugin-import": "~1.8.1", + "eslint": "~2.12.0" + }, + "scripts": { + "dev": "webpack --watch", + "build": "webpack -p", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "BSD" +} diff --git a/hubs/static/client/webpack.config.js b/hubs/static/client/webpack.config.js new file mode 100644 index 0000000..065ce2f --- /dev/null +++ b/hubs/static/client/webpack.config.js @@ -0,0 +1,27 @@ +const webpack = require('webpack'); +const path = require('path'); + +const PATHS = { + app: path.join(__dirname, 'app'), + build: path.join(__dirname, '../js') +}; + +const config = { + entry: { + app: path.join(PATHS.app, 'index.jsx') + }, + output: { + path: PATHS.build, + filename: 'hubs.js' + }, + module : { + loaders : [{ + test : /\.jsx?$/, + loader : 'babel', + exclude: /(node_modules|bowercomponents)/, + include : PATHS.app + }] + } +}; + +module.exports = config; diff --git a/hubs/tests/test_widgets/test_feed.py b/hubs/tests/test_widgets/test_feed.py new file mode 100644 index 0000000..88e4913 --- /dev/null +++ b/hubs/tests/test_widgets/test_feed.py @@ -0,0 +1,74 @@ +import json + +import hubs.tests.test_widgets +from hubs.widgets.feed import PythonObjectEncoder + + +encode = lambda o: json.dumps(o, cls=PythonObjectEncoder) + +class TestPythonObjectEncoder(hubs.tests.test_widgets.WidgetTest): + def test_python_object_encoder_encodes_set(self): + data = set([1,2,3, 'bar', 'foo']) + expected_str = '[1, 2, 3, "bar", "foo"]' + expected_list = list(data) + + encoded = encode(data) + self.assertEqual(encoded, expected_str) + + decoded = json.loads(encoded) + self.assertEqual(expected_list, decoded) + + def test_python_object_encoder_encodes_tuple(self): + data = ('foo', 'bar', 'baz', 1) + expected_str = '["foo", "bar", "baz", 1]' + expected_list = list(data) + + encoded = encode(data) + self.assertEqual(encoded, expected_str) + + decoded = json.loads(encoded) + self.assertEqual(expected_list, decoded) + + + def test_python_object_encoder_encodes_object(self): + class Test(object): + def __init__(self): + self.x = 'x' + + data = Test() + expected_str = '{"x": "x"}' + expected_dict = data.__dict__ + encoded = encode(data) + self.assertEqual(encoded, expected_str) + + decoded = json.loads(encoded) + self.assertEqual(expected_dict, decoded) + + def test_python_object_encoder_encodes_deep_dict(self): + data = { + 'some_dict': { + 'one': 1, + 'aset': set(['two', 'three']), + 'four': 4.0, + 'tupe': (5,6), + }, + 'regular': 'string' + } + expected_str = '{"some_dict": {"four": 4.0, "tupe": [5, 6], "aset": ["two", "three"], "one": 1}, "regular": "string"}' + + expected_dict = { + 'some_dict': { + 'one': 1, + 'aset': ['two', 'three'], + 'four': 4.0, + 'tupe': [5,6], + }, + 'regular': 'string' + } + + encoded = encode(data) + self.assertEqual(expected_str, encoded) + + decoded = json.loads(encoded) + self.assertEqual(expected_dict, decoded) + diff --git a/hubs/widgets/feed.py b/hubs/widgets/feed.py index f40e453..c6e3797 100755 --- a/hubs/widgets/feed.py +++ b/hubs/widgets/feed.py @@ -21,6 +21,7 @@ import flask import hubs.validators as validators import datetime +import json import logging import uuid @@ -30,13 +31,12 @@ import fedmsg.config config = fedmsg.config.load_config() fmn_url = config['fmn.url'] - bazillion = 1000 paths = fmn.lib.load_rules(root='fmn.rules') # No chrome around the feed. -#from hubs.widgets.chrome import panel -#chrome = panel() +# from hubs.widgets.chrome import panel +# chrome = panel() template = templating.environment.get_template('templates/feed.html') position = 'left' @@ -50,7 +50,7 @@ def apply_markup(match): )) markup = markup.replace(*args) # TODO -- have to add these to conglomerate first - #match['long_form'] = match['long_form'].replace(*args) + # match['long_form'] = match['long_form'].replace(*args) for package in match['packages']: args = (package, '{package}'.format( @@ -59,11 +59,25 @@ def apply_markup(match): )) markup = markup.replace(*args) # TODO -- have to add these to conglomerate first - #match['long_form'] = match['long_form'].replace(*args) + # match['long_form'] = match['long_form'].replace(*args) return markup +class PythonObjectEncoder(json.JSONEncoder): + ''' A JSON encoder that handles python data structures in encoding''' + def default(self, obj): + if isinstance(obj, (list, dict, str, unicode, int, float, bool, + type(None))): + return json.JSONEncoder.default(self, obj) + if isinstance(obj, set) or isinstance(obj, tuple): + return list(obj) + if isinstance(obj, object): + return obj.__dict__ + else: + raise TypeError + + @argument(name="username", default=None, validator=validators.username, @@ -95,7 +109,7 @@ def data(session, widget, username, fmn_context): rules = filter['rules'] fmn_hinting = fmn.lib.hinting.gather_hinting(config, rules, paths) total, pages, rows = datanommer.models.Message.grep( - start=end-delta, + start=end - delta, end=end, rows_per_page=bazillion, page=page, @@ -136,10 +150,8 @@ def data(session, widget, username, fmn_context): # And tack on a unique identifier for each top level entry. match['dom_id'] = six.text_type(uuid.uuid4()) - - return dict( - matches=matches, - ) + matches = json.dumps(matches, cls=PythonObjectEncoder) + return dict(matches=matches) @hint(ubiquitous=True) diff --git a/hubs/widgets/templates/feed.html b/hubs/widgets/templates/feed.html index 534a51a..00fc7e2 100644 --- a/hubs/widgets/templates/feed.html +++ b/hubs/widgets/templates/feed.html @@ -1,74 +1,15 @@ -{% for match in matches %} -
    -
    -
    -
    - - - -
    -
    -

    {{match['markup']}}

    - {{match['human_time']}} +
    +
    + +