Skip to content

Commit

Permalink
misc rss updates
Browse files Browse the repository at this point in the history
for #124
  • Loading branch information
snarfed committed Feb 26, 2019
1 parent debfbfc commit b26dc69
Show file tree
Hide file tree
Showing 9 changed files with 95 additions and 42 deletions.
16 changes: 14 additions & 2 deletions api.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
instagram,
jsonfeed,
microformats2,
rss,
source,
twitter,
)
Expand Down Expand Up @@ -71,6 +72,7 @@
'json-mf2',
'jsonfeed',
'mf2-json',
'rss',
'xml',
)

Expand Down Expand Up @@ -163,7 +165,8 @@ def get(self):

self.write_response(response, actor=actor, url=src.BASE_URL)

def write_response(self, response, actor=None, url=None, title=None):
def write_response(self, response, actor=None, url=None, title=None,
hfeed=None):
"""Converts ActivityStreams activities and writes them out.
Args:
Expand All @@ -172,7 +175,8 @@ def write_response(self, response, actor=None, url=None, title=None):
actor: optional ActivityStreams actor dict for current user. Only used
for Atom and JSON Feed output.
url: the input URL
title: string, Used in Atom and JSON Feed output
title: string, used in feed output (Atom, JSON Feed, RSS)
hfeed: dict, parsed mf2 h-feed, if available
"""
format = self.request.get('format') or self.request.get('output') or 'json'
if format not in FORMATS:
Expand Down Expand Up @@ -214,6 +218,14 @@ def write_response(self, response, actor=None, url=None, title=None):
self.response.headers.add('Link', str('<%s>; rel="self"' % self.request.url))
if hub:
self.response.headers.add('Link', str('<%s>; rel="hub"' % hub))
elif format == 'rss':
self.response.headers['Content-Type'] = 'application/rss+xml'
if not title:
title = 'Feed for %s' % url
self.response.out.write(rss.from_activities(
activities, actor, title=title,
feed_url=self.request.url, hfeed=hfeed,
home_page_url=util.base_url(url)))
elif format in ('as1-xml', 'xml'):
self.response.headers['Content-Type'] = 'application/xml'
self.response.out.write(XML_TEMPLATE % util.to_xml(response))
Expand Down
5 changes: 4 additions & 1 deletion app.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from google.appengine.ext import ndb
import mf2py
import mf2util
from oauth_dropins import (
facebook,
flickr,
Expand Down Expand Up @@ -154,6 +155,7 @@ def get(self):

actor = None
title = None
hfeed = None
if mf2:
def fetch_mf2_func(url):
if util.domain_or_parent_in(urlparse.urlparse(url).netloc, SILO_DOMAINS):
Expand All @@ -164,6 +166,7 @@ def fetch_mf2_func(url):
try:
actor = microformats2.find_author(mf2, fetch_mf2_func=fetch_mf2_func)
title = microformats2.get_title(mf2)
hfeed = mf2util.find_first_entry(mf2, ['h-feed'])
except (KeyError, ValueError) as e:
raise exc.HTTPBadRequest('Could not parse %s as %s: %s' % (url, input, e))

Expand Down Expand Up @@ -191,7 +194,7 @@ def fetch_mf2_func(url):
raise exc.HTTPBadRequest('Could not parse %s as JSON Feed' % url)

self.write_response(source.Source.make_activities_base_response(activities),
url=url, actor=actor, title=title)
url=url, actor=actor, title=title, hfeed=hfeed)

def _fetch(self, url):
"""Fetches url and returns (string final url, unicode body)."""
Expand Down
72 changes: 47 additions & 25 deletions granary/rss.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
"""Convert between ActivityStreams and RSS 2.0.
RSS 2.0 spec: http://www.rssboard.org/rss-specification
Apple iTunes Podcasts feed requirements:
https://help.apple.com/itc/podcasts_connect/#/itc1723472cb
Notably:
* Valid RSS 2.0.
* Each podcast item requires <guid>.
* Images should be JPEG or PNG, 1400x1400 to 3000x3000.
* HTTP server that hosts assets and files should support range requests.
"""
from __future__ import absolute_import, unicode_literals
from builtins import str
from past.builtins import basestring

from datetime import date, datetime, time
import mimetypes

from feedgen.feed import FeedGenerator
Expand All @@ -18,18 +28,17 @@
ENCLOSURE_TYPES = {'audio', 'video'}


def from_activities(activities, actor=None, title=None, description=None,
feed_url=None, home_page_url=None, image_url=None):
def from_activities(activities, actor=None, title=None, feed_url=None,
home_page_url=None, hfeed=None):
"""Converts ActivityStreams activities to an RSS 2.0 feed.
Args:
activities: sequence of ActivityStreams activity dicts
actor: ActivityStreams actor dict, the author of the feed
title: string, the feed title
description, the feed description
feed_url: string, the URL for this RSS feed
home_page_url: string, the home page URL
# feed_url: the URL of this RSS feed, if any
image_url: the URL of an image representing this feed
hfeed: dict, parsed mf2 h-feed, if available
Returns:
unicode string with RSS 2.0 XML
Expand All @@ -44,15 +53,28 @@ def from_activities(activities, actor=None, title=None, description=None,

fg = FeedGenerator()
fg.id(feed_url)
assert feed_url
fg.link(href=feed_url, rel='self')
fg.link(href=home_page_url, rel='alternate')
fg.title(title)
fg.description(description)
# TODO: parse language from lang attribute:
# https://github.com/microformats/mf2py/issues/150
fg.language('en')
fg.generator('granary', uri='https://granary.io/')
if image_url:
fg.image(image_url)

hfeed = hfeed or {}
actor = actor or {}
image = util.get_url(hfeed, 'image') or util.get_url(actor, 'image')
if image:
fg.image(image)

props = hfeed.get('properties') or {}
content = microformats2.get_text(util.get_first(props, 'content', ''))
summary = util.get_first(props, 'summary', '')
fg.description(content or summary or '-') # required

latest = None
enclosures = False
for activity in activities:
obj = activity.get('object') or activity
if obj.get('objectType') == 'person':
Expand All @@ -64,7 +86,7 @@ def from_activities(activities, actor=None, title=None, description=None,
item.link(href=url)
item.guid(url, permalink=True)

item.title(obj.get('title') or obj.get('displayName'))
item.title(obj.get('title') or obj.get('displayName') or '-') # required
content = microformats2.render_content(
obj, include_location=True, render_attachments=False) or obj.get('summary')
if content:
Expand All @@ -79,15 +101,17 @@ def from_activities(activities, actor=None, title=None, description=None,
'uri': author.get('url'),
})

for prop in 'published', 'updated':
val = obj.get(prop)
if val:
dt = util.parse_iso8601(val)
getattr(item, prop)(dt)
if not latest or dt > latest:
latest = dt
published = obj.get('published') or obj.get('updated')
if published:
dt = mf2util.parse_datetime(published)
if not isinstance(dt, datetime):
dt = datetime.combine(dt, time.min)
if not dt.tzinfo:
dt = dt.replace(tzinfo=util.UTC)
item.published(dt)
if not latest or dt > latest:
latest = dt

enclosures = False
for att in obj.get('attachments', []):
stream = util.get_first(att, 'stream') or att
if not stream:
Expand All @@ -98,7 +122,7 @@ def from_activities(activities, actor=None, title=None, description=None,
if (att.get('objectType') in ENCLOSURE_TYPES or
mime and mime.split('/')[0] in ENCLOSURE_TYPES):
enclosures = True
item.enclosure(url=url, type=mime) # TODO: length (bytes)
item.enclosure(url=url, type=mime, length='REMOVEME') # TODO: length (bytes)

item.load_extension('podcast')
duration = stream.get('duration')
Expand All @@ -107,15 +131,13 @@ def from_activities(activities, actor=None, title=None, description=None,

if enclosures:
fg.load_extension('podcast')
if actor:
fg.podcast.itunes_author(actor.get('displayName') or actor.get('username'))
fg.podcast.itunes_image(image_url)
if description:
fg.podcast.itunes_subtitle(description)
fg.podcast.itunes_author(actor.get('displayName') or actor.get('username'))
if summary:
fg.podcast.itunes_summary(summary)
fg.podcast.itunes_explicit('no')
fg.podcast.itunes_block(False)

if latest:
fg.lastBuildDate(dt)
fg.lastBuildDate(latest)

return fg.rss_str(pretty=True)
return fg.rss_str(pretty=True).decode().replace(' length="REMOVEME"', '')
2 changes: 2 additions & 0 deletions granary/templates/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,7 @@ <h1>
<option value="html">html</option>
<option value="mf2-json">mf2-json</option>
<option value="jsonfeed">jsonfeed</option>
<option value="rss">rss</option>
</select>

{% if site != 'github' %}
Expand Down Expand Up @@ -302,6 +303,7 @@ <h1>
<option value="html">html</option>
<option value="mf2-json">mf2-json</option>
<option value="jsonfeed">jsonfeed</option>
<option value="rss">rss</option>
</select>
<label>& url = </label>
<input id="url" name="url" type="url" required class="form-control"
Expand Down
18 changes: 13 additions & 5 deletions granary/tests/test_testdata.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,16 +81,24 @@ def html_to_activity(html):
return microformats2.html_to_activities(html)[0]['object']

def rss_from_activities(activities):
hfeed = {
'properties': {
'content': [{'value': 'some stuff by meee'}],
},
}
return rss.from_activities(
activities, actor=ACTOR, title='Stuff', description='some stuff by meee',
feed_url='http://site/feed', home_page_url='http://site/',
image_url='http://site/logo.png').decode('utf-8')
activities, actor=ACTOR, title='Stuff', feed_url='http://site/feed',
home_page_url='http://site/', hfeed=hfeed)

# source extension, destination extension, conversion function, exclude prefix
mappings = (
('as.json', ['mf2-from-as.json', 'mf2.json'], microformats2.object_to_json, ()),
('as.json', ['mf2-from-as.json', 'mf2.json'], microformats2.object_to_json,
# not ready yet
('feed_with_audio_video')),
('as.json', ['mf2-from-as.html', 'mf2.html'], microformats2.object_to_html, ()),
('mf2.json', ['as-from-mf2.json', 'as.json'], microformats2.json_to_object, ()),
('mf2.json', ['as-from-mf2.json', 'as.json'], microformats2.json_to_object,
# not ready yet
('feed_with_audio_video')),
('mf2.json', ['mf2-from-json.html', 'mf2.html'], microformats2.json_to_html,
# we do not format h-media photos properly in html
('note_with_composite_photo',)),
Expand Down
4 changes: 2 additions & 2 deletions granary/tests/testdata/feed_with_audio_video.as.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"objectType": "article",
"displayName": "i'm ready to speak",
"content": "<p>some HTML</p>",
"published": "2012-12-05T00:58:26+00:00",
"published": "2012-12-05T00:58:26+07:00",
"attachments": [{
"stream": {
"url": "http://a/podcast.mp3",
Expand All @@ -17,7 +17,7 @@
"objectType": "article",
"displayName": "i'm ready to perform",
"summary": "other thing",
"updated": "2012-12-06T00:58:26+00:00",
"updated": "2012-12-04",
"attachments": [{
"stream": {
"url": "http://a/vidjo.mov",
Expand Down
14 changes: 7 additions & 7 deletions granary/tests/testdata/feed_with_audio_video.rss.xml
Original file line number Diff line number Diff line change
Expand Up @@ -8,23 +8,23 @@
<docs>http://www.rssboard.org/rss-specification</docs>
<generator>granary</generator>
<image>
<url>http://site/logo.png</url>
<url>http://example.com/martin/image</url>
<title>Stuff</title>
<link>http://site/</link>
</image>
<lastBuildDate>Thu, 06 Dec 2012 00:58:26 +0000</lastBuildDate>
<language>en</language>
<lastBuildDate>Wed, 05 Dec 2012 00:58:26 +0700</lastBuildDate>
<itunes:author>Martin Smith</itunes:author>
<itunes:block>no</itunes:block>
<itunes:image href="http://site/logo.png"/>
<itunes:explicit>no</itunes:explicit>
<itunes:subtitle>some stuff by meee</itunes:subtitle>

<item>
<title>i'm ready to perform</title>
<link>http://vidjo/post</link>
<description>other thing</description>
<guid isPermaLink="true">http://vidjo/post</guid>
<enclosure url="http://a/vidjo.mov" length="0" type="video/quicktime"/>
<enclosure url="http://a/vidjo.mov" type="video/quicktime"/>
<pubDate>Tue, 04 Dec 2012 00:00:00 +0000</pubDate>
<itunes:duration>428</itunes:duration>
</item>

Expand All @@ -33,8 +33,8 @@
<link>http://podcast/post</link>
<description>&lt;p&gt;some HTML&lt;/p&gt;</description>
<guid isPermaLink="true">http://podcast/post</guid>
<enclosure url="http://a/podcast.mp3" length="0" type="audio/mpeg"/>
<pubDate>Wed, 05 Dec 2012 00:58:26 +0000</pubDate>
<enclosure url="http://a/podcast.mp3" type="audio/mpeg"/>
<pubDate>Wed, 05 Dec 2012 00:58:26 +0700</pubDate>
<itunes:duration>328</itunes:duration>
</item>

Expand Down
1 change: 1 addition & 0 deletions requirements.freeze.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ oauth2client==4.1.2
oauthlib==2.0.2
pyasn1==0.4.4
pyasn1-modules==0.2.2
python-dateutil==2.8.0
python-tumblpy==1.0.4
requests==2.19.1
requests-oauthlib==0.8.0
Expand Down
5 changes: 5 additions & 0 deletions test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,11 @@ def test_html_format(self):
self.assertEquals(200, resp.status_int)
self.assertEquals('text/html; charset=utf-8', resp.headers['Content-Type'])

def test_rss_format(self):
resp = self.get_response('/fake?format=rss')
self.assertEquals(200, resp.status_int)
self.assertEquals('application/rss+xml; charset=utf-8', resp.headers['Content-Type'])

def test_unknown_format(self):
resp = self.get_response('/fake?format=bad')
self.assertEquals(400, resp.status_int)
Expand Down

0 comments on commit b26dc69

Please sign in to comment.