1 |
|
import straw, error, re, string, sgmllib, email.Utils |
2 |
from mx import DateTime |
from mx import DateTime |
3 |
from mx.DateTime.Parser import DateTimeFromString |
from mx.DateTime.Parser import DateTimeFromString |
|
import xml.sax.saxutils as XSS |
|
|
import xml.sax.xmlreader as XSX |
|
|
import HTMLParser |
|
|
import straw, error |
|
|
import re, string, StringIO, cgi |
|
|
|
|
|
# let's try to be a bit forgiving about the HTML contained in description. |
|
|
# like allow unquoted attributes containing commas |
|
|
HTMLParser.HTMLParser.attrfind = re.compile( |
|
|
r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*' |
|
|
r'(\'[^\']*\'|"[^"]*"|[-a-zA-Z0-9.,/:;+*%?!&$\(\)_#=~]*))?') |
|
|
|
|
|
class SXMLGenerator(XSS.XMLGenerator): |
|
|
def direct_characters(self, chars): |
|
|
self._out.write(chars) |
|
4 |
|
|
5 |
class ItemDescriptionHandler(HTMLParser.HTMLParser): |
class ImageParser(sgmllib.SGMLParser): |
6 |
def __init__(self, feed): |
def __init__(self, feed): |
7 |
HTMLParser.HTMLParser.__init__(self) |
sgmllib.SGMLParser.__init__(self) |
|
self._images = [] |
|
8 |
self._feed = feed |
self._feed = feed |
9 |
self._outstream = StringIO.StringIO() |
self._image_urls = [] |
|
self._xg = SXMLGenerator(self._outstream, encoding='utf-8') |
|
10 |
|
|
11 |
def handle_starttag(self, tag, attrs): |
def do_img(self, attrs): |
12 |
def t2d(t): |
for name, value in attrs: |
13 |
d = {} |
if name == 'src': |
14 |
for i in t: |
url = straw.utils.complete_url(value, self._feed.location) |
15 |
d[i[0]] = i[1] |
self._image_urls.append(url) |
|
return d |
|
|
if tag == 'img': |
|
|
attrs = filter(lambda pair: pair[0] == 'src', attrs) |
|
|
if len(attrs) > 0: |
|
|
url = attrs[0][1] |
|
|
curl = self._feed.complete_url(url) |
|
|
self._images.append(curl) |
|
|
if tag != 'font': |
|
|
self._xg.startElement(tag, XSX.AttributesImpl(t2d(attrs))) |
|
|
return |
|
|
|
|
|
def handle_endtag(self, tag): |
|
|
if tag != 'font': |
|
|
self._xg.endElement(tag) |
|
|
|
|
|
def handle_data(self, data): |
|
|
self._xg.characters(data) |
|
|
|
|
|
def handle_charref(self, ref): |
|
|
text = "&#%s;" % ref |
|
|
self._xg.direct_characters(text) |
|
|
|
|
|
def handle_entityref(self, ref): |
|
|
text = "&%s;" % ref |
|
|
self._xg.direct_characters(text) |
|
16 |
|
|
17 |
def get_images(self): |
def get_image_urls(self): |
18 |
return self._images |
return self._image_urls |
19 |
|
|
20 |
def get_data(self): |
def unicode_field(dict, key, enc, default=''): |
|
return self._outstream.getvalue() |
|
|
|
|
|
iso_date_re = re.compile('^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d') |
|
|
def unicode_field(dict, key, enc='utf-8', default=''): |
|
21 |
v = dict.get(key, default) |
v = dict.get(key, default) |
22 |
if v != '': |
if v != '': |
23 |
try: |
try: |
28 |
v = unicode(v, enc).strip() |
v = unicode(v, enc).strip() |
29 |
return v |
return v |
30 |
|
|
31 |
|
iso_date_re = re.compile('^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d') |
32 |
def read_date(s): |
def read_date(s): |
33 |
s = s.strip() |
s = s.strip() |
34 |
try: |
try: |
40 |
d = DateTime.now() |
d = DateTime.now() |
41 |
return d |
return d |
42 |
|
|
|
|
|
|
# encoding regex |
|
43 |
xmlheaderRe = re.compile(r'<\?.*encoding=(["\'])(?P<encoding>.*?)\1.*\?>') |
xmlheaderRe = re.compile(r'<\?.*encoding=(["\'])(?P<encoding>.*?)\1.*\?>') |
44 |
|
def parse(content, feed): |
45 |
|
parser = straw.FeedParser() |
46 |
|
parsed = straw.ParsedSummary() |
47 |
|
parser.feed(content) |
48 |
|
|
49 |
|
enc = parser.channel.get('encoding', '') |
50 |
|
|
51 |
|
if enc == '': |
52 |
|
match = xmlheaderRe.match(content) |
53 |
|
if match: |
54 |
|
enc = parser.channel['encoding'] = match.groups()[0].lower() |
55 |
|
else: |
56 |
|
enc = straw.utils.get_encoding() |
57 |
|
|
58 |
class SummaryParser: |
parsed.title = unicode_field(parser.channel, "title", enc) |
59 |
def __init__(self): |
parsed.description = unicode_field(parser.channel, "description", enc) |
60 |
pass |
parsed.link = unicode_field(parser.channel, "link", enc) |
61 |
|
parsed.copyright = unicode_field(parser.channel, "rights", enc) |
62 |
def parse(self, content, feed): |
parsed.last_build_date = read_date(parser.channel.get("date", "")) |
63 |
parser = straw.FeedParser() |
|
64 |
parser.feed(content) |
if parser.channel.has_key('creator'): |
65 |
parsed = straw.ParsedSummary() |
parsed.creator = unicode_field(parser.channel, "creator", enc) |
66 |
|
|
67 |
enc = parser.channel.get('encoding', '').lower() |
# item properties |
68 |
|
for idict in parser.items: |
69 |
if enc == '': |
item = straw.SummaryItem() |
70 |
match = xmlheaderRe.match(content) |
description = '' |
71 |
if match: |
item.feed = feed |
72 |
enc = parser.channel['encoding'] = match.group('encoding').lower() |
item.title = unicode_field(idict, 'title', enc) |
73 |
|
|
74 |
parsed.title = unicode_field(parser.channel, "title", enc) |
if idict.has_key('link'): |
75 |
parsed.description = unicode_field(parser.channel, "description", enc) |
item.link = unicode_field(idict, 'link', enc).strip() |
76 |
parsed.link = unicode_field(parser.channel, "link", enc) |
if idict.has_key('content_encoded'): |
77 |
parsed.copyright = unicode_field(parser.channel, "rights", enc) |
description = unicode_field(idict, 'content_encoded', enc) |
78 |
parsed.last_build_date = read_date(parser.channel.get("date", "")) |
elif idict.has_key('description'): |
79 |
|
description = unicode_field(idict, 'description', enc) |
80 |
creator = unicode_field(parser.channel, "creator", enc) |
|
81 |
if creator is not None and (len(creator) > 0): |
if len(description): |
82 |
parsed.creator = creator |
imp = ImageParser(feed) |
83 |
|
item.description = description |
84 |
# blogchannel |
try: |
85 |
blogroll = unicode_field(parser.channel, "blogRoll", enc) |
imp.feed(description) |
86 |
if blogroll is not None: |
image_urls = imp.get_image_urls() |
87 |
parsed.blogroll = blogroll |
for im in image_urls: |
88 |
subscriptions = unicode_field(parser.channel, "mySubscriptions", enc) |
item.add_image(im) |
89 |
if subscriptions is not None: |
except Exception, ex: |
90 |
parsed.subscriptions = subscriptions |
error.log(_("Exception occurred %s in feed %s") % (ex, item.feed.title)) |
91 |
blink = unicode_field(parser.channel, "blink", enc) |
item.feed.error = _("Error parsing item %s: %s") % (item.title, ex) |
92 |
if blink is not None: |
|
93 |
parsed.blink = blink |
# dc:creator |
94 |
|
if idict.has_key('creator'): |
95 |
# item properties |
item.creator = unicode_field(idict, 'creator', enc) |
|
for idict in parser.items: |
|
|
item = straw.SummaryItem() |
|
|
item.feed = feed |
|
|
item.title = unicode_field(idict, 'title', enc) |
|
|
item.link = idict.get('link', '') |
|
|
if item.link != '': |
|
|
item.link = item.link.strip() |
|
|
|
|
|
item.description = unicode_field(idict, 'description', enc) |
|
|
|
|
|
# dc:creator |
|
|
if idict.has_key('creator'): |
|
|
item.creator = unicode_field(idict, 'creator', enc) |
|
|
|
|
|
# content_encoded |
|
|
if idict.has_key('content_encoded'): |
|
|
ce = unicode_field(idict, 'content_encoded', enc) |
|
|
if ce is not None: |
|
|
item.description = ce |
|
96 |
|
|
97 |
|
if idict.has_key('guid'): |
98 |
item.guid = unicode_field(idict, 'guid', enc) |
item.guid = unicode_field(idict, 'guid', enc) |
99 |
|
|
100 |
|
if idict.has_key('date'): |
101 |
item.pub_date = read_date(idict.get('date', "")) |
item.pub_date = read_date(idict.get('date', "")) |
102 |
|
|
103 |
if idict.has_key('source'): |
if idict.has_key('source'): |
104 |
st = unicode_field(idict, 'source', enc) |
st = unicode_field(idict, 'source', enc) |
105 |
# source format: "url,name" e.g. 'http://blah.foo,BlahFoo' |
# source format: "url,name" e.g. 'http://foo.com,Foo' |
106 |
# .. the first comma is all that matters |
url, text = string.split(st, ',', 1 ) |
107 |
url, text = string.split(st, ',', 1 ) |
item.source = {'url': url.strip(), 'text': text} |
108 |
item.source = {'url': url.strip(), 'text': text} |
|
109 |
|
if idict.has_key('license'): |
110 |
if item.description is not None: |
# freshmeat |
111 |
idh = ItemDescriptionHandler(feed) |
if parser.namespacemap.has_key('fm'): |
112 |
try: |
item.fm_license = unicode_field(idict, 'license', enc) |
113 |
idh.feed(item.description) |
item.fm_changes = unicode_field(idict, 'changes', enc) |
114 |
images = idh.get_images() |
else: |
115 |
for i in images: |
license = unicode_field(idict, "license", enc) |
116 |
item.add_image(i) |
item.license_urls.append(license) |
117 |
item.description = idh.get_data() |
|
118 |
except Exception, ex: |
# prism |
119 |
error.log(_("Exception Occurred: %s") % ex) |
if idict.has_key('publicationName'): |
120 |
item.feed.error = _("Error parsing item %s: %s") % (item.title, ex) |
item.publication_name = unicode_field(idict,'publicationName', enc) |
121 |
|
item.publication_volume = unicode_field(idict, 'volume', enc) |
122 |
if idict.has_key('license'): |
item.publication_number = unicode_field(idict, 'number', enc) |
123 |
# freshmeat |
item.publication_section = unicode_field(idict, 'section', enc) |
124 |
if parser.namespacemap.has_key('fm'): |
item.publication_starting_page = unicode_field(idict, 'startingPage', enc) |
|
item.fm_license = unicode_field(idict, 'license', enc) |
|
|
item.fm_changes = unicode_field(idict, 'changes', enc) |
|
|
# assume it's creativeCommons or other 'license' |
|
|
else: |
|
|
license = unicode_field(idict, "license", enc) |
|
|
item.license_urls.append(license) |
|
|
|
|
|
|
|
|
# prism |
|
|
if idict.has_key('publicationName'): |
|
|
item.publication_name = unicode_field(idict,'publicationName', enc) |
|
|
item.publication_volume = unicode_field(idict, 'volume', enc) |
|
|
item.publication_number = unicode_field(idict, 'number', enc) |
|
|
item.publication_section = unicode_field(idict, 'section', enc) |
|
|
item.publication_starting_page = unicode_field(idict, 'startingPage', enc) |
|
125 |
|
|
126 |
parsed.addItem(item) |
parsed.addItem(item) |
127 |
|
|
128 |
parser.reset() |
parser.reset() |
129 |
|
|
130 |
return parsed |
return parsed |
131 |
|
|