40 |
d = DateTime.now() |
d = DateTime.now() |
41 |
return d |
return d |
42 |
|
|
43 |
|
ws_re = re.compile("\s+") |
44 |
|
def dewhitespacify(s): |
45 |
|
return ws_re.sub(" ", s.strip()) |
46 |
|
|
47 |
xmlheaderRe = re.compile(r'<\?.*encoding=(["\'])(?P<encoding>.*?)\1.*\?>') |
xmlheaderRe = re.compile(r'<\?.*encoding=(["\'])(?P<encoding>.*?)\1.*\?>') |
48 |
def parse(content, feed): |
def parse(content, feed): |
49 |
parser = straw.FeedParser() |
parser = straw.FeedParser() |
59 |
else: |
else: |
60 |
enc = straw.utils.get_encoding() |
enc = straw.utils.get_encoding() |
61 |
|
|
62 |
parsed.title = unicode_field(parser.channel, "title", enc) |
parsed.title = dewhitespacify(unicode_field(parser.channel, "title", enc)) |
63 |
parsed.description = unicode_field(parser.channel, "description", enc) |
parsed.description = dewhitespacify( |
64 |
parsed.link = unicode_field(parser.channel, "link", enc) |
unicode_field(parser.channel, "description", enc)) |
65 |
parsed.copyright = unicode_field(parser.channel, "rights", enc) |
parsed.link = dewhitespacify(unicode_field(parser.channel, "link", enc)) |
66 |
|
parsed.copyright = dewhitespacify( |
67 |
|
unicode_field(parser.channel, "rights", enc)) |
68 |
parsed.last_build_date = read_date(parser.channel.get("date", "")) |
parsed.last_build_date = read_date(parser.channel.get("date", "")) |
69 |
|
|
70 |
if parser.channel.has_key('creator'): |
if parser.channel.has_key('creator'): |
71 |
parsed.creator = unicode_field(parser.channel, "creator", enc) |
parsed.creator = dewhitespacify( |
72 |
|
unicode_field(parser.channel, "creator", enc)) |
73 |
|
|
74 |
# item properties |
# item properties |
75 |
for idict in parser.items: |
for idict in parser.items: |
76 |
item = straw.SummaryItem() |
item = straw.SummaryItem() |
77 |
description = '' |
description = '' |
78 |
item.feed = feed |
item.feed = feed |
79 |
item.title = unicode_field(idict, 'title', enc) |
item.title = dewhitespacify(unicode_field(idict, 'title', enc)) |
80 |
|
|
81 |
if idict.has_key('link'): |
if idict.has_key('link'): |
82 |
item.link = unicode_field(idict, 'link', enc).strip() |
item.link = dewhitespacify(unicode_field(idict, 'link', enc)) |
83 |
if idict.has_key('content_encoded'): |
if idict.has_key('content_encoded'): |
84 |
description = unicode_field(idict, 'content_encoded', enc) |
description = unicode_field(idict, 'content_encoded', enc) |
85 |
elif idict.has_key('description'): |
elif idict.has_key('description'): |
99 |
|
|
100 |
# dc:creator |
# dc:creator |
101 |
if idict.has_key('creator'): |
if idict.has_key('creator'): |
102 |
item.creator = unicode_field(idict, 'creator', enc) |
item.creator = dewhitespacify(unicode_field(idict, 'creator', enc)) |
103 |
|
|
104 |
if idict.has_key('guid'): |
if idict.has_key('guid'): |
105 |
item.guid = unicode_field(idict, 'guid', enc) |
item.guid = unicode_field(idict, 'guid', enc) |
108 |
item.pub_date = read_date(idict.get('date', "")) |
item.pub_date = read_date(idict.get('date', "")) |
109 |
|
|
110 |
if idict.has_key('source'): |
if idict.has_key('source'): |
111 |
st = unicode_field(idict, 'source', enc) |
st = dewhitespacify(unicode_field(idict, 'source', enc)) |
112 |
# source format: "url,name" e.g. 'http://foo.com,Foo' |
# source format: "url,name" e.g. 'http://foo.com,Foo' |
113 |
url, text = string.split(st, ',', 1 ) |
url, text = string.split(st, ',', 1 ) |
114 |
item.source = {'url': url.strip(), 'text': text} |
item.source = {'url': url.strip(), 'text': text} |