8 |
|
|
9 |
iso_date_re = re.compile('^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d') |
iso_date_re = re.compile('^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d') |
10 |
|
|
11 |
|
def unicode_field(dict, key, default = None, enc = 'utf-8'): |
12 |
|
v = dict.get(key, default) |
13 |
|
if v is not None: |
14 |
|
try: |
15 |
|
v = unicode(v, enc) |
16 |
|
except UnicodeError: |
17 |
|
# try with iso-8859-1, usually we get at least something |
18 |
|
v = unicode(v, "iso-8859-1") |
19 |
|
enc = "iso-8859-1" |
20 |
|
v = v.strip() |
21 |
|
return (v, enc) |
22 |
|
|
23 |
|
def read_date(s): |
24 |
|
s = s.strip() |
25 |
|
try: |
26 |
|
if iso_date_re.match(s): |
27 |
|
d = DateTimeFromString(s, ('iso',)) |
28 |
|
else: |
29 |
|
d = DateTimeFromString(s, ('lit',)) |
30 |
|
except ValueError: |
31 |
|
d = DateTime.now() |
32 |
|
return d |
33 |
|
|
34 |
|
|
35 |
class SummaryParserMP: |
class SummaryParserMP: |
36 |
def __init__(self): |
def __init__(self): |
37 |
pass |
pass |
38 |
|
|
39 |
def parse(self, content, feed): |
def parse(self, content, feed): |
40 |
parser = straw.RSSParser() |
parser = straw.FeedParser() |
41 |
enc = None |
enc = None |
42 |
|
|
43 |
try: |
try: |
44 |
enc = SummaryParserCommon.read_encoding(content) |
enc = SummaryParserCommon.read_encoding(content) |
45 |
if enc is None: |
except (ValueError, Exception): |
|
raise ValueError |
|
|
except ValueError: |
|
|
pass |
|
|
if not enc: |
|
46 |
enc = "utf-8" |
enc = "utf-8" |
47 |
|
|
48 |
parser.feed(content) |
parser.feed(content) |
49 |
parsed = straw.ParsedSummary() |
parsed = straw.ParsedSummary() |
50 |
|
|
|
def unicode_field(dict, key, default = None, enc = enc): |
|
|
v = dict.get(key, default) |
|
|
if v is not None: |
|
|
try: |
|
|
v = unicode(v, enc) |
|
|
except UnicodeError: |
|
|
# try with iso-8859-1, usually we get at least something |
|
|
v = unicode(v, "iso8859-1") |
|
|
enc = "iso8859-1" |
|
|
v = v.strip() |
|
|
return (v, enc) |
|
|
|
|
|
def read_date(s): |
|
|
s = s.strip() |
|
|
try: |
|
|
if iso_date_re.match(s): |
|
|
d = DateTimeFromString(s, ('iso',)) |
|
|
else: |
|
|
d = DateTimeFromString(s, ('lit',)) |
|
|
except ValueError: |
|
|
d = DateTime.now() |
|
|
return d |
|
|
|
|
51 |
parsed.title, enc = unicode_field( |
parsed.title, enc = unicode_field( |
52 |
parser.channel, "title", "", enc = enc) |
parser.channel, "title", "", enc = enc) |
53 |
parsed.description, enc = unicode_field( |
parsed.description, enc = unicode_field( |
54 |
parser.channel, "description", "", enc = enc) |
parser.channel, "description", "", enc = enc) |
55 |
parsed.link, enc = unicode_field(parser.channel, "link", "", enc = enc) |
parsed.link, enc = unicode_field( |
56 |
|
parser.channel, "link", "", enc = enc) |
57 |
parsed.copyright, enc = unicode_field( |
parsed.copyright, enc = unicode_field( |
58 |
parser.channel, "rights", "", enc = enc) |
parser.channel, "rights", "", enc = enc) |
59 |
parsed.last_build_date = read_date(parser.channel.get("date", "")) |
parsed.last_build_date = read_date( |
60 |
|
parser.channel.get("date", "")) |
61 |
|
|
62 |
|
creator, enc = unicode_field(parser.channel, "creator", enc=enc) |
63 |
|
if creator is not None: |
64 |
|
parsed.creator = creator |
65 |
|
|
66 |
|
# blogchannel |
67 |
|
blogroll, enc = unicode_field(parser.channel, "blogRoll", enc=enc) |
68 |
|
if blogroll is not None: |
69 |
|
parsed.blogroll = blogroll |
70 |
|
subscriptions, enc = unicode_field(parser.channel, "mySubscriptions", enc=enc) |
71 |
|
if subscriptions is not None: |
72 |
|
parsed.subscriptions = subscriptions |
73 |
|
blink, enc = unicode_field(parser.channel, "blink", enc=enc) |
74 |
|
if blink is not None: |
75 |
|
parsed.blink = blink |
76 |
|
|
77 |
|
# item properties |
78 |
for idict in parser.items: |
for idict in parser.items: |
79 |
item = straw.SummaryItem() |
item = straw.SummaryItem() |
80 |
item.feed = feed |
item.feed = feed |
82 |
item.link = idict.get('link', None) |
item.link = idict.get('link', None) |
83 |
if item.link: |
if item.link: |
84 |
item.link = item.link.strip() |
item.link = item.link.strip() |
85 |
|
|
86 |
item.description, enc = unicode_field(idict, 'description', enc = enc) |
item.description, enc = unicode_field(idict, 'description', enc = enc) |
87 |
ce, enc = unicode_field(idict, 'content_encoded', enc = enc) |
|
88 |
if ce is not None: |
# content_encoded |
89 |
item.description = ce |
if idict.has_key('content_encoded'): |
90 |
|
ce, enc = unicode_field(idict, 'content_encoded', enc = enc) |
91 |
|
if ce is not None: |
92 |
|
item.description = ce |
93 |
|
|
94 |
item.guid, enc = unicode_field(idict, 'guid', enc = enc) |
item.guid, enc = unicode_field(idict, 'guid', enc = enc) |
95 |
item.pub_date = read_date(idict.get('date', "")) |
item.pub_date = read_date(idict.get('date', "")) |
96 |
|
|
97 |
if idict.has_key('source_url'): |
if idict.has_key('source_url'): |
98 |
st, enc = unicode_field(idict, 'source', '', enc = enc) |
st, enc = unicode_field(idict, 'source', '', enc = enc) |
99 |
item.source = {'url': idict['source_url'].strip(), 'text': st} |
item.source = {'url': idict['source_url'].strip(), 'text': st} |
100 |
|
|
101 |
if item.description is not None: |
if item.description is not None: |
102 |
idh = SummaryParserCommon.ItemDescriptionHandler(feed) |
idh = SummaryParserCommon.ItemDescriptionHandler(feed) |
103 |
try: |
try: |
105 |
images = idh.get_images() |
images = idh.get_images() |
106 |
for i in images: |
for i in images: |
107 |
item.add_image(i) |
item.add_image(i) |
108 |
|
item.description = idh.get_data() |
109 |
except Exception, ex: |
except Exception, ex: |
110 |
raise |
raise |
111 |
|
|
112 |
|
# cc stuff |
113 |
|
if idict.has_key('license'): |
114 |
|
license, enc = unicode_field(idict, "license", enc=enc) |
115 |
|
item.license_urls.append(license) |
116 |
|
|
117 |
|
# fm stuff FIXME |
118 |
|
#item.fm_license, enc = unicode_field(idict, 'license', enc=enc) |
119 |
|
#item.fm_changes, enc = unicode_field(idict, 'changes', enc=enc) |
120 |
|
|
121 |
|
# prism |
122 |
|
if idict.has_key('publicationName'): |
123 |
|
item.publication_name, enc = unicode_field(idict,'publicationName', enc=enc) |
124 |
|
item.publication_volume, enc = unicode_field(idict, 'volume', enc=enc) |
125 |
|
item.publication_number, enc = unicode_field(idict, 'number', enc=enc) |
126 |
|
item.publication_section, enc = unicode_field(idict, 'section', enc=enc) |
127 |
|
item.publication_starting_page, enc = unicode_field(idict, 'startingPage', enc=enc) |
128 |
|
|
129 |
parsed.addItem(item) |
parsed.addItem(item) |
130 |
|
|
131 |
|
|
132 |
return parsed |
return parsed |
133 |
|
|