zs - zs - Zeitungsschau rss to email converter
(HTM) git clone git://r-36.net/zs
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) README
(DIR) LICENSE
---
zs (6320B)
---
1 #!/usr/bin/env python
2 # coding=utf-8
3 #
4 # Copy me if you can.
5 # by 20h
6 #
7
8 import sys
9 import os
10 import zeitungsschau.feed as feed
11 import zeitungsschau.feeddb as feeddb
12 import zeitungsschau.opml as opml
13 import zeitungsschau.feedemail as feedemail
14 import socket
15 import http.client
16 import ssl
17 import getopt
18 import pprint
19 import requests.exceptions
20 import requests
21 import smtplib
22 import lxml
23
24 dodebug = False
25
26 def debug(msg):
27 global dodebug
28 if dodebug == True:
29 print("debug: %s" % (msg))
30
31 def sendfeed(db, ufeed):
32 feedemail.send(ufeed, db.cfg["email"], db.cfg["smtphost"], \
33 db.cfg["smtpport"], db.cfg["smtpssl"], \
34 db.cfg["smtpstarttls"], db.cfg["smtpuser"], \
35 db.cfg["smtppassword"], db.cfg["smtpcmd"], \
36 db.cfg["smtpuselocal"])
37
38 def run(db, selfeed=None, dryrun=False, onlychanges=False):
39 feeduris = db.listfeeds()
40
41 if feeduris != None and selfeed in feeduris:
42 feeduris = [selfeed]
43
44 for feeduri in feeduris:
45 if db.ispaused(feeduri):
46 print("pause %s" % (feeduri))
47 continue
48
49 retries = db.getretry(feeduri)
50 estr = None
51 if onlychanges != True:
52 print("fetch %s" % (feeduri))
53 curfeed = None
54 rcode = 0
55
56 """
57 # All errors.
58 (rcode, curfeed) = feed.fetch(feeduri)
59 """
60 try:
61 (rcode, curfeed) = feed.fetch(feeduri)
62 except socket.gaierror:
63 continue
64 except socket.timeout:
65 continue
66 except TimeoutError:
67 continue
68 except ConnectionResetError:
69 estr = "connreset"
70 retries += 1
71 except requests.exceptions.ConnectionError:
72 estr = "connreset"
73 retries += 1
74 except requests.exceptions.ReadTimeout:
75 continue
76 except requests.exceptions.ChunkedEncodingError:
77 continue
78 except lxml.etree.XMLSyntaxError:
79 estr = "xml error"
80 retries += 1
81 except requests.exceptions.TooManyRedirects:
82 estr = "redirects"
83 retries += 1
84 except OSError as err:
85 estr = err.strerror
86 retries += 1
87
88 if rcode == 404:
89 estr = "404"
90 retries += 1
91
92 if curfeed == None:
93 continue
94
95 # retry handling
96 if estr != None:
97 if retries > 2:
98 sys.stderr.write("pause %s %s\n" % \
99 (estr, feeduri))
100 db.pause(feeduri)
101 db.setretry(feeduri, retries)
102 continue
103 elif retries > 0:
104 db.setretry(feeduri, 0)
105
106 try:
107 clen = len(curfeed["articles"])
108 except AttributeError:
109 continue
110
111 if clen == 0:
112 # This is no target anymore. Thanks NATO for your
113 # crappy RSS feed!
114 #print("0 articles -> pause %s" % (feeduri))
115 #db.pause(feeduri)
116 continue
117
118 db.mergefeed(feeduri, curfeed)
119 ufeed = db.unreadarticles(feeduri)
120 if len(ufeed["articles"]) > 0 and onlychanges != True:
121 print("cur %d unread %d" % (clen, \
122 len(ufeed["articles"])))
123 debug(ufeed)
124 if dryrun == False:
125 try:
126 sendfeed(db, ufeed)
127 db.setreadarticles(feeduri, ufeed)
128 except smtplib.SMTPDataError:
129 return
130 except smtplib.SMTPSenderRefused:
131 return
132
133 class ExceptionHook:
134 instance = None
135
136 def __call__(self, *args, **kwargs):
137 if self.instance is None:
138 from IPython.core import ultratb
139 self.instance = ultratb.FormattedTB(mode='Verbose',
140 color_scheme='Linux', call_pdb=1)
141 return self.instance(*args, **kwargs)
142
143 def usage(app):
144 app = os.path.basename(app)
145 sys.stderr.write("usage: %s [-dhs] cmd\n" % (app))
146 sys.exit(1)
147
148 def main(args):
149 global dodebug
150 retval = 0
151
152 try:
153 opts, largs = getopt.getopt(args[1:], "hds")
154 except getopt.GetoptError as err:
155 print(str(err))
156 usage(args[0])
157
158 silent = False
159 for o, a in opts:
160 if o == "-h":
161 usage(args[0])
162 elif o == "-d":
163 dodebug = True
164 elif o == "-s":
165 silent = True
166 else:
167 usage(args[0])
168
169 if len(largs) < 1:
170 usage(args[0])
171
172 if dodebug == True:
173 sys.excepthook = ExceptionHook()
174
175 if largs[0] == "testfeed":
176 if len(largs) < 2:
177 print("usage: %s testfeed URI\n" % \
178 (os.path.basename(args[0])))
179 return 1
180
181 fe = feed.fetch(largs[1])
182 pprint.pprint(fe)
183 return 0
184
185 db = feeddb.feeddb()
186
187 if largs[0] == "run":
188 if len(largs) > 1:
189 run(db, largs[1], onlychanges=silent)
190 else:
191 run(db, onlychanges=silent)
192
193 elif largs[0] == "dryrun":
194 if len(largs) > 1:
195 run(db, largs[1], dryrun=True, onlychanges=silent)
196 else:
197 run(db, dryrun=True, onlychanges=silent)
198
199 elif largs[0] == "cfg":
200 if len(largs) < 2:
201 for k in db.cfg:
202 print("%s = '%s'" % (k, db.cfg[k]))
203 elif len(args) < 3:
204 if largs[1] in db.cfg:
205 print("%s = '%s'" % (largs[1], \
206 db.cfg[largs[1]]))
207 else:
208 retval = 1
209 else:
210 db.cfg[largs[1]] = largs[2]
211 print("%s = '%s'" % (largs[1], db.cfg[largs[1]]))
212 db.default = False
213
214 elif largs[0] == "cfgdel":
215 if len(largs) < 2:
216 usage(args[0])
217 if largs[1] in db.cfg:
218 del db.cfg[largs[1]]
219
220 elif largs[0] == "add":
221 if len(largs) < 2:
222 usage(args[0])
223 db.addfeed(largs[1])
224
225 elif largs[0] == "list":
226 print("\n".join(db.listfeeds()))
227
228 elif largs[0] == "listuuids":
229 if len(largs) < 2:
230 usage(args[0])
231 rfeed = db.readfeed(largs[1])
232 for art in rfeed["articles"]:
233 print("%s: %s: %s" % (art["uuid"], art["link"],\
234 art["title"]))
235
236 elif largs[0] == "unread":
237 if len(largs) < 3:
238 usage(args[0])
239 db.setarticleunread(largs[1], largs[2])
240
241 elif largs[0] == "resend":
242 if len(largs) < 2:
243 usage(args[0])
244 ufeed = db.unreadarticles(largs[1])
245 sendfeed(db, ufeed)
246 db.setreadarticles(largs[1], ufeed)
247
248 elif largs[0] == "del":
249 if len(largs) < 2:
250 usage(args[0])
251 if db.delfeed(largs[1]) == True:
252 print("'%s' has been deleted." % (largs[1]))
253
254 elif largs[0] == "reset":
255 if len(largs) < 2:
256 usage(args[0])
257 db.resetarticles(largs[1])
258
259 elif largs[0] == "retry":
260 if len(largs) < 3:
261 usage(args[0])
262 db.setretry(largs[1], int(largs[2]))
263
264 elif largs[0] == "pause":
265 if len(largs) < 2:
266 usage(args[0])
267 db.pause(largs[1])
268
269 elif largs[0] == "unpause":
270 if len(largs) < 2:
271 usage(args[0])
272 db.unpause(largs[1])
273
274 elif largs[0] == "opmlexport":
275 if len(largs) > 1:
276 filen = open(largs[1], "w")
277 else:
278 filen = sys.stdout
279 filen.write(opml.write(db.listfeeds()))
280
281 elif largs[0] == "opmlimport":
282 if len(largs) > 1:
283 filen = open(largs[1], "r")
284 else:
285 filen = sys.stdin
286 feedlist = db.listfeeds()
287 nfeedlist = opml.read(filen.read().encode("utf-8"))
288 for f in nfeedlist:
289 if not f in feedlist:
290 print("import feed: %s" % (f))
291 db.addfeed(f)
292
293 del db
294 return retval
295
296 if __name__ == "__main__":
297 sys.exit(main(sys.argv))
298