From 0df39b89d4625e1671603fbc7e2a94b045673378 Mon Sep 17 00:00:00 2001 From: zoidberg10 Date: Thu, 5 Jan 2012 18:26:40 +0100 Subject: fix post urlencode regression --- module/network/HTTPRequest.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/network/HTTPRequest.py') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 40f18f2a5..d4c33bbff 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -31,6 +31,7 @@ def myquote(url): return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") def myurlencode(data): + data = dict(data) return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ y.encode('utf_8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) -- cgit v1.2.3 From 1bb6ebf544b43cacf7c0755c5a8608b79b95e2d6 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 20:11:16 +0100 Subject: MultiHoster plugin type, some fixes, new documentation structure --- module/network/HTTPRequest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/network/HTTPRequest.py') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index d4c33bbff..cd13dd01f 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -25,7 +25,7 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Plugin import Abort +from module.plugins.Hoster import Abort def myquote(url): return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") -- cgit v1.2.3 From c654f31efc548957f10e3f4c1a5060dcea1dcdec Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 10 Jan 2012 00:04:24 +0100 Subject: changed HEAD request --- module/network/HTTPRequest.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'module/network/HTTPRequest.py') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index cd13dd01f..8d65b025f 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -193,11 +193,19 @@ class HTTPRequest(): if just_header: self.c.setopt(pycurl.FOLLOWLOCATION, 0) self.c.setopt(pycurl.NOBODY, 1) + + # overwrite HEAD request, we want a common request type + if post: + self.c.setopt(pycurl.CUSTOMREQUEST, "POST") + else: + self.c.setopt(pycurl.CUSTOMREQUEST, "GET") + self.c.perform() rep = self.header self.c.setopt(pycurl.FOLLOWLOCATION, 1) self.c.setopt(pycurl.NOBODY, 0) + self.c.setopt(pycurl.CUSTOMREQUEST, 0) else: self.c.perform() -- cgit v1.2.3 From 692d015627ecf03fbc23cfdb4afcf398b9a09a51 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 12 Jan 2012 17:26:28 +0100 Subject: scripts for testing and syntax unit test --- module/network/HTTPRequest.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'module/network/HTTPRequest.py') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 8d65b025f..4684397d9 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -39,7 +39,7 @@ bad_headers = range(400, 404) + range(405, 418) + range(500, 506) class BadHeader(Exception): def __init__(self, code, content=""): - Exception.__init__(self, "Bad server response: %s %s" % (code, responses[int(code)])) + Exception.__init__(self, "Bad server response: %s %s" % (code, responses.get(int(code), "Unknown Header"))) self.code = code self.content = content @@ -200,12 +200,13 @@ class HTTPRequest(): else: self.c.setopt(pycurl.CUSTOMREQUEST, "GET") - self.c.perform() - rep = self.header - - self.c.setopt(pycurl.FOLLOWLOCATION, 1) - self.c.setopt(pycurl.NOBODY, 0) - self.c.setopt(pycurl.CUSTOMREQUEST, 0) + try: + self.c.perform() + rep = self.header + finally: + self.c.setopt(pycurl.FOLLOWLOCATION, 1) + self.c.setopt(pycurl.NOBODY, 0) + self.c.setopt(pycurl.CUSTOMREQUEST, 0) else: self.c.perform() -- cgit v1.2.3 From c7ad1cc5b4a5d190a060e3ddd9274c3065da6708 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Fri, 13 Jan 2012 23:24:21 +0100 Subject: plugin unit test, closed #499, #500 --- module/network/HTTPRequest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/network/HTTPRequest.py') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 4684397d9..7887081e7 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -206,7 +206,7 @@ class HTTPRequest(): finally: self.c.setopt(pycurl.FOLLOWLOCATION, 1) self.c.setopt(pycurl.NOBODY, 0) - self.c.setopt(pycurl.CUSTOMREQUEST, 0) + self.c.unsetopt(pycurl.CUSTOMREQUEST) else: self.c.perform() -- cgit v1.2.3 From 17b3595dc5db8b3270e6bcd07176ed4b7b47930a Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 15 Jan 2012 19:28:12 +0100 Subject: improved handling of content-disposition --- module/network/HTTPRequest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/network/HTTPRequest.py') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 7887081e7..a0b419763 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -271,7 +271,7 @@ class HTTPRequest(): #TODO: html_unescape as default except LookupError: - self.log.debug("No Decoder foung for %s" % encoding) + self.log.debug("No Decoder found for %s" % encoding) except Exception: self.log.debug("Error when decoding string from %s." % encoding) -- cgit v1.2.3 From 995b6ace8598f22fe8b21b67c587797baa6f7f21 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 16 Jan 2012 18:31:59 +0100 Subject: correct utf8 conversion for urls --- module/network/HTTPRequest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/network/HTTPRequest.py') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index a0b419763..2f084efb5 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -28,12 +28,12 @@ from cStringIO import StringIO from module.plugins.Hoster import Abort def myquote(url): - return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") + return quote(url.encode('utf8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") def myurlencode(data): data = dict(data) - return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ - y.encode('utf_8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) + return urlencode(dict((x.encode('utf8') if isinstance(x, unicode) else x, \ + y.encode('utf8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) bad_headers = range(400, 404) + range(405, 418) + range(500, 506) -- cgit v1.2.3 From 50d4df8b4d48b855bd18e9922355b7f3f2b4da4e Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 20 Mar 2012 14:57:45 +0100 Subject: captcha decrypting for all plugin types, new interaction manager --- module/network/HTTPRequest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/network/HTTPRequest.py') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 2f084efb5..774c0e64d 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -25,7 +25,7 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Hoster import Abort +from module.plugins.Base import Abort def myquote(url): return quote(url.encode('utf8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") -- cgit v1.2.3 From c64d929c6339226e6c29e3bfba195fbc5610865b Mon Sep 17 00:00:00 2001 From: zoidberg10 Date: Fri, 17 Aug 2012 01:02:44 +0200 Subject: fix multipart post encoding --- module/network/HTTPRequest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/network/HTTPRequest.py') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 774c0e64d..290a27a40 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -167,7 +167,7 @@ class HTTPRequest(): self.c.setopt(pycurl.POSTFIELDS, post) else: - post = [(x, str(quote(y)) if type(y) in (str, unicode) else y ) for x, y in post.iteritems()] + post = [(x, y.encode('utf8') if type(y) == unicode else y ) for x, y in post.iteritems()] self.c.setopt(pycurl.HTTPPOST, post) else: self.c.setopt(pycurl.POST, 0) -- cgit v1.2.3 From 7fe05500cc738eda76216510e110eb118ad04dee Mon Sep 17 00:00:00 2001 From: RaNaN Date: Wed, 12 Sep 2012 09:56:44 +0200 Subject: higher low speed time, easier way to set curl options --- module/network/HTTPRequest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'module/network/HTTPRequest.py') diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 290a27a40..4b45e10a2 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -62,6 +62,7 @@ class HTTPRequest(): self.initHandle() self.setInterface(options) + self.setOptions(options) self.c.setopt(pycurl.WRITEFUNCTION, self.write) self.c.setopt(pycurl.HEADERFUNCTION, self.writeHeader) @@ -79,7 +80,8 @@ class HTTPRequest(): if hasattr(pycurl, "AUTOREFERER"): self.c.setopt(pycurl.AUTOREFERER, 1) self.c.setopt(pycurl.SSL_VERIFYPEER, 0) - self.c.setopt(pycurl.LOW_SPEED_TIME, 30) + # Interval for low speed, detects connection loss, but can abort dl if hoster stalls the download + self.c.setopt(pycurl.LOW_SPEED_TIME, 45) self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5) #self.c.setopt(pycurl.VERBOSE, 1) @@ -127,6 +129,11 @@ class HTTPRequest(): if "timeout" in options: self.c.setopt(pycurl.LOW_SPEED_TIME, options["timeout"]) + def setOptions(self, options): + """ Sets same options as available in pycurl """ + for k, v in options.iteritems(): + if hasattr(pycurl, k): + self.c.setopt(getattr(pycurl, k), v) def addCookies(self): """ put cookies from curl handle to cj """ -- cgit v1.2.3