diff options
Diffstat (limited to 'module/network/HTTPRequest.py')
| -rw-r--r-- | module/network/HTTPRequest.py | 104 |
1 files changed, 50 insertions, 54 deletions
diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 4747d937f..eac03a365 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -1,21 +1,7 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" +# @author: RaNaN + +from __future__ import with_statement import pycurl @@ -25,15 +11,17 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Plugin import Abort +from pyload.plugin.Plugin import Abort, Fail + +from pyload.utils import encode + def myquote(url): - return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") - + return quote(encode(url), safe="%/:=&?~#+!$,;'@()*[]") + def myurlencode(data): data = dict(data) - return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ - y.encode('utf_8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) + return urlencode(dict((encode(x), encode(y)) for x, y in data.iteritems())) bad_headers = range(400, 404) + range(405, 418) + range(500, 506) @@ -44,7 +32,7 @@ class BadHeader(Exception): self.content = content -class HTTPRequest(): +class HTTPRequest(object): def __init__(self, cookies=None, options=None): self.c = pycurl.Curl() self.rep = StringIO() @@ -79,8 +67,9 @@ class HTTPRequest(): if hasattr(pycurl, "AUTOREFERER"): self.c.setopt(pycurl.AUTOREFERER, 1) self.c.setopt(pycurl.SSL_VERIFYPEER, 0) - self.c.setopt(pycurl.LOW_SPEED_TIME, 30) + self.c.setopt(pycurl.LOW_SPEED_TIME, 60) self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5) + self.c.setopt(pycurl.USE_SSL, pycurl.CURLUSESSL_TRY) #self.c.setopt(pycurl.VERBOSE, 1) @@ -89,8 +78,8 @@ class HTTPRequest(): if pycurl.version_info()[7]: self.c.setopt(pycurl.ENCODING, "gzip, deflate") self.c.setopt(pycurl.HTTPHEADER, ["Accept: */*", - "Accept-Language: en-US,en", - "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language: en-US, en", + "Accept-Charset: ISO-8859-1, utf-8;q=0.7,*;q=0.7", "Connection: keep-alive", "Keep-Alive: 300", "Expect:"]) @@ -167,7 +156,7 @@ class HTTPRequest(): self.c.setopt(pycurl.POSTFIELDS, post) else: - post = [(x, y.encode('utf8') if type(y) == unicode else y ) for x, y in post.iteritems()] + post = [(x, encode(y)) for x, y in post.iteritems()] self.c.setopt(pycurl.HTTPPOST, post) else: self.c.setopt(pycurl.POST, 0) @@ -181,7 +170,7 @@ class HTTPRequest(): self.getCookies() - def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False): + def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False, follow_location=True, save_cookies=True): """ load and returns a given page """ self.setRequestContext(url, get, post, referer, cookies, multipart) @@ -190,24 +179,32 @@ class HTTPRequest(): self.c.setopt(pycurl.HTTPHEADER, self.headers) - if just_header: + if post: + self.c.setopt(pycurl.POST, 1) + else: + self.c.setopt(pycurl.HTTPGET, 1) + + if not follow_location: self.c.setopt(pycurl.FOLLOWLOCATION, 0) + + if just_header: self.c.setopt(pycurl.NOBODY, 1) - self.c.perform() - rep = self.header + self.c.perform() + rep = self.header if just_header else self.getResponse() + + if not follow_location: self.c.setopt(pycurl.FOLLOWLOCATION, 1) - self.c.setopt(pycurl.NOBODY, 0) - else: - self.c.perform() - rep = self.getResponse() + if just_header: + self.c.setopt(pycurl.NOBODY, 0) self.c.setopt(pycurl.POSTFIELDS, "") self.lastEffectiveURL = self.c.getinfo(pycurl.EFFECTIVE_URL) self.code = self.verifyHeader() - self.addCookies() + if save_cookies: + self.addCookies() if decode: rep = self.decodeResponse(rep) @@ -228,11 +225,13 @@ class HTTPRequest(): def getResponse(self): """ retrieve response from string io """ - if self.rep is None: return "" - value = self.rep.getvalue() - self.rep.close() - self.rep = StringIO() - return value + if self.rep is None: + return "" + else: + value = self.rep.getvalue() + self.rep.close() + self.rep = StringIO() + return value def decodeResponse(self, rep): """ decode with correct encoding, relies on header """ @@ -255,7 +254,7 @@ class HTTPRequest(): #self.log.debug("Decoded %s" % encoding ) if lookup(encoding).name == 'utf-8' and rep.startswith(BOM_UTF8): encoding = 'utf-8-sig' - + decoder = getincrementaldecoder(encoding)("replace") rep = decoder.decode(rep, True) @@ -263,6 +262,7 @@ class HTTPRequest(): except LookupError: self.log.debug("No Decoder foung for %s" % encoding) + except Exception: self.log.debug("Error when decoding string from %s." % encoding) @@ -272,13 +272,15 @@ class HTTPRequest(): """ writes response """ if self.rep.tell() > 1000000 or self.abort: rep = self.getResponse() - if self.abort: raise Abort() - f = open("response.dump", "wb") - f.write(rep) - f.close() - raise Exception("Loaded Url exceeded limit") - self.rep.write(buf) + if self.abort: + raise Abort + + with open("response.dump", "wb") as f: + f.write(rep) + raise Fail("Loaded url exceeded size limit") + else: + self.rep.write(buf) def writeHeader(self, buf): """ writes header """ @@ -298,9 +300,3 @@ class HTTPRequest(): if hasattr(self, "c"): self.c.close() del self.c - -if __name__ == "__main__": - url = "http://pyload.org" - c = HTTPRequest() - print c.load(url) - |