diff options
Diffstat (limited to 'module')
| -rw-r--r-- | module/gui/MainWindow.py | 46 | ||||
| -rw-r--r-- | module/gui/Queue.py | 16 | ||||
| -rw-r--r-- | module/network/Browser.py | 80 | ||||
| -rw-r--r-- | module/network/Bucket.py | 53 | ||||
| -rw-r--r-- | module/network/FTPBase.py | 200 | ||||
| -rw-r--r-- | module/network/HTTPBase.py | 382 | ||||
| -rw-r--r-- | module/network/HTTPChunk.py | 201 | ||||
| -rw-r--r-- | module/network/HTTPDownload.py | 305 | ||||
| -rw-r--r-- | module/network/MultipartPostHandler.py | 2 | ||||
| -rw-r--r-- | module/network/helper.py | 111 | ||||
| -rw-r--r-- | module/network/socks.py | 442 |
11 files changed, 1826 insertions, 12 deletions
diff --git a/module/gui/MainWindow.py b/module/gui/MainWindow.py index c0fe6e804..a02ea7989 100644 --- a/module/gui/MainWindow.py +++ b/module/gui/MainWindow.py @@ -62,16 +62,40 @@ class MainWindow(QMainWindow): self.setCentralWidget(lw) #status - #@TODO: build a fancy box - - #self.statusw = QFrame() - #self.statusw.setFrameStyle(QFrame.StyledPanel | QFrame.Raised) - #self.statusw.setLineWidth(2) - #self.statusw.setLayout(QGridLayout()) - #l = self.statusw.layout() - #l.addWidget(QLabel("Status:"), 0, 0) - #l.addWidget(QLabel("Speed:"), 0, 3) - #l.addWidget(QLabel("Space:"), 0, 5) + self.statusw = QFrame() + self.statusw.setFrameStyle(QFrame.StyledPanel | QFrame.Raised) + self.statusw.setLineWidth(2) + self.statusw.setLayout(QGridLayout()) + l = self.statusw.layout() + l.addWidget(QLabel(_("packages:")), 0, 0) + l.addWidget(QLabel(_("files:")), 1, 0) + self.packageCount = QLabel("0") + self.fileCount = QLabel("0") + l.addWidget(self.packageCount, 0, 1) + l.addWidget(self.fileCount, 1, 1) + + l.addWidget(QLabel(_("status:")), 0, 3) + l.addWidget(QLabel(_("ip:")), 1, 3) + self.status = QLabel("running") + self.ip = QLabel("") + l.addWidget(self.status, 0, 4) + l.addWidget(self.ip, 1, 4) + + l.addWidget(QLabel(_("speed:")), 0, 5) + l.addWidget(QLabel(_("space:")), 1, 5) + self.speed = QLabel("") + self.space = QLabel("") + l.addWidget(self.speed, 0, 6) + l.addWidget(self.space, 1, 6) + + l.addWidget(QLabel(_("max. downloads:")), 0, 7) + l.addWidget(QLabel(_("max. chunks:")), 1, 7) + self.maxDownloads = QSpinBox() + self.maxDownloads.setEnabled(False) + self.maxChunks = QSpinBox() + self.maxChunks.setEnabled(False) + l.addWidget(self.maxDownloads, 0, 8) + l.addWidget(self.maxChunks, 1, 8) #set menubar and statusbar self.menubar = self.menuBar() @@ -126,7 +150,7 @@ class MainWindow(QMainWindow): #layout self.masterlayout.addWidget(self.tabw) - #self.masterlayout.addWidget(self.statusw) + self.masterlayout.addWidget(self.statusw) #signals.. self.connect(self.mactions["manager"], SIGNAL("triggered()"), self.slotShowConnector) diff --git a/module/gui/Queue.py b/module/gui/Queue.py index d60858e34..7c5c59f15 100644 --- a/module/gui/Queue.py +++ b/module/gui/Queue.py @@ -61,6 +61,22 @@ class QueueModel(CollectorModel): self._data.append(package) self._data = sorted(self._data, key=lambda p: p.data["order"]) self.endInsertRows() + self.updateCount() + + def insertEvent(self, event): + CollectorModel.insertEvent(self, event) + self.updateCount() + + def removeEvent(self, event): + CollectorModel.removeEvent(self, event) + self.updateCount() + + def updateCount(self): + packageCount = len(self._data) + fileCount = 0 + for p in self._data: + fileCount += len(p.children) + self.emit(SIGNAL("updateCount"), packageCount, fileCount) def update(self): locker = QMutexLocker(self.mutex) diff --git a/module/network/Browser.py b/module/network/Browser.py new file mode 100644 index 000000000..65aefbae8 --- /dev/null +++ b/module/network/Browser.py @@ -0,0 +1,80 @@ +from HTTPBase import HTTPBase +from HTTPDownload import HTTPDownload + +from os.path import exists + +import zlib +from cookielib import CookieJar +from FTPBase import FTPDownload + +class Browser(): + def __init__(self, interface=None, cookieJar=CookieJar(), bucket=None, proxies={}): + self.lastURL = None + self.interface = interface + self.cookieJar = cookieJar + self.bucket = bucket + + self.http = HTTPBase(interface=interface, proxies=proxies) + self.http.cookieJar = cookieJar + self.proxies = proxies + + def clearReferer(self): + self.lastURL = None + + def getPage(self, url, get={}, post={}, referer=None, cookies=True, customHeaders={}): + if not referer: + referer = self.lastURL + self.http.followRedirect = True + resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders) + data = resp.read() + try: + if resp.info()["Content-Encoding"] == "gzip": + data = zlib.decompress(data, 16+zlib.MAX_WBITS) + elif resp.info()["Content-Encoding"] == "deflate": + data = zlib.decompress(data, -zlib.MAX_WBITS) + except: + pass + self.lastURL = resp.geturl() + return data + + def getRedirectLocation(self, url, get={}, post={}, referer=None, cookies=True, customHeaders={}): + if not referer: + referer = self.lastURL + self.http.followRedirect = False + resp = self.http.getResponse(url, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders) + resp.close() + self.lastURL = resp.geturl() + location = None + try: + location = resp.info()["Location"] + except: + pass + return location + + def httpDownload(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, chunks=1, resume=False): + if not referer: + referer = self.lastURL + + dwnld = HTTPDownload(url, filename, get=get, post=post, referer=referer, cookies=cookies, customHeaders=customHeaders, bucket=self.bucket, interface=self.interface, proxies=self.proxies) + dwnld.cookieJar = self.cookieJar + + d = dwnld.download(chunks=chunks, resume=resume) + return d + + def ftpDownload(self, url, filename, resume=False): + dwnld = FTPDownload(url, filename, bucket=self.bucket, interface=self.interface, proxies=self.proxies) + + d = dwnld.download(resume=resume) + return d + +if __name__ == "__main__": + browser = Browser(proxies={"socks5": "localhost:5000"}) + ip = "http://www.whatismyip.com/automation/n09230945.asp" + #browser.getPage("http://google.com/search?q=bar") + #browser.getPage("https://encrypted.google.com/") + print browser.getPage(ip) + #print browser.getRedirectLocation("http://google.com/") + #browser.getPage("https://encrypted.google.com/") + #browser.getPage("http://google.com/search?q=bar") + + #browser.downloadFile("https://bitbucket.org/spoob/pyload/downloads/Logo_neu.png", "logo.png") diff --git a/module/network/Bucket.py b/module/network/Bucket.py new file mode 100644 index 000000000..35e27bcd4 --- /dev/null +++ b/module/network/Bucket.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: mkaay +""" + +from time import time, sleep +from threading import Lock + +class Bucket: + def __init__(self): + self.content = 0 + self.rate = 0 + self.lastDrip = time() + self.lock = Lock() + + def setRate(self, rate): + self.lock.acquire() + self.rate = rate + self.lock.release() + + def add(self, amount): + self.lock.acquire() + self.drip() + allowable = min(amount, self.rate - self.content) + if allowable > 0: + sleep(0.005) #@XXX: high sysload without?! + + self.content += allowable + self.lock.release() + return allowable + + def drip(self): + if self.rate == 0: + self.content = 0 + else: + now = time() + deltaT = now - self.lastDrip + self.content = long(max(0, self.content - deltaT * self.rate)) + self.lastDrip = now diff --git a/module/network/FTPBase.py b/module/network/FTPBase.py new file mode 100644 index 000000000..9bcb9b45e --- /dev/null +++ b/module/network/FTPBase.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: mkaay +""" + +from ftplib import FTP + +import socket +import socks + +from os.path import getsize +from urlparse import urlparse +from urllib2 import _parse_proxy + +from helper import * + +class FTPBase(FTP): + sourceAddress = ('', 0) + + def setSourceAddress(self, host): + self.sourceAddress = (host, 0) + + def connect(self, host='', port=0, timeout=30, proxies={}): + if host != '': + self.host = host + if port > 0: + self.port = port + self.timeout = timeout + + proxytype = None + proxy = None + if proxies.has_key("socks5"): + proxytype = socks.PROXY_TYPE_SOCKS5 + proxy = proxies["socks5"] + elif proxies.has_key("socks4"): + proxytype = socks.PROXY_TYPE_SOCKS4 + proxy = proxies["socks4"] + if proxytype: + self.sock = socks.socksocket() + t = _parse_proxy(proxy) + self.sock.setproxy(proxytype, addr=t[3].split(":")[0], port=int(t[3].split(":")[1]), username=t[1], password=t[2]) + else: + self.sock = socket.socket() + self.sock.settimeout(self.timeout) + self.sock.bind(self.sourceAddress) + self.sock.connect((self.host, self.port)) + self.af = self.sock.family + self.file = self.sock.makefile('rb') + self.welcome = self.getresp() + return self.welcome + +class FTPDownload(): + def __init__(self, url, filename, interface=None, bucket=None, proxies={}): + self.url = url + self.filename = filename + + self.bucket = bucket + self.interface = interface + self.proxies = proxies + + self.deferred = Deferred() + + self.finished = False + self.size = None + + self.speed = 0 + + self.abort = False + + self.arrived = 0 + + self.startTime = None + self.endTime = None + + self.speed = 0 #byte/sec + self.speedCalcTime = None + self.speedCalcLen = 0 + + self.bufferSize = 16*1024 #tune if performance is poor + + self.ftp = FTPBase() + self.fh = None + + @threaded + def _download(self, offset): + remotename = self.url.split("/")[-1] + cmd = "RETR %s" % remotename + + self.startTime = inttime() + self.arrived = offset + conn, size = self.ftp.ntransfercmd(cmd, None if offset == 0 else offset) #explicit None + if size: + self.size = size + offset + while True: + if self.abort: + self.ftp.abort() + break + count = self.bufferSize + if self.bucket: + count = self.bucket.add(count) + if count == 0: + sleep(0.01) + continue + + try: + data = conn.recv(count) + except: + self.deferred.error("timeout") + + if self.speedCalcTime < inttime(): + self.speed = self.speedCalcLen + self.speedCalcTime = inttime() + self.speedCalcLen = 0 + size = len(data) + self.speedCalcLen += size + self.arrived += size + + if not data: + break + + self.fh.write(data) + conn.close() + self.endTime = inttime() + if not self.abort: + print self.ftp.voidresp() #debug + + self.ftp.quit() + if self.abort: + self.deferred.error("abort") + elif self.size is None or self.size == self.arrived: + self.deferred.callback(resp) + else: + self.deferred.error("wrong content lenght") + + def download(self, resume=False): + self.fh = open("%s.part" % self.filename, "ab" if resume else "wb") + offset = 0 + if resume: + offset = getsize("%s.part" % self.filename) + + up = urlparse(self.url) + + self.ftp.connect(up.hostname, up.port if up.port else 21, proxies=self.proxies) + self.ftp.login(up.username, up.password) + self.ftp.cwd("/".join(up.path.split("/")[:-1])) + self.ftp.voidcmd('TYPE I') + self.size = self.ftp.size(self.url.split("/")[-1]) + + self._download(offset) + return self.deferred + +if __name__ == "__main__": + import sys + from Bucket import Bucket + bucket = Bucket() + bucket.setRate(200*1000) + #bucket = None + + url = "ftp://mirror.sov.uk.goscomb.net/ubuntu-releases/maverick/ubuntu-10.10-desktop-i386.iso" + + finished = False + def err(*a, **b): + print a, b + def callb(*a, **b): + global finished + finished = True + print a, b + + print "starting" + + dwnld = FTPDownload(url, "ubuntu_ftp.iso") + d = dwnld.download(resume=True) + d.addCallback(callb) + d.addErrback(err) + + try: + while True: + if not dwnld.finished: + print dwnld.speed/1024, "kb/s", "size", dwnld.arrived, "/", dwnld.size#, int(float(dwnld.arrived)/dwnld.size*100), "%" + if finished: + print "- finished" + break + sleep(1) + except KeyboardInterrupt: + dwnld.abort = True + sys.exit() diff --git a/module/network/HTTPBase.py b/module/network/HTTPBase.py new file mode 100644 index 000000000..e8c7d07ad --- /dev/null +++ b/module/network/HTTPBase.py @@ -0,0 +1,382 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: mkaay +""" + +from urllib import urlencode +from urlparse import urlparse + +from urllib2 import Request +from urllib2 import OpenerDirector + +from urllib2 import BaseHandler +from urllib2 import HTTPHandler +from urllib2 import HTTPRedirectHandler +from urllib2 import HTTPCookieProcessor +from urllib2 import HTTPSHandler +from urllib2 import HTTPDefaultErrorHandler +from urllib2 import HTTPErrorProcessor +from urllib2 import ProxyHandler + +from urllib2 import URLError + +from urllib2 import addinfourl +from urllib2 import _parse_proxy + +from httplib import HTTPConnection +from httplib import HTTPResponse +from httplib import responses as HTTPStatusCodes +from httplib import ResponseNotReady + +from cookielib import CookieJar + +import socket +import socks + +from MultipartPostHandler import MultipartPostHandler + +DEBUG = 1 +HANDLE_ERRORS = 1 + +class PyLoadHTTPResponse(HTTPResponse): + def __init__(self, sock, debuglevel=0, strict=0, method=None): + if method: # the httplib in python 2.3 uses the method arg + HTTPResponse.__init__(self, sock, debuglevel, method) + else: # 2.2 doesn't + HTTPResponse.__init__(self, sock, debuglevel) + self.fileno = sock.fileno + self._rbuf = '' + self._rbufsize = 8096 + self._handler = None # inserted by the handler later + self._host = None # (same) + self._url = None # (same) + + _raw_read = HTTPResponse.read + + def close_connection(self): + self.close() + self._handler._remove_connection(self._host, close=1) + + def info(self): + return self.msg + + def geturl(self): + return self._url + + def read(self, amt=None): + # the _rbuf test is only in this first if for speed. It's not + # logically necessary + if self._rbuf and not amt is None: + L = len(self._rbuf) + if amt > L: + amt -= L + else: + s = self._rbuf[:amt] + self._rbuf = self._rbuf[amt:] + return s + + s = self._rbuf + self._raw_read(amt) + self._rbuf = '' + return s + + def readline(self, limit=-1): + data = "" + i = self._rbuf.find('\n') + while i < 0 and not (0 < limit <= len(self._rbuf)): + new = self._raw_read(self._rbufsize) + if not new: break + i = new.find('\n') + if i >= 0: i = i + len(self._rbuf) + self._rbuf = self._rbuf + new + if i < 0: i = len(self._rbuf) + else: i = i+1 + if 0 <= limit < len(self._rbuf): i = limit + data, self._rbuf = self._rbuf[:i], self._rbuf[i:] + return data + + def readlines(self, sizehint = 0): + total = 0 + list = [] + while 1: + line = self.readline() + if not line: break + list.append(line) + total += len(line) + if sizehint and total >= sizehint: + break + return list + + @property + def code(self): + return self.status + + def getcode(self): + return self.status + +class PyLoadHTTPConnection(HTTPConnection): + sourceAddress = ('', 0) + socksProxy = None + response_class = PyLoadHTTPResponse + + def connect(self): + if self.socksProxy: + self.sock = socks.socksocket() + t = _parse_proxy(self.socksProxy[1]) + self.sock.setproxy(self.socksProxy[0], addr=t[3].split(":")[0], port=int(t[3].split(":")[1]), username=t[1], password=t[2]) + else: + self.sock = socket.socket() + self.sock.settimeout(30) + self.sock.bind(self.sourceAddress) + self.sock.connect((self.host, self.port)) + + try: + if self._tunnel_host: + self._tunnel() + except: #python2.5 + pass + +class PyLoadHTTPHandler(HTTPHandler): + sourceAddress = ('', 0) + socksProxy = None + + def __init__(self): + self._connections = {} + + def setInterface(self, interface): + if interface is None: + interface = "" + self.sourceAddress = (interface, 0) + + def setSocksProxy(self, *t): + self.socksProxy = t + + def close_connection(self, host): + """close connection to <host> + host is the host:port spec, as in 'www.cnn.com:8080' as passed in. + no error occurs if there is no connection to that host.""" + self._remove_connection(host, close=1) + + def open_connections(self): + """return a list of connected hosts""" + return self._connections.keys() + + def close_all(self): + """close all open connections""" + for host, conn in self._connections.items(): + conn.close() + self._connections = {} + + def _remove_connection(self, host, close=0): + if self._connections.has_key(host): + if close: self._connections[host].close() + del self._connections[host] + + def _start_connection(self, h, req): + try: + if req.has_data(): + data = req.get_data() + h.putrequest('POST', req.get_selector()) + if not req.headers.has_key('Content-type'): + h.putheader('Content-type', + 'application/x-www-form-urlencoded') + if not req.headers.has_key('Content-length'): + h.putheader('Content-length', '%d' % len(data)) + else: + h.putrequest('GET', req.get_selector(), skip_accept_encoding=1) + except socket.error, err: + raise urllib2.URLError(err) + + for args in self.parent.addheaders: + h.putheader(*args) + for k, v in req.headers.items(): + h.putheader(k, v) + h.endheaders() + if req.has_data(): + h.send(data) + + def do_open(self, http_class, req): + host = req.get_host() + if not host: + raise URLError('no host given') + + try: + need_new_connection = 1 + h = self._connections.get(host) + if not h is None: + try: + self._start_connection(h, req) + except socket.error, e: + r = None + else: + try: r = h.getresponse() + except ResponseNotReady, e: r = None + + if r is None or r.version == 9: + # httplib falls back to assuming HTTP 0.9 if it gets a + # bad header back. This is most likely to happen if + # the socket has been closed by the server since we + # last used the connection. + if DEBUG: print "failed to re-use connection to %s" % host + h.close() + else: + if DEBUG: print "re-using connection to %s" % host + need_new_connection = 0 + if need_new_connection: + if DEBUG: print "creating new connection to %s" % host + h = http_class(host) + h.sourceAddress = self.sourceAddress + h.socksProxy = self.socksProxy + self._connections[host] = h + self._start_connection(h, req) + r = h.getresponse() + except socket.error, err: + raise URLError(err) + + # if not a persistent connection, don't try to reuse it + if r.will_close: self._remove_connection(host) + + if DEBUG: + print "STATUS: %s, %s" % (r.status, r.reason) + r._handler = self + r._host = host + r._url = req.get_full_url() + + if r.status in (200, 206) or not HANDLE_ERRORS: + return r + else: + return self.parent.error('http', req, r, r.status, r.reason, r.msg) + + def http_open(self, req): + return self.do_open(PyLoadHTTPConnection, req) + +class NoRedirectHandler(BaseHandler): #supress error + def http_error_302(self, req, fp, code, msg, headers): + resp = addinfourl(fp, headers, req.get_full_url()) + resp.code = code + resp.msg = msg + return resp + + http_error_301 = http_error_303 = http_error_307 = http_error_302 + +class HTTPBase(): + def __init__(self, interface=None, proxies={}): + self.followRedirect = True + self.interface = interface + self.proxies = proxies + + self.size = None + + self.referer = None + + self.cookieJar = None + + self.userAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.10" + + self.handler = PyLoadHTTPHandler() + self.handler.setInterface(interface) + if proxies.has_key("socks5"): + self.handler.setSocksProxy(socks.PROXY_TYPE_SOCKS5, proxies["socks5"]) + elif proxies.has_key("socks4"): + self.handler.setSocksProxy(socks.PROXY_TYPE_SOCKS4, proxies["socks4"]) + + self.cookieJar = CookieJar() + + self.debug = True + + def createOpener(self, cookies=True): + opener = OpenerDirector() + opener.add_handler(self.handler) + opener.add_handler(MultipartPostHandler()) + opener.add_handler(HTTPSHandler()) + opener.add_handler(HTTPDefaultErrorHandler()) + opener.add_handler(HTTPErrorProcessor()) + if self.proxies.has_key("http") or self.proxies.has_key("https"): + opener.add_handler(ProxyHandler(self.proxies)) + opener.add_handler(HTTPRedirectHandler() if self.followRedirect else NoRedirectHandler()) + if cookies: + opener.add_handler(HTTPCookieProcessor(self.cookieJar)) + opener.version = self.userAgent + opener.addheaders[0] = ("User-Agent", self.userAgent) + return opener + + def createRequest(self, url, get={}, post={}, referer=None, cookies=True, customHeaders={}): + if get: + if isinstance(get, dict): + get = urlencode(get) + url = "%s?%s" % (url, get) + + req = Request(url) + + if post: + if isinstance(post, dict): + post = urlencode(post) + req.add_data(post) + + req.add_header("Accept", "application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5") + + if referer: + req.add_header("Referer", referer) + + req.add_header("Accept-Encoding", "gzip, deflate") + for key, val in customHeaders.iteritems(): + req.add_header(key, val) + + return req + + def getResponse(self, url, get={}, post={}, referer=None, cookies=True, customHeaders={}): + req = self.createRequest(url, get, post, referer, cookies, customHeaders) + opener = self.createOpener(cookies) + + if self.debug: + print "[HTTP] ----" + print "[HTTP] creating request" + print "[HTTP] URL:", url + print "[HTTP] GET" + for key, value in get.iteritems(): + print "[HTTP] \t", key, ":", value + if post: + print "[HTTP] POST" + for key, value in post.iteritems(): + print "[HTTP] \t", key, ":", value + print "[HTTP] headers" + for key, value in opener.addheaders: + print "[HTTP] \t", key, ":", value + for key, value in req.headers.iteritems(): + print "[HTTP] \t", key, ":", value + print "[HTTP] ----" + + resp = opener.open(req) + resp.getcode = lambda: resp.code + + if self.debug: + print "[HTTP] ----" + print "[HTTP] got response" + print "[HTTP] status:", resp.getcode() + print "[HTTP] headers" + for key, value in resp.info().dict.iteritems(): + print "[HTTP] \t", key, ":", value + print "[HTTP] ----" + try: + self.size = int(resp.info()["Content-Length"]) + except: #chunked transfer + pass + return resp + +if __name__ == "__main__": + base = HTTPBase() + resp = base.getResponse("http://python.org/") + print resp.read() diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py new file mode 100644 index 000000000..37c28f685 --- /dev/null +++ b/module/network/HTTPChunk.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: mkaay +""" + +from HTTPBase import HTTPBase +from urllib2 import HTTPError +from threading import Lock +from helper import * +from time import sleep +from traceback import print_exc + +class HTTPChunk(HTTPBase): + def __init__(self, url, fh, get={}, post={}, referer=None, cookies=True, customHeaders={}, range=None, bucket=None, interface=None, proxies={}): + HTTPBase.__init__(self, interface=interface, proxies=proxies) + + self.url = url + self.bucket = bucket + self.range = range + self.noRangeHeader = False + self.fh = fh + + self.get = get + self.post = post + self.referer = referer + self.cookies = cookies + self.customHeaders = customHeaders + + self.deferred = Deferred() + + self.lock = Lock() + self.abort = False + self.finished = False + + self.arrived = 0 + + self.startTime = None + self.endTime = None + + self.speed = 0 #byte/sec + self.speedCalcTime = None + self.speedCalcLen = 0 + + self.bufferSize = 16*1024 #tune if performance is poor + self.resp = None + + def getSpeed(self): + self.lock.acquire() + speed = self.speed + self.lock.release() + return speed + + @threaded + def _download(self, resp): + self.arrived = 0 + self.lastSpeed = self.startTime = inttime() + + if self.noRangeHeader and not self.range[0] == 0: + self.deferred.error("range starts not at 0") + + running = True + while running: + if self.abort: + break + count = self.bufferSize + if self.noRangeHeader: + if self.range[1] <= self.arrived+count: + count = min(count, self.arrived+count - self.range[1]) + running = False + if self.bucket: + count = self.bucket.add(count) + if count == 0: + sleep(0.01) + continue + + try: + data = resp.read(count) + except: + self.deferred.error("timeout") + break + + if self.speedCalcTime < inttime(): + self.lock.acquire() + self.speed = self.speedCalcLen + self.lock.release() + self.speedCalcTime = inttime() + self.speedCalcLen = 0 + size = len(data) + self.speedCalcLen += size + self.arrived += size + if self.noRangeHeader: + if self.range[1] <= self.arrived: + self.fh.write(data[:-(self.arrived-self.range[1])]) + break + + if data: + self.fh.write(data) + else: + break + + self.speed = 0 + self.endTime = inttime() + self.finished = True + self.fh.close() + + if self.abort: + self.deferred.error("abort") + elif self.size == self.arrived: + self.deferred.callback(resp) + else: + self.deferred.error("wrong content lenght") + + def getEncoding(self): + try: + if self.resp.info()["Content-Encoding"] in ("gzip", "deflate"): + return self.resp.info()["Content-Encoding"] + except: + pass + return "plain" + + def download(self): + if self.range: + self.customHeaders["Range"] = "bytes=%i-%i" % self.range + try: + print "req" + resp = self.getResponse(self.url, self.get, self.post, self.referer, self.cookies, self.customHeaders) + self.resp = resp + except HTTPError, e: + print_exc() + self.deferred.error(e) + return self.deferred + + if (self.range and resp.getcode() == 206) or (not self.range and resp.getcode() == 200): + self._download(resp) + else: + self.deferred.error(resp.getcode(), resp) + return self.deferred + +if __name__ == "__main__": + import sys + from Bucket import Bucket + bucket = Bucket() + #bucket.setRate(200*1000) + bucket = None + + url = "http://download.fedoraproject.org/pub/fedora/linux/releases/13/Live/x86_64/Fedora-13-x86_64-Live.iso" + + finished = 0 + def err(*a, **b): + print a, b + def callb(*a, **b): + global finished + finished += 1 + print a, b + + print "starting" + + conns = 4 + + chunks = [] + for a in range(conns): + fh = open("file.part%d" % a, "wb") + chunk = HTTPChunk(url, fh, bucket=bucket, range=(a*5*1024*1024, (a+1)*5*1024*1024)) + print "fireing chunk #%d" % a + d = chunk.download() + d.addCallback(callb) + d.addErrback(err) + chunks.append(chunk) + + try: + while True: + aspeed = 0 + for a, chunk in enumerate(chunks): + if not chunk.finished: + print "#%d" % a, chunk.getSpeed()/1024, "kb/s" + else: + print "#%d" % a, "finished" + aspeed += chunk.getSpeed() + print "sum", aspeed/1024 + if finished == conns: + print "- finished" + break + sleep(1) + except KeyboardInterrupt: + for chunk in chunks: + chunk.abort = True + sys.exit() diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py new file mode 100644 index 000000000..78dc00d72 --- /dev/null +++ b/module/network/HTTPDownload.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: mkaay +""" + +from HTTPChunk import HTTPChunk +from helper import * +from os.path import exists, getsize +from os import remove +from shutil import move, copyfileobj + +from cookielib import CookieJar + +class WrongFormat(Exception): + pass + +class ChunkInfo(): + def __init__(self, name): + self.name = name + self.size = None + self.loaded = False + self.chunks = [] + + def setSize(self, size): + self.size = int(size) + + def addChunk(self, name, range, encoding): + self.chunks.append((name, range, encoding)) + + def clear(self): + self.chunks = [] + self.loaded = False + + def save(self): + fh = open("%s.chunks" % self.name, "w") + fh.write("name:%s\n" % self.name) + fh.write("size:%s\n" % self.size) + for i, c in enumerate(self.chunks): + fh.write("#%d:\n" % i) + fh.write("\tname:%s\n" % c[0]) + fh.write("\tencoding:%s\n" % c[2]) + fh.write("\trange:%i-%i\n" % c[1]) + + @staticmethod + def load(name): + if not exists("%s.chunks" % name): + raise IOError() + fh = open("%s.chunks" % name, "r") + name = fh.readline()[:-1] + size = fh.readline()[:-1] + if name.startswith("name:") and size.startswith("size:"): + name = name[5:] + size = size[5:] + else: + raise WrongFormat() + ci = ChunkInfo(name) + ci.loaded = True + ci.setSize(size) + while True: + if not fh.readline(): #skip line + break + name = fh.readline()[1:-1] + encoding = fh.readline()[1:-1] + range = fh.readline()[1:-1] + if name.startswith("name:") and encoding.startswith("encoding:") and range.startswith("range:"): + name = name[5:] + encoding = encoding[9:] + range = range[6:].split("-") + else: + raise WrongFormat() + + ci.addChunk(name, (long(range[0]), long(range[1])), encoding) + return ci + + def removeInfo(self): + remove("%s.chunks" % self.name) + + def getCount(self): + return len(self.chunks) + + def getChunkName(self, index): + return self.chunks[index][0] + + def getChunkRange(self, index): + return self.chunks[index][1] + + def getChunkEncoding(self, index): + return self.chunks[index][2] + +class HTTPDownload(): + def __init__(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, bucket=None, interface=None, proxies={}): + self.url = url + self.filename = filename + self.interface = interface + self.proxies = proxies + + self.get = get + self.post = post + + self.referer = referer + self.cookies = cookies + + self.customHeaders = customHeaders + + self.bucket = bucket + + self.deferred = Deferred() + + self.finished = False + self.size = None + + self.cookieJar = CookieJar() + + self.chunks = [] + try: + self.info = ChunkInfo.load(filename) + except IOError: + self.info = ChunkInfo(filename) + self.noChunkSupport = False + + @property + def arrived(self): + arrived = 0 + for i in range(self.info.getCount()): + arrived += getsize(self.info.getChunkName(i)) #ugly, but difficult to calc otherwise due chunk resume + return arrived + + def abort(self): + for chunk in self.chunks: + chunk.abort = True + + def getSpeed(self): + speed = 0 + for chunk in self.chunks: + speed += chunk.getSpeed() + return speed + + @property + def speed(self): + return self.getSpeed() + + def _copyChunks(self): + fo = open(self.filename, "wb") + for i in range(self.info.getCount()): + encoding = self.info.getChunkEncoding(i) + + decompress = lambda data: data + if encoding == "gzip": + gz = zlib.decompressobj(16+zlib.MAX_WBITS) + decompress = lambda data: gz.decompress(data) + if encoding == "deflate": + df = zlib.decompressobj(-zlib.MAX_WBITS) + decompress = lambda data: df.decompress(data) + + fname = "%s.chunk%d" % (self.filename, i) + fi = open(fname, "rb") + while True: + data = fi.read(512*1024) + if not data: + break + fo.write(decompress(data)) + fi.close() + remove(fname) + fo.close() + self.info.removeInfo() + self.deferred.callback() + + def _createChunk(self, fh, range=None): + chunk = HTTPChunk(self.url, fh, get=self.get, post=self.post, + referer=self.referer, cookies=self.cookies, + customHeaders=self.customHeaders, + bucket=self.bucket, range=range, + interface=self.interface, proxies=self.proxies) + chunk.cookieJar = self.cookieJar + return chunk + + def download(self, chunks=1, resume=False): + if chunks > 0: + dg = DeferredGroup() + if self.info.loaded and not self.info.getCount() == chunks: + self.info.clear() + crange = None + if resume: + if self.info.getCount() == chunks and exists("%s.chunk0" % (self.filename, )): + crange = self.info.getChunkRange(0) + crange = (crange[0]+getsize("%s.chunk0" % (self.filename, )), crange[1]) + + if crange is None or crange[1]-crange[0] > 0: + fh = open("%s.chunk0" % (self.filename, ), "ab" if crange else "wb") + chunk = self._createChunk(fh, range=crange) + self.chunks.append(chunk) + d = chunk.download() + dg.addDeferred(d) + + if not self.info.loaded: + size = chunk.size + chunksize = size/chunks + lastchunk = chunksize + + chunk.range = (0, chunksize-1) + chunk.noRangeHeader = True + self.size = chunk.size + self.info.setSize(self.size) + chunk.size = chunksize + self.info.addChunk("%s.chunk0" % (self.filename, ), chunk.range, chunk.getEncoding()) + + lastchunk = size - chunksize*(chunks-1) + else: + self.size = self.info.size + self.firstchunk = chunk + + for i in range(1, chunks): + cont = False + if not self.info.loaded: #first time load + if i+1 == chunks: + rng = (i*chunksize, i*chunksize+lastchunk) + else: + rng = (i*chunksize, (i+1)*chunksize-1) + else: #not finished + rng = self.info.getChunkRange(i) + if resume and exists("%s.chunk%d" % (self.filename, i)): #continue chunk + rng = (rng[0]+getsize("%s.chunk%d" % (self.filename, i)), rng[1]) + cont = True + + if rng[1]-rng[0] <= 0: #chunk done + continue + + fh = open("%s.chunk%d" % (self.filename, i), "ab" if cont else "wb") + chunk = self._createChunk(fh, range=rng) + self.chunks.append(chunk) + d = chunk.download() + if not chunk.resp.getcode() == 206 and i == 1: #no range supported, tell chunk0 to download everything + chunk.abort = True + self.noChunkSupport = True + self.firstchunk.size = self.size + self.firstchunk.range = None + self.info.clear() + self.info.addChunk("%s.chunk0" % (self.filename, ), (0, self.firstchunk.size), chunk.getEncoding()) + break + dg.addDeferred(d) + + if not self.info.loaded: + self.info.addChunk("%s.chunk%d" % (self.filename, i), chunk.range, chunk.getEncoding()) + + self.info.save() + dg.addCallback(self._copyChunks) + if len(self.chunks) == 0: + dg.callback() + return self.deferred + else: + raise Exception("no chunks") + +if __name__ == "__main__": + import sys + from Bucket import Bucket + bucket = Bucket() + #bucket.setRate(200*1000) + bucket = None + + url = "http://mirror.sov.uk.goscomb.net/ubuntu-releases/maverick/ubuntu-10.10-desktop-i386.iso" + + finished = False + def err(*a, **b): + print a, b + def callb(*a, **b): + global finished + finished = True + print a, b + + print "starting" + + dwnld = HTTPDownload(url, "ubuntu.iso") + d = dwnld.download(chunks=1, resume=True) + d.addCallback(callb) + d.addErrback(err) + + try: + while True: + for a, chunk in enumerate(dwnld.chunks): + if not chunk.finished: + print "#%d" % a, chunk.getSpeed()/1024, "kb/s", "size", int(float(chunk.arrived)/chunk.size*100), "%" + else: + print "#%d" % a, "finished" + print "sum", dwnld.speed/1024, dwnld.arrived, "/", dwnld.size, int(float(dwnld.arrived)/dwnld.size*100), "%" + if finished: + print "- finished" + break + sleep(1) + except KeyboardInterrupt: + dwnld.abort() + sys.exit() diff --git a/module/network/MultipartPostHandler.py b/module/network/MultipartPostHandler.py index 6804bcc90..113fd7cf9 100644 --- a/module/network/MultipartPostHandler.py +++ b/module/network/MultipartPostHandler.py @@ -136,4 +136,4 @@ def main(): validateFile("http://www.google.com") if __name__=="__main__": - main()
\ No newline at end of file + main() diff --git a/module/network/helper.py b/module/network/helper.py new file mode 100644 index 000000000..4b7119a2b --- /dev/null +++ b/module/network/helper.py @@ -0,0 +1,111 @@ +from threading import Thread +from time import time, sleep + +def inttime(): + return int(time()) + +class AlreadyCalled(Exception): + pass + +def callInThread(f, *args, **kwargs): + class FThread(Thread): + def __init__(self): + Thread.__init__(self) + self.d = Deferred() + def run(self): + ret = f(*args, **kwargs) + self.d.callback(ret) + t = FThread() + t.start() + return t.d + +class Deferred(): + def __init__(self): + self.call = [] + self.err = [] + self.result = () + self.errresult = () + + def addCallback(self, f, *cargs, **ckwargs): + self.call.append((f, cargs, ckwargs)) + if self.result: + args, kwargs = self.result + args+=tuple(cargs) + kwargs.update(ckwargs) + callInThread(f, *args, **kwargs) + + def addErrback(self, f, *cargs, **ckwargs): + self.err.append((f, cargs, ckwargs)) + if self.errresult: + args, kwargs = self.errresult + args+=tuple(cargs) + kwargs.update(ckwargs) + callInThread(f, *args, **kwargs) + + def callback(self, *args, **kwargs): + if self.result: + raise AlreadyCalled + self.result = (args, kwargs) + for f, cargs, ckwargs in self.call: + args+=tuple(cargs) + kwargs.update(ckwargs) + callInThread(f, *args, **kwargs) + + def error(self, *args, **kwargs): + self.errresult = (args, kwargs) + for f, cargs, ckwargs in self.err: + args+=tuple(cargs) + kwargs.update(ckwargs) + callInThread(f, *args, **kwargs) + +#decorator +def threaded(f): + def ret(*args, **kwargs): + return callInThread(f, *args, **kwargs) + return ret + +def waitFor(d): + class Waiter(): + waiting = True + args = () + err = None + + def wait(self): + d.addCallback(self.callb) + d.addErrback(self.errb) + while self.waiting: + sleep(0.5) + if self.err: + raise Exception(self.err) + return self.args + + def callb(self, *args, **kwargs): + self.waiting = False + self.args = (args, kwargs) + + def errb(self, *args, **kwargs): + self.waiting = False + self.err = (args, kwargs) + w = Waiter() + return w.wait() + +class DeferredGroup(Deferred): + def __init__(self, group=[]): + Deferred.__init__(self) + self.group = group + self.done = 0 + + for d in self.group: + d.addCallback(self._cb) + d.addErrback(self.error) + + def addDeferred(self, d): + d.addCallback(self._cb) + d.addErrback(self.error) + self.group.append(d) + + def _cb(self, *args, **kwargs): + self.done += 1 + if len(self.group) == self.done: + self.callback() + diff --git a/module/network/socks.py b/module/network/socks.py new file mode 100644 index 000000000..626ffe176 --- /dev/null +++ b/module/network/socks.py @@ -0,0 +1,442 @@ +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +""" + +Minor modifications made by Christopher Gilbert (http://motomastyle.com/) +for use in PyLoris (http://pyloris.sourceforge.net/) + +Minor modifications made by Mario Vilas (http://breakingcode.wordpress.com/) +mainly to merge bug fixes found in Sourceforge + +""" + +import socket + +if getattr(socket, 'socket', None) is None: + raise ImportError('socket.socket missing, proxy support unusable') + +import struct +import sys + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 + +_defaultproxy = None + +# Small hack for Python 2.x +if sys.version_info[0] <= 2: + def bytes(obj, enc=None): + return obj + +class ProxyError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class GeneralProxyError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks5AuthError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks5Error(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks4Error(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class HTTPError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +_generalerrors = ("success", + "invalid data", + "not connected", + "not available", + "bad proxy type", + "bad input") + +_socks5errors = ("succeeded", + "general SOCKS server failure", + "connection not allowed by ruleset", + "Network unreachable", + "Host unreachable", + "Connection refused", + "TTL expired", + "Command not supported", + "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", + "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", + "unknown error") + +_socks4errors = ("request granted", + "request rejected or failed", + ("request rejected because SOCKS server cannot connect to " + "identd on the client"), + ("request rejected because the client program and identd" + " report different user-ids"), + "unknown error") + + +def setdefaultproxy(proxytype=None, addr=None, port=None, rdns=True, + username=None, password=None): + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype, addr, port, rdns, username, password) + + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=0, _sock=None): + socket.socket.__init__(self, family, type, proto, _sock) + if _defaultproxy != None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + + def __decode(self, bytes): + if getattr(bytes, 'decode', False): + try: + bytes = bytes.decode() + except Exception: + pass + return bytes + + def __encode(self, bytes): + if getattr(bytes, 'encode', False): + try: + bytes = bytes.encode() + except Exception: + pass + return bytes + + def __recvall(self, count): + """__recvall(count) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = bytes("") + while len(data) < count: + d = self.recv(count - len(data)) + if not d: + raise GeneralProxyError( + (0, "connection closed unexpectedly")) + data = data + self.__decode(d) + return data + + def sendall(self, bytes): + socket.socket.sendall(self, self.__encode(bytes)) + + def setproxy(self, proxytype=None, addr=None, port=None, rdns=True, + username=None, password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype, addr, port, rdns, username, password) + + def __negotiatesocks5(self, destaddr, destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4] != None) and (self.__proxy[5] != None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall("\x05\x02\x00\x02") + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall("\x05\x01\x00") + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0] != "\x05": + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1] == "\x00": + # No authentication is required + pass + elif chosenauth[1] == "\x02": + # Okay, we need to perform a basic username/password + # authentication. + self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + + chr(len(self.__proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0] != "\x01": + # Bad response + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if authstat[1] != "\x00": + # Authentication failed + self.close() + raise Socks5AuthError((3, _socks5autherrors[3])) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == "\xFF": + raise Socks5AuthError((2, _socks5autherrors[2])) + else: + raise GeneralProxyError((1, _generalerrors[1])) + # Now we can request the actual connection + req = "\x05\x01\x00" + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + "\x01" + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3] == True: + # Resolve remotely + ipaddr = None + req = req + "\x03" + chr(len(destaddr)) + destaddr + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + "\x01" + ipaddr + req = req + self.__decode(struct.pack(">H", destport)) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0] != "\x05": + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + elif resp[1] != "\x00": + # Connection failed + self.close() + if ord(resp[1]) <= 8: + raise Socks5Error((ord(resp[1]), _socks5errors[ord(resp[1])])) + else: + raise Socks5Error((9, _socks5errors[9])) + # Get the bound address/port + elif resp[3] == "\x01": + boundaddr = self.__recvall(4) + elif resp[3] == "\x03": + resp = resp + self.recv(1) + boundaddr = self.__recvall(ord(resp[4])) + else: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + d = bytes(self.__recvall(2), 'utf8') + d = str(d) #python2.5 no unicode in struct + boundport = struct.unpack(">H", d)[0] + self.__proxysockname = boundaddr, boundport + if ipaddr != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr), destport) + else: + self.__proxypeername = (destaddr, destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return socket.socket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self, destaddr, destport): + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3] == True: + ipaddr = "\x00\x00\x00\x01" + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = "\x04\x01" + self.__decode(struct.pack(">H", destport)) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] != None: + req = req + self.__proxy[4] + req = req + "\x00" + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv==True: + req = req + destaddr + "\x00" + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0] != "\x00": + # Bad data + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if resp[1] != "\x5A": + # Server returned an error + self.close() + if ord(resp[1]) in (91,92,93): + self.close() + raise Socks4Error((ord(resp[1]), _socks4errors[ord(resp[1])-90])) + else: + raise Socks4Error((94,_socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]),struct.unpack(">H",bytes(resp[2:4],'utf8'))[0]) + if rmtrslv != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) + else: + self.__proxypeername = (destaddr, destport) + + def __negotiatehttp(self, destaddr, destport): + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if self.__proxy[3] == False: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + self.sendall(("CONNECT %s:%s HTTP/1.1\r\n" + "Host: %s\r\n\r\n") % (addr, destport, destaddr)) + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n") == -1: + resp = resp + self.recv(1) + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ", 2) + if statusline[0] not in ("HTTP/1.0", "HTTP/1.1"): + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1, _generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode, statusline[2])) + self.__proxysockname = ("0.0.0.0", 0) + self.__proxypeername = (addr, destport) + + def connect(self, destpair): + """connect(self,despair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + # TODO(durin42): seriously? type checking? do we care? + if ((not isinstance(destpair, (list, tuple))) or len(destpair) < 2 + or not isinstance(destpair[0], str) or not isinstance(destpair[1], int)): + raise GeneralProxyError((5, _generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + socket.socket.connect(self,(self.__proxy[1], portnum)) + self.__negotiatesocks5(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + socket.socket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatesocks4(destpair[0], destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + socket.socket.connect(self, (self.__proxy[1], portnum)) + self.__negotiatehttp(destpair[0], destpair[1]) + elif self.__proxy[0] == None: + socket.socket.connect(self, (destpair[0], destpair[1])) + else: + raise GeneralProxyError((4, _generalerrors[4])) |