diff options
Diffstat (limited to 'module')
| -rw-r--r-- | module/RequestFactory.py | 4 | ||||
| -rw-r--r-- | module/network/FtpRequest.py | 260 | ||||
| -rw-r--r-- | module/network/XdccRequest.py | 6 | ||||
| -rw-r--r-- | module/plugins/container/LinkList.py | 2 | ||||
| -rw-r--r-- | module/plugins/hoster/Ftp.py | 67 |
5 files changed, 335 insertions, 4 deletions
diff --git a/module/RequestFactory.py b/module/RequestFactory.py index 410d5ccd5..f66dfaf5d 100644 --- a/module/RequestFactory.py +++ b/module/RequestFactory.py @@ -20,6 +20,7 @@ from threading import Lock from module.network.Request import Request from module.network.XdccRequest import XdccRequest +from module.network.FtpRequest import FtpRequest import pycurl class RequestFactory(): @@ -40,6 +41,9 @@ class RequestFactory(): elif type == "XDCC": req = XdccRequest() + elif type == "FTP": + req = FtpRequest() + self.requests.append((pluginName, account, req)) self.lock.release() return req diff --git a/module/network/FtpRequest.py b/module/network/FtpRequest.py new file mode 100644 index 000000000..4896ea689 --- /dev/null +++ b/module/network/FtpRequest.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: spoob + @author: RaNaN + @author: mkaay + @author: jeix + @version: v0.3.2 +""" + +import base64 +import time +from os import sep, rename, stat +from os.path import exists +from cStringIO import StringIO +import pycurl + +class AbortDownload(Exception): + pass + +class FtpRequest: + def __init__(self, interface=None): + + self.dl_time = 0 + self.dl_finished = 0 + self.dl_size = 0 + self.dl_arrived = 0 + self.dl = False + + self.abort = False + + self.timeout = 5 + self.auth = False + + bufferBase = 1024 + bufferMulti = 4 + self.bufferSize = bufferBase*bufferMulti + self.canContinue = False + self.offset = 0 + + self.dl_speed = 0.0 + self.averageSpeed = 0.0 + self.averageSpeeds = [] + self.averageSpeedTime = 0.0 + self.averageSpeedCount = 0.0 + + self.speedLimitActive = False + self.maxSpeed = 0 + self.isSlow = False + self.interface = interface + + # change this for connection information + self.debug = True + + self.init_curl() + + def set_timeout(self, timeout): + self.timeout = int(timeout) + + def init_curl(self): + self.rep = StringIO() + self.header = "" + + self.pycurl = pycurl.Curl() + self.pycurl.setopt(pycurl.FOLLOWLOCATION, 1) + self.pycurl.setopt(pycurl.MAXREDIRS, 5) + self.pycurl.setopt(pycurl.TIMEOUT, (self.timeout*3600)) + self.pycurl.setopt(pycurl.CONNECTTIMEOUT, 30) + self.pycurl.setopt(pycurl.NOSIGNAL, 1) + self.pycurl.setopt(pycurl.NOPROGRESS, 0) + self.pycurl.setopt(pycurl.PROGRESSFUNCTION, self.progress) + self.pycurl.setopt(pycurl.AUTOREFERER, 1) + self.pycurl.setopt(pycurl.BUFFERSIZE, self.bufferSize) + self.pycurl.setopt(pycurl.SSL_VERIFYPEER, 0) + if self.debug: + self.pycurl.setopt(pycurl.VERBOSE, 1) + if self.interface: + self.pycurl.setopt(pycurl.INTERFACE, self.interface) + + + def add_auth(self, user, pw): + self.auth = True + self.pycurl.setopt(pycurl.USERNAME, user) + self.pycurl.setopt(pycurl.PASSWORD, pw) + + def add_proxy(self, protocol, adress): + # @TODO: pycurl proxy protocoll selection + self.pycurl.setopt(pycurl.PROXY, adress.split(":")[0]) + self.pycurl.setopt(pycurl.PROXYPORT, adress.split(":")[1]) + + def download(self, url, file_name): + file_temp = self.get_free_name(file_name) + ".part" + self.fp = open(file_temp, 'wb') + + self.init_curl() + self.pycurl.setopt(pycurl.URL, url) + + self.dl_arrived = self.offset + + if self.auth: + self.add_auth(self.user, self.pw) + + self.dl_time = time.time() + self.dl = True + + self.chunkSize = 0 + self.chunkRead = 0 + self.subStartTime = 0 + self.maxChunkSize = 0 + + def restLimit(): + subTime = time.time() - self.subStartTime + if subTime <= 1: + if self.speedLimitActive: + return self.maxChunkSize + else: + return -1 + else: + self.updateCurrentSpeed(float(self.chunkRead/1024) / subTime) + + self.subStartTime = time.time() + self.chunkRead = 0 + if self.maxSpeed > 0: + self.maxChunkSize = self.maxSpeed + else: + self.maxChunkSize = 0 + return 0 + + def writefunc(buf): + if self.abort: + return False + chunkSize = len(buf) + while chunkSize > restLimit() > -1: + time.sleep(0.05) + self.maxChunkSize -= chunkSize + self.fp.write(buf) + self.chunkRead += chunkSize + self.dl_arrived += chunkSize + + self.pycurl.setopt(pycurl.WRITEFUNCTION, writefunc) + + try: + self.pycurl.perform() + except Exception, e: + code, msg = e + if not code == 23: + raise Exception, e + + self.fp.close() + + if self.abort: + raise AbortDownload + + free_name = self.get_free_name(file_name) + rename(file_temp, free_name) + + self.dl = False + self.dl_finished = time.time() + + return free_name + + def updateCurrentSpeed(self, speed): + self.dl_speed = speed + if self.averageSpeedTime + 10 < time.time(): + self.averageSpeeds = [] + self.averageSpeeds.append(self.averageSpeed) + self.averageSpeeds.append(speed) + self.averageSpeed = (speed + self.averageSpeed)/2 + self.averageSpeedTime = time.time() + self.averageSpeedCount = 2 + else: + self.averageSpeeds.append(speed) + self.averageSpeedCount += 1 + allspeed = 0.0 + for s in self.averageSpeeds: + allspeed += s + self.averageSpeed = allspeed / self.averageSpeedCount + + def write_header(self, string): + self.header += string + + def get_rep(self): + value = self.rep.getvalue() + self.rep.close() + self.rep = StringIO() + return value + + def get_header(self): + h = self.header + self.header = "" + return h + + def get_speed(self): + try: + return self.dl_speed + except: + return 0 + + def get_ETA(self): + try: + return (self.dl_size - self.dl_arrived) / (self.dl_arrived / (time.time() - self.dl_time)) + except: + return 0 + + def kB_left(self): + return (self.dl_size - self.dl_arrived) / 1024 + + def progress(self, dl_t, dl_d, up_t, up_d): + if self.abort: + return False + self.dl_arrived = int(dl_d) + self.dl_size = int(dl_t) + + def get_free_name(self, file_name): + file_count = 0 + while exists(file_name): + file_count += 1 + if "." in file_name: + file_split = file_name.split(".") + temp_name = "%s-%i.%s" % (".".join(file_split[:-1]), file_count, file_split[-1]) + else: + temp_name = "%s-%i" % (file_name, file_count) + if not exists(temp_name): + file_name = temp_name + return file_name + + def __del__(self): + self.clean() + + def clean(self): + try: + self.pycurl.close() + except: + pass + +# def getURL(url): + # """ + # currently used for update check + # """ + # req = Request() + # c = req.load(url) + # req.pycurl.close() + # return c + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/module/network/XdccRequest.py b/module/network/XdccRequest.py index c7c4bc893..57fdced66 100644 --- a/module/network/XdccRequest.py +++ b/module/network/XdccRequest.py @@ -192,7 +192,7 @@ class XdccRequest: if len(m.groups()) > 3:
self.dl_size = int(m.group(4))
dcc_packname = self.get_free_name(path + '\\' + dcc_packname)
- dcc_fpointer = open(dcc_packname, "wb")
+ dcc_fpointer = open(dcc_packname + ".part", "wb")
dcc_total = 0
# recv loop for dcc socket
@@ -225,8 +225,8 @@ class XdccRequest: # acknowledge data by sending number of recceived bytes
dcc.send(struct.pack('!I', dcc_total))
########################
- free_name = self.get_free_name(file_name)
- rename(file_temp, free_name)
+ free_name = self.get_free_name(dcc_packname)
+ rename(dcc_packname + ".part", free_name)
self.dl = False
self.dl_finished = time.time()
diff --git a/module/plugins/container/LinkList.py b/module/plugins/container/LinkList.py index 6cc30ecc6..1651b91ed 100644 --- a/module/plugins/container/LinkList.py +++ b/module/plugins/container/LinkList.py @@ -31,7 +31,7 @@ class LinkList(Container): continue if link.startswith("[") and link.endswith("]"): # new package - curPack = link[1:-2] + curPack = link[1:-1] packages[curPack] = [] continue packages[curPack].append(link.replace("\n", "")) diff --git a/module/plugins/hoster/Ftp.py b/module/plugins/hoster/Ftp.py new file mode 100644 index 000000000..dc536fa1f --- /dev/null +++ b/module/plugins/hoster/Ftp.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*-
+
+"""
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+ @author: jeix
+"""
+
+import logging
+from os.path import exists
+from os.path import join
+from os.path import exists
+from os import makedirs
+import re
+import sys
+
+from module.plugins.Hoster import Hoster
+
+
+class Ftp(Hoster):
+ __name__ = "Ftp"
+ __version__ = "0.1"
+ __pattern__ = r'ftp://(.*?:.*?@)?.*?/.*' # ftp://user:password@ftp.server.org/path/to/file
+ __type__ = "hoster"
+ __description__ = """A Plugin that allows you to download from an from an ftp directory"""
+ __author_name__ = ("jeix")
+ __author_mail__ = ("jeix@hasnomail.com")
+
+ def __init__(self, parent):
+ self.parent = parent
+ self.req = parent.core.requestFactory.getRequest(self.__name__, type="FTP")
+ self.want_reconnect = False
+ self.multi_dl = True
+ self.logger = logging.getLogger("log")
+ self.pyfile = self.parent
+
+ def prepare(self, thread):
+ self.pyfile.status.url = self.parent.url
+ self.pyfile.status.filename = self.get_file_name()
+ thread.wait(self.parent)
+ return True
+
+ def get_file_name(self):
+ return self.parent.url.rpartition('/')[2]
+
+ def proceed(self, url, location):
+ download_folder = self.parent.core.config['general']['download_folder']
+ if self.pyfile.package.data["package_name"] != (self.parent.core.config['general']['link_file']) and self.parent.core.xmlconfig.get("general", "folder_per_package", False):
+ self.pyfile.folder = self.pyfile.package.data["package_name"]
+ location = join(download_folder, self.pyfile.folder.decode(sys.getfilesystemencoding()))
+ if not exists(location): makedirs(location)
+ file_path = join(location.decode(sys.getfilesystemencoding()), self.pyfile.status.filename.decode(sys.getfilesystemencoding()))
+ else:
+ file_path = join(download_folder, self.pyfile.status.filename.decode(sys.getfilesystemencoding()))
+
+ self.pyfile.status.filename = self.req.download(url, file_path)
|