summaryrefslogtreecommitdiffstats
path: root/module
diff options
context:
space:
mode:
Diffstat (limited to 'module')
-rw-r--r--module/PyFile.py8
-rw-r--r--module/config/default.conf1
-rw-r--r--module/gui/Queue.py1
-rw-r--r--module/network/Browser.py26
-rw-r--r--module/network/HTTPChunk.py7
-rw-r--r--module/network/HTTPDownload.py2
-rw-r--r--module/network/helper.py45
-rw-r--r--module/plugins/Plugin.py19
-rw-r--r--module/plugins/hoster/RapidshareCom.py3
-rw-r--r--module/plugins/hoster/UploadedTo.py5
10 files changed, 94 insertions, 23 deletions
diff --git a/module/PyFile.py b/module/PyFile.py
index 62588e116..1723808c7 100644
--- a/module/PyFile.py
+++ b/module/PyFile.py
@@ -161,11 +161,13 @@ class PyFile():
"""abort pyfile if possible"""
while self.id in self.m.core.threadManager.processingIds():
self.abort = True
- if self.plugin and self.plugin.req: self.plugin.req.abort = True
+ if self.plugin and self.plugin.req:
+ self.plugin.req.abortDownloads()
sleep(0.1)
self.abort = False
- if hasattr(self, "plugin") and self.plugin and self.plugin.req: self.plugin.req.abort = False
+ if hasattr(self, "plugin") and self.plugin and self.plugin.req:
+ self.plugin.req.abortDownloads()
self.release()
def finishIfDone(self):
@@ -205,7 +207,7 @@ class PyFile():
def getSpeed(self):
""" calculates speed """
try:
- return self.plugin.req.get_speed()
+ return self.plugin.req.speed/1024 #kb/s
except:
return 0
diff --git a/module/config/default.conf b/module/config/default.conf
index cd23b3ca1..884891375 100644
--- a/module/config/default.conf
+++ b/module/config/default.conf
@@ -32,6 +32,7 @@ general - "General":
bool skip_existing : "Skip already existing files" = False
ip download_interface : "Outgoing IP address for downloads" = None
int renice : "CPU Priority" = 0
+ int chunks : "Max connections for one download" = 4
permission - "Permissions":
bool change_user : "Change user of running process" = False
str user : "Username" = user
diff --git a/module/gui/Queue.py b/module/gui/Queue.py
index 7c5c59f15..aa1afbd3c 100644
--- a/module/gui/Queue.py
+++ b/module/gui/Queue.py
@@ -88,6 +88,7 @@ class QueueModel(CollectorModel):
child = pack.getChild(d["id"])
if child:
child.data["downloading"] = d
+ child.data["progress"] = child.data["downloading"]["percent"]
k = pack.getChildKey(d["id"])
self.emit(SIGNAL("dataChanged(const QModelIndex &, const QModelIndex &)"), self.index(k, 0, self.index(p, 0)), self.index(k, self.cols, self.index(p, self.cols)))
diff --git a/module/network/Browser.py b/module/network/Browser.py
index b68c39c53..28edf8e9d 100644
--- a/module/network/Browser.py
+++ b/module/network/Browser.py
@@ -23,6 +23,9 @@ class Browser(object):
self.http = HTTPBase(interface=interface, proxies=proxies)
self.setCookieJar(cookieJar)
self.proxies = proxies
+ self.abort = property(lambda: False, lambda val: self.abortDownloads())
+
+ self.downloadConnections = []
def setCookieJar(self, cookieJar):
self.cookieJar = cookieJar
@@ -78,7 +81,22 @@ class Browser(object):
except:
pass
return location
-
+
+ def _removeConnection(self, d):
+ i = self.downloadConnections.index(d)
+ del self.downloadConnections[i]
+
+ def abortDownloads(self):
+ for d in self.downloadConnections:
+ d.abort = True
+
+ @property
+ def speed(self):
+ speed = 0
+ for d in self.downloadConnections:
+ speed += d.speed
+ return speed
+
def httpDownload(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, chunks=1,
resume=False):
if not referer:
@@ -90,12 +108,16 @@ class Browser(object):
dwnld.cookieJar = self.cookieJar
d = dwnld.download(chunks=chunks, resume=resume)
+ self.downloadConnections.append(d)
+ d.addCallback(self._removeConnection, d)
return d
def ftpDownload(self, url, filename, resume=False):
dwnld = FTPDownload(url, filename, bucket=self.bucket, interface=self.interface, proxies=self.proxies)
d = dwnld.download(resume=resume)
+ self.downloadConnections.append(d)
+ d.addCallback(self._removeConnection, d)
return d
def xdccDownload(self, server, port, channel, bot, pack, filename, nick="pyload_%d" % randint(1000, 9999),
@@ -104,6 +126,8 @@ class Browser(object):
interface=self.interface, proxies=self.proxies)
d = dwnld.download()
+ self.downloadConnections.append(d)
+ d.addCallback(self._removeConnection, d)
return d
#compatibility wrapper
diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py
index 509ff3983..f26e681b6 100644
--- a/module/network/HTTPChunk.py
+++ b/module/network/HTTPChunk.py
@@ -22,6 +22,8 @@ from urllib2 import HTTPError
from helper import *
from time import sleep
from traceback import print_exc
+from module.plugins.Plugin import Abort
+from module.plugins.Plugin import Fail
class HTTPChunk(HTTPBase):
def __init__(self, url, fh, get={}, post={}, referer=None, cookies=True, customHeaders={}, range=None, bucket=None, interface=None, proxies={}):
@@ -109,11 +111,11 @@ class HTTPChunk(HTTPBase):
self.fh.close()
if self.abort:
- self.deferred.error("abort")
+ self.deferred.error(Abort)
elif self.size == self.arrived:
self.deferred.callback(resp)
else:
- self.deferred.error("wrong content lenght")
+ self.deferred.error(Fail)
def getEncoding(self):
try:
@@ -127,7 +129,6 @@ class HTTPChunk(HTTPBase):
if self.range:
self.customHeaders["Range"] = "bytes=%i-%i" % self.range
try:
- print "req"
resp = self.getResponse(self.url, self.get, self.post, self.referer, self.cookies, self.customHeaders)
self.resp = resp
except HTTPError, e:
diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py
index ebc8c8e65..e38e1ba1b 100644
--- a/module/network/HTTPDownload.py
+++ b/module/network/HTTPDownload.py
@@ -274,7 +274,7 @@ class HTTPDownload(object):
dg.addCallback(self._copyChunks)
if not len(self.chunks):
dg.callback()
- return WrappedHTTPDeferred(self, self.deferred)
+ return WrappedHTTPDeferred(self, dg)
else:
raise Exception("no chunks")
diff --git a/module/network/helper.py b/module/network/helper.py
index 8cc61d3ff..ed9a88798 100644
--- a/module/network/helper.py
+++ b/module/network/helper.py
@@ -24,6 +24,7 @@ class Deferred():
def __init__(self):
self.call = []
self.err = []
+ self.prgr = {}
self.result = ()
self.errresult = ()
@@ -35,6 +36,13 @@ class Deferred():
kwargs.update(ckwargs)
callInThread(f, *args, **kwargs)
+ #test not in use
+ def addProgress(self, chain, f):
+ if self.prgr.has_key(chain):
+ self.prgr[chain].append(f)
+ else:
+ self.prgr[chain] = [f]
+
def addErrback(self, f, *cargs, **ckwargs):
self.err.append((f, cargs, ckwargs))
if self.errresult:
@@ -58,6 +66,13 @@ class Deferred():
args+=tuple(cargs)
kwargs.update(ckwargs)
callInThread(f, *args, **kwargs)
+
+ #test not in use
+ def progress(self, chain, *args, **kwargs):
+ if not self.prgr.has_key(chain):
+ return
+ for f in self.prgr[chain]:
+ callInThread(f, *args, **kwargs)
#decorator
def threaded(f):
@@ -71,12 +86,17 @@ def waitFor(d):
args = ()
err = None
+ def __init__(self, d):
+ self.d = d
+
def wait(self):
- d.addCallback(self.callb)
- d.addErrback(self.errb)
+ self.d.addCallback(self.callb)
+ self.d.addErrback(self.errb)
while self.waiting:
sleep(0.5)
if self.err:
+ if issubclass(self.err[0][0], Exception):
+ raise self.err[0][0]()
raise Exception(self.err)
return self.args
@@ -87,7 +107,7 @@ def waitFor(d):
def errb(self, *args, **kwargs):
self.waiting = False
self.err = (args, kwargs)
- w = Waiter()
+ w = Waiter(d)
return w.wait()
class DeferredGroup(Deferred):
@@ -110,16 +130,25 @@ class DeferredGroup(Deferred):
if len(self.group) == self.done:
self.callback()
-class WrappedDeferred():
+class WrappedDeferred(object):
def __init__(self, download, d):
- self.download = download
- self.d = d
+ self.__dict__["download"] = download
+ self.__dict__["d"] = d
def addCallback(self, *args, **kwargs):
- self.d.addCallback(*args, **kwargs)
+ self.__dict__["d"].addCallback(*args, **kwargs)
def addErrback(self, *args, **kwargs):
- self.d.addErrback(*args, **kwargs)
+ self.__dict__["d"].addErrback(*args, **kwargs)
+
+ def callback(self, *args, **kwargs):
+ self.__dict__["d"].callback(*args, **kwargs)
+
+ def error(self, *args, **kwargs):
+ self.__dict__["d"].error(*args, **kwargs)
def __getattr__(self, attr):
return getattr(self.download, attr)
+
+ def __setattr__(self, attr, val):
+ return setattr(self.download, attr, val)
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index f2c3b18eb..4af3d0d8b 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -42,6 +42,8 @@ from mimetypes import guess_type
from itertools import islice
+from module.network.helper import waitFor
+
def chunks(iterable, size):
it = iter(iterable)
item = list(islice(it, size))
@@ -86,6 +88,8 @@ class Plugin(object):
self.wantReconnect = False
self.multiDL = True
self.limitDL = 0
+ self.chunkLimit = 1
+ self.resumeDownload = False
self.waitUntil = 0 # time() + wait in seconds
self.waiting = False
@@ -113,7 +117,12 @@ class Plugin(object):
self.js = self.core.js # js engine
#self.setup()
-
+
+ def getChunkCount(self):
+ if self.chunkLimit <= 0:
+ return self.config["general"]["chunks"]
+ return min(self.config["general"]["chunks"], self.chunkLimit)
+
def __call__(self):
return self.__name__
@@ -283,7 +292,7 @@ class Plugin(object):
""" returns the content loaded """
if self.pyfile.abort: raise Abort
- res = self.req.load(url, get, post, ref, cookies, just_header, no_post_encode, raw_cookies)
+ res = self.req.getPage(url, get=get, post=post, cookies=cookies)
if self.core.debug:
from inspect import currentframe
frame = currentframe()
@@ -320,8 +329,10 @@ class Plugin(object):
self.log.warning(_("Setting User and Group failed: %s") % str(e))
name = self.pyfile.name.encode(sys.getfilesystemencoding(), "replace")
- newname = self.req.download(url, name, location, get, post, ref, cookies)
- newname = basename(newname)
+ filename = join(location, name)
+ d = self.req.httpDownload(url, filename, get=get, post=post, chunks=self.getChunkCount(), resume=self.resumeDownload)
+ waitFor(d)
+ newname = basename(filename)
self.pyfile.size = self.req.dl_size
diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py
index 8370aebb7..8163ffc44 100644
--- a/module/plugins/hoster/RapidshareCom.py
+++ b/module/plugins/hoster/RapidshareCom.py
@@ -66,7 +66,8 @@ class RapidshareCom(Hoster):
if self.account:
self.multiDL = True
- self.req.canContinue = True
+ self.chunkLimit = -1
+ self.resumeDownload = True
def process(self, pyfile):
self.url = self.pyfile.url
diff --git a/module/plugins/hoster/UploadedTo.py b/module/plugins/hoster/UploadedTo.py
index 5d4798d4e..1bfafc9c6 100644
--- a/module/plugins/hoster/UploadedTo.py
+++ b/module/plugins/hoster/UploadedTo.py
@@ -38,7 +38,8 @@ class UploadedTo(Hoster):
self.url = False
if self.account:
self.multiDL = True
- self.req.canContinue = True
+ self.chunkLimit = -1
+ self.resumeDownload = True
def process(self, pyfile):
@@ -157,4 +158,4 @@ class UploadedTo(Hoster):
url = url.replace("/?id=", "/file/")
url = url.replace("?id=", "file/")
url = re.sub("/\?(.*?)&id=", "/file/", url, 1)
- return url \ No newline at end of file
+ return url
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-31 20:36:59 +0100