diff options
Diffstat (limited to 'module')
| -rw-r--r-- | module/network/Browser.py | 5 | ||||
| -rw-r--r-- | module/network/HTTPBase.py | 5 | ||||
| -rw-r--r-- | module/network/HTTPChunk.py | 11 | ||||
| -rw-r--r-- | module/network/HTTPDownload.py | 108 | ||||
| -rw-r--r-- | module/network/helper.py | 46 | ||||
| -rw-r--r-- | module/plugins/Plugin.py | 2 | ||||
| -rw-r--r-- | module/plugins/hoster/NetloadIn.py | 3 |
7 files changed, 101 insertions, 79 deletions
diff --git a/module/network/Browser.py b/module/network/Browser.py index 28edf8e9d..512a437ed 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -82,8 +82,8 @@ class Browser(object): pass return location - def _removeConnection(self, d): - i = self.downloadConnections.index(d) + def _removeConnection(self, *args, **kwargs): + i = self.downloadConnections.index(args[-1]) del self.downloadConnections[i] def abortDownloads(self): @@ -110,6 +110,7 @@ class Browser(object): d = dwnld.download(chunks=chunks, resume=resume) self.downloadConnections.append(d) d.addCallback(self._removeConnection, d) + d.addErrback(self._removeConnection, d) return d def ftpDownload(self, url, filename, resume=False): diff --git a/module/network/HTTPBase.py b/module/network/HTTPBase.py index 2737fd33c..7627c8499 100644 --- a/module/network/HTTPBase.py +++ b/module/network/HTTPBase.py @@ -333,8 +333,9 @@ class HTTPBase(): print "[HTTP] creating request" print "[HTTP] URL:", url print "[HTTP] GET" - for key, value in get.iteritems(): - print "[HTTP] \t", key, ":", value + if get: + for key, value in get.iteritems(): + print "[HTTP] \t", key, ":", value if post: print "[HTTP] POST" for key, value in post.iteritems(): diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index 48ebc13f3..02134ca63 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -85,7 +85,7 @@ class HTTPChunk(HTTPBase): try: data = resp.read(count) except: - self.deferred.error("timeout") + self.deferred.error(Fail, "timeout") break if self.speedCalcTime < inttime(): @@ -104,7 +104,7 @@ class HTTPChunk(HTTPBase): if self.noRangeHeader and self.arrived == self.range[1]: running = False - if data: + if size: self.fh.write(data) else: break @@ -117,9 +117,10 @@ class HTTPChunk(HTTPBase): if self.abort: self.deferred.error(Abort) elif self.size == self.arrived: - self.deferred.callback(resp) + self.deferred.callback() else: - self.deferred.error(Fail) + print self.arrived, self.size + self.deferred.error(Fail, "wrong content-length") def getEncoding(self): try: @@ -140,7 +141,7 @@ class HTTPChunk(HTTPBase): self.deferred.error(e) return self.deferred - if (self.range and resp.getcode() == 206) or (not self.range and resp.getcode() == 200): + if resp.getcode() in (200, 206): self._download(resp) else: self.deferred.error(resp.getcode(), resp) diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index 1baf0ccad..b81008cc5 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -105,7 +105,7 @@ class ChunkInfo(): class WrappedHTTPDeferred(WrappedDeferred): pass -class HTTPDownload(object): +class HTTPDownload(): def __init__(self, url, filename, get={}, post={}, referer=None, cookies=True, customHeaders={}, bucket=None, interface=None, proxies={}): self.url = url self.filename = filename @@ -131,6 +131,7 @@ class HTTPDownload(object): self.cookieJar = CookieJar() self.chunks = [] + self.chunksDone = 0 try: self.info = ChunkInfo.load(filename) except IOError: @@ -170,11 +171,18 @@ class HTTPDownload(object): def calcProgress(self, p): self.deferred.progress("percent", 100-int((self.size - self.arrived)/float(self.size)*100)) + def _chunkDone(self): + self.chunksDone += 1 + print self.chunksDone, "/", len(self.chunks) + if self.chunksDone == len(self.chunks): + self._copyChunks() + def _copyChunks(self): - fo = open(self.filename, "wb") + fo = open(self.filename, "wb") #out file for i in range(self.info.getCount()): encoding = self.info.getChunkEncoding(i) + #decompress method, if any decompress = lambda data: data if encoding == "gzip": gz = decompressobj(16+MAX_WBITS) @@ -183,18 +191,19 @@ class HTTPDownload(object): df = decompressobj(-MAX_WBITS) decompress = lambda data: df.decompress(data) + #input file fname = "%s.chunk%d" % (self.filename, i) fi = open(fname, "rb") - while True: + while True: #copy in chunks, consumes less memory data = fi.read(512*1024) if not data: break - fo.write(decompress(data)) + fo.write(decompress(data)) #decompressing fi.close() - remove(fname) + remove(fname) #remove fo.close() - self.info.removeInfo() - self.deferred.callback() + self.info.removeInfo() #remove info file + self.deferred.callback() #done, emit callbacks def _createChunk(self, fh, range=None): chunk = HTTPChunk(self.url, fh, get=self.get, post=self.post, @@ -205,81 +214,92 @@ class HTTPDownload(object): chunk.cookieJar = self.cookieJar return chunk + def _addChunk(self, chunk, d): + self.chunks.append(chunk) + d.addProgress("percent", self.calcProgress) + d.addCallback(self._chunkDone) + d.addErrback(lambda *args, **kwargs: self.setAbort(True)) + d.addErrback(self.deferred.error) + def download(self, chunks=1, resume=False): + self.chunksDone = 0 if chunks > 0: - dg = DeferredGroup() + #diffentent chunk count in info, resetting if self.info.loaded and not self.info.getCount() == chunks: self.info.clear() + + #if resuming, calculate range with offset crange = None if resume: if self.info.getCount() == chunks and exists("%s.chunk0" % (self.filename, )): crange = self.info.getChunkRange(0) crange = (crange[0]+getsize("%s.chunk0" % (self.filename, )), crange[1]-1) + #if firstpart not done if crange is None or crange[1]-crange[0] > 0: fh = open("%s.chunk0" % (self.filename, ), "ab" if crange else "wb") + chunk = self._createChunk(fh, range=crange) - self.chunks.append(chunk) - d = chunk.download() - dg.addDeferred(d) - d.addProgress("percent", self.calcProgress) + d = chunk.download() #start downloading + self._addChunk(chunk, d) + + #no info file, need to calculate ranges if not self.info.loaded: - size = chunk.size - chunksize = size/chunks - lastchunk = chunksize + size = chunk.size #overall size + chunksize = size/chunks #chunk size - chunk.range = (0, chunksize) + chunk.range = (0, chunksize) #setting range for first part chunk.noRangeHeader = True - self.size = chunk.size - self.info.setSize(self.size) - chunk.size = chunksize - self.info.addChunk("%s.chunk0" % (self.filename, ), chunk.range, chunk.getEncoding()) + chunk.size = chunksize #setting size for first chunk - lastchunk = size - chunksize*(chunks-1) - else: - self.size = self.info.size - self.firstchunk = chunk + self.size = size #setting overall size + self.info.setSize(self.size) #saving overall size + self.info.addChunk("%s.chunk0" % (self.filename, ), chunk.range, chunk.getEncoding()) #add chunk to infofile + + lastchunk = size - chunksize*(chunks-1) #calculating size for last chunk + self.firstchunk = chunk #remeber first chunk + + if self.info.loaded and not self.size: + self.size = self.info.size #setting overall size - for i in range(1, chunks): + for i in range(1, chunks): #other chunks cont = False if not self.info.loaded: #first time load - if i+1 == chunks: - rng = (i*chunksize, i*chunksize+lastchunk) + if i+1 == chunks: #last chunk? + rng = (i*chunksize, i*chunksize+lastchunk-1) else: - rng = (i*chunksize, (i+1)*chunksize-1) - else: #not finished - rng = self.info.getChunkRange(i) + rng = (i*chunksize, (i+1)*chunksize-1) #adjusting range + else: #info available + rng = self.info.getChunkRange(i) #loading previous range if resume and exists("%s.chunk%d" % (self.filename, i)): #continue chunk - rng = (rng[0]+getsize("%s.chunk%d" % (self.filename, i)), rng[1]) - cont = True + rng = (rng[0]+getsize("%s.chunk%d" % (self.filename, i)), rng[1]) #adjusting offset + cont = True #set append mode if rng[1]-rng[0] <= 0: #chunk done continue fh = open("%s.chunk%d" % (self.filename, i), "ab" if cont else "wb") chunk = self._createChunk(fh, range=rng) - d = chunk.download() - if not chunk.resp.getcode() == 206 and i == 1: #no range supported, tell chunk0 to download everything + d = chunk.download() #try + + if not chunk.resp.getcode() == 206 and i == 1: #no range supported, tell first chunk to download everything chunk.abort = True self.noChunkSupport = True self.firstchunk.size = self.size self.firstchunk.range = None - self.info.clear() - self.info.addChunk("%s.chunk0" % (self.filename, ), (0, self.firstchunk.size), chunk.getEncoding()) + self.info.clear() #clear info + self.info.addChunk("%s.chunk0" % (self.filename, ), (0, self.firstchunk.size), chunk.getEncoding()) #re-adding info with correct ranges break - self.chunks.append(chunk) - dg.addDeferred(d) - d.addProgress("percent", self.calcProgress) - if not self.info.loaded: + self._addChunk(chunk, d) + + if not self.info.loaded: #adding info self.info.addChunk("%s.chunk%d" % (self.filename, i), chunk.range, chunk.getEncoding()) - self.info.save() - dg.addCallback(self._copyChunks) + self.info.save() #saving info if not len(self.chunks): - dg.callback() - dg.addErrback(self.deferred.error) + self._copyChunks() return WrappedHTTPDeferred(self, self.deferred) else: raise Exception("no chunks") diff --git a/module/network/helper.py b/module/network/helper.py index ebe0fa8f5..dac0185ce 100644 --- a/module/network/helper.py +++ b/module/network/helper.py @@ -51,13 +51,13 @@ class Deferred(): callInThread(f, *args, **kwargs) def callback(self, *args, **kwargs): - if self.result: - raise AlreadyCalled self.result = (args, kwargs) for f, cargs, ckwargs in self.call: args+=tuple(cargs) kwargs.update(ckwargs) callInThread(f, *args, **kwargs) + self.call = [] + self.result = () def error(self, *args, **kwargs): self.errresult = (args, kwargs) @@ -65,6 +65,8 @@ class Deferred(): args+=tuple(cargs) kwargs.update(ckwargs) callInThread(f, *args, **kwargs) + self.err = [] + self.errresult = () def progress(self, chain, *args, **kwargs): if not self.prgr.has_key(chain): @@ -93,9 +95,12 @@ def waitFor(d): while self.waiting: sleep(0.5) if self.err: + #try: if issubclass(self.err[0][0], Exception): - raise self.err[0][0]() - raise Exception(self.err) + raise self.err[0][0](*self.err[0][1:], **self.err[1]) + #except: + # pass + raise Exception(*self.err[0], **self.err[1]) return self.args def callb(self, *args, **kwargs): @@ -127,32 +132,25 @@ class DeferredGroup(Deferred): self.done += 1 if len(self.group) == self.done: self.callback() + self.group = [] + self.done = 0 + + def error(self, *args, **kwargs): + Deferred.error(self, *args, **kwargs) + self.group = [] + self.done = 0 class WrappedDeferred(object): def __init__(self, download, d): self.__dict__["download"] = download self.__dict__["d"] = d - def addCallback(self, *args, **kwargs): - self.__dict__["d"].addCallback(*args, **kwargs) - - def addErrback(self, *args, **kwargs): - self.__dict__["d"].addErrback(*args, **kwargs) - - def addProgress(self, *args, **kwargs): - self.__dict__["d"].addProgress(*args, **kwargs) - - def callback(self, *args, **kwargs): - self.__dict__["d"].callback(*args, **kwargs) - - def error(self, *args, **kwargs): - self.__dict__["d"].error(*args, **kwargs) - - def progress(self, *args, **kwargs): - self.__dict__["d"].progress(*args, **kwargs) - def __getattr__(self, attr): - return getattr(self.download, attr) + if attr in ("addCallback", "addErrback", "addProgress", "callback", "error", "progress"): + return getattr(self.__dict__["d"], attr) + return getattr(self.__dict__["download"], attr) def __setattr__(self, attr, val): - return setattr(self.download, attr, val) + if attr in ("addCallback", "addErrback", "addProgress", "callback", "error", "progress"): + return setattr(self.__dict__["d"], attr, val) + return setattr(self.__dict__["download"], attr, val) diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 6a7ec5b14..95e937d1c 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -336,7 +336,7 @@ class Plugin(object): waitFor(d) self.pyfile.download = None newname = basename(filename) - + self.pyfile.size = d.size if newname and newname != name: diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py index a0f50cfeb..d7aee2c27 100644 --- a/module/plugins/hoster/NetloadIn.py +++ b/module/plugins/hoster/NetloadIn.py @@ -65,7 +65,8 @@ class NetloadIn(Hoster): self.multiDL = False if self.account: self.multiDL = True - self.req.canContinue = True + self.chunkLimit = -1 + self.resumeDownload = True def process(self, pyfile): self.url = pyfile.url |