diff options
| -rw-r--r-- | module/ThreadManager.py | 5 | ||||
| -rw-r--r-- | module/network/Bucket.py | 28 | ||||
| -rw-r--r-- | module/network/HTTPChunk.py | 26 | ||||
| -rw-r--r-- | module/network/HTTPDownload.py | 8 | ||||
| -rw-r--r-- | module/network/helper.py | 1 |
5 files changed, 33 insertions, 35 deletions
diff --git a/module/ThreadManager.py b/module/ThreadManager.py index eba23f28c..1b37bdfed 100644 --- a/module/ThreadManager.py +++ b/module/ThreadManager.py @@ -178,8 +178,9 @@ class ThreadManager: elif len(self.threads) < self.core.config.get("general", "max_downloads"): self.createThread() else: - #@TODO: close thread - pass + free = [x for x in self.threads if not x.active] + if free: + free[0].put("quit") def cleanPyCurl(self): diff --git a/module/network/Bucket.py b/module/network/Bucket.py index 35e27bcd4..c7eb62a54 100644 --- a/module/network/Bucket.py +++ b/module/network/Bucket.py @@ -22,7 +22,7 @@ from threading import Lock class Bucket: def __init__(self): - self.content = 0 + self.tokens = 0 self.rate = 0 self.lastDrip = time() self.lock = Lock() @@ -32,22 +32,22 @@ class Bucket: self.rate = rate self.lock.release() - def add(self, amount): + def consume(self, amount): + """ consume specified amount, return False if not enough tokens in bucket """ + if not self.rate: return True self.lock.acquire() - self.drip() - allowable = min(amount, self.rate - self.content) - if allowable > 0: - sleep(0.005) #@XXX: high sysload without?! + if amount < self.getTokens(): + self.tokens -= amount + self.lock.release() + return True - self.content += allowable self.lock.release() - return allowable + return False - def drip(self): - if self.rate == 0: - self.content = 0 - else: + def getTokens(self): + if self.tokens < self.rate: now = time() - deltaT = now - self.lastDrip - self.content = long(max(0, self.content - deltaT * self.rate)) + delta = self.rate * (now - self.lastDrip) + self.tokens = min(self.rate, self.tokens + delta) self.lastDrip = now + return self.tokens diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index f9ad3def7..c7d981880 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -42,7 +42,6 @@ class HTTPChunk(HTTPBase): self.deferred = Deferred() - self.lock = Lock() self.abort = False self.finished = False @@ -59,10 +58,7 @@ class HTTPChunk(HTTPBase): self.resp = None def getSpeed(self): - self.lock.acquire() - speed = self.speed - self.lock.release() - return speed + return self.speed @threaded def _download(self, resp): @@ -80,8 +76,8 @@ class HTTPChunk(HTTPBase): if self.noRangeHeader: count = min(count, self.range[1] - self.arrived) if self.bucket: - count = self.bucket.add(count) - if count == 0: + allow = self.bucket.consume(count) + if not allow: sleep(0.01) continue @@ -92,17 +88,17 @@ class HTTPChunk(HTTPBase): break if self.speedCalcTime < inttime(): - self.lock.acquire() self.speed = self.speedCalcLen - self.lock.release() self.speedCalcTime = inttime() self.speedCalcLen = 0 size = len(data) - if self.noRangeHeader and self.arrived+size == self.range[1]: - running = False - self.speedCalcLen += size - self.arrived += size + self.arrived += size + self.speedCalcLen += size + + if self.noRangeHeader and self.arrived == self.range[1]: + running = False + if data: self.fh.write(data) else: @@ -150,8 +146,8 @@ if __name__ == "__main__": import sys from Bucket import Bucket bucket = Bucket() - #bucket.setRate(200*1000) - bucket = None + bucket.setRate(200*1000) + #bucket = None url = "http://speedtest.netcologne.de/test_100mb.bin" diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index fed99ae23..d0e2eeb1f 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -261,7 +261,7 @@ class HTTPDownload(): self.info.save() dg.addCallback(self._copyChunks) - if len(self.chunks) == 0: + if not len(self.chunks): dg.callback() return self.deferred else: @@ -272,9 +272,9 @@ if __name__ == "__main__": from Bucket import Bucket bucket = Bucket() bucket.setRate(200*1024) - #bucket = None + bucket = None - url = "http://speedtest.netcologne.de/test_10mb.bin" + url = "http://speedtest.netcologne.de/test_100mb.bin" finished = False def err(*a, **b): @@ -286,7 +286,7 @@ if __name__ == "__main__": print "starting" - dwnld = HTTPDownload(url, "test_10mb.bin", bucket=bucket) + dwnld = HTTPDownload(url, "test_100mb.bin", bucket=bucket) d = dwnld.download(chunks=5, resume=True) d.addCallback(callb) d.addErrback(err) diff --git a/module/network/helper.py b/module/network/helper.py index 4b7119a2b..6900467f5 100644 --- a/module/network/helper.py +++ b/module/network/helper.py @@ -11,6 +11,7 @@ def callInThread(f, *args, **kwargs): class FThread(Thread): def __init__(self): Thread.__init__(self) + self.setDaemon(True) self.d = Deferred() def run(self): ret = f(*args, **kwargs) |