From 5ee3579572b60bb8f9e6475a517d69462b0cfe29 Mon Sep 17 00:00:00 2001 From: mkaay Date: Wed, 23 Dec 2009 21:04:06 +0100 Subject: download speed limit --- module/XMLConfigParser.py | 10 ++++- module/config/core.xml | 58 -------------------------- module/config/core_default.xml | 59 +++++++++++++++++++++++++++ module/network/Request.py | 34 ++++++++++++--- module/thread_list.py | 93 ++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 184 insertions(+), 70 deletions(-) delete mode 100644 module/config/core.xml create mode 100644 module/config/core_default.xml (limited to 'module') diff --git a/module/XMLConfigParser.py b/module/XMLConfigParser.py index b46f9c959..51741fdfd 100644 --- a/module/XMLConfigParser.py +++ b/module/XMLConfigParser.py @@ -17,18 +17,24 @@ """ from __future__ import with_statement +from os.path import exists + from xml.dom.minidom import parse class XMLConfigParser(): - def __init__(self, data): + def __init__(self, data, default_data=None): self.xml = None self.file = data + self.file_default = default_data self.config = {} self.loadData() self.root = None def loadData(self): - with open(self.file, 'r') as fh: + file = self.file + if not exists(self.file): + file = self.file_default + with open(file, 'r') as fh: self.xml = parse(fh) self._read_config() diff --git a/module/config/core.xml b/module/config/core.xml deleted file mode 100644 index 6da85cf32..000000000 --- a/module/config/core.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - 7227 - 0.0.0.0 - admin - pwhere - - - False - ssl.srt - ssl.key - - - True - 0.0.0.0 - 8000 - - True - None - None - None - None - None - - - True - Logs - 5 - - - de - Downloads - 3 - False - links.txt - failed_links.txt - reconnect_method - False - 5 - - - True - False - - - 0:00 - 0:00 - - - 0:00 - 0:00 - - - False - http://localhost:8080 - http - - diff --git a/module/config/core_default.xml b/module/config/core_default.xml new file mode 100644 index 000000000..56ac2825c --- /dev/null +++ b/module/config/core_default.xml @@ -0,0 +1,59 @@ + + + 7227 + 0.0.0.0 + admin + pwhere + + + False + ssl.srt + ssl.key + + + True + 0.0.0.0 + 8000 + + True + None + None + None + None + None + + + True + Logs + 5 + + + de + Downloads + 3 + False + links.txt + failed_links.txt + reconnect_method + False + 5 + 0 + + + True + False + + + 0:00 + 0:00 + + + 0:00 + 0:00 + + + False + http://localhost:8080 + http + + diff --git a/module/network/Request.py b/module/network/Request.py index 4aca935dc..752c16f04 100755 --- a/module/network/Request.py +++ b/module/network/Request.py @@ -51,14 +51,19 @@ class Request: self.timeout = 5 bufferBase = 1024 - bufferMulti = 2 + bufferMulti = 4 self.bufferSize = bufferBase*bufferMulti self.canContinue = False self.dl_speed = 0.0 + self.averageSpeed = 0.0 + self.averageSpeeds = [] + self.averageSpeedTime = 0.0 + self.averageSpeedCount = 0.0 self.speedLimitActive = False - self.maxSpeed = 100 * 1024 + self.maxSpeed = 0 + self.isSlow = False try: if pycurl: self.curl = True @@ -171,7 +176,7 @@ class Request: self.add_cookies(req) #add cookies - rep = self.opener.open(req, timeout=2) + rep = self.opener.open(req) for cookie in self.cj.make_cookies(rep, req): self.cookies.append(cookie) @@ -287,7 +292,7 @@ class Request: else: return -1 else: - self.dl_speed = float(self.chunkRead/1024) / subTime + self.updateCurrentSpeed(float(self.chunkRead/1024) / subTime) self.subStartTime = time.time() self.chunkRead = 0 @@ -390,7 +395,7 @@ class Request: else: time.sleep(0.05) subTime = time.time() - subStartTime - self.dl_speed = float(chunkRead/1024) / subTime + self.updateCurrentSpeed(float(chunkRead/1024) / subTime) file.close() if self.abort: @@ -399,7 +404,24 @@ class Request: self.dl_finished = time.time() rename(file_temp, self.get_free_name(file_name)) return True - + + def updateCurrentSpeed(self, speed): + self.dl_speed = speed + if self.averageSpeedTime + 10 < time.time(): + self.averageSpeeds = [] + self.averageSpeeds.append(self.averageSpeed) + self.averageSpeeds.append(speed) + self.averageSpeed = (speed + self.averageSpeed)/2 + self.averageSpeedTime = time.time() + self.averageSpeedCount = 2 + else: + self.averageSpeeds.append(speed) + self.averageSpeedCount += 1 + allspeed = 0.0 + for s in self.averageSpeeds: + allspeed += s + self.averageSpeed = allspeed / self.averageSpeedCount + def write_header(self, string): self.header += string diff --git a/module/thread_list.py b/module/thread_list.py index d78a9b95c..9cdb5fc8f 100644 --- a/module/thread_list.py +++ b/module/thread_list.py @@ -21,7 +21,7 @@ from __future__ import with_statement from os.path import exists import re import subprocess -from threading import RLock +from threading import RLock, Thread import time import urllib2 @@ -40,6 +40,7 @@ class Thread_List(object): self.reconnecting = False self.select_thread() + self.speedManager = self.SpeedManager(self) def create_thread(self): """ creates thread for Py_Load_File and append thread to self.threads @@ -207,10 +208,94 @@ class Thread_List(object): out.wait() def scripts_download_finished(self, pluginname, url, filename, location): - map(lambda script: subprocess.Popen([script, pluginname, url, filename, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) + map(lambda script: subprocess.Popen([script, pluginname, url, filename, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) def scripts_package_finished(self, name, location): #@TODO Implement! - map(lambda script: subprocess.Popen([script, name, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) + map(lambda script: subprocess.Popen([script, name, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) def scripts_reconnected(self, ip): - map(lambda script: subprocess.Popen([script, ip], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) + map(lambda script: subprocess.Popen([script, ip], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) + + class SpeedManager(Thread): + def __init__(self, parent): + Thread.__init__(self) + self.parent = parent + self.running = True + self.lastSlowCheck = 0.0 + + stat = {} + stat["slow_downloads"] = None + stat["each_speed"] = None + stat["each_speed_optimized"] = None + self.stat = stat + + self.slowCheckInterval = 60 + self.slowCheckTestTime = 25 + + self.logger = self.parent.parent.logger + self.start() + + def run(self): + while self.running: + time.sleep(1) + self.manageSpeed() + + def getMaxSpeed(self): + return self.parent.parent.getMaxSpeed() + + def manageSpeed(self): + maxSpeed = self.getMaxSpeed() + if maxSpeed <= 0: + for thread in self.parent.py_downloading: + thread.plugin.req.speedLimitActive = False + return + threads = self.parent.py_downloading + threadCount = len(threads) + if threadCount <= 0: + return + eachSpeed = maxSpeed/threadCount + + currentOverallSpeed = 0 + restSpeed = maxSpeed - currentOverallSpeed + speeds = [] + for thread in threads: + currentOverallSpeed += thread.plugin.req.dl_speed + speeds.append((thread.plugin.req.dl_speed, thread.plugin.req.averageSpeed, thread)) + thread.plugin.req.speedLimitActive = True + + if currentOverallSpeed+50 < maxSpeed: + for thread in self.parent.py_downloading: + thread.plugin.req.speedLimitActive = False + return + + print "-----" + + slowCount = 0 + slowSpeed = 0 + if self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime < time.time(): + self.lastSlowCheck = time.time() + if self.lastSlowCheck + self.slowCheckInterval < time.time() < self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime: + for speed in speeds: + speed[2].plugin.req.isSlow = False + else: + for speed in speeds: + if speed[0] <= eachSpeed-7: + if speed[1] < eachSpeed-15: + if speed[2].plugin.req.dl_time > 0 and speed[2].plugin.req.dl_time+30 < time.time(): + speed[2].plugin.req.isSlow = True + if not speed[1]-5 < speed[2].plugin.req.maxSpeed/1024 < speed[1]+5: + speed[2].plugin.req.maxSpeed = (speed[1]+10)*1024 + if speed[2].plugin.req.isSlow: + slowCount += 1 + slowSpeed += speed[2].plugin.req.maxSpeed/1024 + + stat["slow_downloads"] = slowCount + stat["each_speed"] = eachSpeed + eachSpeed = (maxSpeed - slowSpeed) / (threadCount - slowCount) + stat["each_speed_optimized"] = eachSpeed + for speed in speeds: + if speed[2].plugin.req.isSlow: + continue + speed[2].plugin.req.maxSpeed = eachSpeed*1024 + print "max", speed[2].plugin.req.maxSpeed, "current", speed[2].plugin.req.dl_speed + -- cgit v1.2.3