diff options
| -rw-r--r-- | .hgignore | 1 | ||||
| -rw-r--r-- | module/XMLConfigParser.py | 10 | ||||
| -rw-r--r-- | module/config/core_default.xml (renamed from module/config/core.xml) | 1 | ||||
| -rwxr-xr-x | module/network/Request.py | 34 | ||||
| -rw-r--r-- | module/thread_list.py | 93 | ||||
| -rw-r--r-- | pyLoadCore.py | 11 |
6 files changed, 135 insertions, 15 deletions
@@ -13,6 +13,7 @@ Logs/* Plugins/DLC.py failed_links.txt module/config/gui.xml +module/config/core.xml links.txt module/links.pkl module/cookies.txt diff --git a/module/XMLConfigParser.py b/module/XMLConfigParser.py index b46f9c959..51741fdfd 100644 --- a/module/XMLConfigParser.py +++ b/module/XMLConfigParser.py @@ -17,18 +17,24 @@ """ from __future__ import with_statement +from os.path import exists + from xml.dom.minidom import parse class XMLConfigParser(): - def __init__(self, data): + def __init__(self, data, default_data=None): self.xml = None self.file = data + self.file_default = default_data self.config = {} self.loadData() self.root = None def loadData(self): - with open(self.file, 'r') as fh: + file = self.file + if not exists(self.file): + file = self.file_default + with open(file, 'r') as fh: self.xml = parse(fh) self._read_config() diff --git a/module/config/core.xml b/module/config/core_default.xml index 6da85cf32..56ac2825c 100644 --- a/module/config/core.xml +++ b/module/config/core_default.xml @@ -37,6 +37,7 @@ <reconnect_method>reconnect_method</reconnect_method> <debug_mode>False</debug_mode> <max_download_time>5</max_download_time> + <download_speed_limit>0</download_speed_limit> </general> <updates> <search_updates>True</search_updates> diff --git a/module/network/Request.py b/module/network/Request.py index 4aca935dc..752c16f04 100755 --- a/module/network/Request.py +++ b/module/network/Request.py @@ -51,14 +51,19 @@ class Request: self.timeout = 5 bufferBase = 1024 - bufferMulti = 2 + bufferMulti = 4 self.bufferSize = bufferBase*bufferMulti self.canContinue = False self.dl_speed = 0.0 + self.averageSpeed = 0.0 + self.averageSpeeds = [] + self.averageSpeedTime = 0.0 + self.averageSpeedCount = 0.0 self.speedLimitActive = False - self.maxSpeed = 100 * 1024 + self.maxSpeed = 0 + self.isSlow = False try: if pycurl: self.curl = True @@ -171,7 +176,7 @@ class Request: self.add_cookies(req) #add cookies - rep = self.opener.open(req, timeout=2) + rep = self.opener.open(req) for cookie in self.cj.make_cookies(rep, req): self.cookies.append(cookie) @@ -287,7 +292,7 @@ class Request: else: return -1 else: - self.dl_speed = float(self.chunkRead/1024) / subTime + self.updateCurrentSpeed(float(self.chunkRead/1024) / subTime) self.subStartTime = time.time() self.chunkRead = 0 @@ -390,7 +395,7 @@ class Request: else: time.sleep(0.05) subTime = time.time() - subStartTime - self.dl_speed = float(chunkRead/1024) / subTime + self.updateCurrentSpeed(float(chunkRead/1024) / subTime) file.close() if self.abort: @@ -399,7 +404,24 @@ class Request: self.dl_finished = time.time() rename(file_temp, self.get_free_name(file_name)) return True - + + def updateCurrentSpeed(self, speed): + self.dl_speed = speed + if self.averageSpeedTime + 10 < time.time(): + self.averageSpeeds = [] + self.averageSpeeds.append(self.averageSpeed) + self.averageSpeeds.append(speed) + self.averageSpeed = (speed + self.averageSpeed)/2 + self.averageSpeedTime = time.time() + self.averageSpeedCount = 2 + else: + self.averageSpeeds.append(speed) + self.averageSpeedCount += 1 + allspeed = 0.0 + for s in self.averageSpeeds: + allspeed += s + self.averageSpeed = allspeed / self.averageSpeedCount + def write_header(self, string): self.header += string diff --git a/module/thread_list.py b/module/thread_list.py index d78a9b95c..9cdb5fc8f 100644 --- a/module/thread_list.py +++ b/module/thread_list.py @@ -21,7 +21,7 @@ from __future__ import with_statement from os.path import exists import re import subprocess -from threading import RLock +from threading import RLock, Thread import time import urllib2 @@ -40,6 +40,7 @@ class Thread_List(object): self.reconnecting = False self.select_thread() + self.speedManager = self.SpeedManager(self) def create_thread(self): """ creates thread for Py_Load_File and append thread to self.threads @@ -207,10 +208,94 @@ class Thread_List(object): out.wait() def scripts_download_finished(self, pluginname, url, filename, location): - map(lambda script: subprocess.Popen([script, pluginname, url, filename, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) + map(lambda script: subprocess.Popen([script, pluginname, url, filename, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) def scripts_package_finished(self, name, location): #@TODO Implement! - map(lambda script: subprocess.Popen([script, name, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) + map(lambda script: subprocess.Popen([script, name, location], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) def scripts_reconnected(self, ip): - map(lambda script: subprocess.Popen([script, ip], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) + map(lambda script: subprocess.Popen([script, ip], stdout=subprocess.PIPE), self.parent.scripts['download_finished']) + + class SpeedManager(Thread): + def __init__(self, parent): + Thread.__init__(self) + self.parent = parent + self.running = True + self.lastSlowCheck = 0.0 + + stat = {} + stat["slow_downloads"] = None + stat["each_speed"] = None + stat["each_speed_optimized"] = None + self.stat = stat + + self.slowCheckInterval = 60 + self.slowCheckTestTime = 25 + + self.logger = self.parent.parent.logger + self.start() + + def run(self): + while self.running: + time.sleep(1) + self.manageSpeed() + + def getMaxSpeed(self): + return self.parent.parent.getMaxSpeed() + + def manageSpeed(self): + maxSpeed = self.getMaxSpeed() + if maxSpeed <= 0: + for thread in self.parent.py_downloading: + thread.plugin.req.speedLimitActive = False + return + threads = self.parent.py_downloading + threadCount = len(threads) + if threadCount <= 0: + return + eachSpeed = maxSpeed/threadCount + + currentOverallSpeed = 0 + restSpeed = maxSpeed - currentOverallSpeed + speeds = [] + for thread in threads: + currentOverallSpeed += thread.plugin.req.dl_speed + speeds.append((thread.plugin.req.dl_speed, thread.plugin.req.averageSpeed, thread)) + thread.plugin.req.speedLimitActive = True + + if currentOverallSpeed+50 < maxSpeed: + for thread in self.parent.py_downloading: + thread.plugin.req.speedLimitActive = False + return + + print "-----" + + slowCount = 0 + slowSpeed = 0 + if self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime < time.time(): + self.lastSlowCheck = time.time() + if self.lastSlowCheck + self.slowCheckInterval < time.time() < self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime: + for speed in speeds: + speed[2].plugin.req.isSlow = False + else: + for speed in speeds: + if speed[0] <= eachSpeed-7: + if speed[1] < eachSpeed-15: + if speed[2].plugin.req.dl_time > 0 and speed[2].plugin.req.dl_time+30 < time.time(): + speed[2].plugin.req.isSlow = True + if not speed[1]-5 < speed[2].plugin.req.maxSpeed/1024 < speed[1]+5: + speed[2].plugin.req.maxSpeed = (speed[1]+10)*1024 + if speed[2].plugin.req.isSlow: + slowCount += 1 + slowSpeed += speed[2].plugin.req.maxSpeed/1024 + + stat["slow_downloads"] = slowCount + stat["each_speed"] = eachSpeed + eachSpeed = (maxSpeed - slowSpeed) / (threadCount - slowCount) + stat["each_speed_optimized"] = eachSpeed + for speed in speeds: + if speed[2].plugin.req.isSlow: + continue + speed[2].plugin.req.maxSpeed = eachSpeed*1024 + print "max", speed[2].plugin.req.maxSpeed, "current", speed[2].plugin.req.dl_speed + diff --git a/pyLoadCore.py b/pyLoadCore.py index 232adc253..85ca5a5fe 100644 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -90,7 +90,7 @@ class Core(object): self.plugin_folder = join("module", "plugins")
- self.xmlconfig = XMLConfigParser(join(self.path,"module","config","core.xml"))
+ self.xmlconfig = XMLConfigParser(join(self.path,"module","config","core.xml"), join(self.path,"module","config","core_default.xml"))
self.config = self.xmlconfig.getConfig()
self.do_kill = False
@@ -116,7 +116,9 @@ class Core(object): if self.config['ssl']['activated']:
self.check_install("OpenSSL", "OpenSSL for secure connection", True)
self.check_file(self.config['ssl']['cert'], _("ssl certificate"), False, True)
- self.check_file(self.config['ssl']['key'], _("ssl key"), False, True)
+ self.check_file(self.config['ssl']['key'], _("ssl key"), False, True) + + self.downloadSpeedLimit = int(self.xmlconfig.get("general", "download_speed_limit", 0))
if self.config['general']['debug_mode']:
self.init_logger(logging.DEBUG) # logging level
@@ -307,7 +309,10 @@ class Core(object): if start < now and end > now: return True
elif start > end and (now > start or now < end): return True
elif start < now and end < now and start > end: return True
- else: return False
+ else: return False + + def getMaxSpeed(self): + return self.downloadSpeedLimit
####################################
########## XMLRPC Methods ##########
|