better threadmanager

5 files changed, 86 insertions, 177 deletions

diff --git a/module/download_thread.py b/module/DownloadThread.py
index 84f00c99f..4953ffa53 100644
--- a/module/download_thread.py
+++ b/module/DownloadThread.py
@@ -18,18 +18,16 @@
     @author: mkaay
     @author: spoob
     @author: sebnapi
-    @version: v0.3.1
+    @version: v0.3.2
 """
 
-import threading
+from threading import Thread
 import traceback
 from time import sleep, time
 
 from module.network.Request import AbortDownload
 from module.PullEvents import UpdateEvent
 
-
-
 class Status(object):
     """ Saves all status information
     """
@@ -76,59 +74,49 @@ class Checksum(Exception):
 class CaptchaError(Exception):
     pass
 
-class Download_Thread(threading.Thread):
-    def __init__(self, parent):
-        threading.Thread.__init__(self)
-        self.shutdown = False
+class DownloadThread(Thread):
+    def __init__(self, parent, job):
+        Thread.__init__(self)
         self.parent = parent
         self.setDaemon(True)
-        self.loadedPyFile = None
-
-        self.start()
+        self.loadedPyFile = job
 
     def run(self):
-        while not self.shutdown:
-            self.loadedPyFile = self.parent.get_job()
-            if self.loadedPyFile:
-                try:
-                    self.download(self.loadedPyFile)
-                except AbortDownload:
-                    self.loadedPyFile.plugin.req.abort = False
-                    self.loadedPyFile.status.type = "aborted"
-                except Reconnect:
-                    pass
-                except Checksum, e:
-                    self.loadedPyFile.status.type = "failed"
-                    self.loadedPyFile.status.error = "Checksum error: %d" % e.getCode()
-                    f = open("%s.info" % e.getFile(), "w")
-                    f.write("Checksum not matched!")
-                    f.close()
-                except CaptchaError:
-                    self.loadedPyFile.status.type = "failed"
-                    self.loadedPyFile.status.error = "Can't solve captcha"
-                except Exception, e:
-                    try:
-                        if self.parent.parent.config['general']['debug_mode']:
-                            traceback.print_exc()
-                        code, msg = e
-                        if code == 7:
-                            sleep(60)
-                        self.parent.parent.logger.info(_("Hoster unvailable, wait 60 seconds"))
-                    except Exception, f:
-                        self.parent.parent.logger.debug(_("Error getting error code: %s") % f)
-                        if self.parent.parent.config['general']['debug_mode']:
-                            traceback.print_exc()
-                    self.loadedPyFile.status.type = "failed"
-                    self.loadedPyFile.status.error = str(e)
-                finally:
-                    self.parent.job_finished(self.loadedPyFile)
-                    self.parent.parent.pullManager.addEvent(UpdateEvent("file", self.loadedPyFile.id, "queue"))
-            else:
-                sleep(3)
+        try:
+            self.download(self.loadedPyFile)
+        except AbortDownload:
+            self.loadedPyFile.plugin.req.abort = False
+            self.loadedPyFile.status.type = "aborted"
+        except Reconnect:
+            pass
+        except Checksum, e:
+            self.loadedPyFile.status.type = "failed"
+            self.loadedPyFile.status.error = "Checksum error: %d" % e.getCode()
+            f = open("%s.info" % e.getFile(), "w")
+            f.write("Checksum not matched!")
+            f.close()
+        except CaptchaError:
+            self.loadedPyFile.status.type = "failed"
+            self.loadedPyFile.status.error = "Can't solve captcha"
+        except Exception, e:
+            try:
+                if self.parent.parent.config['general']['debug_mode']:
+                    traceback.print_exc()
+                code, msg = e
+                if code == 7:
+                    sleep(60)
+                self.parent.parent.logger.info(_("Hoster unvailable, wait 60 seconds"))
+            except Exception, f:
+                self.parent.parent.logger.debug(_("Error getting error code: %s") % f)
+                if self.parent.parent.config['general']['debug_mode']:
+                    traceback.print_exc()
+            self.loadedPyFile.status.type = "failed"
+            self.loadedPyFile.status.error = str(e)
+        finally:
+            self.parent.jobFinished(self.loadedPyFile)
+            self.parent.parent.pullManager.addEvent(UpdateEvent("file", self.loadedPyFile.id, "queue"))
             sleep(0.8)
-        if self.shutdown:
-            sleep(1)
-            self.parent.remove_thread(self)
+        self.parent.removeThread(self)
 
     def download(self, pyfile):
         status = pyfile.status
diff --git a/module/file_list.py b/module/FileList.py
index eff96fa9e..5282c0be6 100644
--- a/module/file_list.py
+++ b/module/FileList.py
@@ -31,8 +31,8 @@ from threading import RLock
 from time import sleep
 
 import cPickle
-from download_thread import Status
-import module.Plugin
+from module.DownloadThread import Status
+import module.plugins.Plugin
 from module.PullEvents import InsertEvent
 from module.PullEvents import RemoveEvent
 from module.PullEvents import UpdateEvent
@@ -40,7 +40,7 @@ from module.PullEvents import UpdateEvent
 class NoSuchElementException(Exception):
     pass
 
-class File_List(object):
+class FileList(object):
     def __init__(self, core):
         self.core = core
         self.lock = RLock()
@@ -431,8 +431,8 @@ class PyLoadFile():
                     pass
             pluginClass = getattr(self.modul, pluginName)
         else:
-            self.modul = module.Plugin
-            pluginClass = module.Plugin.Plugin
+            self.modul = module.plugins.Plugin
+            pluginClass = module.plugins.Plugin.Plugin
         self.plugin = pluginClass(self)
         self.status = Status(self)
         self.status.filename = self.url
diff --git a/module/thread_list.py b/module/ThreadManager.py
index 8f02536de..ab0f99cfa 100644
--- a/module/thread_list.py
+++ b/module/ThreadManager.py
@@ -19,7 +19,7 @@
     @author: spoob
     @author: sebnapi
     @author: RaNaN
-    @version: v0.3.1
+    @version: v0.3.2
 """
 
 from __future__ import with_statement
@@ -27,53 +27,55 @@ from os.path import exists
 import re
 import subprocess
 from threading import RLock, Thread
-import time
-import urllib2
-from download_thread import Download_Thread
+from time import sleep
+from module.network.Request import getURL
+from module.DownloadThread import DownloadThread
+from module.SpeedManager import SpeedManager
 
-class Thread_List(object):
+class ThreadManager(Thread):
     def __init__(self, parent):
+        Thread.__init__(self)
         self.parent = parent
         self.list = parent.file_list #file list
         self.threads = []
-        self.max_threads = int(self.parent.config['general']['max_downloads'])
         self.lock = RLock()
         self.py_downloading = [] # files downloading
         self.occ_plugins = [] #occupied plugins
         self.pause = True
         self.reconnecting = False
 
-        self.select_thread()
-        if self.parent.config['general']['download_speed_limit'] != 0:
-            self.speedManager = self.SpeedManager(self)
-
-    def create_thread(self):
+        self.speedManager = SpeedManager(self)
+    
+    def run(self):
+        while True:
+            if len(self.threads) < int(self.parent.config['general']['max_downloads']) and not self.pause:
+                job = self.getJob()
+                if job:
+                    thread = self.createThread(job)
+                    thread.start()
+            sleep(1)
+
+    def createThread(self, job):
         """ creates thread for Py_Load_File and append thread to self.threads
         """
-        thread = Download_Thread(self)
+        thread = DownloadThread(self, job)
         self.threads.append(thread)
-        return True
+        return thread
 
-    def remove_thread(self, thread):
+    def removeThread(self, thread):
         self.threads.remove(thread)
 
-    def select_thread(self):
-        """ create all threads
-        """
-        while len(self.threads) < self.max_threads:
-            self.create_thread()
-
-    def get_job(self):
+    def getJob(self):
         """return job if suitable, otherwise send thread idle"""
 
         if not self.parent.server_methods.is_time_download() or self.pause or self.reconnecting or self.list.queueEmpty(): #conditions when threads dont download
             return None
         
         if self.parent.freeSpace() < self.parent.config["general"]["min_free_space"]:
-            self.parent.logger.debug("min free space exceeded")
+            self.parent.logger.debug(_("minimal free space exceeded"))
             return None
 
-        self.init_reconnect()
+        self.initReconnect()
 
         self.lock.acquire()
 
@@ -95,7 +97,7 @@ class Thread_List(object):
         self.lock.release()
         return pyfile
 
-    def job_finished(self, pyfile):
+    def jobFinished(self, pyfile):
         """manage completing download"""
         self.lock.acquire()
 
@@ -164,7 +166,7 @@ class Thread_List(object):
         self.lock.release()
         return True
 
-    def init_reconnect(self):
+    def initReconnect(self):
         """initialise a reonnect"""
         if not self.parent.config['reconnect']['activated'] or self.reconnecting or not self.parent.server_methods.is_time_reconnect():
             return False
@@ -176,7 +178,7 @@ class Thread_List(object):
 
         self.lock.acquire()
 
-        if self.check_reconnect():
+        if self.checkReconnect():
             self.reconnecting = True
             self.reconnect()
             time.sleep(1.1)
@@ -188,7 +190,7 @@ class Thread_List(object):
         self.lock.release()
         return False
 
-    def check_reconnect(self):
+    def checkReconnect(self):
         """checks if all files want reconnect"""
 
         if not self.py_downloading:
@@ -206,7 +208,7 @@ class Thread_List(object):
 
     def reconnect(self):
         self.parent.logger.info(_("Starting reconnect"))
-        ip = re.match(".*Current IP Address: (.*)</body>.*", urllib2.urlopen("http://checkip.dyndns.org/").read()).group(1)
+        ip = re.match(".*Current IP Address: (.*)</body>.*", getURL("http://checkip.dyndns.org/")).group(1)
         self.parent.hookManager.beforeReconnecting(ip)
         reconn = subprocess.Popen(self.parent.config['reconnect']['method'])#, stdout=subprocess.PIPE)
         reconn.wait()
@@ -214,7 +216,7 @@ class Thread_List(object):
         ip = ""
         while ip == "":
             try:
-                ip = re.match(".*Current IP Address: (.*)</body>.*", urllib2.urlopen("http://checkip.dyndns.org/").read()).group(1) #versuchen neue ip aus zu lesen
+                ip = re.match(".*Current IP Address: (.*)</body>.*", getURL("http://checkip.dyndns.org/")).group(1) #versuchen neue ip aus zu lesen
             except:
                 ip = ""
             time.sleep(1)
@@ -225,86 +227,3 @@ class Thread_List(object):
         self.pause = True
         for pyfile in self.py_downloading:
             pyfile.plugin.req.abort = True
-    
-    class SpeedManager(Thread):
-        def __init__(self, parent):
-            Thread.__init__(self)
-            self.parent = parent
-            self.running = True
-            self.lastSlowCheck = 0.0
-            
-            stat = {}
-            stat["slow_downloads"] = None
-            stat["each_speed"] = None
-            stat["each_speed_optimized"] = None
-            self.stat = stat
-            
-            self.slowCheckInterval = 60
-            self.slowCheckTestTime = 25
-            
-            self.logger = self.parent.parent.logger
-            self.start()
-        
-        def run(self):
-            while self.running:
-                time.sleep(1)
-                self.manageSpeed()
-        
-        def getMaxSpeed(self):
-            return self.parent.parent.getMaxSpeed()
-        
-        def manageSpeed(self):
-            maxSpeed = self.getMaxSpeed()
-            if maxSpeed <= 0:
-                for thread in self.parent.py_downloading:
-                    thread.plugin.req.speedLimitActive = False
-                return
-            threads = self.parent.py_downloading
-            threadCount = len(threads)
-            if threadCount <= 0:
-                return
-            eachSpeed = maxSpeed/threadCount
-            
-            currentOverallSpeed = 0
-            restSpeed = maxSpeed - currentOverallSpeed
-            speeds = []
-            for thread in threads:
-                currentOverallSpeed += thread.plugin.req.dl_speed
-                speeds.append((thread.plugin.req.dl_speed, thread.plugin.req.averageSpeed, thread))
-                thread.plugin.req.speedLimitActive = True
-            
-            if currentOverallSpeed+50 < maxSpeed:
-                for thread in self.parent.py_downloading:
-                    thread.plugin.req.speedLimitActive = False
-                return
-            
-            slowCount = 0
-            slowSpeed = 0
-            if self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime < time.time():
-                self.lastSlowCheck = time.time()
-            if self.lastSlowCheck + self.slowCheckInterval < time.time() < self.lastSlowCheck + self.slowCheckInterval + self.slowCheckTestTime:
-                for speed in speeds:
-                    speed[2].plugin.req.isSlow = False
-            else:
-                for speed in speeds:
-                    if speed[0] <= eachSpeed-7:
-                        if speed[1] < eachSpeed-15:
-                            if speed[2].plugin.req.dl_time > 0 and speed[2].plugin.req.dl_time+30 < time.time():
-                                speed[2].plugin.req.isSlow = True
-                                if not speed[1]-5 < speed[2].plugin.req.maxSpeed/1024 < speed[1]+5:
-                                    speed[2].plugin.req.maxSpeed = (speed[1]+10)*1024
-                    if speed[2].plugin.req.isSlow:
-                        slowCount += 1
-                        slowSpeed += speed[2].plugin.req.maxSpeed/1024
-            stat = {}
-            stat["slow_downloads"] = slowCount
-            stat["each_speed"] = eachSpeed
-            eachSpeed = (maxSpeed - slowSpeed) / (threadCount - slowCount)
-            stat["each_speed_optimized"] = eachSpeed
-            self.stat = stat
-            
-            for speed in speeds:
-                if speed[2].plugin.req.isSlow:
-                    continue
-                speed[2].plugin.req.maxSpeed = eachSpeed*1024
-                print "max", speed[2].plugin.req.maxSpeed, "current", speed[2].plugin.req.dl_speed
diff --git a/module/Plugin.py b/module/plugins/Plugin.py
index 1428e6235..c4ac8ee12 100644
--- a/module/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -29,7 +29,7 @@ from os.path import exists
 from module.network.Request import Request
 from os import makedirs
 
-from module.download_thread import CaptchaError
+from module.DownloadThread import CaptchaError
 
 class Plugin():
 
diff --git a/pyLoadCore.py b/pyLoadCore.py
index 8d8793b33..8fcfa6c82 100755
--- a/pyLoadCore.py
+++ b/pyLoadCore.py
@@ -61,16 +61,17 @@ import time
 from time import sleep
 from xmlrpclib import Binary
 
-from module.CaptchaManager import CaptchaManager
-from module.HookManager import HookManager
-from module.PullEvents import PullManager
 from module.XMLConfigParser import XMLConfigParser
-from module.file_list import File_List
 from module.network.Request import getURL
 import module.remote.SecureXMLRPCServer as Server
-from module.thread_list import Thread_List
 from module.web.ServerThread import WebServer
 
+from module.ThreadManager import ThreadManager
+from module.CaptchaManager import CaptchaManager
+from module.HookManager import HookManager
+from module.PullEvents import PullManager
+from module.FileList import FileList
+
 class Core(object):
     """ pyLoad Core """
 
@@ -250,9 +251,9 @@ class Core(object):
         self.lastGuiConnected = 0
         
         self.server_methods = ServerMethods(self)
-        self.file_list = File_List(self)
+        self.file_list = FileList(self)
         self.pullManager = PullManager(self)
-        self.thread_list = Thread_List(self)
+        self.thread_list = ThreadManager(self)
         self.captchaManager = CaptchaManager(self)
         
         self.last_update_check = 0
@@ -295,6 +296,7 @@ class Core(object):
             self.logger.info(_("Free space: %sMB") % freeSpace)
 
         self.thread_list.pause = False
+        self.thread_list.start()
         
         self.hookManager.coreReady()