improvement for hook plugins, new internal plugin type

9 files changed, 131 insertions, 54 deletions

diff --git a/module/Api.py b/module/Api.py
index aad61cd5f..9aa6f86bb 100644
--- a/module/Api.py
+++ b/module/Api.py
@@ -286,15 +286,14 @@ class Api(Iface):
         :return: list of `DownloadStatus`
         """
         data = []
-        for pyfile in [x.active for x in self.core.threadManager.threads + self.core.threadManager.localThreads if
-                       x.active]:
-            if not isinstance(pyfile, PyFile) or not pyfile.hasPlugin():
+        for pyfile in self.core.threadManager.getActiveFiles():
+            if not isinstance(pyfile, PyFile):
                 continue
 
             data.append(DownloadInfo(
                 pyfile.id, pyfile.name, pyfile.getSpeed(), pyfile.getETA(), pyfile.formatETA(),
                 pyfile.getBytesLeft(), pyfile.getSize(), pyfile.formatSize(), pyfile.getPercent(),
-                pyfile.status, pyfile.m.statusMsg[pyfile.status], pyfile.formatWait(),
+                pyfile.status, pyfile.getStatusName(), pyfile.formatWait(),
                 pyfile.waitUntil, pyfile.packageid, pyfile.package().name, pyfile.pluginname))
 
         return data
diff --git a/module/ConfigParser.py b/module/ConfigParser.py
index e0d6af7ee..bcf5bcd2a 100644
--- a/module/ConfigParser.py
+++ b/module/ConfigParser.py
@@ -9,8 +9,8 @@ from shutil import copy
 from traceback import print_exc
 from utils import chmod
 
-IGNORE = ("FreakshareNet", "SpeedManager")
-#ignore this plugin configs
+IGNORE = ("FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx")
+# ignore these plugin configs, mainly because plugins were wiped out
 
 CONF_VERSION = 1
 
@@ -332,6 +332,7 @@ class ConfigParser:
         for item in config:
             if item[0] in conf:
                 conf[item[0]]["type"] = item[1]
+                conf[item[0]]["desc"] = item[2]
             else:
                 conf[item[0]] = {
                     "desc": item[2],
@@ -345,6 +346,12 @@ class ConfigParser:
             if item not in values:
                 del conf[item]
 
+    def deleteConfig(self, name):
+        """Removes a plugin config"""
+        if name in self.plugin:
+            del self.plugin[name]
+
+
     def deleteOldPlugins(self):
         """ remove old plugins from config """
 
diff --git a/module/HookManager.py b/module/HookManager.py
index be5f548d7..88ce62da6 100644
--- a/module/HookManager.py
+++ b/module/HookManager.py
@@ -22,7 +22,6 @@ import __builtin__
 import traceback
 from thread import start_new_thread
 from threading import RLock
-from time import time
 
 from types import MethodType
 
@@ -270,8 +269,8 @@ class HookManager:
     def unrarFinished(self, folder, fname):
         self.dispatchEvent("unrarFinished", folder, fname)
 
-    def startThread(self, function, pyfile):
-        t = HookThread(self.core.threadManager, function, pyfile)
+    def startThread(self, function, *args, **kwargs):
+        t = HookThread(self.core.threadManager, function, args, kwargs)
 
     def activePlugins(self):
         """ returns all active plugins """
diff --git a/module/PluginThread.py b/module/PluginThread.py
index 8d1516bde..d29d609c7 100644
--- a/module/PluginThread.py
+++ b/module/PluginThread.py
@@ -26,6 +26,7 @@ from time import sleep, time, strftime, gmtime
 from traceback import print_exc, format_exc
 from pprint import pformat
 from sys import exc_info, exc_clear
+from copy import copy
 from types import MethodType
 
 from pycurl import error
@@ -185,6 +186,10 @@ class DownloadThread(PluginThread):
                 self.m.core.hookManager.downloadStarts(pyfile)
                 pyfile.plugin.preprocessing(self)
 
+                self.m.log.info(_("Download finished: %s") % pyfile.name)
+                self.m.core.hookManager.downloadFinished(pyfile)
+                self.m.core.files.checkPackageFinished(pyfile)
+
             except NotImplementedError:
                 self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname)
                 pyfile.setStatus("failed")
@@ -312,13 +317,9 @@ class DownloadThread(PluginThread):
                 pyfile.checkIfProcessed()
                 exc_clear()
 
-            self.m.log.info(_("Download finished: %s") % pyfile.name)
+            
             #pyfile.plugin.req.clean()
 
-            self.m.core.hookManager.downloadFinished(pyfile)
-
-            self.m.core.files.checkPackageFinished(pyfile)
-
             self.active = False
             pyfile.finishIfDone()
             self.m.core.files.save()
@@ -337,7 +338,6 @@ class DownloadThread(PluginThread):
 class DecrypterThread(PluginThread):
     """thread for decrypting"""
 
-    #----------------------------------------------------------------------
     def __init__(self, manager, pyfile):
         """constructor"""
         PluginThread.__init__(self, manager)
@@ -349,7 +349,9 @@ class DecrypterThread(PluginThread):
 
         self.start()
 
-    #----------------------------------------------------------------------
+    def getActiveFiles(self):
+        return [self.active]
+
     def run(self):
         """run method"""
 
@@ -422,26 +424,45 @@ class HookThread(PluginThread):
     """thread for hooks"""
 
     #----------------------------------------------------------------------
-    def __init__(self, m, function, pyfile):
+    def __init__(self, m, function, args, kwargs):
         """Constructor"""
         PluginThread.__init__(self, m)
 
         self.f = function
-        self.active = pyfile
+        self.args = args
+        self.kwargs = kwargs
 
-        m.localThreads.append(self)
+        self.active = []
 
-        if isinstance(pyfile, PyFile):
-            pyfile.setStatus("processing")
+        m.localThreads.append(self)
 
         self.start()
 
+    def getActiveFiles(self):
+        return self.active
+
+    def addActive(self, pyfile):
+        """ Adds a pyfile to active list and thus will be displayed on overview"""
+        self.active.append(pyfile)
+
+    def finishFile(self, pyfile):
+        if pyfile in self.active:
+            self.active.remove(pyfile)
+
+        pyfile.finishIfDone()
+
     def run(self):
-        self.f(self.active)
+        try:
+            try:
+                self.f(*self.args, thread=self, **self.kwargs)
+            except TypeError:
+                self.f(*self.args, **self.kwargs)
+        finally:
+            local = copy(self.active)
+            for x in local:
+                self.finishFile(x)
 
-        self.m.localThreads.remove(self)
-        if isinstance(self.active, PyFile):
-            self.active.finishIfDone()
+            self.m.localThreads.remove(self)
 
 
 class InfoThread(PluginThread):
diff --git a/module/PyFile.py b/module/PyFile.py
index de8ed1145..3dede9360 100644
--- a/module/PyFile.py
+++ b/module/PyFile.py
@@ -51,7 +51,7 @@ class PyFile(object):
     Represents a file object at runtime
     """
     __slots__ = ("m", "id", "url", "name", "size", "_size", "status", "pluginname", "packageid",
-                 "error", "order", "lock", "plugin", "waitUntil", "active", "abort",
+                 "error", "order", "lock", "plugin", "waitUntil", "active", "abort", "statusname",
                  "reconnected", "progress", "maxprogress", "pluginmodule", "pluginclass")
 
     def __init__(self, manager, id, url, name, size, status, error, pluginname, package, order):
@@ -79,6 +79,8 @@ class PyFile(object):
         self.active = False #obsolete?
         self.abort = False
         self.reconnected = False
+
+        self.statusname = None
         
         self.progress = 0
         self.maxprogress = 100
@@ -115,6 +117,16 @@ class PyFile(object):
     def setStatus(self, status):
         self.status = statusMap[status]
         self.sync() #@TODO needed aslong no better job approving exists
+
+    def setCustomStatus(self, msg, status="processing"):
+        self.statusname = msg
+        self.setStatus(status)
+
+    def getStatusName(self):
+        if self.status not in (13, 14) or not self.statusname:
+            return self.m.statusMsg[self.status]
+        else:
+            return self.statusname
     
     def hasStatus(self, status):
         return statusMap[status] == self.status
@@ -163,7 +175,7 @@ class PyFile(object):
                 'size': self.getSize(),
                 'format_size': self.formatSize(),
                 'status': self.status,
-                'statusmsg': self.m.statusMsg[self.status],
+                'statusmsg': self.getStatusName(),
                 'package': self.packageid,
                 'error': self.error,
                 'order': self.order
diff --git a/module/ThreadManager.py b/module/ThreadManager.py
index 7b64a2f9a..8937f4a29 100644
--- a/module/ThreadManager.py
+++ b/module/ThreadManager.py
@@ -116,17 +116,19 @@ class ThreadManager:
     def setInfoResults(self, rid, result):
         self.infoResults[rid].update(result)
 
-    def downloadingIds(self):
-        """get a list of the currently downloading pyfile's ids"""
-        return [x.active.id for x in self.threads if x.active and isinstance(x.active, PyFile)]
+    def getActiveFiles(self):
+        active = [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)]
 
+        for t in self.localThreads:
+            active.extend(t.getActiveFiles())
+
+        return active
 
     def processingIds(self):
         """get a id list of all pyfiles processed"""
-        return [x.active.id for x in self.threads + self.localThreads if x.active and isinstance(x.active, PyFile)]
+        return [x.id for x in self.getActiveFiles()]
 
 
-    
     def work(self):
         """run all task which have to be done (this is for repetivive call by core)"""
         try:
@@ -163,7 +165,7 @@ class ThreadManager:
 
         active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active]
 
-        if not (active.count(True) > 0 and len(active) == active.count(True)):
+        if not (0 < active.count(True) == len(active)):
             return False
 
         if not exists(self.core.config['reconnect']['method']):
@@ -241,7 +243,7 @@ class ThreadManager:
 
     def cleanPycurl(self):
         """ make a global curl cleanup (currently ununused) """
-        if self.downloadingIds() or self.processingIds():
+        if self.processingIds():
             return False
         pycurl.global_cleanup()
         pycurl.global_init(pycurl.GLOBAL_DEFAULT)
diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py
index fdcaccfe3..51ebd1aec 100644
--- a/module/plugins/Hook.py
+++ b/module/plugins/Hook.py
@@ -18,7 +18,6 @@
     @interface-version: 0.2
 """
 
-from thread import start_new_thread
 from traceback import print_exc
 
 from Plugin import Base
@@ -32,7 +31,7 @@ class Expose(object):
 
 def threaded(f):
     def run(*args,**kwargs):
-        return start_new_thread(f, args, kwargs)
+        hookManager.startThread(f, *args, **kwargs)
     return run
 
 class Hook(Base):
diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py
index 0bf15f1a8..4ddb1b1b9 100644
--- a/module/plugins/PluginManager.py
+++ b/module/plugins/PluginManager.py
@@ -53,6 +53,11 @@ class PluginManager():
         self.createIndex()
 
 
+    rePattern = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)')
+    reVersion = re.compile(r'__version__.*=.*("|\')([0-9.]+)')
+    reConfig = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE)
+    reDesc = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)')
+
     def createIndex(self):
         """create information for all plugins available"""
 
@@ -69,17 +74,18 @@ class PluginManager():
         self.reConfig = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE)
         self.reDesc = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)')
 
-        self.crypterPlugins = self.parse(_("Crypter"), "crypter", pattern=True)
-        self.containerPlugins = self.parse(_("Container"), "container", pattern=True)
-        self.hosterPlugins = self.parse(_("Hoster"), "hoster", pattern=True)
+        self.crypterPlugins = self.parse("crypter", pattern=True)
+        self.containerPlugins = self.parse("container", pattern=True)
+        self.hosterPlugins = self.parse("hoster", pattern=True)
 
-        self.captchaPlugins = self.parse(_("Captcha"), "captcha")
-        self.accountPlugins = self.parse(_("Account"), "accounts", create=True)
-        self.hookPlugins = self.parse(_("Hook"), "hooks")
+        self.captchaPlugins = self.parse("captcha")
+        self.accountPlugins = self.parse("accounts")
+        self.hookPlugins = self.parse("hooks")
+        self.internalPlugins = self.parse("internal")
 
         self.log.debug("created index of plugins")
 
-    def parse(self, typ, folder, create=False, pattern=False, home={}):
+    def parse(self, folder, pattern=False, home={}):
         """
         returns dict with information 
         home contains parsed plugins from module.
@@ -108,11 +114,11 @@ class PluginManager():
                 content = data.read()
                 data.close()
 
-                if f.endswith("_25.pyc") and not version_info[0:2] == (2, 5):
+                if f.endswith("_25.pyc") and version_info[0:2] != (2, 5):
                     continue
-                elif f.endswith("_26.pyc") and not version_info[0:2] == (2, 6):
+                elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6):
                     continue
-                elif f.endswith("_27.pyc") and not version_info[0:2] == (2, 7):
+                elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7):
                     continue
 
                 name = f[:-3]
@@ -124,6 +130,7 @@ class PluginManager():
                 else:
                     version = 0
 
+                # home contains plugins from pyload root
                 if home and name in home:
                     if home[name]["v"] >= version:
                         continue
@@ -158,15 +165,17 @@ class PluginManager():
                     except:
                         self.log.error(_("%s has a invalid pattern.") % name)
 
-                config = self.reConfig.findall(content)
 
+                # internals have no config
+                if folder == "internal":
+                    self.core.config.deleteConfig(name)
+                    continue
+
+                config = self.reConfig.findall(content)
                 if config:
                     config = literal_eval(config[0].strip().replace("\n", "").replace("\r", ""))
                     desc = self.reDesc.findall(content)
-                    if desc:
-                        desc = desc[0][1]
-                    else:
-                        desc = ""
+                    desc = desc[0][1] if desc else ""
 
                     if type(config[0]) == tuple:
                         config = [list(x) for x in config]
@@ -188,8 +197,18 @@ class PluginManager():
                     except :
                         self.log.error("Invalid config in %s: %s" % (name, config))
 
+                elif folder == "hooks": #force config creation
+                    desc = self.reDesc.findall(content)
+                    desc = desc[0][1] if desc else ""
+                    config = (["activated", "bool", "Activated", False],)
+
+                    try:
+                        self.core.config.addPluginConfig(name, config, desc)
+                    except :
+                        self.log.error("Invalid config in %s: %s" % (name, config))
+
         if not home:
-            temp = self.parse(typ, folder, create, pattern, plugins)
+            temp = self.parse(folder, pattern, plugins)
             plugins.update(temp)
 
         return plugins
@@ -327,6 +346,25 @@ class PluginManager():
 
         return pluginClass
 
+    def getInternalModul(self, name):
+
+        if name not in self.internalPlugins: return None
+
+        value = self.internalPlugins[name]
+
+        if "module" in value:
+            return value["module"]
+
+        try:
+            module = __import__(value["path"], globals(), locals(), [value["name"]], -1)
+        except Exception, e:
+            self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)})
+            if self.core.debug:
+                print_exc()
+            return None
+
+        value["module"] = module
+        return module
     
     def reloadPlugins(self):
         """ reloads and reindexes plugins """
diff --git a/module/plugins/hooks/UnRar.py b/module/plugins/internal/UnRar.py
index 7ace49a76..8ff99b5e8 100644
--- a/module/plugins/hooks/UnRar.py
+++ b/module/plugins/internal/UnRar.py
@@ -37,8 +37,8 @@ import re
 
 class UnRar(Hook):
     __name__ = "UnRar"
-    __version__ = "0.11"
-    __description__ = """unrar"""
+    __version__ = "0.1"
+    __description__ = """Unrar plugin for archive extractor"""
     __config__ = [("activated", "bool", "Activated", False),
         ("fullpath", "bool", "extract full path", True),
         ("overwrite", "bool", "overwrite files", True),