updated plugin api and plugin manager

4 files changed, 103 insertions, 95 deletions

diff --git a/module/plugins/Base.py b/module/plugins/Base.py
index 98573ea63..b166ebae8 100644
--- a/module/plugins/Base.py
+++ b/module/plugins/Base.py
@@ -17,11 +17,16 @@
     @author: RaNaN
 """
 
+# TODO: config format definition
+#       more attributes if needed
+#       get rid of catpcha & container plugins ?! (move to crypter & internals)
+#       adapt old plugins as needed
+
 class Base(object):
     """
     The Base plugin class with all shared methods and every possible attribute for plugin definition.
     """
-    __version__ = "0.4"
+    __version__ = "0.1"
     #: Regexp pattern which will be matched for download plugins
     __pattern__ = r""
     #: Flat config definition
diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py
index 4bf41484a..744e09ef0 100644
--- a/module/plugins/PluginManager.py
+++ b/module/plugins/PluginManager.py
@@ -23,28 +23,27 @@ import sys
 from os import listdir, makedirs
 from os.path import isfile, join, exists, abspath
 from sys import version_info
-from itertools import chain
+from time import time
 from traceback import print_exc
 
 from module.lib.SafeEval import const_eval as literal_eval
 from module.ConfigParser import IGNORE
+from module.plugins.Base import Base
 
 from namedtuple import namedtuple
 
-PluginTuple = namedtuple("PluginTuple", "version pattern desc long_desc deps user name module")
+PluginTuple = namedtuple("PluginTuple", "version re desc long_desc deps user path")
 
 class PluginManager:
     ROOT = "module.plugins."
     USERROOT = "userplugins."
     TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal")
 
-    PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)')
-    VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)')
-    CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE)
-    DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)')
+    SINGLE = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?<!")"(?!")|\'|\().*(?:(?<!")"(?!")|\'|\)))',
+        re.I)
 
-
-    ATTR = re.compile(r'__([a-z0-9_])__\s*=\s*({|\[|\(|"|\')([^__]+)', re.M | re.I)
+    # note the nongreedy character, that means we can not embed list and dicts
+    MULTI = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I)
 
     def __init__(self, core):
         self.core = core
@@ -53,12 +52,18 @@ class PluginManager:
         self.log = core.log
 
         self.plugins = {}
+        self.modules = {} # cached modules
+        self.names = {} # overwritten names
+        self.history = []  # match history to speedup parsing (type, name)
         self.createIndex()
 
         #register for import hook
         sys.meta_path.append(self)
 
 
+    def logDebug(self, type, plugin, msg):
+        self.log.debug("Plugin %s | %s: %s" % (type, plugin, msg))
+
     def createIndex(self):
         """create information for all plugins available"""
 
@@ -70,27 +75,14 @@ class PluginManager:
             f = open(join("userplugins", "__init__.py"), "wb")
             f.close()
 
-        self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True)
-        self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True)
-        self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True)
-
-        self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha")
-        self.plugins["accounts"] = self.accountPlugins = self.parse("accounts")
-        self.plugins["hooks"] = self.hookPlugins = self.parse("hooks")
-        self.plugins["internal"] = self.internalPlugins = self.parse("internal")
+        a = time()
+        for type in self.TYPES:
+            self.plugins[type] = self.parse(type)
 
-        self.log.debug("created index of plugins")
+        self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000 )
 
-    def parse(self, folder, pattern=False, home={}):
-        """
-        returns dict with information 
-        home contains parsed plugins from module.
-        
-        {
-        name : {path, version, config, (pattern, re), (plugin, class)}
-        }
-        
-        """
+    def parse(self, folder, home={}):
+        """  Creates a dict with PluginTuples according to parsed information """
         plugins = {}
         if home:
             pfolder = join("userplugins", folder)
@@ -120,55 +112,56 @@ class PluginManager:
                 name = f[:-3]
                 if name[-1] == ".": name = name[:-4]
 
-                version = self.VERSION.findall(content)
-                if version:
-                    version = float(version[0][1])
+                attrs = {}
+                for m in self.SINGLE.findall(content) + self.MULTI.findall(content):
+                    #replace gettext function and eval result
+                    attrs[m[0]] = literal_eval(m[-1].replace("_(", "("))
+                    if not hasattr(Base, "__%s__" % m[0]):
+                        if m[0] != "type": #TODO remove type from all plugins, its not needed
+                            self.logDebug(folder, name, "Unknown attribute '%s'" % m[0])
+
+                version = 0
+
+                if "version" in attrs:
+                    try:
+                        version = float(attrs["version"])
+                    except ValueError:
+                        self.logDebug(folder, name, "Invalid version %s" % attrs["version"])
+                        version = 9 #TODO remove when plugins are fixed, causing update loops
                 else:
-                    version = 0
+                    self.logDebug(folder, name, "No version attribute")
 
                 # home contains plugins from pyload root
                 if home and name in home:
-                    if home[name]["v"] >= version:
+                    if home[name].version >= version:
                         continue
 
                 if name in IGNORE or (folder, name) in IGNORE:
-                     continue
-
-                plugins[name] = {}
-                plugins[name]["v"] = version
-
-                module = f.replace(".pyc", "").replace(".py", "")
-
-                # the plugin is loaded from user directory
-                plugins[name]["user"] = True if home else False
-                plugins[name]["name"] = module
-
-                if pattern:
-                    pattern = self.PATTERN.findall(content)
-
-                    if pattern:
-                        pattern = pattern[0][1]
-                    else:
-                        pattern = "^unmachtable$"
-
-                    plugins[name]["pattern"] = pattern
+                    continue
 
+                if "pattern" in attrs and attrs["pattern"]:
                     try:
-                        plugins[name]["re"] = re.compile(pattern)
+                        plugin_re = re.compile(attrs["pattern"])
                     except:
-                        self.log.error(_("%s has a invalid pattern.") % name)
+                        self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"])
+                        plugin_re = None
+                else: plugin_re = None
+
+
+                # create plugin tuple
+                plugin = PluginTuple(version, plugin_re, None, None, None, True if home else False,
+                    f.replace(".pyc", "").replace(".py", ""))
 
+                plugins[name] = plugin
 
                 # internals have no config
                 if folder == "internal":
                     self.core.config.deleteConfig(name)
                     continue
 
-                config = self.CONFIG.findall(content)
-                if config:
-                    config = literal_eval(config[0].strip().replace("\n", "").replace("\r", ""))
-                    desc = self.DESC.findall(content)
-                    desc = desc[0][1] if desc else ""
+                if "config" in attrs and attrs["config"]:
+                    config = attrs["config"]
+                    desc = attrs["description"] if "description" in attrs else ""
 
                     if type(config[0]) == tuple:
                         config = [list(x) for x in config]
@@ -189,8 +182,7 @@ class PluginManager:
                         self.log.error("Invalid config in %s: %s" % (name, config))
 
                 elif folder == "hooks": #force config creation
-                    desc = self.DESC.findall(content)
-                    desc = desc[0][1] if desc else ""
+                    desc = attrs["description"] if "description" in attrs else ""
                     config = (["activated", "bool", "Activated", False],)
 
                     try:
@@ -199,39 +191,54 @@ class PluginManager:
                         self.log.error("Invalid config in %s: %s" % (name, config))
 
         if not home:
-            temp = self.parse(folder, pattern, plugins)
+            temp = self.parse(folder, plugins)
             plugins.update(temp)
 
         return plugins
 
+    def parsePlugin(self):
+        pass
+
 
     def parseUrls(self, urls):
         """parse plugins for given list of urls"""
 
-        last = None
         res = [] # tupels of (url, plugin)
 
         for url in urls:
             if type(url) not in (str, unicode, buffer): continue
             found = False
 
-            if last and last[1]["re"].match(url):
-                res.append((url, last[0]))
+            for ptype, name in self.history:
+                if self.plugins[ptype][name].re.match(url):
+                    res.append((url, name))
+                    found = (ptype, name)
+
+            if found:
+                # found match, update history
+                self.history.remove(found)
+                self.history.insert(0, found)
                 continue
 
-            for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(),
-                self.containerPlugins.iteritems()):
-                if value["re"].match(url):
-                    res.append((url, name))
-                    last = (name, value)
-                    found = True
-                    break
+            for ptype in ("crypter", "hoster", "container"):
+                for name, plugin in self.plugins[ptype].iteritems():
+                    if plugin.re.match(url):
+                        res.append((url, name))
+                        self.history.insert(0, (ptype, name))
+                        del self.history[10:] # cut down to size of 10
+                        found = True
+                        break
 
             if not found:
                 res.append((url, "BasePlugin"))
 
         return res
 
+    def getPlugins(self, type):
+        # TODO clean this workaround
+        if type not in self.plugins: type+="s" # append s, so updater can find the plugins
+        return self.plugins[type]
+
     def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")):
         for ptype in pluginlist:
             if name in self.plugins[ptype]:
@@ -244,7 +251,7 @@ class PluginManager:
 
         if not plugin:
             self.log.warning("Plugin %s not found." % name)
-            plugin = self.hosterPlugins["BasePlugin"]
+            plugin = self.plugins["hoster"]["BasePlugin"]
 
         if "new_module" in plugin and not original:
             return plugin["new_module"]
@@ -268,11 +275,11 @@ class PluginManager:
         """
         plugins = self.plugins[type]
         if name in plugins:
-            if "module" in plugins[name]: return plugins[name]["module"]
+            if (type, name) in self.modules: return self.modules[(type, name)]
             try:
-                module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]["name"]), globals(), locals(),
-                    plugins[name]["name"])
-                plugins[name]["module"] = module  #cache import, maybe unneeded
+                module = __import__(self.ROOT + "%s.%s" % (type, plugins[name].path), globals(), locals(),
+                    plugins[name].path)
+                self.modules[(type, name)] = module # cache import, maybne unneeded
                 return module
             except Exception, e:
                 self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)})
@@ -286,7 +293,7 @@ class PluginManager:
 
     def getAccountPlugins(self):
         """return list of account plugin names"""
-        return self.accountPlugins.keys()
+        return self.plugins["accounts"].keys()
 
     def find_module(self, fullname, path=None):
         #redirecting imports if necesarry
@@ -300,10 +307,10 @@ class PluginManager:
 
             if type in self.plugins and name in self.plugins[type]:
                 #userplugin is a newer version
-                if not user and self.plugins[type][name]["user"]:
+                if not user and self.plugins[type][name].user:
                     return self
-                #imported from userdir, but pyloads is newer
-                if user and not self.plugins[type][name]["user"]:
+                    #imported from userdir, but pyloads is newer
+                if user and not self.plugins[type][name].user:
                     return self
 
 
@@ -335,7 +342,7 @@ class PluginManager:
         self.log.debug("Request reload of plugins: %s" % type_plugins)
 
         as_dict = {}
-        for t,n in type_plugins:
+        for t, n in type_plugins:
             if t in as_dict:
                 as_dict[t].append(n)
             else:
@@ -348,16 +355,13 @@ class PluginManager:
         for type in as_dict.iterkeys():
             for plugin in as_dict[type]:
                 if plugin in self.plugins[type]:
-                    if "module" in self.plugins[type][plugin]:
+                    if (type, plugin) in self.modules:
                         self.log.debug("Reloading %s" % plugin)
-                        reload(self.plugins[type][plugin]["module"])
+                        reload(self.modules[(type, plugin)])
 
-        #index creation
-        self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True)
-        self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True)
-        self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True)
-        self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha")
-        self.plugins["accounts"] = self.accountPlugins = self.parse("accounts")
+        # index re-creation
+        for type in ("crypter", "container", "hoster", "captcha", "accounts"):
+            self.plugins[type] = self.parse(type)
 
         if "accounts" in as_dict: #accounts needs to be reloaded
             self.core.accountManager.initPlugins()
@@ -366,7 +370,6 @@ class PluginManager:
         return True
 
 
-
 if __name__ == "__main__":
     _ = lambda x: x
     pypath = "/home/christian/Projekte/pyload-0.4/module/plugins"
diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py
index 920a88060..8cbc56c42 100644
--- a/module/plugins/hooks/UpdateManager.py
+++ b/module/plugins/hooks/UpdateManager.py
@@ -129,10 +129,10 @@ class UpdateManager(Hook):
             else:
                 type = prefix
 
-            plugins = getattr(self.core.pluginManager, "%sPlugins" % type)
+            plugins = self.core.pluginManager.getPlugins(type)
 
             if name in plugins:
-                if float(plugins[name]["v"]) >= float(version):
+                if float(plugins[name].version) >= float(version):
                     continue
 
             if name in IGNORE or (type, name) in IGNORE:
diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py
index 0d927c525..f3011a488 100644
--- a/module/plugins/hoster/RapidshareCom.py
+++ b/module/plugins/hoster/RapidshareCom.py
@@ -52,7 +52,7 @@ class RapidshareCom(Hoster):
     __pattern__ = r"https?://[\w\.]*?rapidshare.com/(?:files/(?P<id>\d*?)/(?P<name>[^?]+)|#!download\|(?:\w+)\|(?P<id_new>\d+)\|(?P<name_new>[^|]+))"
     __version__ = "1.37"
     __description__ = """Rapidshare.com Download Hoster"""
-    __config__ = [["server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None"]] 
+    __config__ = [("server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None")]
     __author_name__ = ("spoob", "RaNaN", "mkaay")
     __author_mail__ = ("spoob@pyload.org", "ranan@pyload.org", "mkaay@mkaay.de")