diff options
Diffstat (limited to 'module/plugins')
| -rw-r--r-- | module/plugins/Account.py | 335 | ||||
| -rw-r--r-- | module/plugins/AccountManager.py | 210 | ||||
| -rw-r--r-- | module/plugins/Base.py | 131 | ||||
| -rw-r--r-- | module/plugins/Hook.py | 2 | ||||
| -rw-r--r-- | module/plugins/Hoster.py | 22 | ||||
| -rw-r--r-- | module/plugins/Plugin.py | 112 | ||||
| -rw-r--r-- | module/plugins/PluginManager.py | 317 | ||||
| -rw-r--r-- | module/plugins/hooks/UpdateManager.py | 6 | ||||
| -rw-r--r-- | module/plugins/hoster/RapidshareCom.py | 2 | ||||
| -rw-r--r-- | module/plugins/internal/MultiHoster.py | 21 |
10 files changed, 571 insertions, 587 deletions
diff --git a/module/plugins/Account.py b/module/plugins/Account.py index c147404e0..9da8d0357 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -1,63 +1,72 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from random import choice from time import time from traceback import print_exc from threading import RLock from Plugin import Base from module.utils import compare_time, parseFileSize, lock +from module.config.converter import from_string +from module.Api import AccountInfo +from module.network.CookieJar import CookieJar class WrongPassword(Exception): pass - -class Account(Base): +#noinspection PyUnresolvedReferences +class Account(Base, AccountInfo): """ Base class for every Account plugin. Just overwrite `login` and cookies will be stored and account becomes accessible in\ associated hoster plugin. Plugin should also provide `loadAccountInfo` """ - __name__ = "Account" - __version__ = "0.2" - __type__ = "account" - __description__ = """Account Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + + # Default values + valid = True + validuntil = None + trafficleft = None + maxtraffic = None + premium = True + activated = True #: after that time [in minutes] pyload will relogin the account login_timeout = 600 #: account data will be reloaded after this time info_threshold = 600 + # known options + known_opt = ["time", "limitDL"] - def __init__(self, manager, accounts): + + def __init__(self, manager, loginname, password, options): Base.__init__(self, manager.core) + if "activated" in options: + activated = from_string(options["activated"], "bool") + else: + activated = Account.activated + + for opt in self.known_opt: + if opt not in options: + options[opt] = "" + + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] + + # default account attributes + AccountInfo.__init__(self, self.__name__, loginname, Account.valid, Account.validuntil, Account.trafficleft, + Account.maxtraffic, Account.premium, activated, options) + self.manager = manager - self.accounts = {} - self.infos = {} # cache for account information + self.lock = RLock() + self.timestamp = 0 + self.login_ts = 0 # timestamp for login + self.cj = CookieJar(self.__name__) + self.password = password + self.error = None - self.timestamps = {} - self.setAccounts(accounts) self.init() def init(self): @@ -70,76 +79,68 @@ class Account(Base): :param data: data dictionary :param req: `Request` instance """ - pass + raise NotImplemented @lock - def _login(self, user, data): + def _login(self): # set timestamp for login - self.timestamps[user] = time() - - req = self.getAccountRequest(user) + self.login_ts = time() + + req = self.getAccountRequest() try: - self.login(user, data, req) + self.login(self.loginname, {"password": self.password}, req) + self.valid = True except WrongPassword: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": _("Wrong Password")}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": _("Wrong Password")}) + self.valid = False except Exception, e: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": e}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": e}) + self.valid = False if self.core.debug: print_exc() finally: - if req: req.close() - - def relogin(self, user): - req = self.getAccountRequest(user) - if req: - req.cj.clear() req.close() - if user in self.infos: - del self.infos[user] #delete old information - - self._login(user, self.accounts[user]) - def setAccounts(self, accounts): - self.accounts = accounts - for user, data in self.accounts.iteritems(): - self._login(user, data) - self.infos[user] = {} + def restoreDefaults(self): + self.valid = Account.valid + self.validuntil = Account.validuntil + self.trafficleft = Account.trafficleft + self.maxtraffic = Account.maxtraffic + self.premium = Account.premium + self.activated = Account.activated - def updateAccounts(self, user, password=None, options={}): + def update(self, password=None, options={}): """ updates account and return true if anything changed """ - if user in self.accounts: - self.accounts[user]["valid"] = True #do not remove or accounts will not login - if password: - self.accounts[user]["password"] = password - self.relogin(user) - return True - if options: - before = self.accounts[user]["options"] - self.accounts[user]["options"].update(options) - return self.accounts[user]["options"] != before - else: - self.accounts[user] = {"password": password, "options": options, "valid": True} - self._login(user, self.accounts[user]) + self.login_ts = 0 + + if "activated" in options: + self.activated = from_string(options["avtivated"], "bool") + + if password: + self.password = password + self._login() return True + if options: + # remove unknown options + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] - def removeAccount(self, user): - if user in self.accounts: - del self.accounts[user] - if user in self.infos: - del self.infos[user] - if user in self.timestamps: - del self.timestamps[user] + before = self.options + self.options.update(options) + return self.options != before + + def getAccountRequest(self): + return self.core.requestFactory.getRequest(self.__name__, self.cj) @lock - def getAccountInfo(self, name, force=False): + def getAccountInfo(self, force=False): """retrieve account infos for an user, do **not** overwrite this method!\\ just use it to retrieve infos in hoster plugins. see `loadAccountInfo` @@ -147,113 +148,55 @@ class Account(Base): :param force: reloads cached account information :return: dictionary with information """ - data = Account.loadAccountInfo(self, name) + if force or self.timestamp + self.info_threshold * 60 < time(): - if force or name not in self.infos: - self.logDebug("Get Account Info for %s" % name) - req = self.getAccountRequest(name) + # make sure to login + self.checkLogin() + self.logDebug("Get Account Info for %s" % self.loginname) + req = self.getAccountRequest() try: - infos = self.loadAccountInfo(name, req) - if not type(infos) == dict: - raise Exception("Wrong return format") + infos = self.loadAccountInfo(self.loginname, req) except Exception, e: infos = {"error": str(e)} - if req: req.close() + req.close() self.logDebug("Account Info: %s" % str(infos)) + self.timestamp = time() + + self.restoreDefaults() # reset to initial state + if type(infos) == dict: # copy result from dict to class + for k, v in infos.iteritems(): + if hasattr(self, k): + setattr(self, k, v) + else: + self.logDebug("Unknown attribute %s=%s" % (k, v)) + + def isPremium(self, user=None): + if user: self.logDebug("Deprecated Argument user for .isPremium()", user) + return self.premium + + def isUsable(self): + """Check several contraints to determine if account should be used""" + if not self.valid or not self.activated: return False + + if self.options["time"]: + time_data = "" + try: + time_data = self.options["time"] + start, end = time_data.split("-") + if not compare_time(start.split(":"), end.split(":")): + return False + except: + self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + + if 0 < self.validuntil < time(): + return False + if self.trafficleft is 0: # test explicity for 0 + return False - infos["timestamp"] = time() - self.infos[name] = infos - elif "timestamp" in self.infos[name] and self.infos[name][ - "timestamp"] + self.info_threshold * 60 < time(): - self.logDebug("Reached timeout for account data") - self.scheduleRefresh(name) - - data.update(self.infos[name]) - return data - - def isPremium(self, user): - info = self.getAccountInfo(user) - return info["premium"] - - def loadAccountInfo(self, name, req=None): - """this should be overwritten in account plugin,\ - and retrieving account information for user - - :param name: - :param req: `Request` instance - :return: - """ - return { - "validuntil": None, # -1 for unlimited - "login": name, - #"password": self.accounts[name]["password"], #@XXX: security - "options": self.accounts[name]["options"], - "valid": self.accounts[name]["valid"], - "trafficleft": None, # in kb, -1 for unlimited - "maxtraffic": None, - "premium": True, #useful for free accounts - "timestamp": 0, #time this info was retrieved - "type": self.__name__, - } - - def getAllAccounts(self, force=False): - return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] - - def getAccountRequest(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - req = self.core.requestFactory.getRequest(self.__name__, user) - return req - - def getAccountCookies(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - cj = self.core.requestFactory.getCookieJar(self.__name__, user) - return cj - - def getAccountData(self, user): - return self.accounts[user] - - def selectAccount(self): - """ returns an valid account name and data""" - usable = [] - for user, data in self.accounts.iteritems(): - if not data["valid"]: continue - - if "time" in data["options"] and data["options"]["time"]: - time_data = "" - try: - time_data = data["options"]["time"][0] - start, end = time_data.split("-") - if not compare_time(start.split(":"), end.split(":")): - continue - except: - self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) - - if user in self.infos: - if "validuntil" in self.infos[user]: - if self.infos[user]["validuntil"] > 0 and time() > self.infos[user]["validuntil"]: - continue - if "trafficleft" in self.infos[user]: - if self.infos[user]["trafficleft"] == 0: - continue - - usable.append((user, data)) - - if not usable: return None, None - return choice(usable) - - def canUse(self): - return False if self.selectAccount() == (None, None) else True + return True def parseTraffic(self, string): #returns kbyte return parseFileSize(string) / 1024 @@ -261,32 +204,36 @@ class Account(Base): def wrongPassword(self): raise WrongPassword - def empty(self, user): - if user in self.infos: - self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) + def empty(self, user=None): + if user: self.logDebug("Deprecated argument user for .empty()", user) + + self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % self.login) - self.infos[user].update({"trafficleft": 0}) - self.scheduleRefresh(user, 30 * 60) + self.trafficleft = 0 + self.scheduleRefresh(30 * 60) def expired(self, user): if user in self.infos: self.logWarning(_("Account %s is expired, checking again in 1h") % user) - self.infos[user].update({"validuntil": time() - 1}) - self.scheduleRefresh(user, 60 * 60) + self.validuntil = time() - 1 + self.scheduleRefresh(60 * 60) - def scheduleRefresh(self, user, time=0, force=True): + def scheduleRefresh(self, time=0, force=True): """ add task to refresh account info to sheduler """ - self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) - self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) + self.logDebug("Scheduled Account refresh for %s in %s seconds." % (self.loginname, time)) + self.core.scheduler.addJob(time, self.getAccountInfo, [force]) @lock - def checkLogin(self, user): + def checkLogin(self): """ checks if user is still logged in """ - if user in self.timestamps: - if self.timestamps[user] + self.login_timeout * 60 < time(): - self.logDebug("Reached login timeout for %s" % user) - self.relogin(user) - return False + if self.login_ts + self.login_timeout * 60 < time(): + if self.login_ts: # seperate from fresh login to have better debug logs + self.logDebug("Reached login timeout for %s" % self.loginname) + else: + self.logDebug("Login with %s" % self.loginname) + + self._login() + return False return True diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index fc521d36c..c718510ed 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -17,169 +17,119 @@ @author: RaNaN """ -from os.path import exists -from shutil import copy - from threading import Lock +from random import choice -from module.PullEvents import AccountUpdateEvent -from module.utils import chmod, lock - -ACC_VERSION = 1 +from module.common.json_layer import json +from module.interaction.PullEvents import AccountUpdateEvent +from module.utils import lock class AccountManager(): """manages all accounts""" - #---------------------------------------------------------------------- def __init__(self, core): """Constructor""" self.core = core self.lock = Lock() - self.initPlugins() - self.saveAccounts() # save to add categories to conf + self.loadAccounts() - def initPlugins(self): - self.accounts = {} # key = ( plugin ) - self.plugins = {} + def loadAccounts(self): + """loads all accounts available""" - self.initAccountPlugins() - self.loadAccounts() + self.accounts = {} + for plugin, loginname, activated, password, options in self.core.db.loadAccounts(): + # put into options as used in other context + options = json.loads(options) if options else {} + options["activated"] = activated - def getAccountPlugin(self, plugin): - """get account instance for plugin or None if anonymous""" - if plugin in self.accounts: - if plugin not in self.plugins: - self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin]) + self.createAccount(plugin, loginname, password, options) + + return - return self.plugins[plugin] - else: - return None - - def getAccountPlugins(self): - """ get all account instances""" - - plugins = [] - for plugin in self.accounts.keys(): - plugins.append(self.getAccountPlugin(plugin)) - - return plugins - #---------------------------------------------------------------------- - def loadAccounts(self): - """loads all accounts available""" - - if not exists("accounts.conf"): - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - - f = open("accounts.conf", "rb") - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - f.close() - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - self.core.log.warning(_("Account settings deleted, due to new config format.")) - return - - - - plugin = "" - name = "" - - for line in content[1:]: - line = line.strip() - - if not line: continue - if line.startswith("#"): continue - if line.startswith("version"): continue - - if line.endswith(":") and line.count(":") == 1: - plugin = line[:-1] - self.accounts[plugin] = {} - - elif line.startswith("@"): - try: - option = line[1:].split() - self.accounts[plugin][name]["options"][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) - except: - pass - - elif ":" in line: - name, sep, pw = line.partition(":") - self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - #---------------------------------------------------------------------- def saveAccounts(self): """save all account information""" - - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION) + "\n") - - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin+":\n") - - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data["password"]) ) - if data["options"]: - for option, values in data["options"].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) - - f.close() - chmod(f.name, 0600) - - - #---------------------------------------------------------------------- - def initAccountPlugins(self): - """init names""" - for name in self.core.pluginManager.getAccountPlugins(): - self.accounts[name] = {} - + + data = [] + for name, plugin in self.accounts.iteritems(): + data.extend([(name, acc.loginname, acc.activated, acc.password, json.dumps(acc.options)) for acc in + plugin.itervalues()]) + self.core.db.saveAccounts(data) + + def createAccount(self, plugin, loginname, password, options): + klass = self.core.pluginManager.loadClass("accounts", plugin) + if not klass: + self.core.log.warning(_("Unknown account plugin %s") % plugin) + return + + if plugin not in self.accounts: + self.accounts[plugin] = {} + + self.core.log.debug("Create account %s:%s" % (plugin, loginname)) + + self.accounts[plugin][loginname] = klass(self, loginname, password, options) + + @lock - def updateAccount(self, plugin , user, password=None, options={}): + def updateAccount(self, plugin, user, password=None, options={}): """add or update account""" - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - updated = p.updateAccounts(user, password, options) - #since accounts is a ref in plugin self.accounts doesnt need to be updated here - + if plugin in self.accounts and user in self.accounts[plugin]: + acc = self.accounts[plugin][user] + updated = acc.update(password, options) + self.saveAccounts() - if updated: p.scheduleRefresh(user, force=False) - + if updated: acc.scheduleRefresh(force=True) + else: + self.createAccount(plugin, user, password, options) + self.saveAccounts() + @lock def removeAccount(self, plugin, user): """remove account""" - + if plugin in self.accounts and user in self.accounts[plugin]: + del self.accounts[plugin][user] + self.core.db.removeAccount(plugin, user) + else: + self.core.log.debug("Remove non existing account %s %s" % (plugin, user)) + + + @lock + def getAccountForPlugin(self, plugin): if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - p.removeAccount(user) + accs = [x for x in self.accounts[plugin].values() if x.isUsable()] + if accs: return choice(accs) - self.saveAccounts() + return None @lock - def getAccountInfos(self, force=True, refresh=False): - data = {} + def getAllAccounts(self, refresh=False): + """ Return account info, refresh afterwards if needed + :param refresh: + :return: + """ if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - force = False - - for p in self.accounts.keys(): - if self.accounts[p]: - p = self.getAccountPlugin(p) - data[p.__name__] = p.getAllAccounts(force) - else: - data[p] = [] + self.core.scheduler.addJob(0, self.core.accountManager.getAllAccounts) + + # load unavailable account info + for p_dict in self.accounts.itervalues(): + for acc in p_dict.itervalues(): + acc.getAccountInfo() + e = AccountUpdateEvent() self.core.pullManager.addEvent(e) - return data - + + return self.accounts + + def refreshAllAccounts(self): + """ Force a refresh of every account """ + for p in self.accounts.itervalues(): + for acc in p.itervalues(): + acc.getAccountInfo(True) + + def sendChange(self): e = AccountUpdateEvent() self.core.pullManager.addEvent(e) diff --git a/module/plugins/Base.py b/module/plugins/Base.py new file mode 100644 index 000000000..36df7e423 --- /dev/null +++ b/module/plugins/Base.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" + +import sys + +# TODO: config format definition +# more attributes if needed +# get rid of catpcha & container plugins ?! (move to crypter & internals) +# adapt old plugins as needed + +class Base(object): + """ + The Base plugin class with all shared methods and every possible attribute for plugin definition. + """ + __version__ = "0.1" + #: Regexp pattern which will be matched for download plugins + __pattern__ = r"" + #: Flat config definition + __config__ = tuple() + #: Short description, one liner + __description__ = "" + #: More detailed text + __long_description__ = """""" + #: List of needed modules + __dependencies__ = tuple() + #: Tags to categorize the plugin + __tags__ = tuple() + #: Base64 encoded .png icon + __icon__ = "" + #: Alternative, link to png icon + __icon_url__ = "" + #: Url with general information/support/discussion + __url__ = "" + __author_name__ = tuple() + __author_mail__ = tuple() + + + def __init__(self, core): + self.__name__ = self.__class__.__name__ + + #: Core instance + self.core = core + #: logging instance + self.log = core.log + #: core config + self.config = core.config + + #log functions + def logInfo(self, *args): + self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logWarning(self, *args): + self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logError(self, *args): + self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logDebug(self, *args): + self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + + def setConf(self, option, value): + """ see `setConfig` """ + self.core.config.set(self.__name__, option, value) + + def setConfig(self, option, value): + """ Set config value for current plugin + + :param option: + :param value: + :return: + """ + self.setConf(option, value) + + def getConf(self, option): + """ see `getConfig` """ + return self.core.config.get(self.__name__, option) + + def getConfig(self, option): + """ Returns config value for current plugin + + :param option: + :return: + """ + return self.getConf(option) + + def setStorage(self, key, value): + """ Saves a value persistently to the database """ + self.core.db.setStorage(self.__name__, key, value) + + def store(self, key, value): + """ same as `setStorage` """ + self.core.db.setStorage(self.__name__, key, value) + + def getStorage(self, key=None, default=None): + """ Retrieves saved value or dict of all saved entries if key is None """ + if key is not None: + return self.core.db.getStorage(self.__name__, key) or default + return self.core.db.getStorage(self.__name__, key) + + def retrieve(self, *args, **kwargs): + """ same as `getStorage` """ + return self.getStorage(*args, **kwargs) + + def delStorage(self, key): + """ Delete entry in db """ + self.core.db.delStorage(self.__name__, key) + + def shell(self): + """ open ipython shell """ + if self.core.debug: + from IPython import embed + #noinspection PyUnresolvedReferences + sys.stdout = sys._stdout + embed() diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index 5efd08bae..860dc76bb 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -119,7 +119,7 @@ class Hook(Base): def isActivated(self): """ checks if hook is activated""" - return self.config.getPlugin(self.__name__, "activated") + return self.config.get(self.__name__, "activated") #event methods - overwrite these if needed diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 814a70949..aa50099fb 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -19,15 +19,15 @@ from module.plugins.Plugin import Plugin -def getInfo(self): - #result = [ .. (name, size, status, url) .. ] - return - class Hoster(Plugin): - __name__ = "Hoster" - __version__ = "0.1" - __pattern__ = None - __type__ = "hoster" - __description__ = """Base hoster plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + + @staticmethod + def getInfo(urls): + """This method is used to retrieve the online status of files for hoster plugins. + It has to *yield* list of tuples with the result in this format (name, size, status, url), + where status is one of API pyfile statusses. + + :param urls: List of urls + :return: + """ + pass
\ No newline at end of file diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 15bf3971f..d78eb162b 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -29,17 +29,9 @@ if os.name != "nt": from pwd import getpwnam from grp import getgrnam -from itertools import islice - -from module.utils import save_join, save_path, fs_encode, fs_decode - -def chunks(iterable, size): - it = iter(iterable) - item = list(islice(it, size)) - while item: - yield item - item = list(islice(it, size)) +from Base import Base +from module.utils import save_join, save_path, fs_encode, fs_decode, chunks class Abort(Exception): """ raised when aborted """ @@ -61,95 +53,11 @@ class SkipDownload(Exception): """ raised when download should be skipped """ -class Base(object): - """ - A Base class with log/config/db methods *all* plugin types can use - """ - - def __init__(self, core): - #: Core instance - self.core = core - #: logging instance - self.log = core.log - #: core config - self.config = core.config - - #log functions - def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - - def setConf(self, option, value): - """ see `setConfig` """ - self.core.config.setPlugin(self.__name__, option, value) - - def setConfig(self, option, value): - """ Set config value for current plugin - - :param option: - :param value: - :return: - """ - self.setConf(option, value) - - def getConf(self, option): - """ see `getConfig` """ - return self.core.config.getPlugin(self.__name__, option) - - def getConfig(self, option): - """ Returns config value for current plugin - - :param option: - :return: - """ - return self.getConf(option) - - def setStorage(self, key, value): - """ Saves a value persistently to the database """ - self.core.db.setStorage(self.__name__, key, value) - - def store(self, key, value): - """ same as `setStorage` """ - self.core.db.setStorage(self.__name__, key, value) - - def getStorage(self, key=None, default=None): - """ Retrieves saved value or dict of all saved entries if key is None """ - if key is not None: - return self.core.db.getStorage(self.__name__, key) or default - return self.core.db.getStorage(self.__name__, key) - - def retrieve(self, *args, **kwargs): - """ same as `getStorage` """ - return self.getStorage(*args, **kwargs) - - def delStorage(self, key): - """ Delete entry in db """ - self.core.db.delStorage(self.__name__, key) - - class Plugin(Base): """ Base plugin for hoster/crypter. Overwrite `process` / `decrypt` in your subclassed plugin. """ - __name__ = "Plugin" - __version__ = "0.4" - __pattern__ = None - __type__ = "hoster" - __config__ = [("name", "type", "desc", "default")] - __description__ = """Base Plugin""" - __author_name__ = ("RaNaN", "spoob", "mkaay") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") - def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) @@ -167,26 +75,26 @@ class Plugin(Base): self.ocr = None #captcha reader instance #: account handler instance, see :py:class:`Account` - self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + self.account = self.core.accountManager.getAccountForPlugin(self.__name__) #: premium status self.premium = False #: username/login self.user = None - if self.account and not self.account.canUse(): self.account = None + if self.account and not self.account.isUsable(): self.account = None if self.account: - self.user, data = self.account.selectAccount() + self.user, data = self.account.loginname, {} #TODO change plugins to not use data anymore #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest(self.user) + self.req = self.account.getAccountRequest() self.chunkLimit = -1 # chunk limit, -1 for unlimited #: enables resume (will be ignored if server dont accept chunks) self.resumeDownload = True self.multiDL = True #every hoster with account should provide multiple downloads #: premium status - self.premium = self.account.isPremium(self.user) + self.premium = self.account.isPremium() else: - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) + self.req = self.core.requestFactory.getRequest(self.__name__) #: associated pyfile instance, see `PyFile` self.pyfile = pyfile @@ -226,7 +134,7 @@ class Plugin(Base): self.thread = thread if self.account: - self.account.checkLogin(self.user) + self.account.checkLogin() else: self.req.clearCookies() @@ -350,7 +258,7 @@ class Plugin(Base): temp_file.write(img) temp_file.close() - has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins + has_plugin = self.__name__ in self.core.pluginManager.getPlugins("captcha") if self.core.captcha: Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index f3f5f47bc..18dea7699 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -21,24 +21,34 @@ import re import sys from os import listdir, makedirs -from os.path import isfile, join, exists, abspath +from os.path import isfile, join, exists, abspath, basename from sys import version_info -from itertools import chain +from time import time from traceback import print_exc from module.lib.SafeEval import const_eval as literal_eval -from module.ConfigParser import IGNORE +from module.plugins.Base import Base + +from new_collections import namedtuple + +# ignore these plugin configs, mainly because plugins were wiped out +IGNORE = ( + "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('hooks', 'UnRar'), + 'EasyShareCom', 'FlyshareCz' + ) + +PluginTuple = namedtuple("PluginTuple", "version re deps user path") class PluginManager: ROOT = "module.plugins." USERROOT = "userplugins." TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal") - PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') - VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)') - CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) - DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') + SINGLE = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?<!")"(?!")|\'|\().*(?:(?<!")"(?!")|\'|\)))', + re.I) + # note the nongreedy character, that means we can not embed list and dicts + MULTI = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I) def __init__(self, core): self.core = core @@ -47,15 +57,24 @@ class PluginManager: self.log = core.log self.plugins = {} + self.modules = {} # cached modules + self.names = {} # overwritten names + self.history = [] # match history to speedup parsing (type, name) self.createIndex() + + self.core.config.parseValues(self.core.config.PLUGIN) + #register for import hook sys.meta_path.append(self) + def logDebug(self, type, plugin, msg): + self.log.debug("Plugin %s | %s: %s" % (type, plugin, msg)) + def createIndex(self): """create information for all plugins available""" - + # add to path, so we can import from userplugins sys.path.append(abspath("")) if not exists("userplugins"): @@ -64,27 +83,14 @@ class PluginManager: f = open(join("userplugins", "__init__.py"), "wb") f.close() - self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - - self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") - self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") - self.plugins["hooks"] = self.hookPlugins = self.parse("hooks") - self.plugins["internal"] = self.internalPlugins = self.parse("internal") + a = time() + for type in self.TYPES: + self.plugins[type] = self.parse(type) - self.log.debug("created index of plugins") + self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000) - def parse(self, folder, pattern=False, home={}): - """ - returns dict with information - home contains parsed plugins from module. - - { - name : {path, version, config, (pattern, re), (plugin, class)} - } - - """ + def parse(self, folder, home=None): + """ Analyze and parses all plugins in folder """ plugins = {} if home: pfolder = join("userplugins", folder) @@ -100,10 +106,6 @@ class PluginManager: for f in listdir(pfolder): if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - data = open(join(pfolder, f)) - content = data.read() - data.close() - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): continue elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): @@ -111,146 +113,174 @@ class PluginManager: elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): continue + # replace suffix and version tag name = f[:-3] if name[-1] == ".": name = name[:-4] - version = self.VERSION.findall(content) - if version: - version = float(version[0][1]) - else: - version = 0 + plugin = self.parsePlugin(join(pfolder, f), folder, name, home) + if plugin: + plugins[name] = plugin - # home contains plugins from pyload root - if home and name in home: - if home[name]["v"] >= version: - continue + if not home: + temp = self.parse(folder, plugins) + plugins.update(temp) - if name in IGNORE or (folder, name) in IGNORE: - continue + return plugins - plugins[name] = {} - plugins[name]["v"] = version + def parsePlugin(self, filename, folder, name, home=None): + """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. - module = f.replace(".pyc", "").replace(".py", "") + :arg home: dict with plugins, of which the found one will be matched against (according version) + :returns PluginTuple""" - # the plugin is loaded from user directory - plugins[name]["user"] = True if home else False - plugins[name]["name"] = module + data = open(filename, "rb") + content = data.read() + data.close() - if pattern: - pattern = self.PATTERN.findall(content) + attrs = {} + for m in self.SINGLE.findall(content) + self.MULTI.findall(content): + #replace gettext function and eval result + try: + attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) + except: + self.logDebug(folder, name, "Error when parsing: %s" % m[-1]) + return + if not hasattr(Base, "__%s__" % m[0]): + if m[0] != "type": #TODO remove type from all plugins, its not needed + self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) - if pattern: - pattern = pattern[0][1] - else: - pattern = "^unmachtable$" + version = 0 - plugins[name]["pattern"] = pattern + if "version" in attrs: + try: + version = float(attrs["version"]) + except ValueError: + self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) + version = 9 #TODO remove when plugins are fixed, causing update loops + else: + self.logDebug(folder, name, "No version attribute") - try: - plugins[name]["re"] = re.compile(pattern) - except: - self.log.error(_("%s has a invalid pattern.") % name) + # home contains plugins from pyload root + if home and name in home: + if home[name].version >= version: + return + if name in IGNORE or (folder, name) in IGNORE: + return - # internals have no config - if folder == "internal": - self.core.config.deleteConfig(name) - continue + if "pattern" in attrs and attrs["pattern"]: + try: + plugin_re = re.compile(attrs["pattern"]) + except: + self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) + plugin_re = None + else: plugin_re = None - config = self.CONFIG.findall(content) - if config: - config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" + deps = attrs.get("dependencies", None) - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] + # create plugin tuple + plugin = PluginTuple(version, plugin_re, deps, bool(home), filename) - if folder == "hooks": - append = True - for item in config: - if item[0] == "activated": append = False - # activated flag missing - if append: config.append(["activated", "bool", "Activated", False]) + # internals have no config + if folder == "internal": + return plugin - try: - self.core.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) + if folder == "hooks" and "config" not in attrs: + attrs["config"] = (["activated", "bool", "Activated", False],) - elif folder == "hooks": #force config creation - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - config = (["activated", "bool", "Activated", False],) + if "config" in attrs and attrs["config"]: + config = attrs["config"] + desc = attrs.get("description", "") + long_desc = attrs.get("long_description", "") - try: - self.core.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] - if not home: - temp = self.parse(folder, pattern, plugins) - plugins.update(temp) + if folder == "hooks": + append = True + for item in config: + if item[0] == "activated": append = False - return plugins + # activated flag missing + if append: config.insert(0, ("activated", "bool", "Activated", False)) + + try: + self.core.config.addConfigSection(name, name, desc, long_desc, config) + except: + self.logDebug(folder, name, "Invalid config %s" % config) + + return plugin def parseUrls(self, urls): """parse plugins for given list of urls""" - last = None res = [] # tupels of (url, plugin) for url in urls: - if type(url) not in (str, unicode, buffer): continue + if type(url) not in (str, unicode, buffer): + self.log.debug("Parsing invalid type %s" % type(url)) + continue found = False - if last and last[1]["re"].match(url): - res.append((url, last[0])) + for ptype, name in self.history: + if self.plugins[ptype][name].re.match(url): + res.append((url, name)) + found = (ptype, name) + + if found and self.history[0] != found: + # found match, update history + self.history.remove(found) + self.history.insert(0, found) continue - for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), - self.containerPlugins.iteritems()): - if value["re"].match(url): - res.append((url, name)) - last = (name, value) - found = True - break + for ptype in ("crypter", "hoster", "container"): + for name, plugin in self.plugins[ptype].iteritems(): + if plugin.re.match(url): + res.append((url, name)) + self.history.insert(0, (ptype, name)) + del self.history[10:] # cut down to size of 10 + found = True + break if not found: res.append((url, "BasePlugin")) return res + def getPlugins(self, type): + # TODO clean this workaround + if type not in self.plugins: type += "s" # append s, so updater can find the plugins + return self.plugins[type] + def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): for ptype in pluginlist: if name in self.plugins[ptype]: - return self.plugins[ptype][name], ptype + return ptype, self.plugins[ptype][name] return None, None def getPlugin(self, name, original=False): """return plugin module from hoster|decrypter|container""" - plugin, type = self.findPlugin(name) + type, plugin = self.findPlugin(name) if not plugin: self.log.warning("Plugin %s not found." % name) - plugin = self.hosterPlugins["BasePlugin"] + name = "BasePlugin" - if "new_module" in plugin and not original: - return plugin["new_module"] + if (type, name) in self.modules and not original: + return self.modules[(type, name)] return self.loadModule(type, name) def getPluginName(self, name): """ used to obtain new name if other plugin was injected""" - plugin, type = self.findPlugin(name) + type, plugin = self.findPlugin(name) - if "new_name" in plugin: - return plugin["new_name"] + if (type, name) in self.names: + return self.names[(type, name)] return name @@ -262,11 +292,12 @@ class PluginManager: """ plugins = self.plugins[type] if name in plugins: - if "module" in plugins[name]: return plugins[name]["module"] + if (type, name) in self.modules: return self.modules[(type, name)] try: - module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]["name"]), globals(), locals(), - plugins[name]["name"]) - plugins[name]["module"] = module #cache import, maybe unneeded + # convert path to python recognizable import + path = basename(plugins[name].path).replace(".pyc", "").replace(".py", "") + module = __import__(self.ROOT + "%s.%s" % (type, path), globals(), locals(), path) + self.modules[(type, name)] = module # cache import, maybe unneeded return module except Exception, e: self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) @@ -278,9 +309,17 @@ class PluginManager: module = self.loadModule(type, name) if module: return getattr(module, name) - def getAccountPlugins(self): - """return list of account plugin names""" - return self.accountPlugins.keys() + def injectPlugin(self, type, name, module, new_name): + """ Overwrite a plugin with a other module. used by Multihoster """ + self.modules[(type, name)] = module + self.names[(type, name)] = new_name + + def restoreState(self, type, name): + """ Restore the state of a plugin after injecting """ + if (type, name) in self.modules: + del self.modules[(type, name)] + if (type, name) in self.names: + del self.names[(type, name)] def find_module(self, fullname, path=None): #redirecting imports if necesarry @@ -294,10 +333,10 @@ class PluginManager: if type in self.plugins and name in self.plugins[type]: #userplugin is a newer version - if not user and self.plugins[type][name]["user"]: + if not user and self.plugins[type][name].user: return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name]["user"]: + #imported from userdir, but pyloads is newer + if user and not self.plugins[type][name].user: return self @@ -329,7 +368,7 @@ class PluginManager: self.log.debug("Request reload of plugins: %s" % type_plugins) as_dict = {} - for t,n in type_plugins: + for t, n in type_plugins: if t in as_dict: as_dict[t].append(n) else: @@ -342,16 +381,13 @@ class PluginManager: for type in as_dict.iterkeys(): for plugin in as_dict[type]: if plugin in self.plugins[type]: - if "module" in self.plugins[type][plugin]: + if (type, plugin) in self.modules: self.log.debug("Reloading %s" % plugin) - reload(self.plugins[type][plugin]["module"]) + reload(self.modules[(type, plugin)]) - #index creation - self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") - self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") + # index re-creation + for type in ("crypter", "container", "hoster", "captcha", "accounts"): + self.plugins[type] = self.parse(type) if "accounts" in as_dict: #accounts needs to be reloaded self.core.accountManager.initPlugins() @@ -359,6 +395,23 @@ class PluginManager: return True + def loadIcons(self): + """Loads all icons from plugins, plugin type is not in result, because its not important here. + + :return: Dict of names mapped to icons + """ + pass + + def loadIcon(self, type, name): + """ load icon for single plugin, base64 encoded""" + pass + + def checkDependencies(self, type, name): + """ Check deps for given plugin + + :return: List of unfullfilled dependencies + """ + pass if __name__ == "__main__": diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py index 920a88060..4324a96ba 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/hooks/UpdateManager.py @@ -24,7 +24,7 @@ from os import stat from os.path import join, exists from time import time -from module.ConfigParser import IGNORE +from module.plugins.PluginManager import IGNORE from module.network.RequestFactory import getURL from module.plugins.Hook import threaded, Expose, Hook @@ -129,10 +129,10 @@ class UpdateManager(Hook): else: type = prefix - plugins = getattr(self.core.pluginManager, "%sPlugins" % type) + plugins = self.core.pluginManager.getPlugins(type) if name in plugins: - if float(plugins[name]["v"]) >= float(version): + if float(plugins[name].version) >= float(version): continue if name in IGNORE or (type, name) in IGNORE: diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py index 0d927c525..f3011a488 100644 --- a/module/plugins/hoster/RapidshareCom.py +++ b/module/plugins/hoster/RapidshareCom.py @@ -52,7 +52,7 @@ class RapidshareCom(Hoster): __pattern__ = r"https?://[\w\.]*?rapidshare.com/(?:files/(?P<id>\d*?)/(?P<name>[^?]+)|#!download\|(?:\w+)\|(?P<id_new>\d+)\|(?P<name_new>[^|]+))" __version__ = "1.37" __description__ = """Rapidshare.com Download Hoster""" - __config__ = [["server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None"]] + __config__ = [("server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None")] __author_name__ = ("spoob", "RaNaN", "mkaay") __author_mail__ = ("spoob@pyload.org", "ranan@pyload.org", "mkaay@mkaay.de") diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index d50df3943..872e0b770 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -5,6 +5,7 @@ import re from module.utils import remove_chars from module.plugins.Hook import Hook +from module.plugins.PluginManager import PluginTuple class MultiHoster(Hook): """ @@ -43,7 +44,7 @@ class MultiHoster(Hook): def coreReady(self): pluginMap = {} - for name in self.core.pluginManager.hosterPlugins.keys(): + for name in self.core.pluginManager.getPlugins("hoster").keys(): pluginMap[name.lower()] = name new_supported = [] @@ -66,25 +67,19 @@ class MultiHoster(Hook): # inject plugin plugin self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - dict["new_module"] = module - dict["new_name"] = self.__name__ + self.core.pluginManager.injectPlugin("hoster", hoster, module, self.__name__) self.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) # create new regexp regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict["pattern"] = regexp - dict["re"] = re.compile(regexp) + hoster = self.core.pluginManager.getPlugins("hoster") + p = hoster[self.__name__] + new = PluginTuple(p.version, re.compile(regexp), p.deps, p.user, p.path) + hoster[self.__name__] = new def unload(self): for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - if "module" in dict: - del dict["module"] - - del dict["new_module"] - del dict["new_name"]
\ No newline at end of file + self.core.pluginManager.restoreState("hoster", hoster)
\ No newline at end of file |