diff options
Diffstat (limited to 'module/plugins')
86 files changed, 2249 insertions, 2486 deletions
diff --git a/module/plugins/Account.py b/module/plugins/Account.py index c147404e0..7c24298e7 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -1,292 +1,288 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from random import choice from time import time from traceback import print_exc from threading import RLock -from Plugin import Base -from module.utils import compare_time, parseFileSize, lock +from module.utils import compare_time, format_size, parseFileSize, lock, from_string +from module.Api import AccountInfo +from module.network.CookieJar import CookieJar + +from Base import Base class WrongPassword(Exception): pass - -class Account(Base): +#noinspection PyUnresolvedReferences +class Account(Base, AccountInfo): """ - Base class for every Account plugin. - Just overwrite `login` and cookies will be stored and account becomes accessible in\ - associated hoster plugin. Plugin should also provide `loadAccountInfo` + Base class for every account plugin. + Just overwrite `login` and cookies will be stored and the account becomes accessible in\ + associated hoster plugin. Plugin should also provide `loadAccountInfo`. \ + An instance of this class is created for every entered account, it holds all \ + fields of AccountInfo ttype, and can be set easily at runtime. """ - __name__ = "Account" - __version__ = "0.2" - __type__ = "account" - __description__ = """Account Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + + # constants for special values + UNKNOWN = -1 + UNLIMITED = -2 + + # Default values + valid = True + validuntil = -1 + trafficleft = -1 + maxtraffic = -1 + premium = True + activated = True #: after that time [in minutes] pyload will relogin the account login_timeout = 600 #: account data will be reloaded after this time info_threshold = 600 + # known options + known_opt = ("time", "limitDL") - def __init__(self, manager, accounts): + def __init__(self, manager, loginname, password, options): Base.__init__(self, manager.core) + if "activated" in options: + activated = from_string(options["activated"], "bool") + else: + activated = Account.activated + + for opt in self.known_opt: + if opt not in options: + options[opt] = "" + + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] + + # default account attributes + AccountInfo.__init__(self, self.__name__, loginname, Account.valid, Account.validuntil, Account.trafficleft, + Account.maxtraffic, Account.premium, activated, options) + self.manager = manager - self.accounts = {} - self.infos = {} # cache for account information + self.lock = RLock() + self.timestamp = 0 + self.login_ts = 0 # timestamp for login + self.cj = CookieJar(self.__name__) + self.password = password + self.error = None - self.timestamps = {} - self.setAccounts(accounts) self.init() def init(self): pass - def login(self, user, data, req): - """login into account, the cookies will be saved so user can be recognized + def login(self, req): + """login into account, the cookies will be saved so the user can be recognized - :param user: loginname - :param data: data dictionary :param req: `Request` instance """ - pass + raise NotImplemented + + def relogin(self): + """ Force a login. """ + req = self.getAccountRequest() + try: + return self._login(req) + finally: + req.close() @lock - def _login(self, user, data): + def _login(self, req): # set timestamp for login - self.timestamps[user] = time() - - req = self.getAccountRequest(user) + self.login_ts = time() + try: - self.login(user, data, req) + try: + self.login(req) + except TypeError: #TODO: temporary + self.logDebug("Deprecated .login(...) signature omit user, data") + self.login(self.loginname, {"password": self.password}, req) + + + self.valid = True except WrongPassword: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": _("Wrong Password")}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": _("Wrong Password")}) + self.valid = False except Exception, e: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": e}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": e}) + self.valid = False if self.core.debug: print_exc() - finally: - if req: req.close() - def relogin(self, user): - req = self.getAccountRequest(user) - if req: - req.cj.clear() - req.close() - if user in self.infos: - del self.infos[user] #delete old information - - self._login(user, self.accounts[user]) - - def setAccounts(self, accounts): - self.accounts = accounts - for user, data in self.accounts.iteritems(): - self._login(user, data) - self.infos[user] = {} - - def updateAccounts(self, user, password=None, options={}): - """ updates account and return true if anything changed """ - - if user in self.accounts: - self.accounts[user]["valid"] = True #do not remove or accounts will not login - if password: - self.accounts[user]["password"] = password - self.relogin(user) - return True - if options: - before = self.accounts[user]["options"] - self.accounts[user]["options"].update(options) - return self.accounts[user]["options"] != before - else: - self.accounts[user] = {"password": password, "options": options, "valid": True} - self._login(user, self.accounts[user]) + return self.valid + + def restoreDefaults(self): + self.validuntil = Account.validuntil + self.trafficleft = Account.trafficleft + self.maxtraffic = Account.maxtraffic + self.premium = Account.premium + + def update(self, password=None, options=None): + """ updates the account and returns true if anything changed """ + + self.login_ts = 0 + self.valid = True #set valid, so the login will be retried + + if "activated" in options: + self.activated = from_string(options["avtivated"], "bool") + + if password: + self.password = password + self.relogin() return True + if options: + # remove unknown options + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] + + before = self.options + self.options.update(options) + return self.options != before + + def getAccountRequest(self): + return self.core.requestFactory.getRequest(self.__name__, self.cj) - def removeAccount(self, user): - if user in self.accounts: - del self.accounts[user] - if user in self.infos: - del self.infos[user] - if user in self.timestamps: - del self.timestamps[user] + def getDownloadSettings(self): + """ Can be overwritten to change download settings. Default is no chunkLimit, max dl limit, resumeDownload + + :return: (chunkLimit, limitDL, resumeDownload) / (int, int ,bool) + """ + return -1, 0, True @lock - def getAccountInfo(self, name, force=False): - """retrieve account infos for an user, do **not** overwrite this method!\\ - just use it to retrieve infos in hoster plugins. see `loadAccountInfo` + def getAccountInfo(self, force=False): + """retrieve account info's for an user, do **not** overwrite this method!\\ + just use it to retrieve info's in hoster plugins. see `loadAccountInfo` :param name: username :param force: reloads cached account information :return: dictionary with information """ - data = Account.loadAccountInfo(self, name) - - if force or name not in self.infos: - self.logDebug("Get Account Info for %s" % name) - req = self.getAccountRequest(name) + if force or self.timestamp + self.info_threshold * 60 < time(): + # make sure to login + req = self.getAccountRequest() + self.checkLogin(req) + self.logDebug("Get Account Info for %s" % self.loginname) try: - infos = self.loadAccountInfo(name, req) - if not type(infos) == dict: - raise Exception("Wrong return format") + try: + infos = self.loadAccountInfo(req) + except TypeError: #TODO: temporary + self.logDebug("Deprecated .loadAccountInfo(...) signature, omit user argument.") + infos = self.loadAccountInfo(self.loginname, req) except Exception, e: infos = {"error": str(e)} - - if req: req.close() + finally: + req.close() self.logDebug("Account Info: %s" % str(infos)) - - infos["timestamp"] = time() - self.infos[name] = infos - elif "timestamp" in self.infos[name] and self.infos[name][ - "timestamp"] + self.info_threshold * 60 < time(): - self.logDebug("Reached timeout for account data") - self.scheduleRefresh(name) - - data.update(self.infos[name]) - return data - - def isPremium(self, user): - info = self.getAccountInfo(user) - return info["premium"] - - def loadAccountInfo(self, name, req=None): - """this should be overwritten in account plugin,\ - and retrieving account information for user - - :param name: - :param req: `Request` instance + self.timestamp = time() + + self.restoreDefaults() # reset to initial state + if type(infos) == dict: # copy result from dict to class + for k, v in infos.iteritems(): + if hasattr(self, k): + setattr(self, k, v) + else: + self.logDebug("Unknown attribute %s=%s" % (k, v)) + + #TODO: remove user + def loadAccountInfo(self, req): + """ Overwrite this method and set account attributes within this method. + + :param user: Deprecated + :param req: Request instance :return: """ - return { - "validuntil": None, # -1 for unlimited - "login": name, - #"password": self.accounts[name]["password"], #@XXX: security - "options": self.accounts[name]["options"], - "valid": self.accounts[name]["valid"], - "trafficleft": None, # in kb, -1 for unlimited - "maxtraffic": None, - "premium": True, #useful for free accounts - "timestamp": 0, #time this info was retrieved - "type": self.__name__, - } - - def getAllAccounts(self, force=False): - return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] - - def getAccountRequest(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - req = self.core.requestFactory.getRequest(self.__name__, user) - return req - - def getAccountCookies(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - cj = self.core.requestFactory.getCookieJar(self.__name__, user) - return cj - - def getAccountData(self, user): - return self.accounts[user] + pass - def selectAccount(self): - """ returns an valid account name and data""" - usable = [] - for user, data in self.accounts.iteritems(): - if not data["valid"]: continue + def getAccountCookies(self, user): + self.logDebug("Deprecated method .getAccountCookies -> use account.cj") + return self.cj - if "time" in data["options"] and data["options"]["time"]: - time_data = "" - try: - time_data = data["options"]["time"][0] - start, end = time_data.split("-") - if not compare_time(start.split(":"), end.split(":")): - continue - except: - self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + def getAccountData(self, user): + self.logDebug("Deprecated method .getAccountData -> use fields directly") + return {"password": self.password} - if user in self.infos: - if "validuntil" in self.infos[user]: - if self.infos[user]["validuntil"] > 0 and time() > self.infos[user]["validuntil"]: - continue - if "trafficleft" in self.infos[user]: - if self.infos[user]["trafficleft"] == 0: - continue + def isPremium(self, user=None): + if user: self.logDebug("Deprecated Argument user for .isPremium()", user) + return self.premium - usable.append((user, data)) + def isUsable(self): + """Check several constraints to determine if account should be used""" + if not self.valid or not self.activated: return False - if not usable: return None, None - return choice(usable) + if self.options["time"]: + time_data = "" + try: + time_data = self.options["time"] + start, end = time_data.split("-") + if not compare_time(start.split(":"), end.split(":")): + return False + except: + self.logWarning(_("Your Time %s has a wrong format, use: 1:22-3:44") % time_data) + + if 0 <= self.validuntil < time(): + return False + if self.trafficleft is 0: # test explicitly for 0 + return False - def canUse(self): - return False if self.selectAccount() == (None, None) else True + return True def parseTraffic(self, string): #returns kbyte return parseFileSize(string) / 1024 + def formatTrafficleft(self): + if self.trafficleft is None: + self.getAccountInfo(force=True) + return format_size(self.trafficleft*1024) + def wrongPassword(self): raise WrongPassword - def empty(self, user): - if user in self.infos: - self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) + def empty(self, user=None): + if user: self.logDebug("Deprecated argument user for .empty()", user) + + self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % self.login) + + self.trafficleft = 0 + self.scheduleRefresh(30 * 60) - self.infos[user].update({"trafficleft": 0}) - self.scheduleRefresh(user, 30 * 60) + def expired(self, user=None): + if user: self.logDebug("Deprecated argument user for .expired()", user) - def expired(self, user): - if user in self.infos: - self.logWarning(_("Account %s is expired, checking again in 1h") % user) + self.logWarning(_("Account %s is expired, checking again in 1h") % user) - self.infos[user].update({"validuntil": time() - 1}) - self.scheduleRefresh(user, 60 * 60) + self.validuntil = time() - 1 + self.scheduleRefresh(60 * 60) - def scheduleRefresh(self, user, time=0, force=True): - """ add task to refresh account info to sheduler """ - self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) - self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) + def scheduleRefresh(self, time=0, force=True): + """ add a task for refreshing the account info to the scheduler """ + self.logDebug("Scheduled Account refresh for %s in %s seconds." % (self.loginname, time)) + self.core.scheduler.addJob(time, self.getAccountInfo, [force]) @lock - def checkLogin(self, user): - """ checks if user is still logged in """ - if user in self.timestamps: - if self.timestamps[user] + self.login_timeout * 60 < time(): - self.logDebug("Reached login timeout for %s" % user) - self.relogin(user) - return False + def checkLogin(self, req): + """ checks if the user is still logged in """ + if self.login_ts + self.login_timeout * 60 < time(): + if self.login_ts: # separate from fresh login to have better debug logs + self.logDebug("Reached login timeout for %s" % self.loginname) + else: + self.logDebug("Login with %s" % self.loginname) + + self._login(req) + return False return True diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py deleted file mode 100644 index fc521d36c..000000000 --- a/module/plugins/AccountManager.py +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from os.path import exists -from shutil import copy - -from threading import Lock - -from module.PullEvents import AccountUpdateEvent -from module.utils import chmod, lock - -ACC_VERSION = 1 - -class AccountManager(): - """manages all accounts""" - - #---------------------------------------------------------------------- - def __init__(self, core): - """Constructor""" - - self.core = core - self.lock = Lock() - - self.initPlugins() - self.saveAccounts() # save to add categories to conf - - def initPlugins(self): - self.accounts = {} # key = ( plugin ) - self.plugins = {} - - self.initAccountPlugins() - self.loadAccounts() - - - def getAccountPlugin(self, plugin): - """get account instance for plugin or None if anonymous""" - if plugin in self.accounts: - if plugin not in self.plugins: - self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin]) - - return self.plugins[plugin] - else: - return None - - def getAccountPlugins(self): - """ get all account instances""" - - plugins = [] - for plugin in self.accounts.keys(): - plugins.append(self.getAccountPlugin(plugin)) - - return plugins - #---------------------------------------------------------------------- - def loadAccounts(self): - """loads all accounts available""" - - if not exists("accounts.conf"): - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - - f = open("accounts.conf", "rb") - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - f.close() - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - self.core.log.warning(_("Account settings deleted, due to new config format.")) - return - - - - plugin = "" - name = "" - - for line in content[1:]: - line = line.strip() - - if not line: continue - if line.startswith("#"): continue - if line.startswith("version"): continue - - if line.endswith(":") and line.count(":") == 1: - plugin = line[:-1] - self.accounts[plugin] = {} - - elif line.startswith("@"): - try: - option = line[1:].split() - self.accounts[plugin][name]["options"][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) - except: - pass - - elif ":" in line: - name, sep, pw = line.partition(":") - self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - #---------------------------------------------------------------------- - def saveAccounts(self): - """save all account information""" - - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION) + "\n") - - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin+":\n") - - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data["password"]) ) - if data["options"]: - for option, values in data["options"].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) - - f.close() - chmod(f.name, 0600) - - - #---------------------------------------------------------------------- - def initAccountPlugins(self): - """init names""" - for name in self.core.pluginManager.getAccountPlugins(): - self.accounts[name] = {} - - @lock - def updateAccount(self, plugin , user, password=None, options={}): - """add or update account""" - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - updated = p.updateAccounts(user, password, options) - #since accounts is a ref in plugin self.accounts doesnt need to be updated here - - self.saveAccounts() - if updated: p.scheduleRefresh(user, force=False) - - @lock - def removeAccount(self, plugin, user): - """remove account""" - - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - p.removeAccount(user) - - self.saveAccounts() - - @lock - def getAccountInfos(self, force=True, refresh=False): - data = {} - - if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - force = False - - for p in self.accounts.keys(): - if self.accounts[p]: - p = self.getAccountPlugin(p) - data[p.__name__] = p.getAllAccounts(force) - else: - data[p] = [] - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) - return data - - def sendChange(self): - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) diff --git a/module/plugins/Addon.py b/module/plugins/Addon.py new file mode 100644 index 000000000..60223dd28 --- /dev/null +++ b/module/plugins/Addon.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" + +from traceback import print_exc + +#from functools import wraps +from module.utils import has_method, to_list + +from Base import Base + +def class_name(p): + return p.rpartition(".")[2] + + +def AddEventListener(event): + """ Used to register method for events. Arguments needs to match parameter of event + + :param event: Name of event or list of them. + """ + class _klass(object): + def __new__(cls, f, *args, **kwargs): + for ev in to_list(event): + addonManager.addEventListener(class_name(f.__module__), f.func_name, ev) + return f + return _klass + +class ConfigHandler(object): + """ Register method as config handler. + + Your method signature has to be: + def foo(value=None): + + value will be passed to use your method to set the config. + When value is None your method needs to return an interaction task for configuration. + """ + + def __new__(cls, f, *args, **kwargs): + addonManager.addConfigHandler(class_name(f.__module__), f.func_name) + return f + +def AddonHandler(desc, media=None): + """ Register Handler for files, packages, or arbitrary callable methods. + To let the method work on packages/files, media must be set and the argument named pid or fid. + + :param desc: verbose description + :param media: if True or bits of media type + """ + pass + +def AddonInfo(desc): + """ Called to retrieve information about the current state. + Decorated method must return anything convertable into string. + + :param desc: verbose description + """ + pass + +def threaded(f): + """ Decorator to run method in a thread. """ + + #@wraps(f) + def run(*args,**kwargs): + addonManager.startThread(f, *args, **kwargs) + return run + +class Addon(Base): + """ + Base class for addon plugins. Use @threaded decorator for all longer running tasks. + + Decorate methods with @Expose, @AddEventListener, @ConfigHandler + + """ + + #: automatically register event listeners for functions, attribute will be deleted don't use it yourself + event_map = None + + # Alternative to event_map + #: List of events the plugin can handle, name the functions exactly like eventname. + event_list = None # dont make duplicate entries in event_map + + #: periodic call interval in seconds + interval = 60 + + def __init__(self, core, manager): + Base.__init__(self, core) + + #: Provide information in dict here, usable by API `getInfo` + self.info = None + + #: Callback of periodical job task, used by addonmanager + self.cb = None + + #: `AddonManager` + self.manager = manager + + #register events + if self.event_map: + for event, funcs in self.event_map.iteritems(): + if type(funcs) in (list, tuple): + for f in funcs: + self.evm.addEvent(event, getattr(self,f)) + else: + self.evm.addEvent(event, getattr(self,funcs)) + + #delete for various reasons + self.event_map = None + + if self.event_list: + for f in self.event_list: + self.evm.addEvent(f, getattr(self,f)) + + self.event_list = None + + self.initPeriodical() + self.init() + self.setup() + + def initPeriodical(self): + if self.interval >=1: + self.cb = self.core.scheduler.addJob(0, self._periodical, threaded=False) + + def _periodical(self): + try: + if self.isActivated(): self.periodical() + except Exception, e: + self.core.log.error(_("Error executing addons: %s") % str(e)) + if self.core.debug: + print_exc() + + self.cb = self.core.scheduler.addJob(self.interval, self._periodical, threaded=False) + + + def __repr__(self): + return "<Addon %s>" % self.__name__ + + def isActivated(self): + """ checks if addon is activated""" + return True if self.__internal__ else self.getConfig("activated") + + def init(self): + pass + + def setup(self): + """ more init stuff if needed """ + pass + + def activate(self): + """ Used to activate the addon """ + if has_method(self.__class__, "coreReady"): + self.logDebug("Deprecated method .coreReady() use activate() instead") + self.coreReady() + + def deactivate(self): + """ Used to deactivate the addon. """ + pass + + def periodical(self): + pass + + def newInteractionTask(self, task): + """ new interaction task for the plugin, it MUST set the handler and timeout or will be ignored """ + pass + + def taskCorrect(self, task): + pass + + def taskInvalid(self, task): + pass + + # public events starts from here + def downloadPreparing(self, pyfile): + pass + + def downloadFinished(self, pyfile): + pass + + def downloadFailed(self, pyfile): + pass + + def packageFinished(self, pypack): + pass + + def beforeReconnecting(self, ip): + pass + + def afterReconnecting(self, ip): + pass
\ No newline at end of file diff --git a/module/plugins/Base.py b/module/plugins/Base.py new file mode 100644 index 000000000..2b9e12653 --- /dev/null +++ b/module/plugins/Base.py @@ -0,0 +1,338 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" + +import sys +from time import time, sleep +from random import randint + +from module.utils import decode +from module.utils.fs import exists, makedirs, join, remove + +# TODO +# more attributes if needed +# get rid of catpcha & container plugins ?! (move to crypter & internals) +# adapt old plugins as needed + +class Fail(Exception): + """ raised when failed """ + +class Retry(Exception): + """ raised when start again from beginning """ + +class Abort(Exception): + """ raised when aborted """ + +class Base(object): + """ + The Base plugin class with all shared methods and every possible attribute for plugin definition. + """ + __version__ = "0.1" + #: Regexp pattern which will be matched for download/crypter plugins + __pattern__ = r"" + #: Internal addon plugin which is always loaded + __internal__ = False + #: Config definition: list of (name, type, label, default_value) or + #: (name, type, label, short_description, default_value) + __config__ = list() + #: Short description, one liner + __label__ = "" + #: More detailed text + __description__ = """""" + #: List of needed modules + __dependencies__ = tuple() + #: Used to assign a category to addon plugins + __category__ = "" + #: Tags to categorize the plugin, see documentation for further info + __tags__ = tuple() + #: Base64 encoded .png icon, please don't use sizes above ~3KB + __icon__ = "" + #: Alternative, link to png icon + __icon_url__ = "" + #: Url with general information/support/discussion + __url__ = "" + #: Url to term of content, user is accepting these when using the plugin + __toc_url__ = "" + #: Url to service (to buy premium) for accounts + __ref_url__ = "" + + __author_name__ = tuple() + __author_mail__ = tuple() + + + def __init__(self, core, user=None): + self.__name__ = self.__class__.__name__ + + #: Core instance + self.core = core + #: logging instance + self.log = core.log + #: core config + self.config = core.config + #: :class:`EventManager` + self.evm = core.eventManager + #: :class:`InteractionManager` + self.im = core.interactionManager + if user: + #: :class:`Api`, user api when user is set + self.api = self.core.api.withUserContext(user) + if self.api: + #: :class:`User`, user related to this plugin + self.user = self.api.user + else: + self.api = self.core.api + self.user = None + else: + self.api = self.core.api + self.user = None + + #: last interaction task + self.task = None + + def logInfo(self, *args, **kwargs): + """ Print args to log at specific level + + :param args: Arbitrary object which should be logged + :param kwargs: sep=(how to separate arguments), default = " | " + """ + self._log("info", *args, **kwargs) + + def logWarning(self, *args, **kwargs): + self._log("warning", *args, **kwargs) + + def logError(self, *args, **kwargs): + self._log("error", *args, **kwargs) + + def logDebug(self, *args, **kwargs): + self._log("debug", *args, **kwargs) + + def _log(self, level, *args, **kwargs): + if "sep" in kwargs: + sep = "%s" % kwargs["sep"] + else: + sep = " | " + + strings = [] + for obj in args: + if type(obj) == unicode: + strings.append(obj) + elif type(obj) == str: + strings.append(decode(obj)) + else: + strings.append(str(obj)) + + getattr(self.log, level)("%s: %s" % (self.__name__, sep.join(strings))) + + def setConfig(self, option, value): + """ Set config value for current plugin """ + self.core.config.set(self.__name__, option, value) + + def getConf(self, option): + """ see `getConfig` """ + return self.core.config.get(self.__name__, option) + + def getConfig(self, option): + """ Returns config value for current plugin """ + return self.getConf(option) + + def setStorage(self, key, value): + """ Saves a value persistently to the database """ + self.core.db.setStorage(self.__name__, key, value) + + def store(self, key, value): + """ same as `setStorage` """ + self.core.db.setStorage(self.__name__, key, value) + + def getStorage(self, key=None, default=None): + """ Retrieves saved value or dict of all saved entries if key is None """ + if key is not None: + return self.core.db.getStorage(self.__name__, key) or default + return self.core.db.getStorage(self.__name__, key) + + def retrieve(self, *args, **kwargs): + """ same as `getStorage` """ + return self.getStorage(*args, **kwargs) + + def delStorage(self, key): + """ Delete entry in db """ + self.core.db.delStorage(self.__name__, key) + + def shell(self): + """ open ipython shell """ + if self.core.debug: + from IPython import embed + #noinspection PyUnresolvedReferences + sys.stdout = sys._stdout + embed() + + def abort(self): + """ Check if plugin is in an abort state, is overwritten by subtypes""" + return False + + def checkAbort(self): + """ Will be overwritten to determine if control flow should be aborted """ + if self.abort(): raise Abort() + + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): + """Load content at url and returns it + + :param url: url as string + :param get: GET as dict + :param post: POST as dict, list or string + :param ref: Set HTTP_REFERER header + :param cookies: use saved cookies + :param just_header: if True only the header will be retrieved and returned as dict + :param decode: Whether to decode the output according to http header, should be True in most cases + :return: Loaded content + """ + if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") + self.checkAbort() + + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) + + if self.core.debug: + from inspect import currentframe + + frame = currentframe() + if not exists(join("tmp", self.__name__)): + makedirs(join("tmp", self.__name__)) + + f = open( + join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + , "wb") + del frame # delete the frame or it wont be cleaned + + try: + tmp = res.encode("utf8") + except: + tmp = res + + f.write(tmp) + f.close() + + if just_header: + #parse header + header = {"code": self.req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: continue + + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() + + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value + res = header + + return res + + def invalidTask(self): + if self.task: + self.task.invalid() + + def invalidCaptcha(self): + self.logDebug("Deprecated method .invalidCaptcha, use .invalidTask") + self.invalidTask() + + def correctTask(self): + if self.task: + self.task.correct() + + def correctCaptcha(self): + self.logDebug("Deprecated method .correctCaptcha, use .correctTask") + self.correctTask() + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', + result_type='textual'): + """ Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time())[-6:].replace(".", "") + temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp_file.write(img) + temp_file.close() + + name = "%sOCR" % self.__name__ + has_plugin = name in self.core.pluginManager.getPlugins("internal") + + if self.core.captcha: + OCR = self.core.pluginManager.loadClass("internal", name) + else: + OCR = None + + if OCR and not forceUser: + sleep(randint(3000, 5000) / 1000.0) + self.checkAbort() + + ocr = OCR() + result = ocr.get_captcha(temp_file.name) + else: + task = self.im.newCaptchaTask(img, imgtype, temp_file.name, result_type) + self.task = task + self.im.handleTask(task) + + while task.isWaiting(): + if self.abort(): + self.im.removeTask(task) + raise Abort() + sleep(1) + + #TODO task handling + self.im.removeTask(task) + + if task.error and has_plugin: #ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropriate time by any of the plugins.")) + + result = task.result + self.log.debug("Received captcha result: %s" % str(result)) + + if not self.core.debug: + try: + remove(temp_file.name) + except: + pass + + return result + + def fail(self, reason): + """ fail and give reason """ + raise Fail(reason)
\ No newline at end of file diff --git a/module/plugins/Container.py b/module/plugins/Container.py deleted file mode 100644 index c233d3710..000000000 --- a/module/plugins/Container.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Crypter import Crypter - -from os.path import join, exists, basename -from os import remove -import re - -class Container(Crypter): - __name__ = "Container" - __version__ = "0.1" - __pattern__ = None - __type__ = "container" - __description__ = """Base container plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - - def preprocessing(self, thread): - """prepare""" - - self.setup() - self.thread = thread - - self.loadToDisk() - - self.decrypt(self.pyfile) - self.deleteTmp() - - self.createPackages() - - - def loadToDisk(self): - """loads container to disk if its stored remotely and overwrite url, - or check existent on several places at disk""" - - if self.pyfile.url.startswith("http"): - self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] - content = self.load(self.pyfile.url) - self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name) - f = open(self.pyfile.url, "wb" ) - f.write(content) - f.close() - - else: - self.pyfile.name = basename(self.pyfile.url) - if not exists(self.pyfile.url): - if exists(join(pypath, self.pyfile.url)): - self.pyfile.url = join(pypath, self.pyfile.url) - else: - self.fail(_("File not exists.")) - - - def deleteTmp(self): - if self.pyfile.name.startswith("tmp_"): - remove(self.pyfile.url) - - diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index d1549fe80..920009f44 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -1,72 +1,269 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Plugin import Plugin - -class Crypter(Plugin): - __name__ = "Crypter" - __version__ = "0.1" - __pattern__ = None - __type__ = "container" - __description__ = """Base crypter plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def __init__(self, pyfile): - Plugin.__init__(self, pyfile) - - #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) - self.packages = [] +from traceback import print_exc + +from module.common.packagetools import parseNames +from module.utils import to_list, has_method, uniqify +from module.utils.fs import exists, remove, fs_encode + +from Base import Base, Retry + +class Package: + """ Container that indicates that a new package should be created """ + def __init__(self, name, urls=None): + self.name = name + self.urls = urls if urls else [] + # nested packages + self.packs = [] + + def addURL(self, url): + self.urls.append(url) + + def addPackage(self, pack): + self.packs.append(pack) + + def getAllURLs(self): + urls = self.urls + for p in self.packs: + urls.extend(p.getAllURLs()) + return urls + + # same name and urls is enough to be equal for packages + def __eq__(self, other): + return self.name == other.name and self.urls == other.urls + + def __repr__(self): + return u"<CrypterPackage name=%s, links=%s, packs=%s" % (self.name, self.urls, self.packs) + + def __hash__(self): + return hash(self.name) ^ hash(frozenset(self.urls)) + +class PyFileMockup: + """ Legacy class needed by old crypter plugins """ + def __init__(self, url, pack): + self.url = url + self.name = url + self._package = pack + self.packageid = pack.id if pack else -1 + + def package(self): + return self._package + +class Crypter(Base): + """ + Base class for (de)crypter plugins. Overwrite decrypt* methods. + + How to use decrypt* methods: + + You have to overwrite at least one method of decryptURL, decryptURLs, decryptFile. + + After decrypting and generating urls/packages you have to return the result. + Valid return Data is: + + :class:`Package` instance Crypter.Package + A **new** package will be created with the name and the urls of the object. + + List of urls and `Package` instances + All urls in the list will be added to the **current** package. For each `Package`\ + instance a new package will be created. + + """ + + #: Prefix to annotate that the submited string for decrypting is indeed file content + CONTENT_PREFIX = "filecontent:" + + @classmethod + def decrypt(cls, core, url_or_urls): + """Static method to decrypt urls or content. Can be used by other plugins. + To decrypt file content prefix the string with ``CONTENT_PREFIX `` as seen above. + + :param core: pyLoad `Core`, needed in decrypt context + :param url_or_urls: List of urls or single url/ file content + :return: List of decrypted urls, all package info removed + """ + urls = to_list(url_or_urls) + p = cls(core) + try: + result = p.processDecrypt(urls) + finally: + p.clean() + + ret = [] + + for url_or_pack in result: + if isinstance(url_or_pack, Package): #package + ret.extend(url_or_pack.getAllURLs()) + else: # single url + ret.append(url_or_pack) + # eliminate duplicates + return uniqify(ret) + + def __init__(self, core, package=None, password=None): + Base.__init__(self, core) + self.req = core.requestFactory.getRequest(self.__name__) + + # Package the plugin was initialized for, don't use this, its not guaranteed to be set + self.package = package + #: Password supplied by user + self.password = password + #: Propose a renaming of the owner package + self.rename = None - #: List of urls, pyLoad will generate packagenames + # For old style decrypter, do not use these! + self.packages = [] self.urls = [] - - self.multiDL = True - self.limitDL = 0 - - - def preprocessing(self, thread): - """prepare""" - self.setup() - self.thread = thread - - self.decrypt(self.pyfile) - - self.createPackages() - - - def decrypt(self, pyfile): + self.pyfile = None + + self.init() + + def init(self): + """More init stuff if needed""" + + def setup(self): + """Called everytime before decrypting. A Crypter plugin will be most likely used for several jobs.""" + + def decryptURL(self, url): + """Decrypt a single url + + :param url: url to decrypt + :return: See :class:`Crypter` Documentation + """ + if url.startswith("http"): # basic method to redirect + return self.decryptFile(self.load(url)) + else: + self.fail(_("Not existing file or unsupported protocol")) + + def decryptURLs(self, urls): + """Decrypt a bunch of urls + + :param urls: list of urls + :return: See :class:`Crypter` Documentation + """ raise NotImplementedError - def createPackages(self): - """ create new packages from self.packages """ - for pack in self.packages: + def decryptFile(self, content): + """Decrypt file content - self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } ) - - links = [x.decode("utf-8") for x in pack[1]] - - pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue) + :param content: content to decrypt as string + :return: See :class:`Crypter` Documentation + """ + raise NotImplementedError + + def generatePackages(self, urls): + """Generates :class:`Package` instances and names from urls. Useful for many different links and no\ + given package name. + + :param urls: list of urls + :return: list of `Package` + """ + return [Package(name, purls) for name, purls in parseNames([(url,url) for url in urls]).iteritems()] + + def _decrypt(self, urls): + """ Internal method to select decrypting method + + :param urls: List of urls/content + :return: + """ + cls = self.__class__ + + # separate local and remote files + content, urls = self.getLocalContent(urls) + + if has_method(cls, "decryptURLs"): + self.setup() + result = to_list(self.decryptURLs(urls)) + elif has_method(cls, "decryptURL"): + result = [] + for url in urls: + self.setup() + result.extend(to_list(self.decryptURL(url))) + elif has_method(cls, "decrypt"): + self.logDebug("Deprecated .decrypt() method in Crypter plugin") + result = [] + for url in urls: + self.pyfile = PyFileMockup(url, self.package) + self.setup() + self.decrypt(self.pyfile) + result.extend(self.convertPackages()) + else: + if not has_method(cls, "decryptFile") or urls: + self.logDebug("No suited decrypting method was overwritten in plugin") + result = [] + + if has_method(cls, "decryptFile"): + for f, c in content: + self.setup() + result.extend(to_list(self.decryptFile(c))) + try: + if f.startswith("tmp_"): remove(f) + except : + pass + + return result + + def processDecrypt(self, urls): + """Catches all exceptions in decrypt methods and return results + + :return: Decrypting results + """ + try: + return self._decrypt(urls) + except Exception: + if self.core.debug: + print_exc() + return [] + + def getLocalContent(self, urls): + """Load files from disk and separate to file content and url list + + :param urls: + :return: list of (filename, content), remote urls + """ + content = [] + # do nothing if no decryptFile method + if hasattr(self.__class__, "decryptFile"): + remote = [] + for url in urls: + path = None + if url.startswith("http"): # skip urls directly + pass + elif url.startswith(self.CONTENT_PREFIX): + path = url + elif exists(url): + path = url + elif exists(self.core.path(url)): + path = self.core.path(url) + + if path: + try: + if path.startswith(self.CONTENT_PREFIX): + content.append(("", path[len(self.CONTENT_PREFIX)])) + else: + f = open(fs_encode(path), "rb") + content.append((f.name, f.read())) + f.close() + except IOError, e: + self.logError("IOError", e) + else: + remote.append(url) + + #swap filtered url list + urls = remote + + return content, urls - if self.pyfile.package().password: - self.core.api.setPackageData(pid, {"password": self.pyfile.package().password}) + def retry(self): + """ Retry decrypting, will only work once. Somewhat deprecated method, should be avoided. """ + raise Retry() - if self.urls: - self.core.api.generateAndAddPackages(self.urls) + def convertPackages(self): + """ Deprecated """ + self.logDebug("Deprecated method .convertPackages()") + res = [Package(name, urls) for name, urls in self.packages] + res.extend(self.urls) + return res + def clean(self): + if hasattr(self, "req"): + self.req.close() + del self.req
\ No newline at end of file diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py deleted file mode 100644 index 5efd08bae..000000000 --- a/module/plugins/Hook.py +++ /dev/null @@ -1,161 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay - @interface-version: 0.2 -""" - -from traceback import print_exc - -from Plugin import Base - -class Expose(object): - """ used for decoration to declare rpc services """ - - def __new__(cls, f, *args, **kwargs): - hookManager.addRPC(f.__module__, f.func_name, f.func_doc) - return f - -def threaded(f): - def run(*args,**kwargs): - hookManager.startThread(f, *args, **kwargs) - return run - -class Hook(Base): - """ - Base class for hook plugins. - """ - __name__ = "Hook" - __version__ = "0.2" - __type__ = "hook" - __threaded__ = [] - __config__ = [ ("name", "type", "desc" , "default") ] - __description__ = """interface for hook""" - __author_name__ = ("mkaay", "RaNaN") - __author_mail__ = ("mkaay@mkaay.de", "RaNaN@pyload.org") - - #: automatically register event listeners for functions, attribute will be deleted dont use it yourself - event_map = None - - # Alternative to event_map - #: List of events the plugin can handle, name the functions exactly like eventname. - event_list = None # dont make duplicate entries in event_map - - - #: periodic call interval in secondc - interval = 60 - - def __init__(self, core, manager): - Base.__init__(self, core) - - #: Provide information in dict here, usable by API `getInfo` - self.info = None - - #: Callback of periodical job task, used by hookmanager - self.cb = None - - #: `HookManager` - self.manager = manager - - #register events - if self.event_map: - for event, funcs in self.event_map.iteritems(): - if type(funcs) in (list, tuple): - for f in funcs: - self.manager.addEvent(event, getattr(self,f)) - else: - self.manager.addEvent(event, getattr(self,funcs)) - - #delete for various reasons - self.event_map = None - - if self.event_list: - for f in self.event_list: - self.manager.addEvent(f, getattr(self,f)) - - self.event_list = None - - self.initPeriodical() - self.setup() - - def initPeriodical(self): - if self.interval >=1: - self.cb = self.core.scheduler.addJob(0, self._periodical, threaded=False) - - def _periodical(self): - try: - if self.isActivated(): self.periodical() - except Exception, e: - self.core.log.error(_("Error executing hooks: %s") % str(e)) - if self.core.debug: - print_exc() - - self.cb = self.core.scheduler.addJob(self.interval, self._periodical, threaded=False) - - - def __repr__(self): - return "<Hook %s>" % self.__name__ - - def setup(self): - """ more init stuff if needed """ - pass - - def unload(self): - """ called when hook was deactivated """ - pass - - def isActivated(self): - """ checks if hook is activated""" - return self.config.getPlugin(self.__name__, "activated") - - - #event methods - overwrite these if needed - def coreReady(self): - pass - - def coreExiting(self): - pass - - def downloadPreparing(self, pyfile): - pass - - def downloadFinished(self, pyfile): - pass - - def downloadFailed(self, pyfile): - pass - - def packageFinished(self, pypack): - pass - - def beforeReconnecting(self, ip): - pass - - def afterReconnecting(self, ip): - pass - - def periodical(self): - pass - - def newCaptchaTask(self, task): - """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """ - pass - - def captchaCorrect(self, task): - pass - - def captchaInvalid(self, task): - pass
\ No newline at end of file diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 814a70949..ad4f8f16b 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -13,21 +13,389 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay + + @author: RaNaN, spoob, mkaay """ -from module.plugins.Plugin import Plugin +import os +from time import time + +if os.name != "nt": + from module.utils.fs import chown + from pwd import getpwnam + from grp import getgrnam + +from Base import Base, Fail, Retry +from module.utils import chunks as _chunks +from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\ + remove, makedirs, chmod, stat, exists, join + +# Import for Hoster Plugins +chunks = _chunks + +class Reconnect(Exception): + """ raised when reconnected """ + +class SkipDownload(Exception): + """ raised when download should be skipped """ + +class Hoster(Base): + """ + Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading. + """ + + @staticmethod + def getInfo(urls): + """This method is used to retrieve the online status of files for hoster plugins. + It has to *yield* list of tuples with the result in this format (name, size, status, url), + where status is one of API pyfile statuses. + + :param urls: List of urls + :return: yield list of tuple with results (name, size, status, url) + """ + pass + + def __init__(self, pyfile): + Base.__init__(self, pyfile.m.core) + + self.wantReconnect = False + #: enables simultaneous processing of multiple downloads + self.limitDL = 0 + #: chunk limit + self.chunkLimit = 1 + #: enables resume (will be ignored if server dont accept chunks) + self.resumeDownload = False + + #: time() + wait in seconds + self.waitUntil = 0 + self.waiting = False + + self.ocr = None #captcha reader instance + #: account handler instance, see :py:class:`Account` + self.account = self.core.accountManager.getAccountForPlugin(self.__name__) + + #: premium status + self.premium = False + #: username/login + self.user = None + + if self.account and not self.account.isUsable(): self.account = None + if self.account: + self.user = self.account.loginname + #: Browser instance, see `network.Browser` + self.req = self.account.getAccountRequest() + # Default: -1, True, True + self.chunkLimit, self.limitDL, self.resumeDownload = self.account.getDownloadSettings() + self.premium = self.account.isPremium() + else: + self.req = self.core.requestFactory.getRequest(self.__name__) + + #: associated pyfile instance, see `PyFile` + self.pyfile = pyfile + self.thread = None # holds thread in future + + #: location where the last call to download was saved + self.lastDownload = "" + #: re match of the last call to `checkDownload` + self.lastCheck = None + #: js engine, see `JsEngine` + self.js = self.core.js + + self.retries = 0 # amount of retries already made + self.html = None # some plugins store html code here + + self.init() + + def getMultiDL(self): + return self.limitDL <= 0 + + def setMultiDL(self, val): + self.limitDL = 0 if val else 1 + + #: virtual attribute using self.limitDL on behind + multiDL = property(getMultiDL, setMultiDL) + + def getChunkCount(self): + if self.chunkLimit <= 0: + return self.config["download"]["chunks"] + return min(self.config["download"]["chunks"], self.chunkLimit) + + def getDownloadLimit(self): + if self.account: + limit = self.account.options.get("limitDL", 0) + if limit == "": limit = 0 + if self.limitDL > 0: # a limit is already set, we use the minimum + return min(int(limit), self.limitDL) + else: + return int(limit) + else: + return self.limitDL + + + def __call__(self): + return self.__name__ + + def init(self): + """initialize the plugin (in addition to `__init__`)""" + pass + + def setup(self): + """ setup for environment and other things, called before downloading (possibly more than one time)""" + pass + + def preprocessing(self, thread): + """ handles important things to do before starting """ + self.thread = thread + + if self.account: + # will force a re-login or reload of account info if necessary + self.account.getAccountInfo() + else: + self.req.clearCookies() + + self.setup() + + self.pyfile.setStatus("starting") + + return self.process(self.pyfile) + + def process(self, pyfile): + """the 'main' method of every plugin, you **have to** overwrite it""" + raise NotImplementedError + + def abort(self): + return self.pyfile.abort + + def resetAccount(self): + """ don't use account and retry download """ + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retry() + + def checksum(self, local_file=None): + """ + return codes: + 0 - checksum ok + 1 - checksum wrong + 5 - can't get checksum + 10 - not implemented + 20 - unknown error + """ + #@TODO checksum check addon + + return True, 10 + + + def setWait(self, seconds, reconnect=False): + """Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + if reconnect: + self.wantReconnect = True + self.pyfile.waitUntil = time() + int(seconds) + + def wait(self): + """ waits the time previously set """ + self.waiting = True + self.pyfile.setStatus("waiting") + + while self.pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) + + self.checkAbort() + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + self.waiting = False + self.pyfile.setStatus("starting") + + def offline(self): + """ fail and indicate file is offline """ + raise Fail("offline") + + def tempOffline(self): + """ fail and indicates file ist temporary offline, the core may take consequences """ + raise Fail("temp. offline") + + def retry(self, max_tries=3, wait_time=1, reason=""): + """Retries and begin again from the beginning + + :param max_tries: number of maximum retries + :param wait_time: time to wait in seconds + :param reason: reason for retrying, will be passed to fail if max_tries reached + """ + if 0 < max_tries <= self.retries: + if not reason: reason = "Max retries reached" + raise Fail(reason) + + self.wantReconnect = False + self.setWait(wait_time) + self.wait() + + self.retries += 1 + raise Retry(reason) + + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + """Downloads the content at url to download folder + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param disposition: if True and server provides content-disposition header\ + the filename will be changed if needed + :return: The location where the file was saved + """ + self.checkForSameFiles() + self.checkAbort() + + self.pyfile.setStatus("downloading") + + download_folder = self.config['general']['download_folder'] + + location = save_join(download_folder, self.pyfile.package().folder) + + if not exists(location): + makedirs(location, int(self.core.config["permission"]["folder"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(location, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + # convert back to unicode + location = fs_decode(location) + name = save_filename(self.pyfile.name) + + filename = join(location, name) + + self.core.addonManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) + + try: + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.setProgress, disposition=disposition) + finally: + self.pyfile.size = self.req.size + + if disposition and newname and newname != name: #triple check, just to be sure + self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) + + fs_filename = fs_encode(filename) + + if self.core.config["permission"]["change_file"]: + chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(fs_filename, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + self.lastDownload = filename + return self.lastDownload + + def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): + """ checks the content of the last downloaded file, re match is saved to `lastCheck` + + :param rules: dict with names and rules to match (compiled regexp or strings) + :param api_size: expected file size + :param max_size: if the file is larger then it wont be checked + :param delete: delete if matched + :param read_size: amount of bytes to read from files larger then max_size + :return: dictionary key of the first rule that matched + """ + lastDownload = fs_encode(self.lastDownload) + if not exists(lastDownload): return None + + size = stat(lastDownload) + size = size.st_size + + if api_size and api_size <= size: return None + elif size > max_size and not read_size: return None + self.log.debug("Download Check triggered") + f = open(lastDownload, "rb") + content = f.read(read_size if read_size else -1) + f.close() + #produces encoding errors, better log to other file in the future? + #self.log.debug("Content: %s" % content) + for name, rule in rules.iteritems(): + if type(rule) in (str, unicode): + if rule in content: + if delete: + remove(lastDownload) + return name + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + if delete: + remove(lastDownload) + self.lastCheck = m + return name + + + def getPassword(self): + """ get the password the user provided in the package""" + password = self.pyfile.package().password + if not password: return "" + return password + + + def checkForSameFiles(self, starting=False): + """ checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises SkipDownload: + """ + + pack = self.pyfile.package() + + for pyfile in self.core.files.cache.values(): + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #finished or downloading + raise SkipDownload(pyfile.pluginname) + elif pyfile.status in ( + 5, 7) and starting: #a download is waiting/starting and was apparently started before + raise SkipDownload(pyfile.pluginname) + + download_folder = self.config['general']['download_folder'] + location = save_join(download_folder, pack.folder, self.pyfile.name) + + if starting and self.core.config['download']['skip_existing'] and exists(location): + size = os.stat(location).st_size + if size >= self.pyfile.size: + raise SkipDownload("File exists.") + + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + if pyfile: + if exists(location): + raise SkipDownload(pyfile[0]) -def getInfo(self): - #result = [ .. (name, size, status, url) .. ] - return + self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) -class Hoster(Plugin): - __name__ = "Hoster" - __version__ = "0.1" - __pattern__ = None - __type__ = "hoster" - __description__ = """Base hoster plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + def clean(self): + """ clean everything and remove references """ + if hasattr(self, "pyfile"): + del self.pyfile + if hasattr(self, "req"): + self.req.close() + del self.req + if hasattr(self, "thread"): + del self.thread + if hasattr(self, "html"): + del self.html diff --git a/module/plugins/MultiHoster.py b/module/plugins/MultiHoster.py new file mode 100644 index 000000000..1936478b4 --- /dev/null +++ b/module/plugins/MultiHoster.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +from time import time + +from module.utils import remove_chars + +from Account import Account + +def normalize(domain): + """ Normalize domain/plugin name, so they are comparable """ + return remove_chars(domain.strip().lower(), "-.") + +#noinspection PyUnresolvedReferences +class MultiHoster(Account): + """ + Base class for MultiHoster services. + This is also an Account instance so you should see :class:`Account` and overwrite necessary methods. + Multihoster becomes only active when an Account was entered and the MultiHoster addon was activated. + You need to overwrite `loadHosterList` and a corresponding :class:`Hoster` plugin with the same name should + be available to make your service working. + """ + + #: List of hoster names that will be replaced so pyLoad will recognize them: (orig_name, pyload_name) + replacements = [("freakshare.net", "freakshare.com")] + + #: Load new hoster list every x seconds + hoster_timeout = 300 + + def __init__(self, *args, **kwargs): + + # Hoster list + self.hoster = [] + # Timestamp + self.ts = 0 + + Account.__init__(self, *args, **kwargs) + + def loadHosterList(self, req): + """Load list of supported hoster + + :return: List of domain names + """ + raise NotImplementedError + + + def isHosterUsuable(self, domain): + """ Determine before downloading if hoster should be used. + + :param domain: domain name + :return: True to let the MultiHoster download, False to fallback to default plugin + """ + return True + + def getHosterList(self, force=False): + if self.ts + self.hoster_timeout < time() or force: + req = self.getAccountRequest() + try: + self.hoster = self.loadHosterList(req) + except Exception, e: + self.logError(e) + return [] + finally: + req.close() + + for rep in self.replacements: + if rep[0] in self.hoster: + self.hoster.remove(rep[0]) + if rep[1] not in self.hoster: + self.hoster.append(rep[1]) + + self.ts = time() + + return self.hoster
\ No newline at end of file diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py deleted file mode 100644 index 15bf3971f..000000000 --- a/module/plugins/Plugin.py +++ /dev/null @@ -1,617 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN, spoob, mkaay -""" - -from time import time, sleep -from random import randint - -import os -from os import remove, makedirs, chmod, stat -from os.path import exists, join - -if os.name != "nt": - from os import chown - from pwd import getpwnam - from grp import getgrnam - -from itertools import islice - -from module.utils import save_join, save_path, fs_encode, fs_decode - -def chunks(iterable, size): - it = iter(iterable) - item = list(islice(it, size)) - while item: - yield item - item = list(islice(it, size)) - - -class Abort(Exception): - """ raised when aborted """ - - -class Fail(Exception): - """ raised when failed """ - - -class Reconnect(Exception): - """ raised when reconnected """ - - -class Retry(Exception): - """ raised when start again from beginning """ - - -class SkipDownload(Exception): - """ raised when download should be skipped """ - - -class Base(object): - """ - A Base class with log/config/db methods *all* plugin types can use - """ - - def __init__(self, core): - #: Core instance - self.core = core - #: logging instance - self.log = core.log - #: core config - self.config = core.config - - #log functions - def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - - def setConf(self, option, value): - """ see `setConfig` """ - self.core.config.setPlugin(self.__name__, option, value) - - def setConfig(self, option, value): - """ Set config value for current plugin - - :param option: - :param value: - :return: - """ - self.setConf(option, value) - - def getConf(self, option): - """ see `getConfig` """ - return self.core.config.getPlugin(self.__name__, option) - - def getConfig(self, option): - """ Returns config value for current plugin - - :param option: - :return: - """ - return self.getConf(option) - - def setStorage(self, key, value): - """ Saves a value persistently to the database """ - self.core.db.setStorage(self.__name__, key, value) - - def store(self, key, value): - """ same as `setStorage` """ - self.core.db.setStorage(self.__name__, key, value) - - def getStorage(self, key=None, default=None): - """ Retrieves saved value or dict of all saved entries if key is None """ - if key is not None: - return self.core.db.getStorage(self.__name__, key) or default - return self.core.db.getStorage(self.__name__, key) - - def retrieve(self, *args, **kwargs): - """ same as `getStorage` """ - return self.getStorage(*args, **kwargs) - - def delStorage(self, key): - """ Delete entry in db """ - self.core.db.delStorage(self.__name__, key) - - -class Plugin(Base): - """ - Base plugin for hoster/crypter. - Overwrite `process` / `decrypt` in your subclassed plugin. - """ - __name__ = "Plugin" - __version__ = "0.4" - __pattern__ = None - __type__ = "hoster" - __config__ = [("name", "type", "desc", "default")] - __description__ = """Base Plugin""" - __author_name__ = ("RaNaN", "spoob", "mkaay") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") - - def __init__(self, pyfile): - Base.__init__(self, pyfile.m.core) - - self.wantReconnect = False - #: enables simultaneous processing of multiple downloads - self.multiDL = True - self.limitDL = 0 - #: chunk limit - self.chunkLimit = 1 - self.resumeDownload = False - - #: time() + wait in seconds - self.waitUntil = 0 - self.waiting = False - - self.ocr = None #captcha reader instance - #: account handler instance, see :py:class:`Account` - self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) - - #: premium status - self.premium = False - #: username/login - self.user = None - - if self.account and not self.account.canUse(): self.account = None - if self.account: - self.user, data = self.account.selectAccount() - #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest(self.user) - self.chunkLimit = -1 # chunk limit, -1 for unlimited - #: enables resume (will be ignored if server dont accept chunks) - self.resumeDownload = True - self.multiDL = True #every hoster with account should provide multiple downloads - #: premium status - self.premium = self.account.isPremium(self.user) - else: - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) - - #: associated pyfile instance, see `PyFile` - self.pyfile = pyfile - self.thread = None # holds thread in future - - #: location where the last call to download was saved - self.lastDownload = "" - #: re match of the last call to `checkDownload` - self.lastCheck = None - #: js engine, see `JsEngine` - self.js = self.core.js - self.cTask = None #captcha task - - self.retries = 0 # amount of retries already made - self.html = None # some plugins store html code here - - self.init() - - def getChunkCount(self): - if self.chunkLimit <= 0: - return self.config["download"]["chunks"] - return min(self.config["download"]["chunks"], self.chunkLimit) - - def __call__(self): - return self.__name__ - - def init(self): - """initialize the plugin (in addition to `__init__`)""" - pass - - def setup(self): - """ setup for enviroment and other things, called before downloading (possibly more than one time)""" - pass - - def preprocessing(self, thread): - """ handles important things to do before starting """ - self.thread = thread - - if self.account: - self.account.checkLogin(self.user) - else: - self.req.clearCookies() - - self.setup() - - self.pyfile.setStatus("starting") - - return self.process(self.pyfile) - - - def process(self, pyfile): - """the 'main' method of every plugin, you **have to** overwrite it""" - raise NotImplementedError - - def resetAccount(self): - """ dont use account and retry download """ - self.account = None - self.req = self.core.requestFactory.getRequest(self.__name__) - self.retry() - - def checksum(self, local_file=None): - """ - return codes: - 0 - checksum ok - 1 - checksum wrong - 5 - can't get checksum - 10 - not implemented - 20 - unknown error - """ - #@TODO checksum check hook - - return True, 10 - - - def setWait(self, seconds, reconnect=False): - """Set a specific wait time later used with `wait` - - :param seconds: wait time in seconds - :param reconnect: True if a reconnect would avoid wait time - """ - if reconnect: - self.wantReconnect = True - self.pyfile.waitUntil = time() + int(seconds) - - def wait(self): - """ waits the time previously set """ - self.waiting = True - self.pyfile.setStatus("waiting") - - while self.pyfile.waitUntil > time(): - self.thread.m.reconnecting.wait(2) - - if self.pyfile.abort: raise Abort - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - self.waiting = False - self.pyfile.setStatus("starting") - - def fail(self, reason): - """ fail and give reason """ - raise Fail(reason) - - def offline(self): - """ fail and indicate file is offline """ - raise Fail("offline") - - def tempOffline(self): - """ fail and indicates file ist temporary offline, the core may take consequences """ - raise Fail("temp. offline") - - def retry(self, max_tries=3, wait_time=1, reason=""): - """Retries and begin again from the beginning - - :param max_tries: number of maximum retries - :param wait_time: time to wait in seconds - :param reason: reason for retrying, will be passed to fail if max_tries reached - """ - if 0 < max_tries <= self.retries: - if not reason: reason = "Max retries reached" - raise Fail(reason) - - self.wantReconnect = False - self.setWait(wait_time) - self.wait() - - self.retries += 1 - raise Retry(reason) - - def invalidCaptcha(self): - if self.cTask: - self.cTask.invalid() - - def correctCaptcha(self): - if self.cTask: - self.cTask.correct() - - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual'): - """ Loads a captcha and decrypts it with ocr, plugin, user input - - :param url: url of captcha image - :param get: get part for request - :param post: post part for request - :param cookies: True if cookies should be enabled - :param forceUser: if True, ocr is not used - :param imgtype: Type of the Image - :param result_type: 'textual' if text is written on the captcha\ - or 'positional' for captcha where the user have to click\ - on a specific region on the captcha - - :return: result of decrypting - """ - - img = self.load(url, get=get, post=post, cookies=cookies) - - id = ("%.2f" % time())[-6:].replace(".", "") - temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - temp_file.write(img) - temp_file.close() - - has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins - - if self.core.captcha: - Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) - else: - Ocr = None - - if Ocr and not forceUser: - sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: raise Abort - - ocr = Ocr() - result = ocr.get_captcha(temp_file.name) - else: - captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) - self.cTask = task - captchaManager.handleCaptcha(task) - - while task.isWaiting(): - if self.pyfile.abort: - captchaManager.removeTask(task) - raise Abort - sleep(1) - - captchaManager.removeTask(task) - - if task.error and has_plugin: #ignore default error message since the user could use OCR - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) - - result = task.result - self.log.debug("Received captcha result: %s" % str(result)) - - if not self.core.debug: - try: - remove(temp_file.name) - except: - pass - - return result - - - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): - """Load content at url and returns it - - :param url: - :param get: - :param post: - :param ref: - :param cookies: - :param just_header: if True only the header will be retrieved and returned as dict - :param decode: Wether to decode the output according to http header, should be True in most cases - :return: Loaded content - """ - if self.pyfile.abort: raise Abort - #utf8 vs decode -> please use decode attribute in all future plugins - if type(url) == unicode: url = str(url) - - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) - - if self.core.debug: - from inspect import currentframe - - frame = currentframe() - if not exists(join("tmp", self.__name__)): - makedirs(join("tmp", self.__name__)) - - f = open( - join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) - , "wb") - del frame # delete the frame or it wont be cleaned - - try: - tmp = res.encode("utf8") - except: - tmp = res - - f.write(tmp) - f.close() - - if just_header: - #parse header - header = {"code": self.req.code} - for line in res.splitlines(): - line = line.strip() - if not line or ":" not in line: continue - - key, none, value = line.partition(":") - key = key.lower().strip() - value = value.strip() - - if key in header: - if type(header[key]) == list: - header[key].append(value) - else: - header[key] = [header[key], value] - else: - header[key] = value - res = header - - return res - - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): - """Downloads the content at url to download folder - - :param url: - :param get: - :param post: - :param ref: - :param cookies: - :param disposition: if True and server provides content-disposition header\ - the filename will be changed if needed - :return: The location where the file was saved - """ - - self.checkForSameFiles() - - self.pyfile.setStatus("downloading") - - download_folder = self.config['general']['download_folder'] - - location = save_join(download_folder, self.pyfile.package().folder) - - if not exists(location): - makedirs(location, int(self.core.config["permission"]["folder"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - - chown(location, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) - - # convert back to unicode - location = fs_decode(location) - name = save_path(self.pyfile.name) - - filename = join(location, name) - - self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) - - try: - newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, - chunks=self.getChunkCount(), resume=self.resumeDownload, - progressNotify=self.pyfile.setProgress, disposition=disposition) - finally: - self.pyfile.size = self.req.size - - if disposition and newname and newname != name: #triple check, just to be sure - self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) - self.pyfile.name = newname - filename = join(location, newname) - - fs_filename = fs_encode(filename) - - if self.core.config["permission"]["change_file"]: - chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - - chown(fs_filename, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) - - self.lastDownload = filename - return self.lastDownload - - def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): - """ checks the content of the last downloaded file, re match is saved to `lastCheck` - - :param rules: dict with names and rules to match (compiled regexp or strings) - :param api_size: expected file size - :param max_size: if the file is larger then it wont be checked - :param delete: delete if matched - :param read_size: amount of bytes to read from files larger then max_size - :return: dictionary key of the first rule that matched - """ - lastDownload = fs_encode(self.lastDownload) - if not exists(lastDownload): return None - - size = stat(lastDownload) - size = size.st_size - - if api_size and api_size <= size: return None - elif size > max_size and not read_size: return None - self.log.debug("Download Check triggered") - f = open(lastDownload, "rb") - content = f.read(read_size if read_size else -1) - f.close() - #produces encoding errors, better log to other file in the future? - #self.log.debug("Content: %s" % content) - for name, rule in rules.iteritems(): - if type(rule) in (str, unicode): - if rule in content: - if delete: - remove(lastDownload) - return name - elif hasattr(rule, "search"): - m = rule.search(content) - if m: - if delete: - remove(lastDownload) - self.lastCheck = m - return name - - - def getPassword(self): - """ get the password the user provided in the package""" - password = self.pyfile.package().password - if not password: return "" - return password - - - def checkForSameFiles(self, starting=False): - """ checks if same file was/is downloaded within same package - - :param starting: indicates that the current download is going to start - :raises SkipDownload: - """ - - pack = self.pyfile.package() - - for pyfile in self.core.files.cache.values(): - if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: - if pyfile.status in (0, 12): #finished or downloading - raise SkipDownload(pyfile.pluginname) - elif pyfile.status in ( - 5, 7) and starting: #a download is waiting/starting and was appenrently started before - raise SkipDownload(pyfile.pluginname) - - download_folder = self.config['general']['download_folder'] - location = save_join(download_folder, pack.folder, self.pyfile.name) - - if starting and self.core.config['download']['skip_existing'] and exists(location): - size = os.stat(location).st_size - if size >= self.pyfile.size: - raise SkipDownload("File exists.") - - pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) - if pyfile: - if exists(location): - raise SkipDownload(pyfile[0]) - - self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) - - def clean(self): - """ clean everything and remove references """ - if hasattr(self, "pyfile"): - del self.pyfile - if hasattr(self, "req"): - self.req.close() - del self.req - if hasattr(self, "thread"): - del self.thread - if hasattr(self, "html"): - del self.html diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py deleted file mode 100644 index f3f5f47bc..000000000 --- a/module/plugins/PluginManager.py +++ /dev/null @@ -1,380 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay, RaNaN -""" - -import re -import sys - -from os import listdir, makedirs -from os.path import isfile, join, exists, abspath -from sys import version_info -from itertools import chain -from traceback import print_exc - -from module.lib.SafeEval import const_eval as literal_eval -from module.ConfigParser import IGNORE - -class PluginManager: - ROOT = "module.plugins." - USERROOT = "userplugins." - TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal") - - PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') - VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)') - CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) - DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') - - - def __init__(self, core): - self.core = core - - #self.config = self.core.config - self.log = core.log - - self.plugins = {} - self.createIndex() - - #register for import hook - sys.meta_path.append(self) - - - def createIndex(self): - """create information for all plugins available""" - - sys.path.append(abspath("")) - - if not exists("userplugins"): - makedirs("userplugins") - if not exists(join("userplugins", "__init__.py")): - f = open(join("userplugins", "__init__.py"), "wb") - f.close() - - self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - - self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") - self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") - self.plugins["hooks"] = self.hookPlugins = self.parse("hooks") - self.plugins["internal"] = self.internalPlugins = self.parse("internal") - - self.log.debug("created index of plugins") - - def parse(self, folder, pattern=False, home={}): - """ - returns dict with information - home contains parsed plugins from module. - - { - name : {path, version, config, (pattern, re), (plugin, class)} - } - - """ - plugins = {} - if home: - pfolder = join("userplugins", folder) - if not exists(pfolder): - makedirs(pfolder) - if not exists(join(pfolder, "__init__.py")): - f = open(join(pfolder, "__init__.py"), "wb") - f.close() - - else: - pfolder = join(pypath, "module", "plugins", folder) - - for f in listdir(pfolder): - if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( - "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - data = open(join(pfolder, f)) - content = data.read() - data.close() - - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): - continue - elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): - continue - elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): - continue - - name = f[:-3] - if name[-1] == ".": name = name[:-4] - - version = self.VERSION.findall(content) - if version: - version = float(version[0][1]) - else: - version = 0 - - # home contains plugins from pyload root - if home and name in home: - if home[name]["v"] >= version: - continue - - if name in IGNORE or (folder, name) in IGNORE: - continue - - plugins[name] = {} - plugins[name]["v"] = version - - module = f.replace(".pyc", "").replace(".py", "") - - # the plugin is loaded from user directory - plugins[name]["user"] = True if home else False - plugins[name]["name"] = module - - if pattern: - pattern = self.PATTERN.findall(content) - - if pattern: - pattern = pattern[0][1] - else: - pattern = "^unmachtable$" - - plugins[name]["pattern"] = pattern - - try: - plugins[name]["re"] = re.compile(pattern) - except: - self.log.error(_("%s has a invalid pattern.") % name) - - - # internals have no config - if folder == "internal": - self.core.config.deleteConfig(name) - continue - - config = self.CONFIG.findall(content) - if config: - config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] - - if folder == "hooks": - append = True - for item in config: - if item[0] == "activated": append = False - - # activated flag missing - if append: config.append(["activated", "bool", "Activated", False]) - - try: - self.core.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) - - elif folder == "hooks": #force config creation - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - config = (["activated", "bool", "Activated", False],) - - try: - self.core.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) - - if not home: - temp = self.parse(folder, pattern, plugins) - plugins.update(temp) - - return plugins - - - def parseUrls(self, urls): - """parse plugins for given list of urls""" - - last = None - res = [] # tupels of (url, plugin) - - for url in urls: - if type(url) not in (str, unicode, buffer): continue - found = False - - if last and last[1]["re"].match(url): - res.append((url, last[0])) - continue - - for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), - self.containerPlugins.iteritems()): - if value["re"].match(url): - res.append((url, name)) - last = (name, value) - found = True - break - - if not found: - res.append((url, "BasePlugin")) - - return res - - def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): - for ptype in pluginlist: - if name in self.plugins[ptype]: - return self.plugins[ptype][name], ptype - return None, None - - def getPlugin(self, name, original=False): - """return plugin module from hoster|decrypter|container""" - plugin, type = self.findPlugin(name) - - if not plugin: - self.log.warning("Plugin %s not found." % name) - plugin = self.hosterPlugins["BasePlugin"] - - if "new_module" in plugin and not original: - return plugin["new_module"] - - return self.loadModule(type, name) - - def getPluginName(self, name): - """ used to obtain new name if other plugin was injected""" - plugin, type = self.findPlugin(name) - - if "new_name" in plugin: - return plugin["new_name"] - - return name - - def loadModule(self, type, name): - """ Returns loaded module for plugin - - :param type: plugin type, subfolder of module.plugins - :param name: - """ - plugins = self.plugins[type] - if name in plugins: - if "module" in plugins[name]: return plugins[name]["module"] - try: - module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]["name"]), globals(), locals(), - plugins[name]["name"]) - plugins[name]["module"] = module #cache import, maybe unneeded - return module - except Exception, e: - self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) - if self.core.debug: - print_exc() - - def loadClass(self, type, name): - """Returns the class of a plugin with the same name""" - module = self.loadModule(type, name) - if module: return getattr(module, name) - - def getAccountPlugins(self): - """return list of account plugin names""" - return self.accountPlugins.keys() - - def find_module(self, fullname, path=None): - #redirecting imports if necesarry - if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins - if fullname.startswith(self.USERROOT): user = 1 - else: user = 0 #used as bool and int - - split = fullname.split(".") - if len(split) != 4 - user: return - type, name = split[2 - user:4 - user] - - if type in self.plugins and name in self.plugins[type]: - #userplugin is a newer version - if not user and self.plugins[type][name]["user"]: - return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name]["user"]: - return self - - - def load_module(self, name, replace=True): - if name not in sys.modules: #could be already in modules - if replace: - if self.ROOT in name: - newname = name.replace(self.ROOT, self.USERROOT) - else: - newname = name.replace(self.USERROOT, self.ROOT) - else: newname = name - - base, plugin = newname.rsplit(".", 1) - - self.log.debug("Redirected import %s -> %s" % (name, newname)) - - module = __import__(newname, globals(), locals(), [plugin]) - #inject under new an old name - sys.modules[name] = module - sys.modules[newname] = module - - return sys.modules[name] - - - def reloadPlugins(self, type_plugins): - """ reloads and reindexes plugins """ - if not type_plugins: return False - - self.log.debug("Request reload of plugins: %s" % type_plugins) - - as_dict = {} - for t,n in type_plugins: - if t in as_dict: - as_dict[t].append(n) - else: - as_dict[t] = [n] - - # we do not reload hooks or internals, would cause to much side effects - if "hooks" in as_dict or "internal" in as_dict: - return False - - for type in as_dict.iterkeys(): - for plugin in as_dict[type]: - if plugin in self.plugins[type]: - if "module" in self.plugins[type][plugin]: - self.log.debug("Reloading %s" % plugin) - reload(self.plugins[type][plugin]["module"]) - - #index creation - self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") - self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") - - if "accounts" in as_dict: #accounts needs to be reloaded - self.core.accountManager.initPlugins() - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - - return True - - - -if __name__ == "__main__": - _ = lambda x: x - pypath = "/home/christian/Projekte/pyload-0.4/module/plugins" - - from time import time - - p = PluginManager(None) - - a = time() - - test = ["http://www.youtube.com/watch?v=%s" % x for x in range(0, 100)] - print p.parseUrls(test) - - b = time() - - print b - a, "s" - diff --git a/module/plugins/UserAddon.py b/module/plugins/UserAddon.py new file mode 100644 index 000000000..c49b1ef41 --- /dev/null +++ b/module/plugins/UserAddon.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" + +from Addon import Addon + +class UserAddon(Addon): + """ + Special type of an addon that only works a specific user. Has a configuration for every user who added it . + """
\ No newline at end of file diff --git a/module/plugins/accounts/FilesonicCom.py b/module/plugins/accounts/FilesonicCom.py index 1207f1b56..1b0104b2a 100644 --- a/module/plugins/accounts/FilesonicCom.py +++ b/module/plugins/accounts/FilesonicCom.py @@ -37,10 +37,10 @@ class FilesonicCom(Account): decode=True) return json_loads(xml)["FSApi_Utility"]["getFilesonicDomainForCurrentIp"]["response"] - def loadAccountInfo(self, user, req): + def loadAccountInfo(self, req): xml = req.load(self.API_URL + "/user?method=getInfo&format=json", - post={"u": user, - "p": self.accounts[user]["password"]}, decode=True) + post={"u": self.loginname, + "p": self.password}, decode=True) self.logDebug("account status retrieved from api %s" % xml) @@ -56,15 +56,16 @@ class FilesonicCom(Account): validuntil = -1 return {"validuntil": validuntil, "trafficleft": -1, "premium": premium} - def login(self, user, data, req): + def login(self, req): domain = self.getDomain(req) post_vars = { - "email": user, - "password": data["password"], + "email": self.loginname, + "password": self.password, "rememberMe": 1 } page = req.load("http://www%s/user/login" % domain, cookies=True, post=post_vars, decode=True) if "Provided password does not match." in page or "You must be logged in to view this page." in page: self.wrongPassword() + diff --git a/module/plugins/accounts/OronCom.py b/module/plugins/accounts/OronCom.py index 793984121..2c1d33162 100755 --- a/module/plugins/accounts/OronCom.py +++ b/module/plugins/accounts/OronCom.py @@ -20,16 +20,17 @@ from module.plugins.Account import Account import re from time import strptime, mktime +from module.utils import formatSize, parseFileSize class OronCom(Account): __name__ = "OronCom" - __version__ = "0.12" + __version__ = "0.13" __type__ = "account" __description__ = """oron.com account plugin""" __author_name__ = ("DHMH") __author_mail__ = ("DHMH@pyload.org") - def loadAccountInfo(self, user, req): + def loadAccountInfo(self, req): req.load("http://oron.com/?op=change_lang&lang=german") src = req.load("http://oron.com/?op=my_account").replace("\n", "") validuntil = re.search(r"<td>Premiumaccount läuft bis:</td>\s*<td>(.*?)</td>", src) @@ -37,7 +38,7 @@ class OronCom(Account): validuntil = validuntil.group(1) validuntil = int(mktime(strptime(validuntil, "%d %B %Y"))) trafficleft = re.search(r'<td>Download Traffic verfügbar:</td>\s*<td>(.*?)</td>', src).group(1) - self.logDebug("Oron left: " + trafficleft) + self.logDebug("Oron left: " + formatSize(parseFileSize(trafficleft))) trafficleft = int(self.parseTraffic(trafficleft)) premium = True else: @@ -47,8 +48,9 @@ class OronCom(Account): tmp = {"validuntil": validuntil, "trafficleft": trafficleft, "premium" : premium} return tmp - def login(self, user, data, req): + def login(self, req): req.load("http://oron.com/?op=change_lang&lang=german") - page = req.load("http://oron.com/login", post={"login": user, "password": data["password"], "op": "login"}) + page = req.load("http://oron.com/login", post={"login": self.loginname, "password": self.password, "op": "login"}) if r'<b class="err">Login oder Passwort falsch</b>' in page: self.wrongPassword() + diff --git a/module/plugins/accounts/Premium4Me.py b/module/plugins/accounts/Premium4Me.py index de4fdc219..6a52cb61a 100644 --- a/module/plugins/accounts/Premium4Me.py +++ b/module/plugins/accounts/Premium4Me.py @@ -1,23 +1,27 @@ -from module.plugins.Account import Account
+# -*- coding: utf-8 -*-
+from module.plugins.MultiHoster import MultiHoster
-class Premium4Me(Account):
+class Premium4Me(MultiHoster):
__name__ = "Premium4Me"
- __version__ = "0.02"
+ __version__ = "0.10"
__type__ = "account"
__description__ = """Premium4.me account plugin"""
__author_name__ = ("RaNaN", "zoidberg")
__author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz")
- def loadAccountInfo(self, user, req):
+ def loadAccountInfo(self, req):
traffic = req.load("http://premium4.me/api/traffic.php?authcode=%s" % self.authcode)
- account_info = {"trafficleft": int(traffic) / 1024,
- "validuntil": -1}
+ account_info = {"trafficleft": int(traffic) / 1024, "validuntil": -1}
return account_info
- def login(self, user, data, req):
- self.authcode = req.load("http://premium4.me/api/getauthcode.php?username=%s&password=%s" % (user, data["password"])).strip()
-
+ def login(self, req):
+ self.authcode = req.load("http://premium4.me/api/getauthcode.php?username=%s&password=%s" % (self.loginname, self.password)).strip()
+
if "wrong username" in self.authcode:
- self.wrongPassword()
\ No newline at end of file + self.wrongPassword()
+
+ def loadHosterList(self, req):
+ page = req.load("http://premium4.me/api/hosters.php?authcode=%s" % self.authcode)
+ return [x.strip() for x in page.replace("\"", "").split(";")]
\ No newline at end of file diff --git a/module/plugins/accounts/RealdebridCom.py b/module/plugins/accounts/RealdebridCom.py index adbd090db..9460fc815 100644 --- a/module/plugins/accounts/RealdebridCom.py +++ b/module/plugins/accounts/RealdebridCom.py @@ -1,25 +1,35 @@ -from module.plugins.Account import Account
-import xml.dom.minidom as dom
-
-class RealdebridCom(Account):
- __name__ = "RealdebridCom"
- __version__ = "0.41"
- __type__ = "account"
- __description__ = """Real-Debrid.com account plugin"""
- __author_name__ = ("Devirex, Hazzard")
- __author_mail__ = ("naibaf_11@yahoo.de")
-
- def loadAccountInfo(self, user, req):
- page = req.load("http://real-debrid.com/api/account.php")
- xml = dom.parseString(page)
- account_info = {"validuntil": int(xml.getElementsByTagName("expiration")[0].childNodes[0].nodeValue),
- "trafficleft": -1}
-
- return account_info
-
- def login(self, user, data, req):
- page = req.load("https://real-debrid.com/ajax/login.php", get = {"user": user, "pass": data["password"]})
- #page = req.load("https://real-debrid.com/login.html", post={"user": user, "pass": data["password"]}, cookies=True)
-
- if "Your login informations are incorrect" in page:
- self.wrongPassword()
+#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.plugins.MultiHoster import MultiHoster +import xml.dom.minidom as dom + +class RealdebridCom(MultiHoster): + __name__ = "RealdebridCom" + __version__ = "0.5" + __type__ = "account" + __description__ = """Real-Debrid.com account plugin""" + __author_name__ = ("Devirex, Hazzard") + __author_mail__ = ("naibaf_11@yahoo.de") + + def loadAccountInfo(self, req): + page = req.load("http://real-debrid.com/api/account.php") + xml = dom.parseString(page) + account_info = {"validuntil": int(xml.getElementsByTagName("expiration")[0].childNodes[0].nodeValue), + "trafficleft": -1} + + return account_info + + def login(self, req): + page = req.load("https://real-debrid.com/ajax/login.php?user=%s&pass=%s" % (self.loginname, self.password)) + #page = req.load("https://real-debrid.com/login.html", post={"user": user, "pass": data["password"]}, cookies=True) + + if "Your login informations are incorrect" in page: + self.wrongPassword() + + + def loadHosterList(self, req): + https = "https" if self.getConfig("https") else "http" + page = req.load(https + "://real-debrid.com/api/hosters.php").replace("\"","").strip() + + return[x.strip() for x in page.split(",") if x.strip()]
\ No newline at end of file diff --git a/module/plugins/accounts/RyushareCom.py b/module/plugins/accounts/RyushareCom.py index 055680ea0..f734eb11b 100644 --- a/module/plugins/accounts/RyushareCom.py +++ b/module/plugins/accounts/RyushareCom.py @@ -3,7 +3,7 @@ from module.plugins.internal.XFSPAccount import XFSPAccount class RyushareCom(XFSPAccount): __name__ = "RyushareCom" - __version__ = "0.02" + __version__ = "0.03" __type__ = "account" __description__ = """ryushare.com account plugin""" __author_name__ = ("zoidberg", "trance4us") @@ -12,6 +12,7 @@ class RyushareCom(XFSPAccount): MAIN_PAGE = "http://ryushare.com/" def login(self, user, data, req): + req.lastURL = "http://ryushare.com/login.python" html = req.load("http://ryushare.com/login.python", post={"login": user, "password": data["password"], "op": "login"}) if 'Incorrect Login or Password' in html or '>Error<' in html: self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/ShareonlineBiz.py b/module/plugins/accounts/ShareonlineBiz.py index cdc4ebb63..4dd398d6d 100644 --- a/module/plugins/accounts/ShareonlineBiz.py +++ b/module/plugins/accounts/ShareonlineBiz.py @@ -23,35 +23,53 @@ import re class ShareonlineBiz(Account): __name__ = "ShareonlineBiz" - __version__ = "0.23" + __version__ = "0.3" __type__ = "account" __description__ = """share-online.biz account plugin""" - __author_name__ = ("mkaay", "zoidberg") - __author_mail__ = ("mkaay@mkaay.de", "zoidberg@mujmail.cz") + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") - def getUserAPI(self, user, req): - return req.load("http://api.share-online.biz/account.php?username=%s&password=%s&act=userDetails" % (user, self.accounts[user]["password"])) - - def loadAccountInfo(self, user, req): - src = self.getUserAPI(user, req) - + def getUserAPI(self, req): + src = req.load("http://api.share-online.biz/account.php?username=%s&password=%s&act=userDetails" % (self.loginname, self.password)) info = {} for line in src.splitlines(): - if "=" in line: - key, value = line.split("=") - info[key] = value - self.logDebug(info) + key, value = line.split("=") + info[key] = value + return info + + def loadAccountInfo(self, user, req): + try: + info = self.getUserAPI(req) + return {"validuntil": int(info["expire_date"]), "trafficleft": -1, "premium": not info["group"] == "Sammler"} + except: + pass + + #fallback + src = req.load("http://www.share-online.biz/members.php?setlang=en") + validuntil = re.search(r'<td align="left"><b>Package Expire Date:</b></td>\s*<td align="left">(\d+/\d+/\d+)</td>', src) + if validuntil: + validuntil = int(mktime(strptime(validuntil.group(1), "%m/%d/%y"))) + else: + validuntil = -1 - if "dl" in info and info["dl"].lower() != "not_available": - req.cj.setCookie("share-online.biz", "dl", info["dl"]) - if "a" in info and info["a"].lower() != "not_available": - req.cj.setCookie("share-online.biz", "a", info["a"]) - - return {"validuntil": int(info["expire_date"]) if "expire_date" in info else -1, - "trafficleft": -1, - "premium": True if ("dl" in info or "a" in info) and (info["group"] != "Sammler") else False} + acctype = re.search(r'<td align="left" ><b>Your Package:</b></td>\s*<td align="left">\s*<b>(.*?)</b>\s*</td>', src) + if acctype: + if acctype.group(1) == "Collector account (free)": + premium = False + else: + premium = True + + tmp = {"validuntil": validuntil, "trafficleft": -1, "premium": premium} + return tmp def login(self, user, data, req): - src = self.getUserAPI(user, req) - if "EXCEPTION" in src: - self.wrongPassword()
\ No newline at end of file + post_vars = { + "act": "login", + "location": "index.php", + "dieseid": "", + "user": user, + "pass": data["password"], + "login": "Login" + } + req.lastURL = "http://www.share-online.biz/" + req.load("https://www.share-online.biz/login.php", cookies=True, post=post_vars) diff --git a/module/plugins/hooks/AlldebridCom.py b/module/plugins/addons/AlldebridCom.py index f9657ed8c..f9657ed8c 100644 --- a/module/plugins/hooks/AlldebridCom.py +++ b/module/plugins/addons/AlldebridCom.py diff --git a/module/plugins/hooks/BypassCaptcha.py b/module/plugins/addons/BypassCaptcha.py index 24ad17dd8..24ad17dd8 100644 --- a/module/plugins/hooks/BypassCaptcha.py +++ b/module/plugins/addons/BypassCaptcha.py diff --git a/module/plugins/hooks/CaptchaBrotherhood.py b/module/plugins/addons/CaptchaBrotherhood.py index a22a5ee1d..a22a5ee1d 100644 --- a/module/plugins/hooks/CaptchaBrotherhood.py +++ b/module/plugins/addons/CaptchaBrotherhood.py diff --git a/module/plugins/hooks/CaptchaTrader.py b/module/plugins/addons/CaptchaTrader.py index 2b8a50a8e..889fa38ef 100644 --- a/module/plugins/hooks/CaptchaTrader.py +++ b/module/plugins/addons/CaptchaTrader.py @@ -27,7 +27,7 @@ from pycurl import FORM_FILE, LOW_SPEED_TIME from module.network.RequestFactory import getURL, getRequest from module.network.HTTPRequest import BadHeader -from module.plugins.Hook import Hook +from module.plugins.Addon import Addon PYLOAD_KEY = "9f65e7f381c3af2b076ea680ae96b0b7" @@ -44,7 +44,7 @@ class CaptchaTraderException(Exception): def __repr__(self): return "<CaptchaTraderException %s>" % self.err -class CaptchaTrader(Hook): +class CaptchaTrader(Addon): __name__ = "CaptchaTrader" __version__ = "0.14" __description__ = """send captchas to captchatrader.com""" diff --git a/module/plugins/hooks/Checksum.py b/module/plugins/addons/Checksum.py index cb6f4bfe8..cb6f4bfe8 100644 --- a/module/plugins/hooks/Checksum.py +++ b/module/plugins/addons/Checksum.py diff --git a/module/plugins/hooks/ClickAndLoad.py b/module/plugins/addons/ClickAndLoad.py index 97e5cd57d..6d6928557 100644 --- a/module/plugins/hooks/ClickAndLoad.py +++ b/module/plugins/addons/ClickAndLoad.py @@ -21,9 +21,9 @@ import socket import thread -from module.plugins.Hook import Hook +from module.plugins.Addon import Addon -class ClickAndLoad(Hook): +class ClickAndLoad(Addon): __name__ = "ClickAndLoad" __version__ = "0.2" __description__ = """Gives abillity to use jd's click and load. depends on webinterface""" @@ -32,7 +32,7 @@ class ClickAndLoad(Hook): __author_name__ = ("RaNaN", "mkaay") __author_mail__ = ("RaNaN@pyload.de", "mkaay@mkaay.de") - def coreReady(self): + def activate(self): self.port = int(self.core.config['webinterface']['port']) if self.core.config['webinterface']['activated']: try: diff --git a/module/plugins/hooks/DeathByCaptcha.py b/module/plugins/addons/DeathByCaptcha.py index 59ff40ded..59ff40ded 100644 --- a/module/plugins/hooks/DeathByCaptcha.py +++ b/module/plugins/addons/DeathByCaptcha.py diff --git a/module/plugins/hooks/DownloadScheduler.py b/module/plugins/addons/DownloadScheduler.py index 7cadede38..7cadede38 100644 --- a/module/plugins/hooks/DownloadScheduler.py +++ b/module/plugins/addons/DownloadScheduler.py diff --git a/module/plugins/hooks/EasybytezCom.py b/module/plugins/addons/EasybytezCom.py index 21a988555..21a988555 100644 --- a/module/plugins/hooks/EasybytezCom.py +++ b/module/plugins/addons/EasybytezCom.py diff --git a/module/plugins/hooks/Ev0InFetcher.py b/module/plugins/addons/Ev0InFetcher.py index 5941cf38c..608baf217 100644 --- a/module/plugins/hooks/Ev0InFetcher.py +++ b/module/plugins/addons/Ev0InFetcher.py @@ -18,18 +18,18 @@ from module.lib import feedparser from time import mktime, time -from module.plugins.Hook import Hook +from module.plugins.Addon import Addon -class Ev0InFetcher(Hook): +class Ev0InFetcher(Addon): __name__ = "Ev0InFetcher" __version__ = "0.2" __description__ = """checks rss feeds for ev0.in""" __config__ = [("activated", "bool", "Activated", "False"), ("interval", "int", "Check interval in minutes", "10"), ("queue", "bool", "Move new shows directly to Queue", False), - ("shows", "str", "Shows to check for (comma seperated)", ""), + ("shows", "str", "Shows to check for (comma separated)", ""), ("quality", "xvid;x264;rmvb", "Video Format", "xvid"), - ("hoster", "str", "Hoster to use (comma seperated)", "NetloadIn,RapidshareCom,MegauploadCom,HotfileCom")] + ("hoster", "str", "Hoster to use (comma separated)", "NetloadIn,RapidshareCom,MegauploadCom,HotfileCom")] __author_name__ = ("mkaay") __author_mail__ = ("mkaay@mkaay.de") @@ -40,7 +40,7 @@ class Ev0InFetcher(Hook): results = self.core.pluginManager.parseUrls(links) sortedLinks = {} - for url, hoster in results: + for url, hoster in results[0]: if hoster not in sortedLinks: sortedLinks[hoster] = [] sortedLinks[hoster].append(url) diff --git a/module/plugins/hooks/ExpertDecoders.py b/module/plugins/addons/ExpertDecoders.py index 2e66e49ca..2e66e49ca 100644 --- a/module/plugins/hooks/ExpertDecoders.py +++ b/module/plugins/addons/ExpertDecoders.py diff --git a/module/plugins/hooks/ExternalScripts.py b/module/plugins/addons/ExternalScripts.py index 2e77f1dae..00fc7c114 100644 --- a/module/plugins/hooks/ExternalScripts.py +++ b/module/plugins/addons/ExternalScripts.py @@ -14,18 +14,17 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - @author: mkaay - @interface-version: 0.1 + @author: RaNaN """ import subprocess -from os import listdir, access, X_OK, makedirs -from os.path import join, exists, basename +from os import access, X_OK, makedirs +from os.path import basename -from module.plugins.Hook import Hook -from module.utils import save_join +from module.plugins.Addon import Addon +from module.utils.fs import save_join, exists, join, listdir -class ExternalScripts(Hook): +class ExternalScripts(Addon): __name__ = "ExternalScripts" __version__ = "0.21" __description__ = """Run external scripts""" diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/addons/ExtractArchive.py index c789495ca..5f749ed0d 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/addons/ExtractArchive.py @@ -3,8 +3,7 @@ import sys import os -from os import remove, chmod, makedirs -from os.path import exists, basename, isfile, isdir, join +from os.path import basename, isfile, isdir, join from traceback import print_exc from copy import copy @@ -49,11 +48,11 @@ if os.name != "nt": from pwd import getpwnam from grp import getgrnam -from module.utils import save_join, fs_encode -from module.plugins.Hook import Hook, threaded, Expose +from module.utils.fs import save_join, fs_encode, exists, remove, chmod, makedirs +from module.plugins.Addon import Addon, threaded, Expose from module.plugins.internal.AbstractExtractor import ArchiveError, CRCError, WrongPassword -class ExtractArchive(Hook): +class ExtractArchive(Addon): """ Provides: unrarFinished (folder, filename) """ diff --git a/module/plugins/hooks/HotFolder.py b/module/plugins/addons/HotFolder.py index ee1031ad5..d05026448 100644 --- a/module/plugins/hooks/HotFolder.py +++ b/module/plugins/addons/HotFolder.py @@ -26,9 +26,9 @@ from os.path import isfile from shutil import move import time -from module.plugins.Hook import Hook +from module.plugins.Addon import Addon -class HotFolder(Hook): +class HotFolder(Addon): __name__ = "HotFolder" __version__ = "0.1" __description__ = """observe folder and file for changes and add container and links""" diff --git a/module/plugins/hooks/IRCInterface.py b/module/plugins/addons/IRCInterface.py index e2737dc3a..ddaa40613 100644 --- a/module/plugins/hooks/IRCInterface.py +++ b/module/plugins/addons/IRCInterface.py @@ -27,13 +27,13 @@ from time import sleep from traceback import print_exc import re -from module.plugins.Hook import Hook +from module.plugins.Addon import Addon from module.network.RequestFactory import getURL from module.utils import formatSize from pycurl import FORM_FILE -class IRCInterface(Thread, Hook): +class IRCInterface(Thread, Addon): __name__ = "IRCInterface" __version__ = "0.1" __description__ = """connect to irc and let owner perform different tasks""" @@ -52,7 +52,7 @@ class IRCInterface(Thread, Hook): def __init__(self, core, manager): Thread.__init__(self) - Hook.__init__(self, core, manager) + Addon.__init__(self, core, manager) self.setDaemon(True) # self.sm = core.server_methods self.api = core.api #todo, only use api diff --git a/module/plugins/hooks/ImageTyperz.py b/module/plugins/addons/ImageTyperz.py index 59b6334a7..59b6334a7 100644 --- a/module/plugins/hooks/ImageTyperz.py +++ b/module/plugins/addons/ImageTyperz.py diff --git a/module/plugins/hooks/LinkdecrypterCom.py b/module/plugins/addons/LinkdecrypterCom.py index ac939afd9..ac939afd9 100644 --- a/module/plugins/hooks/LinkdecrypterCom.py +++ b/module/plugins/addons/LinkdecrypterCom.py diff --git a/module/plugins/hooks/MergeFiles.py b/module/plugins/addons/MergeFiles.py index 02d343096..48f997681 100644 --- a/module/plugins/hooks/MergeFiles.py +++ b/module/plugins/addons/MergeFiles.py @@ -24,11 +24,11 @@ import traceback from os.path import join from module.utils import save_join, fs_encode -from module.plugins.Hook import Hook +from module.plugins.Addon import Addon BUFFER_SIZE = 4096 -class MergeFiles(Hook): +class MergeFiles(Addon): __name__ = "MergeFiles" __version__ = "0.1" __description__ = "Merges parts splitted with hjsplit" diff --git a/module/plugins/hooks/MultiHome.py b/module/plugins/addons/MultiHome.py index f15148538..af3f55416 100644 --- a/module/plugins/hooks/MultiHome.py +++ b/module/plugins/addons/MultiHome.py @@ -17,10 +17,10 @@ @author: mkaay """ -from module.plugins.Hook import Hook +from module.plugins.Addon import Addon from time import time -class MultiHome(Hook): +class MultiHome(Addon): __name__ = "MultiHome" __version__ = "0.1" __description__ = """ip address changer""" diff --git a/module/plugins/addons/MultiHoster.py b/module/plugins/addons/MultiHoster.py new file mode 100644 index 000000000..05d25b958 --- /dev/null +++ b/module/plugins/addons/MultiHoster.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from types import MethodType + +from module.plugins.MultiHoster import MultiHoster as MultiHosterAccount, normalize +from module.plugins.Addon import Addon, AddEventListener +from module.plugins.PluginManager import PluginTuple + +class MultiHoster(Addon): + __version__ = "0.1" + __internal__ = True + __description__ = "Gives ability to use MultiHoster services. You need to add your account first." + __config__ = [] + __author_mail__ = ("pyLoad Team",) + __author_mail__ = ("support@pyload.org",) + + #TODO: multiple accounts - multihoster / config options + + def init(self): + + # overwritten plugins + self.plugins = {} + + def addHoster(self, account): + + self.logDebug("New MultiHoster %s" % account.__name__) + + pluginMap = {} + for name in self.core.pluginManager.getPlugins("hoster").keys(): + pluginMap[name.lower()] = name + + supported = [] + new_supported = [] + + for hoster in account.getHosterList(): + name = normalize(hoster) + + if name in pluginMap: + supported.append(pluginMap[name]) + else: + new_supported.append(hoster) + + if not supported and not new_supported: + account.logError(_("No Hoster loaded")) + return + + klass = self.core.pluginManager.getPluginClass(account.__name__) + + # inject plugin plugin + account.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(supported))) + for hoster in supported: + self.plugins[hoster] = klass + + account.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) + + # create new regexp + regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) + + # recreate plugin tuple for new regexp + hoster = self.core.pluginManager.getPlugins("hoster") + p = hoster[account.__name__] + new = PluginTuple(p.version, re.compile(regexp), p.deps, p.user, p.path) + hoster[account.__name__] = new + + + + @AddEventListener("accountDeleted") + def refreshAccounts(self, plugin=None, user=None): + + self.plugins = {} + + for name, account in self.core.accountManager.iterAccounts(): + if isinstance(account, MultiHosterAccount) and account.isUsable(): + self.addHoster(account) + + @AddEventListener("accountUpdated") + def refreshAccount(self, plugin, user): + + account = self.core.accountManager.getAccount(plugin, user) + if isinstance(account, MultiHosterAccount) and account.isUsable(): + self.addHoster(account) + + def activate(self): + self.refreshAccounts() + + # new method for plugin manager + def getPlugin(self2, name): + if name in self.plugins: + return self.plugins[name] + return self2.getPluginClass(name) + + pm = self.core.pluginManager + pm.getPlugin = MethodType(getPlugin, pm, object) + + + def deactivate(self): + #restore state + pm = self.core.pluginManager + pm.getPlugin = pm.getPluginClass + diff --git a/module/plugins/hooks/MultishareCz.py b/module/plugins/addons/MultishareCz.py index a00c6cb2b..a00c6cb2b 100644 --- a/module/plugins/hooks/MultishareCz.py +++ b/module/plugins/addons/MultishareCz.py diff --git a/module/plugins/hooks/Premium4Me.py b/module/plugins/addons/Premium4Me.py index fc3ce2343..fc3ce2343 100644 --- a/module/plugins/hooks/Premium4Me.py +++ b/module/plugins/addons/Premium4Me.py diff --git a/module/plugins/hooks/PremiumizeMe.py b/module/plugins/addons/PremiumizeMe.py index 3825e9219..3825e9219 100644 --- a/module/plugins/hooks/PremiumizeMe.py +++ b/module/plugins/addons/PremiumizeMe.py diff --git a/module/plugins/hooks/RealdebridCom.py b/module/plugins/addons/RealdebridCom.py index bd3179673..bd3179673 100644 --- a/module/plugins/hooks/RealdebridCom.py +++ b/module/plugins/addons/RealdebridCom.py diff --git a/module/plugins/hooks/RehostTo.py b/module/plugins/addons/RehostTo.py index b16987f5c..b16987f5c 100644 --- a/module/plugins/hooks/RehostTo.py +++ b/module/plugins/addons/RehostTo.py diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/addons/UpdateManager.py index ce75399c5..5bc6ac447 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/addons/UpdateManager.py @@ -23,11 +23,11 @@ from os import stat from os.path import join, exists from time import time -from module.ConfigParser import IGNORE +from module.plugins.PluginManager import IGNORE from module.network.RequestFactory import getURL -from module.plugins.Hook import threaded, Expose, Hook +from module.plugins.Addon import threaded, Expose, Addon -class UpdateManager(Hook): +class UpdateManager(Addon): __name__ = "UpdateManager" __version__ = "0.12" __description__ = """checks for updates""" @@ -60,6 +60,11 @@ class UpdateManager(Hook): @threaded def periodical(self): + + if self.core.version.endswith("-dev"): + self.logDebug("No update check performed on dev version.") + return + update = self.checkForUpdate() if update: self.info["pyload"] = True @@ -129,10 +134,10 @@ class UpdateManager(Hook): else: type = prefix - plugins = getattr(self.core.pluginManager, "%sPlugins" % type) + plugins = self.core.pluginManager.getPlugins(type) if name in plugins: - if float(plugins[name]["v"]) >= float(version): + if float(plugins[name].version) >= float(version): continue if name in IGNORE or (type, name) in IGNORE: diff --git a/module/plugins/hooks/XFileSharingPro.py b/module/plugins/addons/XFileSharingPro.py index 3981db2d0..3981db2d0 100644 --- a/module/plugins/hooks/XFileSharingPro.py +++ b/module/plugins/addons/XFileSharingPro.py diff --git a/module/plugins/hooks/XMPPInterface.py b/module/plugins/addons/XMPPInterface.py index a96adf524..e8ef1d2ca 100644 --- a/module/plugins/hooks/XMPPInterface.py +++ b/module/plugins/addons/XMPPInterface.py @@ -19,12 +19,12 @@ """ from pyxmpp import streamtls -from pyxmpp.all import JID, Message +from pyxmpp.all import JID, Message, Presence from pyxmpp.jabber.client import JabberClient from pyxmpp.interface import implements from pyxmpp.interfaces import * -from module.plugins.hooks.IRCInterface import IRCInterface +from module.plugins.addons.IRCInterface import IRCInterface class XMPPInterface(IRCInterface, JabberClient): __name__ = "XMPPInterface" @@ -121,7 +121,26 @@ class XMPPInterface(IRCInterface, JabberClient): in a client session.""" return [ ("normal", self.message), - ] + ] + + def presence_control(self, stanza): + from_jid = unicode(stanza.get_from_jid()) + stanza_type = stanza.get_type() + self.log.debug("pyLoad XMPP: %s stanza from %s" % (stanza_type, + from_jid)) + + if from_jid in self.getConfig("owners"): + return stanza.make_accept_response() + + return stanza.make_deny_response() + + def session_started(self): + self.stream.send(Presence()) + + self.stream.set_presence_handler("subscribe", self.presence_control) + self.stream.set_presence_handler("subscribed", self.presence_control) + self.stream.set_presence_handler("unsubscribe", self.presence_control) + self.stream.set_presence_handler("unsubscribed", self.presence_control) def message(self, stanza): """Message handler for the component.""" @@ -248,4 +267,10 @@ class VersionHandler(object): q.newTextChild(q.ns(), "name", "Echo component") q.newTextChild(q.ns(), "version", "1.0") return iq -
\ No newline at end of file + + def unload(self): + self.log.debug("pyLoad XMPP: unloading") + self.disconnect() + + def deactivate(self): + self.unload() diff --git a/module/plugins/hooks/ZeveraCom.py b/module/plugins/addons/ZeveraCom.py index 46c752c21..46c752c21 100644 --- a/module/plugins/hooks/ZeveraCom.py +++ b/module/plugins/addons/ZeveraCom.py diff --git a/module/plugins/captcha/__init__.py b/module/plugins/addons/__init__.py index e69de29bb..e69de29bb 100644 --- a/module/plugins/captcha/__init__.py +++ b/module/plugins/addons/__init__.py diff --git a/module/plugins/captcha/GigasizeCom.py b/module/plugins/captcha/GigasizeCom.py deleted file mode 100644 index d31742eb5..000000000 --- a/module/plugins/captcha/GigasizeCom.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from captcha import OCR - -class GigasizeCom(OCR): - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.threshold(2.8) - self.run_tesser(True, False, False, True) - return self.result_captcha - -if __name__ == '__main__': - ocr = GigasizeCom() - import urllib - urllib.urlretrieve('http://www.gigasize.com/randomImage.php', "gigasize_tmp.jpg") - - print ocr.get_captcha('gigasize_tmp.jpg') diff --git a/module/plugins/captcha/LinksaveIn.py b/module/plugins/captcha/LinksaveIn.py deleted file mode 100644 index 3ad7b265a..000000000 --- a/module/plugins/captcha/LinksaveIn.py +++ /dev/null @@ -1,147 +0,0 @@ -from captcha import OCR -import Image -from os import sep -from os.path import dirname -from os.path import abspath -from glob import glob - - -class LinksaveIn(OCR): - __name__ = "LinksaveIn" - def __init__(self): - OCR.__init__(self) - self.data_dir = dirname(abspath(__file__)) + sep + "LinksaveIn" + sep - - def load_image(self, image): - im = Image.open(image) - frame_nr = 0 - - lut = im.resize((256, 1)) - lut.putdata(range(256)) - lut = list(lut.convert("RGB").getdata()) - - new = Image.new("RGB", im.size) - npix = new.load() - while True: - try: - im.seek(frame_nr) - except EOFError: - break - frame = im.copy() - pix = frame.load() - for x in range(frame.size[0]): - for y in range(frame.size[1]): - if lut[pix[x, y]] != (0,0,0): - npix[x, y] = lut[pix[x, y]] - frame_nr += 1 - new.save(self.data_dir+"unblacked.png") - self.image = new.copy() - self.pixels = self.image.load() - self.result_captcha = '' - - def get_bg(self): - stat = {} - cstat = {} - img = self.image.convert("P") - for bgpath in glob(self.data_dir+"bg/*.gif"): - stat[bgpath] = 0 - bg = Image.open(bgpath) - - bglut = bg.resize((256, 1)) - bglut.putdata(range(256)) - bglut = list(bglut.convert("RGB").getdata()) - - lut = img.resize((256, 1)) - lut.putdata(range(256)) - lut = list(lut.convert("RGB").getdata()) - - bgpix = bg.load() - pix = img.load() - for x in range(bg.size[0]): - for y in range(bg.size[1]): - rgb_bg = bglut[bgpix[x, y]] - rgb_c = lut[pix[x, y]] - try: - cstat[rgb_c] += 1 - except: - cstat[rgb_c] = 1 - if rgb_bg == rgb_c: - stat[bgpath] += 1 - max_p = 0 - bg = "" - for bgpath, value in stat.items(): - if max_p < value: - bg = bgpath - max_p = value - return bg - - def substract_bg(self, bgpath): - bg = Image.open(bgpath) - img = self.image.convert("P") - - bglut = bg.resize((256, 1)) - bglut.putdata(range(256)) - bglut = list(bglut.convert("RGB").getdata()) - - lut = img.resize((256, 1)) - lut.putdata(range(256)) - lut = list(lut.convert("RGB").getdata()) - - bgpix = bg.load() - pix = img.load() - orgpix = self.image.load() - for x in range(bg.size[0]): - for y in range(bg.size[1]): - rgb_bg = bglut[bgpix[x, y]] - rgb_c = lut[pix[x, y]] - if rgb_c == rgb_bg: - orgpix[x, y] = (255,255,255) - - def eval_black_white(self): - new = Image.new("RGB", (140, 75)) - pix = new.load() - orgpix = self.image.load() - thresh = 4 - for x in range(new.size[0]): - for y in range(new.size[1]): - rgb = orgpix[x, y] - r, g, b = rgb - pix[x, y] = (255,255,255) - if r > max(b, g)+thresh: - pix[x, y] = (0,0,0) - if g < min(r, b): - pix[x, y] = (0,0,0) - if g > max(r, b)+thresh: - pix[x, y] = (0,0,0) - if b > max(r, g)+thresh: - pix[x, y] = (0,0,0) - self.image = new - self.pixels = self.image.load() - - def get_captcha(self, image): - self.load_image(image) - bg = self.get_bg() - self.substract_bg(bg) - self.eval_black_white() - self.to_greyscale() - self.image.save(self.data_dir+"cleaned_pass1.png") - self.clean(4) - self.clean(4) - self.image.save(self.data_dir+"cleaned_pass2.png") - letters = self.split_captcha_letters() - final = "" - for n, letter in enumerate(letters): - self.image = letter - self.image.save(ocr.data_dir+"letter%d.png" % n) - self.run_tesser(True, True, False, False) - final += self.result_captcha - - return final - -if __name__ == '__main__': - import urllib - ocr = LinksaveIn() - testurl = "http://linksave.in/captcha/cap.php?hsh=2229185&code=ZzHdhl3UffV3lXTH5U4b7nShXj%2Bwma1vyoNBcbc6lcc%3D" - urllib.urlretrieve(testurl, ocr.data_dir+"captcha.gif") - - print ocr.get_captcha(ocr.data_dir+'captcha.gif') diff --git a/module/plugins/captcha/MegauploadCom.py b/module/plugins/captcha/MegauploadCom.py deleted file mode 100644 index 469ee4094..000000000 --- a/module/plugins/captcha/MegauploadCom.py +++ /dev/null @@ -1,14 +0,0 @@ -from captcha import OCR - -class MegauploadCom(OCR): - __name__ = "MegauploadCom" - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.run_tesser(True, True, False, True) - return self.result_captcha - -if __name__ == '__main__': - ocr = MegauploadCom() diff --git a/module/plugins/container/CCF.py b/module/plugins/container/CCF.py index 301b033d4..ab7ff1099 100644 --- a/module/plugins/container/CCF.py +++ b/module/plugins/container/CCF.py @@ -4,13 +4,13 @@ import re from urllib2 import build_opener -from module.plugins.Container import Container +from module.plugins.Crypter import Crypter from module.lib.MultipartPostHandler import MultipartPostHandler from os import makedirs from os.path import exists, join -class CCF(Container): +class CCF(Crypter): __name__ = "CCF" __version__ = "0.2" __pattern__ = r"(?!http://).*\.ccf$" diff --git a/module/plugins/container/LinkList.py b/module/plugins/container/LinkList.py deleted file mode 100644 index fefeaf486..000000000 --- a/module/plugins/container/LinkList.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import codecs -from module.utils import fs_encode -from module.plugins.Container import Container - -class LinkList(Container): - __name__ = "LinkList" - __version__ = "0.12" - __pattern__ = r".+\.txt$" - __description__ = """Read Link Lists in txt format""" - __config__ = [("clear", "bool", "Clear Linklist after adding", False), - ("encoding", "string", "File encoding (default utf-8)", "")] - __author_name__ = ("spoob", "jeix") - __author_mail__ = ("spoob@pyload.org", "jeix@hasnomail.com") - - def decrypt(self, pyfile): - try: - file_enc = codecs.lookup(self.getConfig("encoding")).name - except: - file_enc = "utf-8" - - print repr(pyfile.url) - print pyfile.url - - file_name = fs_encode(pyfile.url) - - txt = codecs.open(file_name, 'r', file_enc) - links = txt.readlines() - curPack = "Parsed links from %s" % pyfile.name - - packages = {curPack:[],} - - for link in links: - link = link.strip() - if not link: continue - - if link.startswith(";"): - continue - if link.startswith("[") and link.endswith("]"): - # new package - curPack = link[1:-1] - packages[curPack] = [] - continue - packages[curPack].append(link) - txt.close() - - # empty packages fix - - delete = [] - - for key,value in packages.iteritems(): - if not value: - delete.append(key) - - for key in delete: - del packages[key] - - if self.getConfig("clear"): - try: - txt = open(file_name, 'wb') - txt.close() - except: - self.log.warning(_("LinkList could not be cleared.")) - - for name, links in packages.iteritems(): - self.packages.append((name, links, name)) diff --git a/module/plugins/container/RSDF.py b/module/plugins/container/RSDF.py index ea5cd67f2..cbc9864b1 100644 --- a/module/plugins/container/RSDF.py +++ b/module/plugins/container/RSDF.py @@ -5,9 +5,9 @@ import base64 import binascii import re -from module.plugins.Container import Container +from module.plugins.Crypter import Crypter -class RSDF(Container): +class RSDF(Crypter): __name__ = "RSDF" __version__ = "0.21" __pattern__ = r".*\.rsdf" diff --git a/module/plugins/crypter/FilesonicComFolder.py b/module/plugins/crypter/FilesonicComFolder.py index b967a74a1..02ae66295 100644 --- a/module/plugins/crypter/FilesonicComFolder.py +++ b/module/plugins/crypter/FilesonicComFolder.py @@ -4,8 +4,6 @@ import re from module.plugins.Crypter import Crypter class FilesonicComFolder(Crypter): - __name__ = "FilesonicComFolder" - __type__ = "crypter" __pattern__ = r"http://(\w*\.)?(sharingmatrix|filesonic|wupload)\.[^/]*/folder/\w+/?" __version__ = "0.11" __description__ = """Filesonic.com/Wupload.com Folder Plugin""" @@ -15,9 +13,8 @@ class FilesonicComFolder(Crypter): FOLDER_PATTERN = r'<table>\s*<caption>Files Folder</caption>(.*?)</table>' LINK_PATTERN = r'<a href="([^"]+)">' - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - + def decryptURL(self, url): + html = self.load(url) new_links = [] folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) @@ -26,6 +23,7 @@ class FilesonicComFolder(Crypter): new_links.extend(re.findall(self.LINK_PATTERN, folder.group(1))) if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) + return new_links else: - self.fail('Could not extract any links')
\ No newline at end of file + self.fail('Could not extract any links') + diff --git a/module/plugins/crypter/HotfileFolderCom.py b/module/plugins/crypter/HotfileFolderCom.py index 00771e2a3..ea7311e3c 100644 --- a/module/plugins/crypter/HotfileFolderCom.py +++ b/module/plugins/crypter/HotfileFolderCom.py @@ -9,17 +9,21 @@ class HotfileFolderCom(Crypter): __name__ = "HotfileFolderCom" __type__ = "crypter" __pattern__ = r"http://(?:www\.)?hotfile.com/list/\w+/\w+" - __version__ = "0.1" + __version__ = "0.2" __description__ = """HotfileFolder Download Plugin""" __author_name__ = ("RaNaN") __author_mail__ = ("RaNaN@pyload.org") - def decrypt(self, pyfile): - html = self.load(pyfile.url) + def decryptURL(self, url): + html = self.load(url) - name = re.findall(r'<img src="/i/folder.gif" width="23" height="14" style="margin-bottom: -2px;" />([^<]+)', html, re.MULTILINE)[0].replace("/", "") - new_links = re.findall(r'href="(http://(www.)?hotfile\.com/dl/\d+/[0-9a-zA-Z]+[^"]+)', html) + new_links = [] + for link in re.findall(r'href="(http://(www.)?hotfile\.com/dl/\d+/[0-9a-zA-Z]+[^"]+)', html): + new_links.append(link[0]) - new_links = [x[0] for x in new_links] + if new_links: + self.logDebug("Found %d new links" % len(new_links)) + return new_links + else: + self.fail('Could not extract any links') - self.packages.append((name, new_links, name))
\ No newline at end of file diff --git a/module/plugins/crypter/LinkList.py b/module/plugins/crypter/LinkList.py new file mode 100644 index 000000000..ebfa373eb --- /dev/null +++ b/module/plugins/crypter/LinkList.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.plugins.Crypter import Crypter, Package + +class LinkList(Crypter): + __name__ = "LinkList" + __version__ = "0.11" + __pattern__ = r".+\.txt$" + __description__ = """Read Link Lists in txt format""" + __author_name__ = ("spoob", "jeix") + __author_mail__ = ("spoob@pyload.org", "jeix@hasnomail.com") + + # method declaration is needed here + def decryptURL(self, url): + return Crypter.decryptURL(self, url) + + def decryptFile(self, content): + links = content.splitlines() + + curPack = "default" + packages = {curPack:[]} + + for link in links: + link = link.strip() + if not link: continue + + if link.startswith(";"): + continue + if link.startswith("[") and link.endswith("]"): + # new package + curPack = link[1:-1] + packages[curPack] = [] + continue + packages[curPack].append(link) + + # empty packages fix + delete = [] + + for key,value in packages.iteritems(): + if not value: + delete.append(key) + + for key in delete: + del packages[key] + + urls = [] + + for name, links in packages.iteritems(): + if name == "default": + urls.extend(links) + else: + urls.append(Package(name, links)) + + return urls
\ No newline at end of file diff --git a/module/plugins/crypter/OronComFolder.py b/module/plugins/crypter/OronComFolder.py index 57b273163..726371966 100755 --- a/module/plugins/crypter/OronComFolder.py +++ b/module/plugins/crypter/OronComFolder.py @@ -8,25 +8,39 @@ class OronComFolder(Crypter): __name__ = "OronComFolder" __type__ = "crypter" __pattern__ = r"http://(?:www\.)?oron.com/folder/\w+" - __version__ = "0.1" + __version__ = "0.2" __description__ = """Oron.com Folder Plugin""" __author_name__ = ("DHMH") __author_mail__ = ("webmaster@pcProfil.de") - FOLDER_PATTERN = r'<table width="100%" cellpadding="7" cellspacing="1" class="tbl2">(.*)</table>\n </div>' + FOLDER_PATTERN = r'<table(?:.*)class="tbl"(?:.*)>(?:.*)<table(?:.*)class="tbl2"(?:.*)>(?P<body>.*)</table>(?:.*)</table>' LINK_PATTERN = r'<a href="([^"]+)" target="_blank">' - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) + def decryptURL(self, url): + html = self.load(url) new_links = [] + if 'No such folder exist' in html: + # Don't fail because if there's more than a folder for this package + # and only one of them fails, no urls at all will be added. + self.logWarning("Folder does not exist") + return new_links + folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if folder is None: self.fail("Parse error (FOLDER)") + if folder is None: + # Don't fail because if there's more than a folder for this package + # and only one of them fails, no urls at all will be added. + self.logWarning("Parse error (FOLDER)") + return new_links new_links.extend(re.findall(self.LINK_PATTERN, folder.group(0))) if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) + self.logDebug("Found %d new links" % len(new_links)) + return new_links else: - self.fail('Could not extract any links')
\ No newline at end of file + # Don't fail because if there's more than a folder for this package + # and only one of them fails, no urls at all will be added. + self.logWarning('Could not extract any links') + return new_links diff --git a/module/plugins/crypter/XfilesharingProFolder.py b/module/plugins/crypter/XfilesharingProFolder.py new file mode 100644 index 000000000..8e58c207d --- /dev/null +++ b/module/plugins/crypter/XfilesharingProFolder.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*-
+
+from module.plugins.Crypter import Crypter, Package
+import re
+
+class XfilesharingProFolder(Crypter):
+ __name__ = "XfilesharingProFolder"
+ __type__ = "crypter"
+ __pattern__ = r"http://(?:www\.)?((easybytez|turboupload|uploadville|file4safe|fileband|filebeep|grupload|247upload)\.com|(muchshare|annonhost).net|bzlink.us)/users/.*"
+ __version__ = "0.01"
+ __description__ = """Generic XfilesharingPro Folder Plugin"""
+ __author_name__ = ("zoidberg")
+ __author_mail__ = ("zoidberg@mujmail.cz")
+
+ LINK_PATTERN = r'<div class="link"><a href="([^"]+)" target="_blank">[^<]*</a></div>'
+ SUBFOLDER_PATTERN = r'<TD width="1%"><img src="[^"]*/images/folder2.gif"></TD><TD><a href="([^"]+)"><b>(?!\. \.<)([^<]+)</b></a></TD>'
+
+ def decryptURL(self, url):
+ return self.decryptFile(self.load(url, decode = True))
+
+ def decryptFile(self, html):
+ new_links = []
+
+ new_links.extend(re.findall(self.LINK_PATTERN, html))
+
+ subfolders = re.findall(self.SUBFOLDER_PATTERN, html)
+ #self.logDebug(subfolders)
+ for (url, name) in subfolders:
+ if self.package: name = "%s/%s" % (self.package.name, name)
+ new_links.append(Package(name, [url]))
+
+ if not new_links: self.fail('Could not extract any links')
+
+ return new_links
\ No newline at end of file diff --git a/module/plugins/hooks/__init__.py b/module/plugins/hooks/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/module/plugins/hooks/__init__.py +++ /dev/null diff --git a/module/plugins/hoster/BezvadataCz.py b/module/plugins/hoster/BezvadataCz.py index 680bbc173..a0717ad64 100644 --- a/module/plugins/hoster/BezvadataCz.py +++ b/module/plugins/hoster/BezvadataCz.py @@ -23,7 +23,7 @@ class BezvadataCz(SimpleHoster): __name__ = "BezvadataCz" __type__ = "hoster" __pattern__ = r"http://(\w*\.)*bezvadata.cz/stahnout/.*" - __version__ = "0.22" + __version__ = "0.23" __description__ = """BezvaData.cz""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") diff --git a/module/plugins/hoster/DdlstorageCom.py b/module/plugins/hoster/DdlstorageCom.py index 0177d9648..1ad5fa6d8 100644 --- a/module/plugins/hoster/DdlstorageCom.py +++ b/module/plugins/hoster/DdlstorageCom.py @@ -5,12 +5,12 @@ class DdlstorageCom(XFileSharingPro): __name__ = "DdlstorageCom" __type__ = "hoster" __pattern__ = r"http://(?:\w*\.)*?ddlstorage.com/\w{12}" - __version__ = "0.05" + __version__ = "0.06" __description__ = """DDLStorage.com hoster plugin""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - FILE_INFO_PATTERN = r'<h2>Download File\s*<span[^>]*>(?P<N>[^>]+)</span></h2>\s*[^\(]*\((?P<S>[^\)]+)\)</h2>' + FILE_INFO_PATTERN = r'<h2>\s*Download File\s*<span[^>]*>(?P<N>[^>]+)</span></h2>\s*[^\(]*\((?P<S>[^\)]+)\)</h2>' HOSTER_NAME = "ddlstorage.com" def setup(self): diff --git a/module/plugins/hoster/DlFreeFr.py b/module/plugins/hoster/DlFreeFr.py index 8b32e5eb4..5b318fd54 100644 --- a/module/plugins/hoster/DlFreeFr.py +++ b/module/plugins/hoster/DlFreeFr.py @@ -103,7 +103,6 @@ class DlFreeFr(SimpleHoster): #FILE_URL_PATTERN = r'href="(?P<url>http://.*?)">Télécharger ce fichier' def setup(self): - self.multiDL = True self.limitDL = 5 self.resumeDownload = True self.chunkLimit = 1 diff --git a/module/plugins/hoster/EuroshareEu.py b/module/plugins/hoster/EuroshareEu.py index a0bfe0ab2..1e1cc0b4b 100644 --- a/module/plugins/hoster/EuroshareEu.py +++ b/module/plugins/hoster/EuroshareEu.py @@ -37,7 +37,7 @@ class EuroshareEu(Hoster): __name__ = "EuroshareEu" __type__ = "hoster" __pattern__ = r"http://(\w*\.)?euroshare.eu/file/.*" - __version__ = "0.2b" + __version__ = "0.30" __description__ = """Euroshare.eu""" __author_name__ = ("zoidberg") diff --git a/module/plugins/hoster/FilesMailRu.py b/module/plugins/hoster/FilesMailRu.py index 6002ab3dc..1284329b5 100644 --- a/module/plugins/hoster/FilesMailRu.py +++ b/module/plugins/hoster/FilesMailRu.py @@ -2,9 +2,8 @@ # -*- coding: utf-8 -*- import re -from module.plugins.Hoster import Hoster +from module.plugins.Hoster import Hoster, chunks from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks def getInfo(urls): result = [] diff --git a/module/plugins/hoster/FilesonicCom.py b/module/plugins/hoster/FilesonicCom.py index 525a99e7a..b35ce1b1f 100644 --- a/module/plugins/hoster/FilesonicCom.py +++ b/module/plugins/hoster/FilesonicCom.py @@ -7,7 +7,7 @@ from urllib import unquote from module.plugins.Hoster import Hoster
from module.plugins.ReCaptcha import ReCaptcha
-from module.plugins.Plugin import chunks
+from module.utils import chunks
from module.network.RequestFactory import getURL
from module.common.json_layer import json_loads
diff --git a/module/plugins/hoster/HotfileCom.py b/module/plugins/hoster/HotfileCom.py index bf4250767..df652edcc 100644 --- a/module/plugins/hoster/HotfileCom.py +++ b/module/plugins/hoster/HotfileCom.py @@ -6,7 +6,7 @@ from module.plugins.Hoster import Hoster from module.plugins.ReCaptcha import ReCaptcha from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks +from module.utils import chunks def getInfo(urls): api_url_base = "http://api.hotfile.com/" diff --git a/module/plugins/hoster/MegauploadCom.py b/module/plugins/hoster/MegauploadCom.py index 336cbfb58..8693e4303 100644 --- a/module/plugins/hoster/MegauploadCom.py +++ b/module/plugins/hoster/MegauploadCom.py @@ -2,17 +2,13 @@ # -*- coding: utf-8 -*-
import re
-from time import sleep
from module.plugins.Hoster import Hoster
from module.network.RequestFactory import getURL
-from module.network.HTTPRequest import BadHeader
from module.utils import html_unescape
-from module.PyFile import statusMap
-
-from pycurl import error
+from datatypes.PyFile import statusMap
def getInfo(urls):
yield [(url, 0, 1, url) for url in urls]
diff --git a/module/plugins/hoster/MultishareCz.py b/module/plugins/hoster/MultishareCz.py index a0dda30b8..af7aa94cf 100644 --- a/module/plugins/hoster/MultishareCz.py +++ b/module/plugins/hoster/MultishareCz.py @@ -24,7 +24,7 @@ class MultishareCz(SimpleHoster): __name__ = "MultishareCz" __type__ = "hoster" __pattern__ = r"http://(?:\w*\.)?multishare.cz/stahnout/(?P<ID>\d+).*" - __version__ = "0.34" + __version__ = "0.40" __description__ = """MultiShare.cz""" __author_name__ = ("zoidberg") @@ -50,11 +50,12 @@ class MultishareCz(SimpleHoster): self.download("http://www.multishare.cz/html/download_free.php?ID=%s" % self.fileID) def handlePremium(self): - if not self.checkCredit(): + if not self.checkTrafficLeft(): self.logWarning("Not enough credit left to download file") self.resetAccount() self.download("http://www.multishare.cz/html/download_premium.php?ID=%s" % self.fileID) + self.checkTrafficLeft() def handleOverriden(self): if not self.premium: @@ -63,18 +64,13 @@ class MultishareCz(SimpleHoster): self.html = self.load('http://www.multishare.cz/html/mms_ajax.php', post = {"link": self.pyfile.url}, decode = True) self.getFileInfo() - if not self.checkCredit(): + if not self.checkTrafficLeft(): self.fail("Not enough credit left to download file") url = "http://dl%d.mms.multishare.cz/html/mms_process.php" % round(random()*10000*random()) params = {"u_ID" : self.acc_info["u_ID"], "u_hash" : self.acc_info["u_hash"], "link" : self.pyfile.url} self.logDebug(url, params) self.download(url, get = params) - - def checkCredit(self): - self.acc_info = self.account.getAccountInfo(self.user, True) - self.logInfo("User %s has %i MB left" % (self.user, self.acc_info["trafficleft"]/1024)) - - return self.pyfile.size / 1024 <= self.acc_info["trafficleft"] + self.checkTrafficLeft() getInfo = create_getInfo(MultishareCz)
\ No newline at end of file diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py index fa2f3ddef..9310b5c34 100644 --- a/module/plugins/hoster/NetloadIn.py +++ b/module/plugins/hoster/NetloadIn.py @@ -5,14 +5,12 @@ import re from time import sleep, time +from module.utils import chunks from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks - - def getInfo(urls): - ## returns list of tupels (name, size (in bytes), status (see FileDatabase), url) + ## returns list of tuples (name, size (in bytes), status (see FileDatabase), url) apiurl = "http://api.netload.in/info.php?auth=Zf9SnQh9WiReEsb18akjvQGqT0I830e8&bz=1&md5=1&file_id=" @@ -202,7 +200,7 @@ class NetloadIn(Hoster): file_id = re.search('<input name="file_id" type="hidden" value="(.*)" />', page).group(1) if not captchawaited: wait = self.get_wait_time(page) - if i == 0: self.pyfile.waitUntil = time() # dont wait contrary to time on website + if i == 0: self.pyfile.waitUntil = time() # don't wait contrary to time on web site else: self.pyfile.waitUntil = t self.log.info(_("Netload: waiting for captcha %d s.") % (self.pyfile.waitUntil - time())) #self.setWait(wait) diff --git a/module/plugins/hoster/Premium4Me.py b/module/plugins/hoster/Premium4Me.py index d029b3df1..cd47a9e91 100644 --- a/module/plugins/hoster/Premium4Me.py +++ b/module/plugins/hoster/Premium4Me.py @@ -6,7 +6,7 @@ from module.plugins.Hoster import Hoster class Premium4Me(Hoster):
__name__ = "Premium4Me"
- __version__ = "0.03"
+ __version__ = "0.10"
__type__ = "hoster"
__pattern__ = r"http://premium4.me/.*"
diff --git a/module/plugins/hoster/PutlockerCom.py b/module/plugins/hoster/PutlockerCom.py index 4de0ca218..8cfcd4d9e 100644 --- a/module/plugins/hoster/PutlockerCom.py +++ b/module/plugins/hoster/PutlockerCom.py @@ -52,7 +52,7 @@ class PutlockerCom(Hoster): __name__ = "PutlockerCom" __type__ = "hoster" __pattern__ = r'http://(www\.)?putlocker\.com/(file|embed)/[A-Z0-9]+' - __version__ = "0.1" + __version__ = "0.2" __description__ = """Putlocker.Com""" __author_name__ = ("jeix") @@ -121,6 +121,6 @@ class PutlockerCom(Hoster): # if link is None: # self.fail("%s: Plugin broken." % self.__name__) - return self.link.group(1) + return self.link.group(1).replace("&", "&") diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py index 8b31dd42c..6aacd684e 100644 --- a/module/plugins/hoster/RapidshareCom.py +++ b/module/plugins/hoster/RapidshareCom.py @@ -52,7 +52,7 @@ class RapidshareCom(Hoster): __pattern__ = r"https?://[\w\.]*?rapidshare.com/(?:files/(?P<id>\d*?)/(?P<name>[^?]+)|#!download\|(?:\w+)\|(?P<id_new>\d+)\|(?P<name_new>[^|]+))" __version__ = "1.38" __description__ = """Rapidshare.com Download Hoster""" - __config__ = [["server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None"]] + __config__ = [("server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None")] __author_name__ = ("spoob", "RaNaN", "mkaay") __author_mail__ = ("spoob@pyload.org", "ranan@pyload.org", "mkaay@mkaay.de") @@ -129,7 +129,7 @@ class RapidshareCom(Hoster): self.handleFree() def handlePremium(self): - info = self.account.getAccountInfo(self.user, True) + info = self.account.getAccountInfo(True) self.log.debug("%s: Use Premium Account" % self.__name__) url = self.api_data["mirror"] self.download(url, get={"directstart":1}) diff --git a/module/plugins/hoster/RealdebridCom.py b/module/plugins/hoster/RealdebridCom.py index ff4843afd..3c796232e 100644 --- a/module/plugins/hoster/RealdebridCom.py +++ b/module/plugins/hoster/RealdebridCom.py @@ -1,88 +1,88 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import re
-from time import time
-from urllib import quote, unquote
-from random import randrange
-
-from module.utils import parseFileSize, remove_chars
-from module.common.json_layer import json_loads
-from module.plugins.Hoster import Hoster
-
-class RealdebridCom(Hoster):
- __name__ = "RealdebridCom"
- __version__ = "0.49"
- __type__ = "hoster"
-
- __pattern__ = r"https?://.*real-debrid\..*"
- __description__ = """Real-Debrid.com hoster plugin"""
- __author_name__ = ("Devirex, Hazzard")
- __author_mail__ = ("naibaf_11@yahoo.de")
-
- def getFilename(self, url):
- try:
- name = unquote(url.rsplit("/", 1)[1])
- except IndexError:
- name = "Unknown_Filename..."
- if not name or name.endswith(".."): #incomplete filename, append random stuff
- name += "%s.tmp" % randrange(100,999)
- return name
-
- def init(self):
- self.tries = 0
- self.chunkLimit = 3
- self.resumeDownload = True
-
-
- def process(self, pyfile):
- if not self.account:
- self.logError(_("Please enter your Real-debrid account or deactivate this plugin"))
- self.fail("No Real-debrid account provided")
-
- self.log.debug("Real-Debrid: Old URL: %s" % pyfile.url)
- if re.match(self.__pattern__, pyfile.url):
- new_url = pyfile.url
- else:
- password = self.getPassword().splitlines()
- if not password: password = ""
- else: password = password[0]
-
- url = "http://real-debrid.com/ajax/unrestrict.php?lang=en&link=%s&password=%s&time=%s" % (quote(pyfile.url, ""), password, int(time()*1000))
- page = self.load(url)
- data = json_loads(page)
-
- self.logDebug("Returned Data: %s" % data)
-
- if data["error"] != 0:
- if data["message"] == "Your file is unavailable on the hoster.":
- self.offline()
- else:
- self.logWarning(data["message"])
- self.tempOffline()
- else:
- if self.pyfile.name is not None and self.pyfile.name.endswith('.tmp') and data["file_name"]:
- self.pyfile.name = data["file_name"]
- self.pyfile.size = parseFileSize(data["file_size"])
- new_url = data['generated_links'][0][-1]
-
- if self.getConfig("https"):
- new_url = new_url.replace("http://", "https://")
- else:
- new_url = new_url.replace("https://", "http://")
-
- self.log.debug("Real-Debrid: New URL: %s" % new_url)
-
- if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown") or pyfile.name.endswith('..'):
- #only use when name wasnt already set
- pyfile.name = self.getFilename(new_url)
-
- self.download(new_url, disposition=True)
-
- check = self.checkDownload(
- {"error": "<title>An error occured while processing your request</title>"})
-
- if check == "error":
- #usual this download can safely be retried
- self.retry(reason="An error occured while generating link.", wait_time=60)
-
+#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from time import time +from urllib import quote, unquote +from random import randrange + +from module.utils import parseFileSize, remove_chars +from module.common.json_layer import json_loads +from module.plugins.Hoster import Hoster + +class RealdebridCom(Hoster): + __name__ = "RealdebridCom" + __version__ = "0.49" + __type__ = "hoster" + + __pattern__ = r"https?://.*real-debrid\..*" + __description__ = """Real-Debrid.com hoster plugin""" + __author_name__ = ("Devirex, Hazzard") + __author_mail__ = ("naibaf_11@yahoo.de") + + def getFilename(self, url): + try: + name = unquote(url.rsplit("/", 1)[1]) + except IndexError: + name = "Unknown_Filename..." + if not name or name.endswith(".."): #incomplete filename, append random stuff + name += "%s.tmp" % randrange(100,999) + return name + + def init(self): + self.tries = 0 + self.chunkLimit = 3 + self.resumeDownload = True + + + def process(self, pyfile): + if not self.account: + self.logError(_("Please enter your Real-debrid account or deactivate this plugin")) + self.fail("No Real-debrid account provided") + + self.log.debug("Real-Debrid: Old URL: %s" % pyfile.url) + if re.match(self.__pattern__, pyfile.url): + new_url = pyfile.url + else: + password = self.getPassword().splitlines() + if not password: password = "" + else: password = password[0] + + url = "http://real-debrid.com/ajax/unrestrict.php?lang=en&link=%s&password=%s&time=%s" % (quote(pyfile.url, ""), password, int(time()*1000)) + page = self.load(url) + data = json_loads(page) + + self.logDebug("Returned Data: %s" % data) + + if data["error"] != 0: + if data["message"] == "Your file is unavailable on the hoster.": + self.offline() + else: + self.logWarning(data["message"]) + self.tempOffline() + else: + if self.pyfile.name is not None and self.pyfile.name.endswith('.tmp') and data["file_name"]: + self.pyfile.name = data["file_name"] + self.pyfile.size = parseFileSize(data["file_size"]) + new_url = data['generated_links'][0][-1] + + if self.getConfig("https"): + new_url = new_url.replace("http://", "https://") + else: + new_url = new_url.replace("https://", "http://") + + self.log.debug("Real-Debrid: New URL: %s" % new_url) + + if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown") or pyfile.name.endswith('..'): + #only use when name wasnt already set + pyfile.name = self.getFilename(new_url) + + self.download(new_url, disposition=True) + + check = self.checkDownload( + {"error": "<title>An error occured while processing your request</title>"}) + + if check == "error": + #usual this download can safely be retried + self.retry(reason="An error occured while generating link.", wait_time=60) + diff --git a/module/plugins/hoster/TurbobitNet.py b/module/plugins/hoster/TurbobitNet.py index 9de7f9bd0..b3b01c92b 100644 --- a/module/plugins/hoster/TurbobitNet.py +++ b/module/plugins/hoster/TurbobitNet.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- """ + Copyright (C) 2012 pyLoad team + Copyright (C) 2012 JD-Team support@jdownloader.org + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, @@ -17,8 +20,13 @@ """ import re +import random +from urllib import quote +from binascii import hexlify, unhexlify +from Crypto.Cipher import ARC4 -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from module.network.RequestFactory import getURL +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp from module.plugins.ReCaptcha import ReCaptcha from pycurl import HTTPHEADER @@ -27,29 +35,38 @@ class TurbobitNet(SimpleHoster): __name__ = "TurbobitNet" __type__ = "hoster" __pattern__ = r"http://(?:\w*\.)?(turbobit.net|unextfiles.com)/(?:download/free/)?(?P<ID>\w+).*" - __version__ = "0.05" + __version__ = "0.07" __description__ = """Turbobit.net plugin""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - + FILE_INFO_PATTERN = r"<span class='file-icon1[^>]*>(?P<N>[^<]+)</span>\s*\((?P<S>[^\)]+)\)\s*</h1>" #long filenames are shortened - FILE_NAME_PATTERN = r'<meta name="keywords" content="\s*(?P<N>[^,]+)' #full name but missing on page2 - FILE_OFFLINE_PATTERN = r'<h2>File Not Found</h2>' - FILE_URL_REPLACEMENTS = [(r"(?<=http://)([^/]+)", "turbobit.net")] + FILE_NAME_PATTERN = r'<meta name="keywords" content="\s+(?P<N>[^,]+)' #full name but missing on page2 + FILE_OFFLINE_PATTERN = r'<h2>File Not Found</h2>|html\(\'File was not found' + FILE_URL_REPLACEMENTS = [(r"http://(?:\w*\.)?(turbobit.net|unextfiles.com)/(?:download/free/)?(?P<ID>\w+).*", "http://turbobit.net/\g<ID>.html")] SH_COOKIES = [("turbobit.net", "user_lang", "en")] - + CAPTCHA_KEY_PATTERN = r'src="http://api\.recaptcha\.net/challenge\?k=([^"]+)"' DOWNLOAD_URL_PATTERN = r'(?P<url>/download/redirect/[^"\']+)' LIMIT_WAIT_PATTERN = r'<div id="time-limit-text">\s*.*?<span id=\'timeout\'>(\d+)</span>' - CAPTCHA_SRC_PATTERN = r'<img alt="Captcha" src="(.*?)"' + CAPTCHA_SRC_PATTERN = r'<img alt="Captcha" src="(.*?)"' - def handleFree(self): + def handleFree(self): self.url = "http://turbobit.net/download/free/%s" % self.file_info['ID'] - if not '/download/free/' in self.pyfile.url: - self.html = self.load(self.url) - - recaptcha = ReCaptcha(self) + self.html = self.load(self.url) + + rtUpdate = self.getRtUpdate() + + self.solveCaptcha() + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) + self.url = self.getDownloadUrl(rtUpdate) + self.wait() + self.html = self.load(self.url) + self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With:"]) + self.downloadFile() + + def solveCaptcha(self): for i in range(5): found = re.search(self.LIMIT_WAIT_PATTERN, self.html) if found: @@ -57,12 +74,13 @@ class TurbobitNet(SimpleHoster): self.setWait(wait_time, wait_time > 60) self.wait() self.retry() - + action, inputs = self.parseHtmlForm("action='#'") - if not inputs: self.parseError("inputs") + if not inputs: self.parseError("captcha form") self.logDebug(inputs) - + if inputs['captcha_type'] == 'recaptcha': + recaptcha = ReCaptcha(self) found = re.search(self.CAPTCHA_KEY_PATTERN, self.html) captcha_key = found.group(1) if found else '6LcTGLoSAAAAAHCWY9TTIrQfjUlxu6kZlTYP50_c' inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) @@ -70,36 +88,82 @@ class TurbobitNet(SimpleHoster): found = re.search(self.CAPTCHA_SRC_PATTERN, self.html) if not found: self.parseError('captcha') captcha_url = found.group(1) - inputs['captcha_response'] = self.decryptCaptcha(captcha_url) + inputs['captcha_response'] = self.decryptCaptcha(captcha_url) self.logDebug(inputs) self.html = self.load(self.url, post = inputs) - + if not "<div class='download-timer-header'>" in self.html: self.invalidCaptcha() else: self.correctCaptcha() break else: self.fail("Invalid captcha") - + + def getRtUpdate(self): + rtUpdate = self.getStorage("rtUpdate") + if not rtUpdate: + if self.getStorage("version") != self.__version__ or int(self.getStorage("timestamp", 0)) + 86400000 < timestamp(): + # that's right, we are even using jdownloader updates + rtUpdate = getURL("http://update0.jdownloader.org/pluginstuff/tbupdate.js") + rtUpdate = self.decrypt(rtUpdate.splitlines()[1]) + # but we still need to fix the syntax to work with other engines than rhino + rtUpdate = re.sub(r'for each\(var (\w+) in(\[[^\]]+\])\)\{',r'zza=\2;for(var zzi=0;zzi<zza.length;zzi++){\1=zza[zzi];',rtUpdate) + rtUpdate = re.sub(r"for\((\w+)=",r"for(var \1=", rtUpdate) + + self.logDebug("rtUpdate") + self.setStorage("rtUpdate", rtUpdate) + self.setStorage("timestamp", timestamp()) + self.setStorage("version", self.__version__) + else: + self.logError("Unable to download, wait for update...") + self.tempOffline() + + return rtUpdate + + def getDownloadUrl(self, rtUpdate): self.req.http.lastURL = self.url - self.req.http.c.setopt(HTTPHEADER, ["X-Requested-With: XMLHttpRequest"]) - - self.setWait(60, False) - self.wait() - - self.html = self.load("http://turbobit.net/download/getLinkTimeout/" + self.file_info['ID']) - self.downloadFile() + + found = re.search("(/\w+/timeout\.js\?\w+=)([^\"\'<>]+)", self.html) + url = "http://turbobit.net%s%s" % (found.groups() if found else ('/files/timeout.js?ver=', ''.join(random.choice('0123456789ABCDEF') for x in range(32)))) + fun = self.load(url) + + self.setWait(65, False) + + for b in [1,3]: + self.jscode = "var id = \'%s\';var b = %d;var inn = \'%s\';%sout" % (self.file_info['ID'], b, quote(fun), rtUpdate) + + try: + out = self.js.eval(self.jscode) + self.logDebug("URL", self.js.engine, out) + if out.startswith('/download/'): + return "http://turbobit.net%s" % out.strip() + except Exception, e: + self.logError(e) + else: + if self.retries >= 2: + # retry with updated js + self.delStorage("rtUpdate") + self.retry() + + def decrypt(self, data): + cipher = ARC4.new(hexlify('E\x15\xa1\x9e\xa3M\xa0\xc6\xa0\x84\xb6H\x83\xa8o\xa0')) + return unhexlify(cipher.encrypt(unhexlify(data))) + def getLocalTimeString(): + lt = time.localtime() + tz = time.altzone if lt.tm_isdst else time.timezone + return "%s GMT%+03d%02d" % (time.strftime("%a %b %d %Y %H:%M:%S", lt), -tz // 3600, tz % 3600) + def handlePremium(self): self.logDebug("Premium download as user %s" % self.user) self.downloadFile() - + def downloadFile(self): found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if not found: self.parseError("download link") + if not found: self.parseError("download link") self.url = "http://turbobit.net" + found.group('url') self.logDebug(self.url) - self.download(self.url) + self.download(self.url) getInfo = create_getInfo(TurbobitNet)
\ No newline at end of file diff --git a/module/plugins/hoster/UploadedTo.py b/module/plugins/hoster/UploadedTo.py index 19ca4ba9d..3cb1e71ff 100644 --- a/module/plugins/hoster/UploadedTo.py +++ b/module/plugins/hoster/UploadedTo.py @@ -2,11 +2,10 @@ import re -from module.utils import html_unescape, parseFileSize +from module.utils import html_unescape, parseFileSize, chunks from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks from module.plugins.ReCaptcha import ReCaptcha key = "bGhGMkllZXByd2VEZnU5Y2NXbHhYVlZ5cEE1bkEzRUw=".decode('base64') diff --git a/module/plugins/hoster/XFileSharingPro.py b/module/plugins/hoster/XFileSharingPro.py index 7ddbeb280..8e213e9bf 100644 --- a/module/plugins/hoster/XFileSharingPro.py +++ b/module/plugins/hoster/XFileSharingPro.py @@ -34,7 +34,7 @@ class XFileSharingPro(SimpleHoster): __name__ = "XFileSharingPro" __type__ = "hoster" __pattern__ = r"^unmatchable$" - __version__ = "0.09" + __version__ = "0.11" __description__ = """XFileSharingPro common hoster base""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") @@ -52,7 +52,6 @@ class XFileSharingPro(SimpleHoster): RECAPTCHA_URL_PATTERN = r'http://[^"\']+?recaptcha[^"\']+?\?k=([^"\']+)"' CAPTCHA_DIV_PATTERN = r'<b>Enter code.*?<div.*?>(.*?)</div>' ERROR_PATTERN = r'class=["\']err["\'][^>]*>(.*?)</' - #DIRECT_LINK_PATTERN = r'This direct link.*?href=["\'](.*?)["\']' def setup(self): self.__pattern__ = self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] @@ -74,25 +73,29 @@ class XFileSharingPro(SimpleHoster): else: self.fail("Only premium users can download from other hosters with %s" % self.HOSTER_NAME) else: - location = None + try: + self.html = self.load(pyfile.url, cookies = False, decode = True) + self.file_info = self.getFileInfo() + except PluginParseError: + self.file_info = None + + self.req.http.lastURL = self.pyfile.url self.req.http.c.setopt(FOLLOWLOCATION, 0) self.html = self.load(self.pyfile.url, cookies = True, decode = True) self.header = self.req.http.header - self.req.http.c.setopt(FOLLOWLOCATION, 1) - + self.req.http.c.setopt(FOLLOWLOCATION, 1) + + self.location = None found = re.search("Location\s*:\s*(.*)", self.header, re.I) if found and re.match(self.DIRECT_LINK_PATTERN, found.group(1)): - location = found.group(1) - self.html = self.load(pyfile.url, cookies = False, decode = True) - - try: - self.file_info = self.getFileInfo() - except PluginParseError: - pyfile.name = html_unescape(unquote(urlparse(location if location else pyfile.url).path.split("/")[-1])) - - if location: - self.startDownload(location) + self.location = found.group(1).strip() + + if not self.file_info: + pyfile.name = html_unescape(unquote(urlparse(self.location if self.location else pyfile.url).path.split("/")[-1])) + + if self.location: + self.startDownload(self.location) elif self.premium: self.handlePremium() else: diff --git a/module/plugins/hoster/YoutubeCom.py b/module/plugins/hoster/YoutubeCom.py index 222e9bf84..3ba40e937 100644 --- a/module/plugins/hoster/YoutubeCom.py +++ b/module/plugins/hoster/YoutubeCom.py @@ -77,8 +77,8 @@ class YoutubeCom(Hoster): self.logDebug("Found links: %s" % fmt_dict) for fmt in fmt_dict.keys(): if fmt not in self.formats: - self.logDebug("FMT not supported: %s" % fmt) - del fmt_dict[fmt] + self.logDebug("FMT not supported: %s" % fmt) + del fmt_dict[fmt] allowed = lambda x: self.getConfig(self.formats[x][0]) sel = lambda x: self.formats[x][3] #select quality index diff --git a/module/plugins/hoster/ZeveraCom.py b/module/plugins/hoster/ZeveraCom.py index cbedfcb68..8be725d2f 100644 --- a/module/plugins/hoster/ZeveraCom.py +++ b/module/plugins/hoster/ZeveraCom.py @@ -1,108 +1,108 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from module.plugins.Hoster import Hoster
-from module.utils import html_unescape
-from urllib import quote, unquote
-from time import sleep
-
-class ZeveraCom(Hoster):
- __name__ = "ZeveraCom"
- __version__ = "0.20"
- __type__ = "hoster"
- __pattern__ = r"http://zevera.com/.*"
- __description__ = """zevera.com hoster plugin"""
- __author_name__ = ("zoidberg")
- __author_mail__ = ("zoidberg@mujmail.cz")
-
- def setup(self):
- self.resumeDownload = self.multiDL = True
- self.chunkLimit = 1
-
- def process(self, pyfile):
- if not self.account:
- self.logError(_("Please enter your zevera.com account or deactivate this plugin"))
- self.fail("No zevera.com account provided")
-
- self.logDebug("zevera.com: Old URL: %s" % pyfile.url)
-
- if self.account.getAPIData(self.req, cmd = "checklink", olink = pyfile.url) != "Alive":
- self.fail("Offline or not downloadable - contact Zevera support")
-
- header = self.account.getAPIData(self.req, just_header = True, cmd="generatedownloaddirect", olink = pyfile.url)
- if not "location" in header:
- self.fail("Unable to initialize download - contact Zevera support")
-
- self.download(header['location'], disposition = True)
-
- check = self.checkDownload({"error" : 'action="ErrorDownload.aspx'})
- if check == "error":
- self.fail("Error response received - contact Zevera support")
-
- """
- # BitAPI not used - defunct, probably abandoned by Zevera
-
- api_url = "http://zevera.com/API.ashx"
-
- def process(self, pyfile):
- if not self.account:
- self.logError(_("Please enter your zevera.com account or deactivate this plugin"))
- self.fail("No zevera.com account provided")
-
- self.logDebug("zevera.com: Old URL: %s" % pyfile.url)
-
- last_size = retries = 0
- olink = self.pyfile.url #quote(self.pyfile.url.encode('utf_8'))
-
- for i in range(100):
- self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_request', olink = olink)
- self.checkAPIErrors(self.retData)
-
- if self.retData['FileInfo']['StatusID'] == 100:
- break
- elif self.retData['FileInfo']['StatusID'] == 99:
- self.fail('Failed to initialize download (99)')
- else:
- if self.retData['FileInfo']['Progress']['BytesReceived'] <= last_size:
- if retries >= 6:
- self.fail('Failed to initialize download (%d)' % self.retData['FileInfo']['StatusID'] )
- retries += 1
- else:
- retries = 0
-
- last_size = self.retData['FileInfo']['Progress']['BytesReceived']
-
- self.setWait(self.retData['Update_Wait'])
- self.wait()
-
- pyfile.name = self.retData['FileInfo']['RealFileName']
- pyfile.size = self.retData['FileInfo']['FileSizeInBytes']
-
- self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_start', FileID = self.retData['FileInfo']['FileID'])
- self.checkAPIErrors(self.retData)
-
- self.download(self.api_url, get = {
- 'cmd': "open_stream",
- 'login': self.account.loginname,
- 'pass': self.account.password,
- 'FileID': self.retData['FileInfo']['FileID'],
- 'startBytes': 0
- }
- )
-
- def checkAPIErrors(self, retData):
- if not retData:
- self.fail('Unknown API response')
-
- if retData['ErrorCode']:
- self.logError(retData['ErrorCode'], retData['ErrorMessage'])
- #self.fail('ERROR: ' + retData['ErrorMessage'])
-
- if self.pyfile.size / 1024000 > retData['AccountInfo']['AvailableTODAYTrafficForUseInMBytes']:
- self.logWarning("Not enough data left to download the file")
-
- def crazyDecode(self, ustring):
- # accepts decoded ie. unicode string - API response is double-quoted, double-utf8-encoded
- # no idea what the proper order of calling these functions would be :-/
- return html_unescape(unquote(unquote(ustring.replace('@DELIMITER@','#'))).encode('raw_unicode_escape').decode('utf-8'))
+#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.plugins.Hoster import Hoster +from module.utils import html_unescape +from urllib import quote, unquote +from time import sleep + +class ZeveraCom(Hoster): + __name__ = "ZeveraCom" + __version__ = "0.20" + __type__ = "hoster" + __pattern__ = r"http://zevera.com/.*" + __description__ = """zevera.com hoster plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + def setup(self): + self.resumeDownload = self.multiDL = True + self.chunkLimit = 1 + + def process(self, pyfile): + if not self.account: + self.logError(_("Please enter your zevera.com account or deactivate this plugin")) + self.fail("No zevera.com account provided") + + self.logDebug("zevera.com: Old URL: %s" % pyfile.url) + + if self.account.getAPIData(self.req, cmd = "checklink", olink = pyfile.url) != "Alive": + self.fail("Offline or not downloadable - contact Zevera support") + + header = self.account.getAPIData(self.req, just_header = True, cmd="generatedownloaddirect", olink = pyfile.url) + if not "location" in header: + self.fail("Unable to initialize download - contact Zevera support") + + self.download(header['location'], disposition = True) + + check = self.checkDownload({"error" : 'action="ErrorDownload.aspx'}) + if check == "error": + self.fail("Error response received - contact Zevera support") + + """ + # BitAPI not used - defunct, probably abandoned by Zevera + + api_url = "http://zevera.com/API.ashx" + + def process(self, pyfile): + if not self.account: + self.logError(_("Please enter your zevera.com account or deactivate this plugin")) + self.fail("No zevera.com account provided") + + self.logDebug("zevera.com: Old URL: %s" % pyfile.url) + + last_size = retries = 0 + olink = self.pyfile.url #quote(self.pyfile.url.encode('utf_8')) + + for i in range(100): + self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_request', olink = olink) + self.checkAPIErrors(self.retData) + + if self.retData['FileInfo']['StatusID'] == 100: + break + elif self.retData['FileInfo']['StatusID'] == 99: + self.fail('Failed to initialize download (99)') + else: + if self.retData['FileInfo']['Progress']['BytesReceived'] <= last_size: + if retries >= 6: + self.fail('Failed to initialize download (%d)' % self.retData['FileInfo']['StatusID'] ) + retries += 1 + else: + retries = 0 + + last_size = self.retData['FileInfo']['Progress']['BytesReceived'] + + self.setWait(self.retData['Update_Wait']) + self.wait() + + pyfile.name = self.retData['FileInfo']['RealFileName'] + pyfile.size = self.retData['FileInfo']['FileSizeInBytes'] + + self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_start', FileID = self.retData['FileInfo']['FileID']) + self.checkAPIErrors(self.retData) + + self.download(self.api_url, get = { + 'cmd': "open_stream", + 'login': self.account.loginname, + 'pass': self.account.password, + 'FileID': self.retData['FileInfo']['FileID'], + 'startBytes': 0 + } + ) + + def checkAPIErrors(self, retData): + if not retData: + self.fail('Unknown API response') + + if retData['ErrorCode']: + self.logError(retData['ErrorCode'], retData['ErrorMessage']) + #self.fail('ERROR: ' + retData['ErrorMessage']) + + if self.pyfile.size / 1024000 > retData['AccountInfo']['AvailableTODAYTrafficForUseInMBytes']: + self.logWarning("Not enough data left to download the file") + + def crazyDecode(self, ustring): + # accepts decoded ie. unicode string - API response is double-quoted, double-utf8-encoded + # no idea what the proper order of calling these functions would be :-/ + return html_unescape(unquote(unquote(ustring.replace('@DELIMITER@','#'))).encode('raw_unicode_escape').decode('utf-8')) """
\ No newline at end of file diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index 2130f910e..3cd635eff 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -13,7 +13,7 @@ class WrongPassword(Exception): class AbtractExtractor: @staticmethod def checkDeps(): - """ Check if system statisfy dependencies + """ Check if system satisfies dependencies :return: boolean """ return True @@ -21,7 +21,7 @@ class AbtractExtractor: @staticmethod def getTargets(files_ids): """ Filter suited targets from list of filename id tuple list - :param files_ids: List of filepathes + :param files_ids: List of file paths :return: List of targets, id tuple list """ raise NotImplementedError @@ -30,10 +30,10 @@ class AbtractExtractor: def __init__(self, m, file, out, fullpath, overwrite, renice): """Initialize extractor for specific file - :param m: ExtractArchive Hook plugin - :param file: Absolute filepath + :param m: ExtractArchive addon plugin + :param file: Absolute file path :param out: Absolute path to destination directory - :param fullpath: extract to fullpath + :param fullpath: Extract to fullpath :param overwrite: Overwrite existing archives :param renice: Renice value """ @@ -52,7 +52,7 @@ class AbtractExtractor: def checkArchive(self): - """Check if password if needed. Raise ArchiveError if integrity is + """Check if password is needed. Raise ArchiveError if integrity is questionable. :return: boolean diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py deleted file mode 100644 index e9e321c06..000000000 --- a/module/plugins/internal/MultiHoster.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.utils import remove_chars -from module.plugins.Hook import Hook - -class MultiHoster(Hook): - """ - Generic MultiHoster plugin - """ - - __version__ = "0.12" - - interval = 0 - hosters = [] - replacements = [] - supported = [] - ignored = [] - - def getHosterCached(self): - if not self.hosters: - - try: - self.hosters = [x.strip() for x in self.getHoster()] - self.hosters = filter(lambda x: x and x not in self.ignored, self.hosters) - except Exception, e: - self.logError("%s" % str(e)) - return [] - - for rep in self.replacements: - if rep[0] in self.hosters: - self.hosters.remove(rep[0]) - if rep[1] not in self.hosters: - self.hosters.append(rep[1]) - - return self.hosters - - - def getHoster(self): - """Load list of supported hoster - - :return: List of domain names - """ - raise NotImplementedError - - def coreReady(self): - pluginMap = {} - for name in self.core.pluginManager.hosterPlugins.keys(): - pluginMap[name.lower()] = name - - new_supported = [] - - for hoster in self.getHosterCached(): - name = remove_chars(hoster.lower(), "-.") - - if name in pluginMap: - self.supported.append(pluginMap[name]) - else: - new_supported.append(hoster) - - if not self.supported and not new_supported: - self.logError(_("No Hoster loaded")) - return - - module = self.core.pluginManager.getPlugin(self.__name__) - klass = getattr(module, self.__name__) - - # inject plugin plugin - self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) - for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - dict["new_module"] = module - dict["new_name"] = self.__name__ - - self.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) - - # create new regexp - if not klass.__pattern__: - regexp = r".*(%s).*" % "|".join([x.replace(".", "\\.") for x in new_supported]) - else: - regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) - - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict["pattern"] = regexp - dict["re"] = re.compile(regexp) - - - def unload(self): - for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - if "module" in dict: - del dict["module"] - - del dict["new_module"] - del dict["new_name"] diff --git a/module/plugins/captcha/NetloadIn.py b/module/plugins/internal/NetloadInOCR.py index 7f2e6a8d1..e50978701 100644 --- a/module/plugins/captcha/NetloadIn.py +++ b/module/plugins/internal/NetloadInOCR.py @@ -1,7 +1,10 @@ -from captcha import OCR +# -*- coding: utf-8 -*- + +from OCR import OCR + +class NetloadInOCR(OCR): + __version__ = 0.1 -class NetloadIn(OCR): - __name__ = "NetloadIn" def __init__(self): OCR.__init__(self) @@ -18,7 +21,7 @@ class NetloadIn(OCR): if __name__ == '__main__': import urllib - ocr = NetloadIn() + ocr = NetloadInOCR() urllib.urlretrieve("http://netload.in/share/includes/captcha.php", "captcha.png") print ocr.get_captcha('captcha.png') diff --git a/module/plugins/captcha/captcha.py b/module/plugins/internal/OCR.py index 4cbb736c1..9f8b7ef8c 100644 --- a/module/plugins/captcha/captcha.py +++ b/module/plugins/internal/OCR.py @@ -33,8 +33,7 @@ import JpegImagePlugin class OCR(object): - - __name__ = "OCR" + __version__ = 0.1 def __init__(self): self.logger = logging.getLogger("log") diff --git a/module/plugins/captcha/ShareonlineBiz.py b/module/plugins/internal/ShareonlineBizOCR.py index b07fb9b0f..c5c2e92e8 100644 --- a/module/plugins/captcha/ShareonlineBiz.py +++ b/module/plugins/internal/ShareonlineBizOCR.py @@ -17,10 +17,10 @@ # along with this program; if not, see <http://www.gnu.org/licenses/>. # ### -from captcha import OCR +from OCR import OCR -class ShareonlineBiz(OCR): - __name__ = "ShareonlineBiz" +class ShareonlineBizOCR(OCR): + __version__ = 0.1 def __init__(self): OCR.__init__(self) @@ -48,6 +48,6 @@ class ShareonlineBiz(OCR): if __name__ == '__main__': import urllib - ocr = ShareonlineBiz() + ocr = ShareonlineBizOCR() urllib.urlretrieve("http://www.share-online.biz/captcha.php", "captcha.jpeg") print ocr.get_captcha('captcha.jpeg') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 566615120..5056b22b2 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -141,7 +141,7 @@ class SimpleHoster(Hoster): or FILE_NAME_INFO = r'(?P<N>file_name)' and FILE_SIZE_INFO = r'(?P<S>file_size) (?P<U>units)' FILE_OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN = r'Server maintainance' + TEMP_OFFLINE_PATTERN = r'Server maintenance' """ FILE_SIZE_REPLACEMENTS = [] diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 240dc0233..53995a083 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -19,11 +19,10 @@ import os import re -from os.path import join from glob import glob from subprocess import Popen, PIPE -from module.utils import save_join, decode +from module.utils.fs import save_join, decode, fs_encode from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError class UnRar(AbtractExtractor): @@ -39,7 +38,7 @@ class UnRar(AbtractExtractor): @staticmethod def checkDeps(): if os.name == "nt": - UnRar.CMD = join(pypath, "UnRAR.exe") + UnRar.CMD = save_join(pypath, "UnRAR.exe") p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) p.communicate() else: @@ -80,7 +79,7 @@ class UnRar(AbtractExtractor): self.password = "" #save the correct password def checkArchive(self): - p = self.call_unrar("l", "-v", self.file) + p = self.call_unrar("l", "-v", fs_encode(self.file)) out, err = p.communicate() if self.re_wrongpwd.search(err): self.passwordProtected = True @@ -102,7 +101,7 @@ class UnRar(AbtractExtractor): def checkPassword(self, password): #at this point we can only verify header protected files if self.headerProtected: - p = self.call_unrar("l", "-v", self.file, password=password) + p = self.call_unrar("l", "-v", fs_encode(self.file), password=password) out, err = p.communicate() if self.re_wrongpwd.search(err): return False @@ -115,7 +114,7 @@ class UnRar(AbtractExtractor): # popen thinks process is still alive (just like pexpect) - very strange behavior # so for now progress can not be determined correctly - p = self.call_unrar(command, self.file, self.out, password=password) + p = self.call_unrar(command, fs_encode(self.file), self.out, password=password) renice(p.pid, self.renice) progress(0) @@ -143,7 +142,7 @@ class UnRar(AbtractExtractor): def listContent(self): command = "vb" if self.fullpath else "lb" - p = self.call_unrar(command, "-v", self.file, password=self.password) + p = self.call_unrar(command, "-v", fs_encode(self.file), password=self.password) out, err = p.communicate() if "Cannot open" in err: @@ -178,7 +177,7 @@ class UnRar(AbtractExtractor): #NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.m.logDebug(" ".join(call)) + self.m.logDebug(" ".join([decode(arg) for arg in call])) p = Popen(call, stdout=PIPE, stderr=PIPE) |