From ef4bc4b73756565e40c7453f6b71bc1021735033 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 19:38:25 +0100 Subject: Revert plugins to stable --- module/plugins/Account.py | 159 ++++++++--------- module/plugins/AccountManager.py | 152 ++++++++-------- module/plugins/Container.py | 71 ++++---- module/plugins/Crypter.py | 133 +++++--------- module/plugins/Hook.py | 87 +++++---- module/plugins/Hoster.py | 32 ++-- module/plugins/Plugin.py | 376 +++++++++++++-------------------------- module/plugins/PluginManager.py | 263 ++++++++++++++------------- 8 files changed, 553 insertions(+), 720 deletions(-) diff --git a/module/plugins/Account.py b/module/plugins/Account.py index 2f7c5d30d..c147404e0 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -1,14 +1,30 @@ # -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + from random import choice from time import time from traceback import print_exc from threading import RLock -from module.plugins.Plugin import Base +from Plugin import Base from module.utils import compare_time, parseFileSize, lock - class WrongPassword(Exception): pass @@ -19,19 +35,17 @@ class Account(Base): Just overwrite `login` and cookies will be stored and account becomes accessible in\ associated hoster plugin. Plugin should also provide `loadAccountInfo` """ - __name__ = "Account" - __type__ = "account" - __version__ = "0.03" - - __description__ = """Base account plugin""" - __license__ = "GPLv3" - __authors__ = [("mkaay", "mkaay@mkaay.de")] + __name__ = "Account" + __version__ = "0.2" + __type__ = "account" + __description__ = """Account Plugin""" + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") - - #: after that time (in minutes) pyload will relogin the account - login_timeout = 10 * 60 - #: after that time (in minutes) account data will be reloaded - info_threshold = 10 * 60 + #: after that time [in minutes] pyload will relogin the account + login_timeout = 600 + #: account data will be reloaded after this time + info_threshold = 600 def __init__(self, manager, accounts): @@ -39,19 +53,16 @@ class Account(Base): self.manager = manager self.accounts = {} - self.infos = {} #: cache for account information + self.infos = {} # cache for account information self.lock = RLock() - self.timestamps = {} - - self.init() + self.timestamps = {} self.setAccounts(accounts) - + self.init() def init(self): pass - def login(self, user, data, req): """login into account, the cookies will be saved so user can be recognized @@ -61,34 +72,29 @@ class Account(Base): """ pass - @lock def _login(self, user, data): # set timestamp for login self.timestamps[user] = time() - + req = self.getAccountRequest(user) try: self.login(user, data, req) except WrongPassword: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user, - "msg": _("Wrong Password")}) - success = data['valid'] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": user + , "msg": _("Wrong Password")}) + data["valid"] = False + except Exception, e: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user, - "msg": e}) - success = data['valid'] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": user + , "msg": e}) + data["valid"] = False if self.core.debug: print_exc() - else: - success = True finally: - if req: - req.close() - return success - + if req: req.close() def relogin(self, user): req = self.getAccountRequest(user) @@ -98,8 +104,7 @@ class Account(Base): if user in self.infos: del self.infos[user] #delete old information - return self._login(user, self.accounts[user]) - + self._login(user, self.accounts[user]) def setAccounts(self, accounts): self.accounts = accounts @@ -107,26 +112,24 @@ class Account(Base): self._login(user, data) self.infos[user] = {} - def updateAccounts(self, user, password=None, options={}): """ updates account and return true if anything changed """ if user in self.accounts: - self.accounts[user]['valid'] = True #do not remove or accounts will not login + self.accounts[user]["valid"] = True #do not remove or accounts will not login if password: - self.accounts[user]['password'] = password + self.accounts[user]["password"] = password self.relogin(user) return True if options: - before = self.accounts[user]['options'] - self.accounts[user]['options'].update(options) - return self.accounts[user]['options'] != before + before = self.accounts[user]["options"] + self.accounts[user]["options"].update(options) + return self.accounts[user]["options"] != before else: self.accounts[user] = {"password": password, "options": options, "valid": True} self._login(user, self.accounts[user]) return True - def removeAccount(self, user): if user in self.accounts: del self.accounts[user] @@ -135,7 +138,6 @@ class Account(Base): if user in self.timestamps: del self.timestamps[user] - @lock def getAccountInfo(self, name, force=False): """retrieve account infos for an user, do **not** overwrite this method!\\ @@ -157,14 +159,12 @@ class Account(Base): raise Exception("Wrong return format") except Exception, e: infos = {"error": str(e)} - print_exc() - if req: - req.close() + if req: req.close() - self.logDebug("Account Info: %s" % infos) + self.logDebug("Account Info: %s" % str(infos)) - infos['timestamp'] = time() + infos["timestamp"] = time() self.infos[name] = infos elif "timestamp" in self.infos[name] and self.infos[name][ "timestamp"] + self.info_threshold * 60 < time(): @@ -174,11 +174,9 @@ class Account(Base): data.update(self.infos[name]) return data - def isPremium(self, user): info = self.getAccountInfo(user) - return info['premium'] - + return info["premium"] def loadAccountInfo(self, name, req=None): """this should be overwritten in account plugin,\ @@ -188,22 +186,22 @@ class Account(Base): :param req: `Request` instance :return: """ - return {"validuntil": None, #: -1 for unlimited - "login": name, - # "password": self.accounts[name]['password'], #: commented due security reason - "options": self.accounts[name]['options'], - "valid": self.accounts[name]['valid'], - "trafficleft": None, #: in kb, -1 for unlimited - "maxtraffic": None, - "premium": None, - "timestamp": 0, #: time this info was retrieved - "type": self.__name__} - + return { + "validuntil": None, # -1 for unlimited + "login": name, + #"password": self.accounts[name]["password"], #@XXX: security + "options": self.accounts[name]["options"], + "valid": self.accounts[name]["valid"], + "trafficleft": None, # in kb, -1 for unlimited + "maxtraffic": None, + "premium": True, #useful for free accounts + "timestamp": 0, #time this info was retrieved + "type": self.__name__, + } def getAllAccounts(self, force=False): return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] - def getAccountRequest(self, user=None): if not user: user, data = self.selectAccount() @@ -213,7 +211,6 @@ class Account(Base): req = self.core.requestFactory.getRequest(self.__name__, user) return req - def getAccountCookies(self, user=None): if not user: user, data = self.selectAccount() @@ -223,21 +220,19 @@ class Account(Base): cj = self.core.requestFactory.getCookieJar(self.__name__, user) return cj - def getAccountData(self, user): return self.accounts[user] - def selectAccount(self): """ returns an valid account name and data""" usable = [] for user, data in self.accounts.iteritems(): - if not data['valid']: continue + if not data["valid"]: continue - if "time" in data['options'] and data['options']['time']: + if "time" in data["options"] and data["options"]["time"]: time_data = "" try: - time_data = data['options']['time'][0] + time_data = data["options"]["time"][0] start, end = time_data.split("-") if not compare_time(start.split(":"), end.split(":")): continue @@ -246,10 +241,10 @@ class Account(Base): if user in self.infos: if "validuntil" in self.infos[user]: - if self.infos[user]['validuntil'] > 0 and time() > self.infos[user]['validuntil']: + if self.infos[user]["validuntil"] > 0 and time() > self.infos[user]["validuntil"]: continue if "trafficleft" in self.infos[user]: - if self.infos[user]['trafficleft'] == 0: + if self.infos[user]["trafficleft"] == 0: continue usable.append((user, data)) @@ -257,19 +252,15 @@ class Account(Base): if not usable: return None, None return choice(usable) - def canUse(self): return False if self.selectAccount() == (None, None) else True - def parseTraffic(self, string): #returns kbyte - return parseFileSize(string) - + return parseFileSize(string) / 1024 def wrongPassword(self): raise WrongPassword - def empty(self, user): if user in self.infos: self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) @@ -277,7 +268,6 @@ class Account(Base): self.infos[user].update({"trafficleft": 0}) self.scheduleRefresh(user, 30 * 60) - def expired(self, user): if user in self.infos: self.logWarning(_("Account %s is expired, checking again in 1h") % user) @@ -285,21 +275,18 @@ class Account(Base): self.infos[user].update({"validuntil": time() - 1}) self.scheduleRefresh(user, 60 * 60) - def scheduleRefresh(self, user, time=0, force=True): """ add task to refresh account info to sheduler """ self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) - @lock def checkLogin(self, user): """ checks if user is still logged in """ if user in self.timestamps: - if self.login_timeout > 0 and self.timestamps[user] + self.login_timeout * 60 < time(): + if self.timestamps[user] + self.login_timeout * 60 < time(): self.logDebug("Reached login timeout for %s" % user) - return self.relogin(user) - else: - return True - else: - return False + self.relogin(user) + return False + + return True diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index 3cb14403a..fc521d36c 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -1,4 +1,21 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" from os.path import exists from shutil import copy @@ -10,11 +27,9 @@ from module.utils import chmod, lock ACC_VERSION = 1 - class AccountManager(): """manages all accounts""" - #---------------------------------------------------------------------- def __init__(self, core): """Constructor""" @@ -25,7 +40,6 @@ class AccountManager(): self.initPlugins() self.saveAccounts() # save to add categories to conf - def initPlugins(self): self.accounts = {} # key = ( plugin ) self.plugins = {} @@ -36,110 +50,97 @@ class AccountManager(): def getAccountPlugin(self, plugin): """get account instance for plugin or None if anonymous""" - try: - if plugin in self.accounts: - if plugin not in self.plugins: - klass = self.core.pluginManager.loadClass("accounts", plugin) - if klass: - self.plugins[plugin] = klass(self, self.accounts[plugin]) - else: - raise - - return self.plugins[plugin] - else: - raise - except: - return None + if plugin in self.accounts: + if plugin not in self.plugins: + self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin]) + return self.plugins[plugin] + else: + return None def getAccountPlugins(self): """ get all account instances""" - + plugins = [] for plugin in self.accounts.keys(): plugins.append(self.getAccountPlugin(plugin)) - + return plugins - - #---------------------------------------------------------------------- def loadAccounts(self): """loads all accounts available""" - - try: - with open("accounts.conf", "a+") as f: - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f.seek(0) - f.write("version: " + str(ACC_VERSION)) - - self.core.log.warning(_("Account settings deleted, due to new config format")) - return - - except IOError, e: - self.logError(e) + + if not exists("accounts.conf"): + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION)) + f.close() + + f = open("accounts.conf", "rb") + content = f.readlines() + version = content[0].split(":")[1].strip() if content else "" + f.close() + + if not version or int(version) < ACC_VERSION: + copy("accounts.conf", "accounts.backup") + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION)) + f.close() + self.core.log.warning(_("Account settings deleted, due to new config format.")) return - + + + plugin = "" name = "" - + for line in content[1:]: line = line.strip() - + if not line: continue if line.startswith("#"): continue if line.startswith("version"): continue - + if line.endswith(":") and line.count(":") == 1: plugin = line[:-1] self.accounts[plugin] = {} - + elif line.startswith("@"): try: option = line[1:].split() - self.accounts[plugin][name]['options'][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) + self.accounts[plugin][name]["options"][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) except: pass - + elif ":" in line: name, sep, pw = line.partition(":") self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - - #---------------------------------------------------------------------- def saveAccounts(self): """save all account information""" - - try: - with open("accounts.conf", "wb") as f: - f.write("version: " + str(ACC_VERSION) + "\n") - - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin+":\n") - - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data['password']) ) - if data['options']: - for option, values in data['options'].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) - - chmod(f.name, 0600) - - except Exception, e: - self.logError(e) - - + + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION) + "\n") + + for plugin, accounts in self.accounts.iteritems(): + f.write("\n") + f.write(plugin+":\n") + + for name,data in accounts.iteritems(): + f.write("\n\t%s:%s\n" % (name,data["password"]) ) + if data["options"]: + for option, values in data["options"].iteritems(): + f.write("\t@%s %s\n" % (option, " ".join(values))) + + f.close() + chmod(f.name, 0600) + + #---------------------------------------------------------------------- def initAccountPlugins(self): """init names""" for name in self.core.pluginManager.getAccountPlugins(): self.accounts[name] = {} - - + @lock def updateAccount(self, plugin , user, password=None, options={}): """add or update account""" @@ -147,22 +148,20 @@ class AccountManager(): p = self.getAccountPlugin(plugin) updated = p.updateAccounts(user, password, options) #since accounts is a ref in plugin self.accounts doesnt need to be updated here - + self.saveAccounts() if updated: p.scheduleRefresh(user, force=False) - - + @lock def removeAccount(self, plugin, user): """remove account""" - + if plugin in self.accounts: p = self.getAccountPlugin(plugin) p.removeAccount(user) self.saveAccounts() - @lock def getAccountInfos(self, force=True, refresh=False): data = {} @@ -170,7 +169,7 @@ class AccountManager(): if refresh: self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) force = False - + for p in self.accounts.keys(): if self.accounts[p]: p = self.getAccountPlugin(p) @@ -180,8 +179,7 @@ class AccountManager(): e = AccountUpdateEvent() self.core.pullManager.addEvent(e) return data - - + def sendChange(self): e = AccountUpdateEvent() self.core.pullManager.addEvent(e) diff --git a/module/plugins/Container.py b/module/plugins/Container.py index 913c80481..c233d3710 100644 --- a/module/plugins/Container.py +++ b/module/plugins/Container.py @@ -1,25 +1,36 @@ # -*- coding: utf-8 -*- -import re - -from os import remove -from os.path import basename, exists +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" from module.plugins.Crypter import Crypter -from module.utils import save_join +from os.path import join, exists, basename +from os import remove +import re class Container(Crypter): - __name__ = "Container" - __type__ = "container" - __version__ = "0.01" - + __name__ = "Container" + __version__ = "0.1" __pattern__ = None - __config__ = [] #: [("name", "type", "desc", "default")] - - __description__ = """Base container decrypter plugin""" - __license__ = "GPLv3" - __authors__ = [("mkaay", "mkaay@mkaay.de")] + __type__ = "container" + __description__ = """Base container plugin""" + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") def preprocessing(self, thread): @@ -27,38 +38,38 @@ class Container(Crypter): self.setup() self.thread = thread - + self.loadToDisk() self.decrypt(self.pyfile) self.deleteTmp() - + self.createPackages() - + def loadToDisk(self): - """loads container to disk if its stored remotely and overwrite url, + """loads container to disk if its stored remotely and overwrite url, or check existent on several places at disk""" - + if self.pyfile.url.startswith("http"): self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] content = self.load(self.pyfile.url) - self.pyfile.url = save_join(self.config['general']['download_folder'], self.pyfile.name) - try: - with open(self.pyfile.url, "wb") as f: - f.write(content) - except IOError, e: - self.fail(str(e)) - + self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name) + f = open(self.pyfile.url, "wb" ) + f.write(content) + f.close() + else: self.pyfile.name = basename(self.pyfile.url) if not exists(self.pyfile.url): - if exists(save_join(pypath, self.pyfile.url)): - self.pyfile.url = save_join(pypath, self.pyfile.url) + if exists(join(pypath, self.pyfile.url)): + self.pyfile.url = join(pypath, self.pyfile.url) else: - self.fail(_("File not exists")) - + self.fail(_("File not exists.")) + def deleteTmp(self): if self.pyfile.name.startswith("tmp_"): remove(self.pyfile.url) + + diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index 8888dc4e6..d1549fe80 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -1,107 +1,72 @@ # -*- coding: utf-8 -*- -from module.plugins.Plugin import Plugin -from module.utils import save_path +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" +from module.plugins.Plugin import Plugin class Crypter(Plugin): - __name__ = "Crypter" - __type__ = "crypter" - __version__ = "0.05" - + __name__ = "Crypter" + __version__ = "0.1" __pattern__ = None - __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] - ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] - - __description__ = """Base decrypter plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - html = None #: last html loaded - - + __type__ = "container" + __description__ = """Base crypter plugin""" + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") + def __init__(self, pyfile): + Plugin.__init__(self, pyfile) + #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) self.packages = [] #: List of urls, pyLoad will generate packagenames self.urls = [] - - Plugin.__init__(self, pyfile) - - - def process(self, pyfile): - """ main method """ - - self.decrypt(pyfile) - - if self.urls: - self.generatePackages() - - elif not self.packages: - self.error(_("No link extracted"), "decrypt") - + + self.multiDL = True + self.limitDL = 0 + + + def preprocessing(self, thread): + """prepare""" + self.setup() + self.thread = thread + + self.decrypt(self.pyfile) + self.createPackages() - + def decrypt(self, pyfile): raise NotImplementedError - - def generatePackages(self): - """ generate new packages from self.urls """ - - packages = map(lambda name, links: (name, links, None), self.core.api.generatePackages(self.urls).iteritems()) - self.packages.extend(packages) - - def createPackages(self): """ create new packages from self.packages """ - - package_folder = self.pyfile.package().folder - package_password = self.pyfile.package().password - package_queue = self.pyfile.package().queue - - folder_per_package = self.config['general']['folder_per_package'] - try: - use_subfolder = self.getConfig('use_subfolder') - except: - use_subfolder = folder_per_package - try: - subfolder_per_package = self.getConfig('subfolder_per_package') - except: - subfolder_per_package = True - for pack in self.packages: - name, links, folder = pack - self.logDebug("Parsed package: %s" % name, - "%d links" % len(links), - "Saved to folder: %s" % folder if folder else "Saved to download folder") + self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } ) + + links = [x.decode("utf-8") for x in pack[1]] + + pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue) - links = map(lambda x: x.decode("utf-8"), links) + if self.pyfile.package().password: + self.core.api.setPackageData(pid, {"password": self.pyfile.package().password}) - pid = self.core.api.addPackage(name, links, package_queue) - - if package_password: - self.core.api.setPackageData(pid, {"password": package_password}) - - setFolder = lambda x: self.core.api.setPackageData(pid, {"folder": x or ""}) #: Workaround to do not break API addPackage method - - if use_subfolder: - if not subfolder_per_package: - setFolder(package_folder) - self.logDebug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) - - elif not folder_per_package or name != folder: - if not folder: - folder = name.replace("http://", "").replace(":", "").replace("/", "_").replace("\\", "_") - - folder = save_path(folder) #@TODO: move to core code - - setFolder(folder) - self.logDebug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder}) - - elif folder_per_package: - setFolder(None) + if self.urls: + self.core.api.generateAndAddPackages(self.urls) + diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index 424417752..5efd08bae 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -1,9 +1,26 @@ # -*- coding: utf-8 -*- -from traceback import print_exc +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay + @interface-version: 0.2 +""" -from module.plugins.Plugin import Base +from traceback import print_exc +from Plugin import Base class Expose(object): """ used for decoration to declare rpc services """ @@ -12,29 +29,23 @@ class Expose(object): hookManager.addRPC(f.__module__, f.func_name, f.func_doc) return f - def threaded(f): - def run(*args,**kwargs): hookManager.startThread(f, *args, **kwargs) return run - class Hook(Base): """ Base class for hook plugins. """ - __name__ = "Hook" - __type__ = "hook" - __version__ = "0.03" - - __config__ = [] #: [("name", "type", "desc", "default")] - - __description__ = """Interface for hook""" - __license__ = "GPLv3" - __authors__ = [("mkaay", "mkaay@mkaay.de"), - ("RaNaN", "RaNaN@pyload.org")] - + __name__ = "Hook" + __version__ = "0.2" + __type__ = "hook" + __threaded__ = [] + __config__ = [ ("name", "type", "desc" , "default") ] + __description__ = """interface for hook""" + __author_name__ = ("mkaay", "RaNaN") + __author_mail__ = ("mkaay@mkaay.de", "RaNaN@pyload.org") #: automatically register event listeners for functions, attribute will be deleted dont use it yourself event_map = None @@ -43,15 +54,15 @@ class Hook(Base): #: List of events the plugin can handle, name the functions exactly like eventname. event_list = None # dont make duplicate entries in event_map + #: periodic call interval in secondc interval = 60 - def __init__(self, core, manager): Base.__init__(self, core) #: Provide information in dict here, usable by API `getInfo` - self.info = {} + self.info = None #: Callback of periodical job task, used by hookmanager self.cb = None @@ -80,17 +91,15 @@ class Hook(Base): self.initPeriodical() self.setup() - def initPeriodical(self): if self.interval >=1: self.cb = self.core.scheduler.addJob(0, self._periodical, threaded=False) - def _periodical(self): try: if self.isActivated(): self.periodical() except Exception, e: - self.core.log.error(_("Error executing hooks: ") + str(e)) + self.core.log.error(_("Error executing hooks: %s") % str(e)) if self.core.debug: print_exc() @@ -99,68 +108,54 @@ class Hook(Base): def __repr__(self): return "" % self.__name__ - - + def setup(self): """ more init stuff if needed """ pass - def unload(self): """ called when hook was deactivated """ pass - - + def isActivated(self): """ checks if hook is activated""" return self.config.getPlugin(self.__name__, "activated") + - - #event methods - overwrite these if needed + #event methods - overwrite these if needed def coreReady(self): pass - def coreExiting(self): pass - - + def downloadPreparing(self, pyfile): pass - - + def downloadFinished(self, pyfile): pass - - + def downloadFailed(self, pyfile): pass - - + def packageFinished(self, pypack): pass - def beforeReconnecting(self, ip): pass - - + def afterReconnecting(self, ip): pass - - + def periodical(self): pass - def newCaptchaTask(self, task): """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """ pass - def captchaCorrect(self, task): pass - def captchaInvalid(self, task): - pass + pass \ No newline at end of file diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index be7d75bb9..814a70949 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -1,21 +1,33 @@ # -*- coding: utf-8 -*- -from module.plugins.Plugin import Plugin +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from module.plugins.Plugin import Plugin def getInfo(self): #result = [ .. (name, size, status, url) .. ] return - class Hoster(Plugin): - __name__ = "Hoster" - __type__ = "hoster" - __version__ = "0.02" - + __name__ = "Hoster" + __version__ = "0.1" __pattern__ = None - __config__ = [] #: [("name", "type", "desc", "default")] - + __type__ = "hoster" __description__ = """Base hoster plugin""" - __license__ = "GPLv3" - __authors__ = [("mkaay", "mkaay@mkaay.de")] + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 550169034..15bf3971f 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -1,5 +1,22 @@ # -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN, spoob, mkaay +""" + from time import time, sleep from random import randint @@ -13,11 +30,8 @@ if os.name != "nt": from grp import getgrnam from itertools import islice -from traceback import print_exc -from urlparse import urlparse - -from module.utils import fs_decode, fs_encode, save_join, save_path +from module.utils import save_join, save_path, fs_encode, fs_decode def chunks(iterable, size): it = iter(iterable) @@ -60,39 +74,24 @@ class Base(object): #: core config self.config = core.config - - def _log(self, type, args): - msg = " | ".join([str(a).strip() for a in args if a]) - logger = getattr(self.log, type) - logger("%s: %s" % (self.__name__, msg or _("%s MARK" % type.upper()))) - - - def logDebug(self, *args): - if self.core.debug: - return self._log("debug", args) - - + #log functions def logInfo(self, *args): - return self._log("info", args) - + self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) def logWarning(self, *args): - return self._log("warning", args) - + self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) def logError(self, *args): - return self._log("error", args) + self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logCritical(self, *args): - return self._log("critical", args) + def logDebug(self, *args): + self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) def setConf(self, option, value): """ see `setConfig` """ self.core.config.setPlugin(self.__name__, option, value) - def setConfig(self, option, value): """ Set config value for current plugin @@ -102,12 +101,10 @@ class Base(object): """ self.setConf(option, value) - def getConf(self, option): """ see `getConfig` """ return self.core.config.getPlugin(self.__name__, option) - def getConfig(self, option): """ Returns config value for current plugin @@ -116,29 +113,24 @@ class Base(object): """ return self.getConf(option) - def setStorage(self, key, value): """ Saves a value persistently to the database """ self.core.db.setStorage(self.__name__, key, value) - def store(self, key, value): """ same as `setStorage` """ self.core.db.setStorage(self.__name__, key, value) - def getStorage(self, key=None, default=None): """ Retrieves saved value or dict of all saved entries if key is None """ - if key: + if key is not None: return self.core.db.getStorage(self.__name__, key) or default return self.core.db.getStorage(self.__name__, key) - def retrieve(self, *args, **kwargs): """ same as `getStorage` """ return self.getStorage(*args, **kwargs) - def delStorage(self, key): """ Delete entry in db """ self.core.db.delStorage(self.__name__, key) @@ -149,22 +141,14 @@ class Plugin(Base): Base plugin for hoster/crypter. Overwrite `process` / `decrypt` in your subclassed plugin. """ - __name__ = "Plugin" - __type__ = "hoster" - __version__ = "0.07" - + __name__ = "Plugin" + __version__ = "0.4" __pattern__ = None - __config__ = [] #: [("name", "type", "desc", "default")] - - __description__ = """Base plugin""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("spoob", "spoob@pyload.org"), - ("mkaay", "mkaay@mkaay.de")] - - - info = {} #: file info dict - + __type__ = "hoster" + __config__ = [("name", "type", "desc", "default")] + __description__ = """Base Plugin""" + __author_name__ = ("RaNaN", "spoob", "mkaay") + __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) @@ -190,9 +174,7 @@ class Plugin(Base): #: username/login self.user = None - if self.account and not self.account.canUse(): - self.account = None - + if self.account and not self.account.canUse(): self.account = None if self.account: self.user, data = self.account.selectAccount() #: Browser instance, see `network.Browser` @@ -218,32 +200,27 @@ class Plugin(Base): self.js = self.core.js self.cTask = None #captcha task - self.html = None #@TODO: Move to hoster class self.retries = 0 # amount of retries already made + self.html = None # some plugins store html code here self.init() - def getChunkCount(self): if self.chunkLimit <= 0: - return self.config['download']['chunks'] - return min(self.config['download']['chunks'], self.chunkLimit) - + return self.config["download"]["chunks"] + return min(self.config["download"]["chunks"], self.chunkLimit) def __call__(self): return self.__name__ - def init(self): """initialize the plugin (in addition to `__init__`)""" pass - def setup(self): """ setup for enviroment and other things, called before downloading (possibly more than one time)""" pass - def preprocessing(self, thread): """ handles important things to do before starting """ self.thread = thread @@ -264,14 +241,12 @@ class Plugin(Base): """the 'main' method of every plugin, you **have to** overwrite it""" raise NotImplementedError - def resetAccount(self): """ dont use account and retry download """ self.account = None self.req = self.core.requestFactory.getRequest(self.__name__) self.retry() - def checksum(self, local_file=None): """ return codes: @@ -286,114 +261,46 @@ class Plugin(Base): return True, 10 - def setReconnect(self, reconnect): - reconnect = bool(reconnect) - self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) - self.wantReconnect = reconnect - - - def setWait(self, seconds, reconnect=None): + def setWait(self, seconds, reconnect=False): """Set a specific wait time later used with `wait` - + :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ - wait_time = int(seconds) + 1 - wait_until = time() + wait_time - - self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), - "Wait: %d seconds" % wait_time) + if reconnect: + self.wantReconnect = True + self.pyfile.waitUntil = time() + int(seconds) - self.pyfile.waitUntil = wait_until - - if reconnect is not None: - self.setReconnect(reconnect) - - - def wait(self, seconds=None, reconnect=None): + def wait(self): """ waits the time previously set """ - - pyfile = self.pyfile - - if seconds is not None: - self.setWait(seconds) - - if reconnect is not None: - self.setReconnect(reconnect) - self.waiting = True + self.pyfile.setStatus("waiting") - status = pyfile.status - pyfile.setStatus("waiting") + while self.pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) - self.logInfo(_("Wait: %d seconds") % (pyfile.waitUntil - time()), - _("Reconnect: %s") % self.wantReconnect) - - if self.account: - self.logDebug("Ignore reconnection due account logged") - - while pyfile.waitUntil > time(): - if pyfile.abort: - self.abort() - - sleep(1) - else: - while pyfile.waitUntil > time(): - self.thread.m.reconnecting.wait(2) - - if pyfile.abort: - self.abort() - - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - sleep(1) + if self.pyfile.abort: raise Abort + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect self.waiting = False - - pyfile.status = status - + self.pyfile.setStatus("starting") def fail(self, reason): """ fail and give reason """ raise Fail(reason) - - def abort(self, reason=""): - """ abort and give reason """ - if reason: - self.pyfile.error = str(reason) - raise Abort #@TODO: Use raise Abort(reason) in 0.4.10 - - - def error(self, reason="", type=""): - if not reason and not type: - type = "unknown" - - msg = _("%s error") % _(type.strip().capitalize()) if type else _("Error") - msg += ": " + reason.strip() if reason else "" - msg += _(" | Plugin may be out of date") - - raise Fail(msg) - - - def offline(self, reason=""): + def offline(self): """ fail and indicate file is offline """ - if reason: - self.pyfile.error = str(reason) - raise Fail("offline") #@TODO: Use raise Fail("offline", reason) in 0.4.10 + raise Fail("offline") - - def tempOffline(self, reason=""): + def tempOffline(self): """ fail and indicates file ist temporary offline, the core may take consequences """ - if reason: - self.pyfile.error = str(reason) - raise Fail("temp. offline") #@TODO: Use raise Fail("temp. offline", reason) in 0.4.10 - + raise Fail("temp. offline") - def retry(self, max_tries=5, wait_time=1, reason=""): + def retry(self, max_tries=3, wait_time=1, reason=""): """Retries and begin again from the beginning :param max_tries: number of maximum retries @@ -401,28 +308,26 @@ class Plugin(Base): :param reason: reason for retrying, will be passed to fail if max_tries reached """ if 0 < max_tries <= self.retries: - self.error(reason or _("Max retries reached"), "retry") + if not reason: reason = "Max retries reached" + raise Fail(reason) - self.wait(wait_time, False) + self.wantReconnect = False + self.setWait(wait_time) + self.wait() self.retries += 1 raise Retry(reason) - def invalidCaptcha(self): - self.logError(_("Invalid captcha")) if self.cTask: self.cTask.invalid() - def correctCaptcha(self): - self.logInfo(_("Correct captcha")) if self.cTask: self.cTask.correct() - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual', timeout=290): + result_type='textual'): """ Loads a captcha and decrypts it with ocr, plugin, user input :param url: url of captcha image @@ -434,16 +339,16 @@ class Plugin(Base): :param result_type: 'textual' if text is written on the captcha\ or 'positional' for captcha where the user have to click\ on a specific region on the captcha - + :return: result of decrypting """ img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time())[-6:].replace(".", "") - - with open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: - tmpCaptcha.write(img) + temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp_file.write(img) + temp_file.close() has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins @@ -454,21 +359,20 @@ class Plugin(Base): if Ocr and not forceUser: sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: - self.abort() + if self.pyfile.abort: raise Abort ocr = Ocr() - result = ocr.get_captcha(tmpCaptcha.name) + result = ocr.get_captcha(temp_file.name) else: captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) + task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) self.cTask = task - captchaManager.handleCaptcha(task, timeout) + captchaManager.handleCaptcha(task) while task.isWaiting(): if self.pyfile.abort: captchaManager.removeTask(task) - self.abort() + raise Abort sleep(1) captchaManager.removeTask(task) @@ -478,21 +382,21 @@ class Plugin(Base): elif task.error: self.fail(task.error) elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) + self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) result = task.result - self.logDebug("Received captcha result: " + str(result)) + self.log.debug("Received captcha result: %s" % str(result)) if not self.core.debug: try: - remove(tmpCaptcha.name) + remove(temp_file.name) except: pass return result - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True): + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """Load content at url and returns it :param url: @@ -500,44 +404,35 @@ class Plugin(Base): :param post: :param ref: :param cookies: - :param just_header: If True only the header will be retrieved and returned as dict + :param just_header: if True only the header will be retrieved and returned as dict :param decode: Wether to decode the output according to http header, should be True in most cases - :param follow_location: If True follow location else not - :param save_cookies: If True saves received cookies else discard them :return: Loaded content """ - if self.pyfile.abort: - self.abort() - - if not url: - self.fail(_("No url given")) - - if type(url) == unicode: # utf8 vs decode -> please use decode attribute in all future plugins - url = str(url) #: encode('utf8') - - url = url.strip() - - if self.core.debug: - self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + if self.pyfile.abort: raise Abort + #utf8 vs decode -> please use decode attribute in all future plugins + if type(url) == unicode: url = str(url) - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies) + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.core.debug: from inspect import currentframe frame = currentframe() - framefile = save_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + if not exists(join("tmp", self.__name__)): + makedirs(join("tmp", self.__name__)) + + f = open( + join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + , "wb") + del frame # delete the frame or it wont be cleaned + try: - if not exists(join("tmp", self.__name__)): - makedirs(join("tmp", self.__name__)) + tmp = res.encode("utf8") + except: + tmp = res - with open(framefile, "wb") as f: - del frame #: delete the frame or it wont be cleaned - if decode: - res = res.encode('utf-8') - f.write(res) - except IOError, e: - self.logError(e) + f.write(tmp) + f.close() if just_header: #parse header @@ -547,7 +442,7 @@ class Plugin(Base): if not line or ":" not in line: continue key, none, value = line.partition(":") - key = key.strip().lower() + key = key.lower().strip() value = value.strip() if key in header: @@ -561,7 +456,6 @@ class Plugin(Base): return res - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder @@ -574,19 +468,6 @@ class Plugin(Base): the filename will be changed if needed :return: The location where the file was saved """ - if self.pyfile.abort: - self.abort() - - if not url: - self.fail(_("No url given")) - - if type(url) == unicode: - url = str(url) - - url = url.strip() - - if self.core.debug: - self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) self.checkForSameFiles() @@ -597,16 +478,16 @@ class Plugin(Base): location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): - try: - makedirs(location, int(self.core.config['permission']['folder'], 8)) + makedirs(location, int(self.core.config["permission"]["folder"], 8)) - if self.core.config['permission']['change_dl'] and os.name != "nt": - uid = getpwnam(self.config['permission']['user'])[2] - gid = getgrnam(self.config['permission']['group'])[2] - chown(location, uid, gid) + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] - except Exception, e: - self.fail(e) + chown(location, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) # convert back to unicode location = fs_decode(location) @@ -623,38 +504,31 @@ class Plugin(Base): finally: self.pyfile.size = self.req.size - if newname: - newname = urlparse(newname).path.split("/")[-1] - - if disposition and newname != name: - self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) - self.pyfile.name = newname - filename = join(location, newname) + if disposition and newname and newname != name: #triple check, just to be sure + self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) fs_filename = fs_encode(filename) - if self.core.config['permission']['change_file']: - try: - chmod(fs_filename, int(self.core.config['permission']['file'], 8)) - except Exception, e: - self.logWarning(_("Setting file mode failed"), e) + if self.core.config["permission"]["change_file"]: + chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) - if self.core.config['permission']['change_dl'] and os.name != "nt": + if self.core.config["permission"]["change_dl"] and os.name != "nt": try: - uid = getpwnam(self.config['permission']['user'])[2] - gid = getgrnam(self.config['permission']['group'])[2] - chown(fs_filename, uid, gid) + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + chown(fs_filename, uid, gid) except Exception, e: - self.logWarning(_("Setting User and Group failed"), e) + self.log.warning(_("Setting User and Group failed: %s") % str(e)) self.lastDownload = filename return self.lastDownload - def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): """ checks the content of the last downloaded file, re match is saved to `lastCheck` - + :param rules: dict with names and rules to match (compiled regexp or strings) :param api_size: expected file size :param max_size: if the file is larger then it wont be checked @@ -663,21 +537,19 @@ class Plugin(Base): :return: dictionary key of the first rule that matched """ lastDownload = fs_encode(self.lastDownload) - if not exists(lastDownload): - return None + if not exists(lastDownload): return None size = stat(lastDownload) size = size.st_size if api_size and api_size <= size: return None elif size > max_size and not read_size: return None - self.logDebug("Download Check triggered") - - with open(lastDownload, "rb") as f: - content = f.read(read_size if read_size else -1) - + self.log.debug("Download Check triggered") + f = open(lastDownload, "rb") + content = f.read(read_size if read_size else -1) + f.close() #produces encoding errors, better log to other file in the future? - #self.logDebug("Content: %s" % content) + #self.log.debug("Content: %s" % content) for name, rule in rules.iteritems(): if type(rule) in (str, unicode): if rule in content: @@ -723,27 +595,23 @@ class Plugin(Base): if starting and self.core.config['download']['skip_existing'] and exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: - raise SkipDownload("File exists") + raise SkipDownload("File exists.") pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: if exists(location): raise SkipDownload(pyfile[0]) - self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name) - + self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) def clean(self): """ clean everything and remove references """ if hasattr(self, "pyfile"): del self.pyfile - if hasattr(self, "req"): self.req.close() del self.req - if hasattr(self, "thread"): del self.thread - if hasattr(self, "html"): del self.html diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index 59ba47410..f3f5f47bc 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -1,26 +1,43 @@ # -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay, RaNaN +""" + import re import sys -from itertools import chain from os import listdir, makedirs from os.path import isfile, join, exists, abspath from sys import version_info +from itertools import chain from traceback import print_exc from module.lib.SafeEval import const_eval as literal_eval - +from module.ConfigParser import IGNORE class PluginManager: - ROOT = "module.plugins." + ROOT = "module.plugins." USERROOT = "userplugins." - TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal") + TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal") - PATTERN = re.compile(r'__pattern__\s*=\s*u?r("|\')([^"\']+)') - VERSION = re.compile(r'__version__\s*=\s*("|\')([\d.]+)') - CONFIG = re.compile(r'__config__\s*=\s*\[([^\]]+)', re.M) - DESC = re.compile(r'__description__\s*=\s*("|"""|\')([^"\']+)') + PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') + VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)') + CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) + DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') def __init__(self, core): @@ -41,40 +58,41 @@ class PluginManager: sys.path.append(abspath("")) - #@NOTE: In 0.4.10 directory "accounts" changes to "account" and "hooks" changes to "addon" - self.plugins['accounts'] = self.accountPlugins = self.parse("accounts") - self.plugins['hooks'] = self.hookPlugins = self.parse("hooks") + if not exists("userplugins"): + makedirs("userplugins") + if not exists(join("userplugins", "__init__.py")): + f = open(join("userplugins", "__init__.py"), "wb") + f.close() - for type in set(self.TYPES) - set(('accounts', 'hooks')): - self.plugins[type] = self.parse(type) - setattr(self, "%sPlugins" % type, self.plugins[type]) + self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) + self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) + self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - self.log.debug("Created index of plugins") + self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") + self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") + self.plugins["hooks"] = self.hookPlugins = self.parse("hooks") + self.plugins["internal"] = self.internalPlugins = self.parse("internal") + self.log.debug("created index of plugins") - def parse(self, folder, rootplugins={}): + def parse(self, folder, pattern=False, home={}): """ - returns dict with information + returns dict with information home contains parsed plugins from module. + + { + name : {path, version, config, (pattern, re), (plugin, class)} + } + """ - plugins = {} - - if rootplugins: - try: - pfolder = join("userplugins", folder) - if not exists(pfolder): - makedirs(pfolder) - - for ifile in (join("userplugins", "__init__.py"), - join(pfolder, "__init__.py")): - if not exists(ifile): - f = open(ifile, "wb") - f.close() - - except IOError, e: - self.logCritical(e) - return rootplugins + if home: + pfolder = join("userplugins", folder) + if not exists(pfolder): + makedirs(pfolder) + if not exists(join(pfolder, "__init__.py")): + f = open(join(pfolder, "__init__.py"), "wb") + f.close() else: pfolder = join(pypath, "module", "plugins", folder) @@ -82,27 +100,19 @@ class PluginManager: for f in listdir(pfolder): if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): + data = open(join(pfolder, f)) + content = data.read() + data.close() - try: - with open(join(pfolder, f)) as data: - content = data.read() - - except IOError, e: - self.logError(e) + if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): continue - - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): #@TODO: Remove in 0.4.10 + elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): continue - - elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): #@TODO: Remove in 0.4.10 - continue - - elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): #@TODO: Remove in 0.4.10 + elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): continue name = f[:-3] - if name[-1] == ".": - name = name[:-4] + if name[-1] == ".": name = name[:-4] version = self.VERSION.findall(content) if version: @@ -110,33 +120,38 @@ class PluginManager: else: version = 0 - if rootplugins and name in rootplugins: - if rootplugins[name]['version'] >= version: + # home contains plugins from pyload root + if home and name in home: + if home[name]["v"] >= version: continue + if name in IGNORE or (folder, name) in IGNORE: + continue + plugins[name] = {} - plugins[name]['version'] = version + plugins[name]["v"] = version module = f.replace(".pyc", "").replace(".py", "") # the plugin is loaded from user directory - plugins[name]['user'] = True if rootplugins else False - plugins[name]['name'] = module - - pattern = self.PATTERN.findall(content) + plugins[name]["user"] = True if home else False + plugins[name]["name"] = module if pattern: - pattern = pattern[0][1] + pattern = self.PATTERN.findall(content) + + if pattern: + pattern = pattern[0][1] + else: + pattern = "^unmachtable$" + + plugins[name]["pattern"] = pattern try: - regexp = re.compile(pattern) + plugins[name]["re"] = re.compile(pattern) except: - self.log.error(_("%s has a invalid pattern") % name) - pattern = r'^unmatchable$' - regexp = re.compile(pattern) + self.log.error(_("%s has a invalid pattern.") % name) - plugins[name]['pattern'] = pattern - plugins[name]['re'] = regexp # internals have no config if folder == "internal": @@ -145,19 +160,24 @@ class PluginManager: config = self.CONFIG.findall(content) if config: - try: - config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" + config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) + desc = self.DESC.findall(content) + desc = desc[0][1] if desc else "" - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] - if folder not in ("accounts", "internal") and not [True for item in config if item[0] == "activated"]: - config.insert(0, ["activated", "bool", "Activated", False if folder == "hooks" else True]) + if folder == "hooks": + append = True + for item in config: + if item[0] == "activated": append = False + # activated flag missing + if append: config.append(["activated", "bool", "Activated", False]) + + try: self.core.config.addPluginConfig(name, config, desc) except: self.log.error("Invalid config in %s: %s" % (name, config)) @@ -172,8 +192,9 @@ class PluginManager: except: self.log.error("Invalid config in %s: %s" % (name, config)) - if not rootplugins and plugins: #: Double check - plugins.update(self.parse(folder, plugins)) + if not home: + temp = self.parse(folder, pattern, plugins) + plugins.update(temp) return plugins @@ -188,13 +209,13 @@ class PluginManager: if type(url) not in (str, unicode, buffer): continue found = False - if last and last[1]['re'].match(url): + if last and last[1]["re"].match(url): res.append((url, last[0])) continue for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), - self.containerPlugins.iteritems()): - if 're' in value and value['re'].match(url): #@TODO: Rewrite this check to report missing __pattern__ attribute alert + self.containerPlugins.iteritems()): + if value["re"].match(url): res.append((url, name)) last = (name, value) found = True @@ -205,38 +226,34 @@ class PluginManager: return res - def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): for ptype in pluginlist: if name in self.plugins[ptype]: return self.plugins[ptype][name], ptype return None, None - def getPlugin(self, name, original=False): """return plugin module from hoster|decrypter|container""" plugin, type = self.findPlugin(name) if not plugin: - self.log.warning("Plugin %s not found" % name) - plugin = self.hosterPlugins['BasePlugin'] + self.log.warning("Plugin %s not found." % name) + plugin = self.hosterPlugins["BasePlugin"] if "new_module" in plugin and not original: - return plugin['new_module'] + return plugin["new_module"] return self.loadModule(type, name) - def getPluginName(self, name): """ used to obtain new name if other plugin was injected""" plugin, type = self.findPlugin(name) if "new_name" in plugin: - return plugin['new_name'] + return plugin["new_name"] return name - def loadModule(self, type, name): """ Returns loaded module for plugin @@ -245,41 +262,26 @@ class PluginManager: """ plugins = self.plugins[type] if name in plugins: - if "module" in plugins[name]: - return plugins[name]['module'] - + if "module" in plugins[name]: return plugins[name]["module"] try: - module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]['name']), globals(), locals(), - plugins[name]['name']) - + module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]["name"]), globals(), locals(), + plugins[name]["name"]) + plugins[name]["module"] = module #cache import, maybe unneeded + return module except Exception, e: - self.log.error(_("Error importing plugin: [%(type)s] %(name)s (v%(version).2f) | %(errmsg)s") - % {'name': name, 'type': type, 'version': plugins[name]['version'], "errmsg": str(e)}) + self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) if self.core.debug: print_exc() - else: - plugins[name]['module'] = module #: cache import, maybe unneeded - - self.log.debug(_("Loaded plugin: [%(type)s] %(name)s (v%(version).2f)") - % {'name': name, 'type': type, 'version': plugins[name]['version']}) - return module - - def loadClass(self, type, name): """Returns the class of a plugin with the same name""" module = self.loadModule(type, name) - if module: - return getattr(module, name) - else: - return None - + if module: return getattr(module, name) def getAccountPlugins(self): """return list of account plugin names""" return self.accountPlugins.keys() - def find_module(self, fullname, path=None): #redirecting imports if necesarry if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins @@ -292,10 +294,10 @@ class PluginManager: if type in self.plugins and name in self.plugins[type]: #userplugin is a newer version - if not user and self.plugins[type][name]['user']: + if not user and self.plugins[type][name]["user"]: return self #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name]['user']: + if user and not self.plugins[type][name]["user"]: return self @@ -306,8 +308,7 @@ class PluginManager: newname = name.replace(self.ROOT, self.USERROOT) else: newname = name.replace(self.USERROOT, self.ROOT) - else: - newname = name + else: newname = name base, plugin = newname.rsplit(".", 1) @@ -323,45 +324,40 @@ class PluginManager: def reloadPlugins(self, type_plugins): """ reloads and reindexes plugins """ - if not type_plugins: - return None + if not type_plugins: return False self.log.debug("Request reload of plugins: %s" % type_plugins) - flag = True as_dict = {} - for t,n in type_plugins: if t in as_dict: as_dict[t].append(n) else: as_dict[t] = [n] - for type in as_dict.iterkeys(): - # we do not reload hooks or internals, would cause to much side effects - if type in ("hooks", "internal"): - flag = False - continue + # we do not reload hooks or internals, would cause to much side effects + if "hooks" in as_dict or "internal" in as_dict: + return False + for type in as_dict.iterkeys(): for plugin in as_dict[type]: if plugin in self.plugins[type]: if "module" in self.plugins[type][plugin]: - self.log.debug("Reloading module %s" % plugin) - reload(self.plugins[type][plugin]['module']) + self.log.debug("Reloading %s" % plugin) + reload(self.plugins[type][plugin]["module"]) - #index creation - self.plugins[type] = self.parse(type) - - if type is "accounts": #@TODO: Remove this check in 0.4.10 - self.accountPlugins = self.plugins[type] - else: - setattr(self, "%sPlugins" % type, self.plugins[type]) + #index creation + self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) + self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) + self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) + self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") + self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") - if "accounts" in as_dict: #: accounts needs to be reloaded + if "accounts" in as_dict: #accounts needs to be reloaded self.core.accountManager.initPlugins() self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - return flag + return True @@ -375,9 +371,10 @@ if __name__ == "__main__": a = time() - test = ["http://www.youtube.com/watch?v=%s" % x for x in xrange(0, 100)] + test = ["http://www.youtube.com/watch?v=%s" % x for x in range(0, 100)] print p.parseUrls(test) b = time() print b - a, "s" + -- cgit v1.2.3 From 0de6e675bb0c5a4adb79d16df55ada3071825ad5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 20:04:29 +0100 Subject: Revert remaining modules --- module/Api.py | 24 ++++--- module/CaptchaManager.py | 16 +++-- module/HookManager.py | 19 ++++-- module/PluginThread.py | 59 ++++++++--------- module/PyFile.py | 2 +- module/common/packagetools.py | 132 +++++++++++++++++++-------------------- module/network/HTTPChunk.py | 4 +- module/network/HTTPDownload.py | 9 ++- module/network/HTTPRequest.py | 60 ++++++++---------- module/network/RequestFactory.py | 15 +---- module/network/XDCCRequest.py | 2 +- module/utils.py | 6 +- module/web/cnl_app.py | 47 +++++++------- module/web/pyload_app.py | 2 +- 14 files changed, 186 insertions(+), 211 deletions(-) diff --git a/module/Api.py b/module/Api.py index 551085b51..f0bf5e264 100644 --- a/module/Api.py +++ b/module/Api.py @@ -22,13 +22,11 @@ from os.path import join from time import time import re -from urlparse import urlparse - from PyFile import PyFile +from utils import freeSpace, compare_time from common.packagetools import parseNames from network.RequestFactory import getURL from remote import activated -from utils import compare_time, freeSpace, html_unescape, save_path if activated: try: @@ -51,7 +49,7 @@ def permission(bits): def __new__(cls, func, *args, **kwargs): permMap[func.__name__] = bits return func - + return _Dec @@ -162,7 +160,7 @@ class Api(Iface): @permission(PERMS.SETTINGS) def getConfig(self): """Retrieves complete config of core. - + :return: list of `ConfigSection` """ return self._convertConfigFormat(self.core.config.config) @@ -221,7 +219,7 @@ class Api(Iface): @permission(PERMS.LIST) def statusServer(self): """Some general information about the current status of pyLoad. - + :return: `ServerStatus` """ serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), @@ -319,11 +317,11 @@ class Api(Iface): :return: package id of the new package """ if self.core.config['general']['folder_per_package']: - folder = urlparse(html_unescape(name)).path.split("/")[-1] + folder = name else: folder = "" - folder = save_path(folder) + folder = folder.replace("http://", "").replace(":", "").replace("/", "_").replace("\\", "_") pid = self.core.files.addPackage(name, folder, dest) @@ -486,7 +484,7 @@ class Api(Iface): :return: `PackageData` with .fid attribute """ data = self.core.files.getPackageData(int(pid)) - + if not data: raise PackageDoesNotExists(pid) @@ -513,7 +511,7 @@ class Api(Iface): @permission(PERMS.DELETE) def deleteFiles(self, fids): """Deletes several file entries from pyload. - + :param fids: list of file ids """ for id in fids: @@ -586,7 +584,7 @@ class Api(Iface): @permission(PERMS.ADD) def addFiles(self, pid, links): """Adds files to specific package. - + :param pid: package id :param links: list of urls """ @@ -708,7 +706,7 @@ class Api(Iface): """Gives a package a new position. :param pid: package id - :param position: + :param position: """ self.core.files.reorderPackage(pid, position) @@ -920,7 +918,7 @@ class Api(Iface): :param username: :param password: - :param remoteip: + :param remoteip: :return: dict with info, empty when login is incorrect """ if self.core.config["remote"]["nolocalauth"] and remoteip == "127.0.0.1": diff --git a/module/CaptchaManager.py b/module/CaptchaManager.py index b7b81a5b7..02cd10a11 100644 --- a/module/CaptchaManager.py +++ b/module/CaptchaManager.py @@ -58,11 +58,11 @@ class CaptchaManager(): self.lock.release() return None - def handleCaptcha(self, task, timeout=50): + def handleCaptcha(self, task): cli = self.core.isClientConnected() if cli: #client connected -> should solve the captcha - task.setWaiting(timeout) #wait 50 sec for response + task.setWaiting(50) #wait 50 sec for response for plugin in self.core.hookManager.activePlugins(): try: @@ -125,10 +125,10 @@ class CaptchaTask(): self.status = "waiting" def isWaiting(self): - if self.result or self.error or self.timedOut(): + if self.result or self.error or time() > self.waitUntil: return False - else: - return True + + return True def isTextual(self): """ returns if text is written on the captcha """ @@ -149,12 +149,10 @@ class CaptchaTask(): def invalid(self): """ indicates the captcha was not correct """ - for x in self.handler: - x.captchaInvalid(self) + [x.captchaInvalid(self) for x in self.handler] def correct(self): - for x in self.handler: - x.captchaCorrect(self) + [x.captchaCorrect(self) for x in self.handler] def __str__(self): return "" % self.id diff --git a/module/HookManager.py b/module/HookManager.py index e4b9a33a0..16f692d76 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -141,8 +141,8 @@ class HookManager: if self.core.debug: traceback.print_exc() - self.log.info(_("Activated addons: %s") % ", ".join(sorted(active))) - self.log.info(_("Deactivated addons: %s") % ", ".join(sorted(deactive))) + self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active))) + self.log.info(_("Deactivate plugins: %s") % ", ".join(sorted(deactive))) self.plugins = plugins @@ -219,7 +219,10 @@ class HookManager: def downloadFinished(self, pyfile): for plugin in self.plugins: if plugin.isActivated(): - plugin.downloadFinished(pyfile) + if "downloadFinished" in plugin.__threaded__: + self.startThread(plugin.downloadFinished, pyfile) + else: + plugin.downloadFinished(pyfile) self.dispatchEvent("downloadFinished", pyfile) @@ -228,7 +231,10 @@ class HookManager: def downloadFailed(self, pyfile): for plugin in self.plugins: if plugin.isActivated(): - plugin.downloadFailed(pyfile) + if "downloadFailed" in plugin.__threaded__: + self.startThread(plugin.downloadFinished, pyfile) + else: + plugin.downloadFailed(pyfile) self.dispatchEvent("downloadFailed", pyfile) @@ -236,7 +242,10 @@ class HookManager: def packageFinished(self, package): for plugin in self.plugins: if plugin.isActivated(): - plugin.packageFinished(package) + if "packageFinished" in plugin.__threaded__: + self.startThread(plugin.packageFinished, package) + else: + plugin.packageFinished(package) self.dispatchEvent("packageFinished", package) diff --git a/module/PluginThread.py b/module/PluginThread.py index 051236c3e..56c36c778 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -155,7 +155,7 @@ class DownloadThread(PluginThread): """Constructor""" PluginThread.__init__(self, manager) - self.queue = Queue() #: job queue + self.queue = Queue() # job queue self.active = False self.start() @@ -176,8 +176,7 @@ class DownloadThread(PluginThread): return True try: - if not pyfile.hasPlugin(): - continue + if not pyfile.hasPlugin(): continue #this pyfile was deleted while queueing pyfile.plugin.checkForSameFiles(starting=True) @@ -185,7 +184,6 @@ class DownloadThread(PluginThread): # start download self.m.core.hookManager.downloadPreparing(pyfile) - pyfile.error = "" pyfile.plugin.preprocessing(self) self.m.log.info(_("Download finished: %s") % pyfile.name) @@ -207,9 +205,6 @@ class DownloadThread(PluginThread): pyfile.setStatus("aborted") - if self.m.core.debug: - print_exc() - self.clean(pyfile) continue @@ -242,9 +237,6 @@ class DownloadThread(PluginThread): self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) pyfile.error = msg - if self.m.core.debug: - print_exc() - self.m.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) continue @@ -325,7 +317,7 @@ class DownloadThread(PluginThread): pyfile.checkIfProcessed() exc_clear() - + #pyfile.plugin.req.clean() self.active = False @@ -367,46 +359,41 @@ class DecrypterThread(PluginThread): retry = False try: - self.m.log.info(_("Decrypting starts: %s") % pyfile.name) - pyfile.error = "" - pyfile.plugin.preprocessing(self) + self.m.log.info(_("Decrypting starts: %s") % self.active.name) + self.active.plugin.preprocessing(self) except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) return except Fail, e: msg = e.args[0] if msg == "offline": - pyfile.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % pyfile.name) + self.active.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % self.active.name) else: - pyfile.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) - pyfile.error = msg + self.active.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": msg}) + self.active.error = msg - if self.m.core.debug: - print_exc() return except Abort: self.m.log.info(_("Download aborted: %s") % pyfile.name) pyfile.setStatus("aborted") - if self.m.core.debug: - print_exc() return except Retry: - self.m.log.info(_("Retrying %s") % pyfile.name) + self.m.log.info(_("Retrying %s") % self.active.name) retry = True return self.run() except Exception, e: - pyfile.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) - pyfile.error = str(e) + self.active.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": str(e)}) + self.active.error = str(e) if self.m.core.debug: print_exc() @@ -414,14 +401,21 @@ class DecrypterThread(PluginThread): return + finally: if not retry: - pyfile.release() + self.active.release() self.active = False self.m.core.files.save() self.m.localThreads.remove(self) exc_clear() + + #self.m.core.hookManager.downloadFinished(pyfile) + + + #self.m.localThreads.remove(self) + #self.active.finishIfDone() if not retry: pyfile.delete() @@ -467,7 +461,7 @@ class HookThread(PluginThread): #dirty method to filter out exceptions if "unexpected keyword argument 'thread'" not in e.args[0]: raise - + del self.kwargs["thread"] self.f(*self.args, **self.kwargs) finally: @@ -630,10 +624,9 @@ class InfoThread(PluginThread): self.m.log.debug("Run Info Fetching for %s" % pluginname) for result in plugin.getInfo(process): #result = [ .. (name, size, status, url) .. ] - if not type(result) == list: - result = [result] + if not type(result) == list: result = [result] - for res in result: #: why don't assign res dict directly? + for res in result: self.m.infoCache[res[3]] = res cb(pluginname, result) diff --git a/module/PyFile.py b/module/PyFile.py index 9e20ceab4..3dede9360 100644 --- a/module/PyFile.py +++ b/module/PyFile.py @@ -251,7 +251,7 @@ class PyFile(object): def getBytesLeft(self): """ gets bytes left """ try: - return self.getSize() - self.plugin.req.arrived + return self.plugin.req.size - self.plugin.req.arrived except: return 0 diff --git a/module/common/packagetools.py b/module/common/packagetools.py index d930157e1..5bfbcba95 100644 --- a/module/common/packagetools.py +++ b/module/common/packagetools.py @@ -1,57 +1,17 @@ -# -*- coding: utf-8 -*- +#!/usr/bin/env python -import re +# JDownloader/src/jd/controlling/LinkGrabberPackager.java +import re from urlparse import urlparse - -endings = ("jdeatme", "3gp", "7zip", "7z", "abr", "ac3", "aiff", "aifc", "aif", "ai", - "au", "avi", "apk", "bin", "bmp", "bat", "bz2", "cbr", "cbz", "ccf", "chm", - "cr2", "cso", "cue", "cvd", "dta", "deb", "divx", "djvu", "dlc", "dmg", "doc", - "docx", "dot", "eps", "epub", "exe", "ff", "flv", "flac", "f4v", "gsd", "gif", - "gpg", "gz", "iwd", "idx", "iso", "ipa", "ipsw", "java", "jar", "jpe?g", "load", - "m2ts", "m4v", "m4a", "md5", "mkv", "mp2", "mp3", "mp4", "mobi", "mov", "movie", - "mpeg", "mpe", "mpg", "mpq", "msi", "msu", "msp", "mv", "mws", "nfo", "npk", "oga", - "ogg", "ogv", "otrkey", "par2", "pkg", "png", "pdf", "pptx?", "ppsx?", "ppz", "pot", - "psd", "qt", "rmvb", "rm", "rar", "ram", "ra", "rev", "rnd", "rpm", "run", "rsdf", - "reg", "rtf", "shnf", "sh(?!tml)", "ssa", "smi", "sub", "srt", "snd", "sfv", "sfx", - "swf", "swc", "tar\.(gz|bz2|xz)", "tar", "tgz", "tiff?", "ts", "txt", "viv", "vivo", - "vob", "vtt", "webm", "wav", "wmv", "wma", "xla", "xls", "xpi", "zeno", "zip", - "[r-z]\d{2}", "_[_a-z]{2}", "\d{3,4}(?=\?|$|\"|\r|\n)") - -rarPats = [re.compile(r'(.*)(\.|_|-)pa?r?t?\.?\d+.(rar|exe)$', re.I), - re.compile(r'(.*)(\.|_|-)part\.?[0]*[1].(rar|exe)$', re.I), - re.compile(r'(.*)\.rar$', re.I), - re.compile(r'(.*)\.r\d+$', re.I), - re.compile(r'(.*)(\.|_|-)\d+$', re.I)] - -zipPats = [re.compile(r'(.*)\.zip$', re.I), - re.compile(r'(.*)\.z\d+$', re.I), - re.compile(r'(?is).*\.7z\.[\d]+$', re.I), - re.compile(r'(.*)\.a.$', re.I)] - -ffsjPats = [re.compile(r'(.*)\._((_[a-z])|([a-z]{2}))(\.|$)'), - re.compile(r'(.*)(\.|_|-)[\d]+(\.(' + '|'.join(endings) + ')$)', re.I)] - -iszPats = [re.compile(r'(.*)\.isz$', re.I), - re.compile(r'(.*)\.i\d{2}$', re.I)] - -pat0 = re.compile(r'www\d*\.', re.I) - -pat1 = re.compile(r'(\.?CD\d+)', re.I) -pat2 = re.compile(r'(\.?part\d+)', re.I) - -pat3 = re.compile(r'(.+)[\.\-_]+$') -pat4 = re.compile(r'(.+)\.\d+\.xtm$') - - def matchFirst(string, *args): - """ matches against list of regexp and returns first match """ + """ matches against list of regexp and returns first match""" for patternlist in args: for pattern in patternlist: - m = pattern.search(string) - if m is not None: - name = m.group(1) + r = pattern.search(string) + if r is not None: + name = r.group(1) return name return string @@ -61,10 +21,35 @@ def parseNames(files): """ Generates packages names from name, data lists :param files: list of (name, data) - :return: packagenames mapped to data lists (eg. urls) + :return: packagenames mapt to data lists (eg. urls) """ packs = {} + endings = "\\.(3gp|7zip|7z|abr|ac3|aiff|aifc|aif|ai|au|avi|bin|bz2|cbr|cbz|ccf|cue|cvd|chm|dta|deb|divx|djvu|dlc|dmg|doc|docx|dot|eps|exe|ff|flv|f4v|gsd|gif|gz|iwd|iso|ipsw|java|jar|jpg|jpeg|jdeatme|load|mws|mw|m4v|m4a|mkv|mp2|mp3|mp4|mov|movie|mpeg|mpe|mpg|msi|msu|msp|nfo|npk|oga|ogg|ogv|otrkey|pkg|png|pdf|pptx|ppt|pps|ppz|pot|psd|qt|rmvb|rm|rar|ram|ra|rev|rnd|r\\d+|rpm|run|rsdf|rtf|sh(!?tml)|srt|snd|sfv|swf|tar|tif|tiff|ts|txt|viv|vivo|vob|wav|wmv|xla|xls|xpi|zeno|zip|z\\d+|_[_a-z]{2}|\\d+$)" + + rarPats = [re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I), + re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I), + re.compile("(.*)\\.rar$", re.I), + re.compile("(.*)\\.r\\d+$", re.I), + re.compile("(.*)(\\.|_|-)\\d+$", re.I)] + + zipPats = [re.compile("(.*)\\.zip$", re.I), + re.compile("(.*)\\.z\\d+$", re.I), + re.compile("(?is).*\\.7z\\.[\\d]+$", re.I), + re.compile("(.*)\\.a.$", re.I)] + + ffsjPats = [re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)"), + re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I)] + + iszPats = [re.compile("(.*)\\.isz$", re.I), + re.compile("(.*)\\.i\\d{2}$", re.I)] + + pat1 = re.compile("(\\.?CD\\d+)", re.I) + pat2 = re.compile("(\\.?part\\d+)", re.I) + + pat3 = re.compile("(.+)[\\.\\-_]+$") + pat4 = re.compile("(.+)\\.\\d+\\.xtm$") + for file, url in files: patternMatch = False @@ -79,7 +64,7 @@ def parseNames(files): if len(split) > 1: name = split.pop(1) - #check if an already existing package may be ok for this file + #check if a already existing package may be ok for this file # found = False # for pack in packs: # if pack in file: @@ -87,8 +72,7 @@ def parseNames(files): # found = True # break # - # if found: - # continue + # if found: continue # unrar pattern, 7zip/zip and hjmerge pattern, isz pattern, FFSJ pattern before = name @@ -97,19 +81,19 @@ def parseNames(files): patternMatch = True # xtremsplit pattern - m = pat4.search(name) - if m is not None: - name = m.group(1) + r = pat4.search(name) + if r is not None: + name = r.group(1) # remove part and cd pattern - m = pat1.search(name) - if m is not None: - name = name.replace(m.group(0), "") + r = pat1.search(name) + if r is not None: + name = name.replace(r.group(0), "") patternMatch = True - m = pat2.search(name) - if m is not None: - name = name.replace(m.group(0), "") + r = pat2.search(name) + if r is not None: + name = name.replace(r.group(0), "") patternMatch = True # additional checks if extension pattern matched @@ -124,9 +108,9 @@ def parseNames(files): name = name[:-length] # remove endings like . _ - - m = pat3.search(name) - if m is not None: - name = m.group(1) + r = pat3.search(name) + if r is not None: + name = r.group(1) # replace . and _ with space name = name.replace(".", " ") @@ -139,12 +123,11 @@ def parseNames(files): # fallback: package by hoster if not name: name = urlparse(file).hostname - if name: - name = pat0.sub("", name) + if name: name = name.replace("www.", "") # fallback : default name if not name: - name = _("Unnamed package") + name = "unknown" # build mapping if name in packs: @@ -153,3 +136,20 @@ def parseNames(files): packs[name] = [url] return packs + + +if __name__ == "__main__": + from os.path import join + from pprint import pprint + + f = open(join("..", "..", "testlinks2.txt"), "rb") + urls = [(x.strip(), x.strip()) for x in f.readlines() if x.strip()] + f.close() + + print "Having %d urls." % len(urls) + + packs = parseNames(urls) + + pprint(packs) + + print "Got %d urls." % sum([len(x) for x in packs.itervalues()]) diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index d42744cf4..b637aef32 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -208,7 +208,7 @@ class HTTPChunk(HTTPRequest): # as first chunk, we will parse the headers if not self.range and self.header.endswith("\r\n\r\n"): self.parseHeader() - elif not self.range and buf.startswith("150") and "data connection" in buf.lower(): #: ftp file size parsing + elif not self.range and buf.startswith("150") and "data connection" in buf: #ftp file size parsing size = search(r"(\d+) bytes", buf) if size: self.p.size = int(size.group(1)) @@ -290,4 +290,4 @@ class HTTPChunk(HTTPRequest): """ closes everything, unusable after this """ if self.fp: self.fp.close() self.c.close() - if hasattr(self, "p"): del self.p + if hasattr(self, "p"): del self.p \ No newline at end of file diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index 3f32295b4..fe8075539 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -63,7 +63,7 @@ class HTTPDownload(): except IOError: self.info = ChunkInfo(filename) - self.chunkSupport = True + self.chunkSupport = None self.m = pycurl.CurlMulti() #needed for speed calculation @@ -130,7 +130,7 @@ class HTTPDownload(): except pycurl.error, e: #code 33 - no resume code = e.args[0] - if resume is True and code == 33: + if code == 33: # try again without resume self.log.debug("Errno 33 -> Restart without resume") @@ -151,7 +151,6 @@ class HTTPDownload(): if not resume: self.info.clear() self.info.addChunk("%s.chunk0" % self.filename, (0, 0)) #create an initial entry - self.info.save() self.chunks = [] @@ -165,8 +164,8 @@ class HTTPDownload(): chunksDone = set() # list of curl handles that are finished chunksCreated = False done = False - if self.info.getCount() is 0: # This is a resume, if we were chunked originally assume still can - self.chunkSupport = False + if self.info.getCount() > 1: # This is a resume, if we were chunked originally assume still can + self.chunkSupport = True while 1: #need to create chunks diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 67ede2854..4747d937f 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - + @author: RaNaN """ @@ -25,11 +25,11 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Plugin import Abort, Fail +from module.plugins.Plugin import Abort def myquote(url): return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") - + def myurlencode(data): data = dict(data) return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ @@ -79,7 +79,7 @@ class HTTPRequest(): if hasattr(pycurl, "AUTOREFERER"): self.c.setopt(pycurl.AUTOREFERER, 1) self.c.setopt(pycurl.SSL_VERIFYPEER, 0) - self.c.setopt(pycurl.LOW_SPEED_TIME, 60) + self.c.setopt(pycurl.LOW_SPEED_TIME, 30) self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5) #self.c.setopt(pycurl.VERBOSE, 1) @@ -181,7 +181,7 @@ class HTTPRequest(): self.getCookies() - def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False, follow_location=True, save_cookies=True): + def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False): """ load and returns a given page """ self.setRequestContext(url, get, post, referer, cookies, multipart) @@ -190,27 +190,24 @@ class HTTPRequest(): self.c.setopt(pycurl.HTTPHEADER, self.headers) - if not follow_location: - self.c.setopt(pycurl.FOLLOWLOCATION, 0) - if just_header: + self.c.setopt(pycurl.FOLLOWLOCATION, 0) self.c.setopt(pycurl.NOBODY, 1) + self.c.perform() + rep = self.header - self.c.perform() - rep = self.header if just_header else self.getResponse() - - if not follow_location: self.c.setopt(pycurl.FOLLOWLOCATION, 1) - - if just_header: self.c.setopt(pycurl.NOBODY, 0) + else: + self.c.perform() + rep = self.getResponse() + self.c.setopt(pycurl.POSTFIELDS, "") self.lastEffectiveURL = self.c.getinfo(pycurl.EFFECTIVE_URL) self.code = self.verifyHeader() - if save_cookies: - self.addCookies() + self.addCookies() if decode: rep = self.decodeResponse(rep) @@ -231,13 +228,11 @@ class HTTPRequest(): def getResponse(self): """ retrieve response from string io """ - if self.rep is None: - return "" - else: - value = self.rep.getvalue() - self.rep.close() - self.rep = StringIO() - return value + if self.rep is None: return "" + value = self.rep.getvalue() + self.rep.close() + self.rep = StringIO() + return value def decodeResponse(self, rep): """ decode with correct encoding, relies on header """ @@ -260,7 +255,7 @@ class HTTPRequest(): #self.log.debug("Decoded %s" % encoding ) if lookup(encoding).name == 'utf-8' and rep.startswith(BOM_UTF8): encoding = 'utf-8-sig' - + decoder = getincrementaldecoder(encoding)("replace") rep = decoder.decode(rep, True) @@ -268,7 +263,6 @@ class HTTPRequest(): except LookupError: self.log.debug("No Decoder foung for %s" % encoding) - except Exception: self.log.debug("Error when decoding string from %s." % encoding) @@ -278,15 +272,13 @@ class HTTPRequest(): """ writes response """ if self.rep.tell() > 1000000 or self.abort: rep = self.getResponse() + if self.abort: raise Abort() + f = open("response.dump", "wb") + f.write(rep) + f.close() + raise Exception("Loaded Url exceeded limit") - if self.abort: - raise Abort() - - with open("response.dump", "wb") as f: - f.write(rep) - raise Fail("Loaded url exceeded size limit") - else: - self.rep.write(buf) + self.rep.write(buf) def writeHeader(self, buf): """ writes header """ @@ -311,4 +303,4 @@ if __name__ == "__main__": url = "http://pyload.org" c = HTTPRequest() print c.load(url) - + diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 750f37dc9..5b1528281 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -62,23 +62,12 @@ class RequestFactory(): def getURL(self, *args, **kwargs): """ see HTTPRequest for argument list """ - cj = None - - if 'cookies' in kwargs: - if isinstance(kwargs['cookies'], CookieJar): - cj = kwargs['cookies'] - elif isinstance(kwargs['cookies'], list): - cj = CookieJar(None) - for cookie in kwargs['cookies']: - if isinstance(cookie, tuple) and len(cookie) == 3: - cj.setCookie(*cookie) - - h = HTTPRequest(cj, self.getOptions()) + h = HTTPRequest(None, self.getOptions()) try: rep = h.load(*args, **kwargs) finally: h.close() - + return rep def getCookieJar(self, pluginName, account=None): diff --git a/module/network/XDCCRequest.py b/module/network/XDCCRequest.py index e395e01f0..f03798c17 100644 --- a/module/network/XDCCRequest.py +++ b/module/network/XDCCRequest.py @@ -127,7 +127,7 @@ class XDCCRequest(): return filename - def _keepAlive(self, sock, *readbuffer): + def _keepAlive(self, sock, readbuffer): fdset = select([sock], [], [], 0) if sock not in fdset[0]: return diff --git a/module/utils.py b/module/utils.py index 4406eafde..8748b7693 100644 --- a/module/utils.py +++ b/module/utils.py @@ -36,9 +36,9 @@ def remove_chars(string, repl): def save_path(name): #remove some chars if os.name == 'nt': - return remove_chars(name, '/?%*|"<>') + return remove_chars(name, '/\\?%*:|"<>') else: - return remove_chars(name, '\\"') + return remove_chars(name, '/\\"') def save_join(*args): @@ -54,7 +54,7 @@ if sys.getfilesystemencoding().startswith('ANSI'): try: string = string.encode('utf-8') finally: - return save_path(string) + return string fs_decode = decode #decode utf8 diff --git a/module/web/cnl_app.py b/module/web/cnl_app.py index 718499dfb..d8f7c1180 100644 --- a/module/web/cnl_app.py +++ b/module/web/cnl_app.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*- from os.path import join import re @@ -8,7 +9,6 @@ from binascii import unhexlify from bottle import route, request, HTTPError from webinterface import PYLOAD, DL_ROOT, JS - try: from Crypto.Cipher import AES except: @@ -17,8 +17,8 @@ except: def local_check(function): def _view(*args, **kwargs): - if request.environ.get("REMOTE_ADDR", "0") in ("127.0.0.1", "localhost") \ - or request.environ.get("HTTP_HOST", "0") in ("127.0.0.1:9666", "localhost:9666"): + if request.environ.get('REMOTE_ADDR', "0") in ('127.0.0.1', 'localhost') \ + or request.environ.get('HTTP_HOST','0') == '127.0.0.1:9666': return function(*args, **kwargs) else: return HTTPError(403, "Forbidden") @@ -26,15 +26,14 @@ def local_check(function): return _view -@route('/flash') -@route('/flash/') -@route('/flash', method='POST') +@route("/flash") +@route("/flash/:id") +@route("/flash", method="POST") @local_check def flash(id="0"): return "JDownloader\r\n" - -@route('/flash/add', method='POST') +@route("/flash/add", method="POST") @local_check def add(request): package = request.POST.get('referer', None) @@ -47,10 +46,10 @@ def add(request): return "" - -@route('/flash/addcrypted', method='POST') +@route("/flash/addcrypted", method="POST") @local_check def addcrypted(): + package = request.forms.get('referer', 'ClickAndLoad Package') dlc = request.forms['crypted'].replace(" ", "+") @@ -66,10 +65,10 @@ def addcrypted(): else: return "success\r\n" - -@route('/flash/addcrypted2', method='POST') +@route("/flash/addcrypted2", method="POST") @local_check def addcrypted2(): + package = request.forms.get("source", None) crypted = request.forms["crypted"] jk = request.forms["jk"] @@ -83,7 +82,7 @@ def addcrypted2(): try: jk = re.findall(r"return ('|\")(.+)('|\")", jk)[0][1] except: - ## Test for some known js functions to decode + ## Test for some known js functions to decode if jk.find("dec") > -1 and jk.find("org") > -1: org = re.findall(r"var org = ('|\")([^\"']+)", jk)[0][1] jk = list(org) @@ -101,7 +100,7 @@ def addcrypted2(): IV = Key obj = AES.new(Key, AES.MODE_CBC, IV) - result = obj.decrypt(crypted).replace("\x00", "").replace("\r", "").split("\n") + result = obj.decrypt(crypted).replace("\x00", "").replace("\r","").split("\n") result = filter(lambda x: x != "", result) @@ -115,14 +114,13 @@ def addcrypted2(): else: return "success\r\n" - -@route('/flashgot_pyload') -@route('/flashgot_pyload', method='POST') -@route('/flashgot') -@route('/flashgot', method='POST') +@route("/flashgot_pyload") +@route("/flashgot_pyload", method="POST") +@route("/flashgot") +@route("/flashgot", method="POST") @local_check def flashgot(): - if request.environ['HTTP_REFERER'] not in ("http://localhost:9666/flashgot", "http://127.0.0.1:9666/flashgot"): + if request.environ['HTTP_REFERER'] != "http://localhost:9666/flashgot" and request.environ['HTTP_REFERER'] != "http://127.0.0.1:9666/flashgot": return HTTPError() autostart = int(request.forms.get('autostart', 0)) @@ -137,8 +135,7 @@ def flashgot(): return "" - -@route('/crossdomain.xml') +@route("/crossdomain.xml") @local_check def crossdomain(): rep = "\n" @@ -149,17 +146,17 @@ def crossdomain(): return rep -@route('/flash/checkSupportForUrl') +@route("/flash/checkSupportForUrl") @local_check def checksupport(): + url = request.GET.get("url") res = PYLOAD.checkURLs([url]) supported = (not res[0][1] is None) return str(supported).lower() - -@route('/jdcheck.js') +@route("/jdcheck.js") @local_check def jdcheck(): rep = "jdownloader=true;\n" diff --git a/module/web/pyload_app.py b/module/web/pyload_app.py index f3515f05f..df4a4b3d4 100644 --- a/module/web/pyload_app.py +++ b/module/web/pyload_app.py @@ -261,7 +261,7 @@ def config(): elif not data.trafficleft: data.trafficleft = _("not available") else: - data.trafficleft = formatSize(data.trafficleft) + data.trafficleft = formatSize(data.trafficleft * 1024) if data.validuntil == -1: data.validuntil = _("unlimited") -- cgit v1.2.3 From 8e605a8b234974a89c93d38bccae9d19f86e965d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 20:20:49 +0100 Subject: [XFSAccount] Fix trafficleft --- module/plugins/internal/XFSAccount.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 627dc75c4..4a3d0696b 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.26" + __version__ = "0.27" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -71,6 +71,7 @@ class XFSAccount(Account): else: if validuntil > mktime(gmtime()): premium = True + trafficleft = -1 else: premium = False validuntil = None #: registered account type (not premium) @@ -97,9 +98,6 @@ class XFSAccount(Account): except Exception, e: self.logError(e) - else: - if premium: - trafficleft = -1 return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} -- cgit v1.2.3 From d094edc31d18b67ffff8e38610ee091df2e61322 Mon Sep 17 00:00:00 2001 From: jojo141185 Date: Sat, 22 Nov 2014 20:23:19 +0100 Subject: [FilecryptCc] Tiny fixup --- module/plugins/crypter/FilecryptCc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/plugins/crypter/FilecryptCc.py b/module/plugins/crypter/FilecryptCc.py index db4a8c4ab..dedd284b3 100644 --- a/module/plugins/crypter/FilecryptCc.py +++ b/module/plugins/crypter/FilecryptCc.py @@ -12,7 +12,7 @@ from module.plugins.Crypter import Crypter class FilecryptCc(Crypter): __name__ = "FilecryptCc" __type__ = "crypter" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'https?://(?:www\.)?filecrypt\.cc/Container/\w+' @@ -58,7 +58,7 @@ class FilecryptCc(Crypter): mirror = re.findall(self.MIRROR_PAGE_PATTERN, self.siteWithLinks) - self.logInfo(_("Found %d mirrors") % len(m)) + self.logInfo(_("Found %d mirrors") % len(mirror)) for i in mirror[1:]: self.siteWithLinks = self.siteWithLinks + self.load(i, cookies=True).decode("utf-8", "replace") -- cgit v1.2.3 From e2e96af3b35192107d32cc48d25c6c2aa6f822ba Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 01:27:42 +0100 Subject: [BasePlugin] Improve a bit --- module/plugins/hoster/BasePlugin.py | 85 ++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 63714c437..27e8d050a 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -6,20 +6,27 @@ from urllib import unquote from urlparse import urlparse from module.network.HTTPRequest import BadHeader +from module.plugins.internal.SimpleHoster import create_getInfo from module.plugins.Hoster import Hoster -from module.utils import html_unescape, remove_chars +from module.utils import remove_chars class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.20" + __version__ = "0.21" __pattern__ = r'^unmatchable$' __description__ = """Base Plugin when any other didnt fit""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org")] + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")]] + + + @classmethod + def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10 + return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url or ""} def setup(self): @@ -30,40 +37,42 @@ class BasePlugin(Hoster): def process(self, pyfile): """main function""" - #: debug part, for api exerciser - if pyfile.url.startswith("DEBUG_API"): - self.multiDL = False - return + self.getInfo(pyfile.url) if pyfile.url.startswith("http"): + for _i in xrange(2): + try: + self.downloadFile(pyfile) - try: - self.downloadFile(pyfile) - except BadHeader, e: - if e.code in (401, 403): - self.logDebug("Auth required") + except BadHeader, e: + if e.code is 404: + self.offline() - account = self.core.accountManager.getAccountPlugin('Http') - servers = [x['login'] for x in account.getAllAccounts()] - server = urlparse(pyfile.url).netloc + elif e.code in (401, 403): + self.logDebug("Auth required") - if server in servers: - self.logDebug("Logging on to %s" % server) - self.req.addAuth(account.accounts[server]['password']) - else: - for pwd in pyfile.package().password.splitlines(): - if ":" in pwd: - self.req.addAuth(pwd.strip()) - break - else: - self.fail(_("Authorization required (username:password)")) + account = self.core.accountManager.getAccountPlugin('Http') + servers = [x['login'] for x in account.getAllAccounts()] + server = urlparse(pyfile.url).netloc - self.downloadFile(pyfile) + if server in servers: + self.logDebug("Logging on to %s" % server) + self.req.addAuth(account.accounts[server]['password']) + else: + for pwd in pyfile.package().password.splitlines(): + if ":" in pwd: + self.req.addAuth(pwd.strip()) + break + else: + self.fail(_("Authorization required (username:password)")) + else: + self.fail(e) else: - raise - + break + else: + self.fail(_("No file downloaded")) #@TODO: Move to hoster class (check if self.lastDownload) in 0.4.10 else: - self.fail(_("No Plugin matched and not a downloadable url")) + self.fail(_("No plugin matched")) def downloadFile(self, pyfile): @@ -78,31 +87,37 @@ class BasePlugin(Hoster): if 'location' in header: self.logDebug("Location: " + header['location']) + base = re.match(r'https?://[^/]+', url).group(0) + if header['location'].startswith("http"): url = header['location'] + elif header['location'].startswith("/"): url = base + unquote(header['location']) + else: url = '%s/%s' % (base, unquote(header['location'])) else: break - name = html_unescape(unquote(urlparse(url).path.split("/")[-1])) - if 'content-disposition' in header: self.logDebug("Content-Disposition: " + header['content-disposition']) + m = re.search("filename(?P=|\*=(?P.+)'')(?P.*)", header['content-disposition']) if m: disp = m.groupdict() + self.logDebug(disp) + if not disp['enc']: disp['enc'] = 'utf-8' + name = remove_chars(disp['name'], "\"';").strip() name = unicode(unquote(name), disp['enc']) - if not name: - name = url - pyfile.name = name - self.logDebug("Filename: %s" % pyfile.name) + pyfile.name = name + + self.logDebug("Filename changed to: " + name) + self.download(url, disposition=True) -- cgit v1.2.3 From 9804b57e02d663328af048a96e60a330ac27b60d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 01:30:45 +0100 Subject: Fixes about create_getInfo --- module/plugins/internal/DeadCrypter.py | 13 ++++++++++++- module/plugins/internal/DeadHoster.py | 21 ++++++++++++--------- module/plugins/internal/SimpleCrypter.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 5 +++-- module/plugins/internal/XFSHoster.py | 4 ++-- 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 4c635e366..c721c8390 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -1,12 +1,15 @@ # -*- coding: utf-8 -*- +from urlparse import urlparse + +from module.plugins.internal.SimpleCrypter import create_getInfo from module.plugins.Crypter import Crypter as _Crypter class DeadCrypter(_Crypter): __name__ = "DeadCrypter" __type__ = "crypter" - __version__ = "0.02" + __version__ = "0.03" __pattern__ = r'^unmatchable$' @@ -15,6 +18,14 @@ class DeadCrypter(_Crypter): __authors__ = [("stickell", "l.stickell@yahoo.it")] + @classmethod + def getInfo(cls, url="", html=""): + return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 1, 'url': url or ""} + + def setup(self): self.pyfile.error = "Crypter is no longer available" self.offline() #@TODO: self.offline("Crypter is no longer available") + + +getInfo = create_getInfo(DeadCrypter) diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index b78068069..b85aea3f9 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -1,20 +1,15 @@ # -*- coding: utf-8 -*- -from module.plugins.Hoster import Hoster as _Hoster - - -def create_getInfo(plugin): +from urlparse import urlparse - def getInfo(urls): - yield map(lambda url: ('#N/A: ' + url, 0, 1, url), urls) - - return getInfo +from module.plugins.internal.SimpleHoster import create_getInfo +from module.plugins.Hoster import Hoster as _Hoster class DeadHoster(_Hoster): __name__ = "DeadHoster" __type__ = "hoster" - __version__ = "0.12" + __version__ = "0.13" __pattern__ = r'^unmatchable$' @@ -23,6 +18,14 @@ class DeadHoster(_Hoster): __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + @classmethod + def getInfo(cls, url="", html=""): + return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 1, 'url': url or ""} + + def setup(self): self.pyfile.error = "Hoster is no longer available" self.offline() #@TODO: self.offline("Hoster is no longer available") + + +getInfo = create_getInfo(DeadHoster) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 5d00daa3c..53ffaf4a6 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -5,14 +5,14 @@ import re from urlparse import urlparse from module.plugins.Crypter import Crypter -from module.plugins.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies from module.utils import fixup class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.31" + __version__ = "0.32" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ea298cc39..01702d423 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -100,6 +100,7 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 +#@NOTE: Every plugin must have own parseInfo classmethod to work with 0.4.10 def create_getInfo(plugin): return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfo(urls)] @@ -126,7 +127,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.60" + __version__ = "0.61" __pattern__ = r'^unmatchable$' @@ -197,7 +198,7 @@ class SimpleHoster(Hoster): @classmethod def getInfo(cls, url="", html=""): - info = {'name': url or _("Unknown"), 'size': 0, 'status': 3, 'url': url} + info = {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url or ""} if not html: if url: diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index d5bbf5c63..2cc4833e6 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.22" + __version__ = "0.23" __pattern__ = r'^unmatchable$' @@ -108,7 +108,7 @@ class XFSHoster(SimpleHoster): def getDownloadLink(self): - for i in xrange(1, 5): + for i in xrange(1, 6): self.logDebug("Getting download link: #%d" % i) self.checkErrors() -- cgit v1.2.3 From 6b4527b5603b84dd26e40d85264c4d35778ad275 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 02:08:51 +0100 Subject: [ZShareNet] Dead hoster --- module/plugins/hoster/ZShareNet.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 module/plugins/hoster/ZShareNet.py diff --git a/module/plugins/hoster/ZShareNet.py b/module/plugins/hoster/ZShareNet.py new file mode 100644 index 000000000..dc96facbe --- /dev/null +++ b/module/plugins/hoster/ZShareNet.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.DeadHoster import DeadHoster, create_getInfo + + +class ZShareNet(DeadHoster): + __name__ = "ZShareNet" + __type__ = "hoster" + __version__ = "0.21" + + __pattern__ = r'https?://(?:ww[2w]\.)?zshares?\.net/.+' + + __description__ = """ZShare.net hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("espes", None), + ("Cptn Sandwich", None)] + + +getInfo = create_getInfo(ZShareNet) -- cgit v1.2.3 From 37858ae9675f3363e9e7314707a08d79406d6c6a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 02:10:18 +0100 Subject: Fix missing create_getInfo in dead crypters --- module/plugins/crypter/C1neonCom.py | 5 ++++- module/plugins/crypter/CryptItCom.py | 5 ++++- module/plugins/crypter/DdlstorageComFolder.py | 2 +- module/plugins/crypter/DuploadOrgFolder.py | 5 ++++- module/plugins/crypter/FilebeerInfoFolder.py | 5 ++++- module/plugins/crypter/FilesonicComFolder.py | 5 ++++- module/plugins/crypter/FiredriveComFolder.py | 5 ++++- module/plugins/crypter/HotfileComFolder.py | 5 ++++- module/plugins/crypter/ILoadTo.py | 5 ++++- module/plugins/crypter/LofCc.py | 5 ++++- module/plugins/crypter/MBLinkInfo.py | 5 ++++- module/plugins/crypter/MegauploadComFolder.py | 5 ++++- module/plugins/crypter/Movie2kTo.py | 5 ++++- module/plugins/crypter/MultiuploadCom.py | 5 ++++- module/plugins/crypter/OronComFolder.py | 5 ++++- module/plugins/crypter/RSLayerCom.py | 5 ++++- module/plugins/crypter/SecuredIn.py | 5 ++++- module/plugins/crypter/SharingmatrixComFolder.py | 5 ++++- module/plugins/crypter/SpeedLoadOrgFolder.py | 5 ++++- module/plugins/crypter/StealthTo.py | 5 ++++- module/plugins/crypter/TrailerzoneInfo.py | 5 ++++- module/plugins/crypter/WiiReloadedOrg.py | 5 ++++- module/plugins/crypter/WuploadComFolder.py | 5 ++++- 23 files changed, 89 insertions(+), 23 deletions(-) diff --git a/module/plugins/crypter/C1neonCom.py b/module/plugins/crypter/C1neonCom.py index 18c0f1e73..cf1d2a211 100644 --- a/module/plugins/crypter/C1neonCom.py +++ b/module/plugins/crypter/C1neonCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class C1neonCom(DeadCrypter): @@ -14,3 +14,6 @@ class C1neonCom(DeadCrypter): __description__ = """C1neon.com decrypter plugin""" __license__ = "GPLv3" __authors__ = [("godofdream", "soilfiction@gmail.com")] + + +getInfo = create_getInfo(C1neonCom) diff --git a/module/plugins/crypter/CryptItCom.py b/module/plugins/crypter/CryptItCom.py index 2c91535bd..2cf4e9f62 100644 --- a/module/plugins/crypter/CryptItCom.py +++ b/module/plugins/crypter/CryptItCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class CryptItCom(DeadCrypter): @@ -14,3 +14,6 @@ class CryptItCom(DeadCrypter): __description__ = """Crypt-it.com decrypter plugin""" __license__ = "GPLv3" __authors__ = [("jeix", "jeix@hasnomail.de")] + + +getInfo = create_getInfo(CryptItCom) diff --git a/module/plugins/crypter/DdlstorageComFolder.py b/module/plugins/crypter/DdlstorageComFolder.py index 6ad586434..e02e77fda 100644 --- a/module/plugins/crypter/DdlstorageComFolder.py +++ b/module/plugins/crypter/DdlstorageComFolder.py @@ -17,4 +17,4 @@ class DdlstorageComFolder(DeadCrypter): ("stickell", "l.stickell@yahoo.it")] -getInfo = create_getInfo(SpeedLoadOrg) +getInfo = create_getInfo(DdlstorageComFolder) diff --git a/module/plugins/crypter/DuploadOrgFolder.py b/module/plugins/crypter/DuploadOrgFolder.py index af83dffd0..066fbe3d7 100644 --- a/module/plugins/crypter/DuploadOrgFolder.py +++ b/module/plugins/crypter/DuploadOrgFolder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class DuploadOrgFolder(DeadCrypter): @@ -14,3 +14,6 @@ class DuploadOrgFolder(DeadCrypter): __description__ = """Dupload.org folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] + + +getInfo = create_getInfo(DuploadOrgFolder) diff --git a/module/plugins/crypter/FilebeerInfoFolder.py b/module/plugins/crypter/FilebeerInfoFolder.py index fd2843873..d0f3547bc 100644 --- a/module/plugins/crypter/FilebeerInfoFolder.py +++ b/module/plugins/crypter/FilebeerInfoFolder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class FilebeerInfoFolder(DeadCrypter): @@ -14,3 +14,6 @@ class FilebeerInfoFolder(DeadCrypter): __description__ = """Filebeer.info folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + +getInfo = create_getInfo(FilebeerInfoFolder) diff --git a/module/plugins/crypter/FilesonicComFolder.py b/module/plugins/crypter/FilesonicComFolder.py index 2d0540d26..d58516986 100644 --- a/module/plugins/crypter/FilesonicComFolder.py +++ b/module/plugins/crypter/FilesonicComFolder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class FilesonicComFolder(DeadCrypter): @@ -13,3 +13,6 @@ class FilesonicComFolder(DeadCrypter): __description__ = """Filesonic.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + +getInfo = create_getInfo(FilesonicComFolder) diff --git a/module/plugins/crypter/FiredriveComFolder.py b/module/plugins/crypter/FiredriveComFolder.py index b6b22a4e1..7d3a357fd 100644 --- a/module/plugins/crypter/FiredriveComFolder.py +++ b/module/plugins/crypter/FiredriveComFolder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class FiredriveComFolder(DeadCrypter): @@ -14,3 +14,6 @@ class FiredriveComFolder(DeadCrypter): __description__ = """Firedrive.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + +getInfo = create_getInfo(FiredriveComFolder) diff --git a/module/plugins/crypter/HotfileComFolder.py b/module/plugins/crypter/HotfileComFolder.py index 3fc8f6f63..a294c04e0 100644 --- a/module/plugins/crypter/HotfileComFolder.py +++ b/module/plugins/crypter/HotfileComFolder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class HotfileComFolder(DeadCrypter): @@ -14,3 +14,6 @@ class HotfileComFolder(DeadCrypter): __description__ = """Hotfile.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("RaNaN", "RaNaN@pyload.org")] + + +getInfo = create_getInfo(HotfileComFolder) diff --git a/module/plugins/crypter/ILoadTo.py b/module/plugins/crypter/ILoadTo.py index 3dc806b06..f3415706d 100644 --- a/module/plugins/crypter/ILoadTo.py +++ b/module/plugins/crypter/ILoadTo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class ILoadTo(DeadCrypter): @@ -14,3 +14,6 @@ class ILoadTo(DeadCrypter): __description__ = """Iload.to decrypter plugin""" __license__ = "GPLv3" __authors__ = [("hzpz", None)] + + +getInfo = create_getInfo(ILoadTo) diff --git a/module/plugins/crypter/LofCc.py b/module/plugins/crypter/LofCc.py index 5819e0375..65c9b18bd 100644 --- a/module/plugins/crypter/LofCc.py +++ b/module/plugins/crypter/LofCc.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class LofCc(DeadCrypter): @@ -14,3 +14,6 @@ class LofCc(DeadCrypter): __description__ = """Lof.cc decrypter plugin""" __license__ = "GPLv3" __authors__ = [("mkaay", "mkaay@mkaay.de")] + + +getInfo = create_getInfo(LofCc) diff --git a/module/plugins/crypter/MBLinkInfo.py b/module/plugins/crypter/MBLinkInfo.py index 680aa49c0..82c2d9719 100644 --- a/module/plugins/crypter/MBLinkInfo.py +++ b/module/plugins/crypter/MBLinkInfo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class MBLinkInfo(DeadCrypter): @@ -15,3 +15,6 @@ class MBLinkInfo(DeadCrypter): __license__ = "GPLv3" __authors__ = [("Gummibaer", "Gummibaer@wiki-bierkiste.de"), ("stickell", "l.stickell@yahoo.it")] + + +getInfo = create_getInfo(MBLinkInfo) diff --git a/module/plugins/crypter/MegauploadComFolder.py b/module/plugins/crypter/MegauploadComFolder.py index 1d487ed61..08f96700d 100644 --- a/module/plugins/crypter/MegauploadComFolder.py +++ b/module/plugins/crypter/MegauploadComFolder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class MegauploadComFolder(DeadCrypter): @@ -13,3 +13,6 @@ class MegauploadComFolder(DeadCrypter): __description__ = """Megaupload.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + +getInfo = create_getInfo(MegauploadComFolder) diff --git a/module/plugins/crypter/Movie2kTo.py b/module/plugins/crypter/Movie2kTo.py index 2acaa5a28..0be7eb7eb 100644 --- a/module/plugins/crypter/Movie2kTo.py +++ b/module/plugins/crypter/Movie2kTo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class Movie2kTo(DeadCrypter): @@ -14,3 +14,6 @@ class Movie2kTo(DeadCrypter): __description__ = """Movie2k.to decrypter plugin""" __license__ = "GPLv3" __authors__ = [("4Christopher", "4Christopher@gmx.de")] + + +getInfo = create_getInfo(Movie2kTo) diff --git a/module/plugins/crypter/MultiuploadCom.py b/module/plugins/crypter/MultiuploadCom.py index 2cfb9537f..347b7e5af 100644 --- a/module/plugins/crypter/MultiuploadCom.py +++ b/module/plugins/crypter/MultiuploadCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class MultiuploadCom(DeadCrypter): @@ -14,3 +14,6 @@ class MultiuploadCom(DeadCrypter): __description__ = """ MultiUpload.com decrypter plugin """ __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + +getInfo = create_getInfo(MultiuploadCom) diff --git a/module/plugins/crypter/OronComFolder.py b/module/plugins/crypter/OronComFolder.py index 30709ee60..9e06bdf32 100755 --- a/module/plugins/crypter/OronComFolder.py +++ b/module/plugins/crypter/OronComFolder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class OronComFolder(DeadCrypter): @@ -14,3 +14,6 @@ class OronComFolder(DeadCrypter): __description__ = """Oron.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("DHMH", "webmaster@pcProfil.de")] + + +getInfo = create_getInfo(OronComFolder) diff --git a/module/plugins/crypter/RSLayerCom.py b/module/plugins/crypter/RSLayerCom.py index 1bcfd05fe..cc3b23bbc 100644 --- a/module/plugins/crypter/RSLayerCom.py +++ b/module/plugins/crypter/RSLayerCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class RSLayerCom(DeadCrypter): @@ -14,3 +14,6 @@ class RSLayerCom(DeadCrypter): __description__ = """RS-Layer.com decrypter plugin""" __license__ = "GPLv3" __authors__ = [("hzpz", None)] + + +getInfo = create_getInfo(RSLayerCom) diff --git a/module/plugins/crypter/SecuredIn.py b/module/plugins/crypter/SecuredIn.py index d2cd6ff4d..cbfa919ac 100644 --- a/module/plugins/crypter/SecuredIn.py +++ b/module/plugins/crypter/SecuredIn.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class SecuredIn(DeadCrypter): @@ -14,3 +14,6 @@ class SecuredIn(DeadCrypter): __description__ = """Secured.in decrypter plugin""" __license__ = "GPLv3" __authors__ = [("mkaay", "mkaay@mkaay.de")] + + +getInfo = create_getInfo(SecuredIn) diff --git a/module/plugins/crypter/SharingmatrixComFolder.py b/module/plugins/crypter/SharingmatrixComFolder.py index ddc3bdb37..e16bdf814 100644 --- a/module/plugins/crypter/SharingmatrixComFolder.py +++ b/module/plugins/crypter/SharingmatrixComFolder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class SharingmatrixComFolder(DeadCrypter): @@ -13,3 +13,6 @@ class SharingmatrixComFolder(DeadCrypter): __description__ = """Sharingmatrix.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + +getInfo = create_getInfo(SharingmatrixComFolder) diff --git a/module/plugins/crypter/SpeedLoadOrgFolder.py b/module/plugins/crypter/SpeedLoadOrgFolder.py index f11260345..1d3a7df91 100644 --- a/module/plugins/crypter/SpeedLoadOrgFolder.py +++ b/module/plugins/crypter/SpeedLoadOrgFolder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class SpeedLoadOrgFolder(DeadCrypter): @@ -14,3 +14,6 @@ class SpeedLoadOrgFolder(DeadCrypter): __description__ = """Speedload decrypter plugin""" __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] + + +getInfo = create_getInfo(SpeedLoadOrgFolder) diff --git a/module/plugins/crypter/StealthTo.py b/module/plugins/crypter/StealthTo.py index a6e86fb8b..e4da3e7e4 100644 --- a/module/plugins/crypter/StealthTo.py +++ b/module/plugins/crypter/StealthTo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class StealthTo(DeadCrypter): @@ -14,3 +14,6 @@ class StealthTo(DeadCrypter): __description__ = """Stealth.to decrypter plugin""" __license__ = "GPLv3" __authors__ = [("spoob", "spoob@pyload.org")] + + +getInfo = create_getInfo(StealthTo) diff --git a/module/plugins/crypter/TrailerzoneInfo.py b/module/plugins/crypter/TrailerzoneInfo.py index 66a00bc74..abdb2307e 100644 --- a/module/plugins/crypter/TrailerzoneInfo.py +++ b/module/plugins/crypter/TrailerzoneInfo.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class TrailerzoneInfo(DeadCrypter): @@ -14,3 +14,6 @@ class TrailerzoneInfo(DeadCrypter): __description__ = """TrailerZone.info decrypter plugin""" __license__ = "GPLv3" __authors__ = [("godofdream", "soilfiction@gmail.com")] + + +getInfo = create_getInfo(TrailerzoneInfo) diff --git a/module/plugins/crypter/WiiReloadedOrg.py b/module/plugins/crypter/WiiReloadedOrg.py index 9d6f12174..c3c5b8222 100644 --- a/module/plugins/crypter/WiiReloadedOrg.py +++ b/module/plugins/crypter/WiiReloadedOrg.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class WiiReloadedOrg(DeadCrypter): @@ -14,3 +14,6 @@ class WiiReloadedOrg(DeadCrypter): __description__ = """Wii-Reloaded.org decrypter plugin""" __license__ = "GPLv3" __authors__ = [("hzpz", None)] + + +getInfo = create_getInfo(WiiReloadedOrg) diff --git a/module/plugins/crypter/WuploadComFolder.py b/module/plugins/crypter/WuploadComFolder.py index b77bc3e67..873c71fad 100644 --- a/module/plugins/crypter/WuploadComFolder.py +++ b/module/plugins/crypter/WuploadComFolder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.DeadCrypter import DeadCrypter +from module.plugins.internal.DeadCrypter import DeadCrypter, create_getInfo class WuploadComFolder(DeadCrypter): @@ -13,3 +13,6 @@ class WuploadComFolder(DeadCrypter): __description__ = """Wupload.com folder decrypter plugin""" __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + +getInfo = create_getInfo(WuploadComFolder) -- cgit v1.2.3 From 2ca8cc959a587992dc389d6baf71dd3e4f66df1c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 04:41:17 +0100 Subject: Fix some URL_REPLACEMENTS --- module/plugins/crypter/TusfilesNetFolder.py | 4 ++-- module/plugins/hoster/DataportCz.py | 4 +--- module/plugins/hoster/DepositfilesCom.py | 12 ++++++------ module/plugins/hoster/NowVideoAt.py | 4 ++-- module/plugins/hoster/TurbobitNet.py | 4 ++-- module/plugins/hoster/VeohCom.py | 6 +++--- module/plugins/hoster/VimeoCom.py | 8 ++++---- 7 files changed, 20 insertions(+), 22 deletions(-) diff --git a/module/plugins/crypter/TusfilesNetFolder.py b/module/plugins/crypter/TusfilesNetFolder.py index e8da59736..9533a6e76 100644 --- a/module/plugins/crypter/TusfilesNetFolder.py +++ b/module/plugins/crypter/TusfilesNetFolder.py @@ -10,7 +10,7 @@ from module.plugins.internal.XFSCrypter import XFSCrypter class TusfilesNetFolder(XFSCrypter): __name__ = "TusfilesNetFolder" __type__ = "crypter" - __version__ = "0.06" + __version__ = "0.07" __pattern__ = r'https?://(?:www\.)?tusfiles\.net/go/(?P\w+)' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), @@ -26,7 +26,7 @@ class TusfilesNetFolder(XFSCrypter): PAGES_PATTERN = r'>\((\d+) \w+\)<' - URL_REPLACEMENTS = [(__pattern__, r'https://www.tusfiles.net/go/\g/')] + URL_REPLACEMENTS = [(__pattern__ + ".*", r'https://www.tusfiles.net/go/\g/')] def loadPage(self, page_n): diff --git a/module/plugins/hoster/DataportCz.py b/module/plugins/hoster/DataportCz.py index 6bea4afac..b9e6fd370 100644 --- a/module/plugins/hoster/DataportCz.py +++ b/module/plugins/hoster/DataportCz.py @@ -6,7 +6,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class DataportCz(SimpleHoster): __name__ = "DataportCz" __type__ = "hoster" - __version__ = "0.39" + __version__ = "0.40" __pattern__ = r'http://(?:www\.)?dataport\.cz/file/(.*)' @@ -19,8 +19,6 @@ class DataportCz(SimpleHoster): SIZE_PATTERN = r'Velikost\s*(?P[^<]+)' OFFLINE_PATTERN = r'

Soubor nebyl nalezen

' - URL_REPLACEMENTS = [(__pattern__, r'http://www.dataport.cz/file/\1')] - CAPTCHA_PATTERN = r'
\s*(\d+)
' diff --git a/module/plugins/hoster/DepositfilesCom.py b/module/plugins/hoster/DepositfilesCom.py index d414e7021..6588a3b37 100644 --- a/module/plugins/hoster/DepositfilesCom.py +++ b/module/plugins/hoster/DepositfilesCom.py @@ -11,7 +11,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class DepositfilesCom(SimpleHoster): __name__ = "DepositfilesCom" __type__ = "hoster" - __version__ = "0.50" + __version__ = "0.51" __pattern__ = r'https?://(?:www\.)?(depositfiles\.com|dfiles\.(eu|ru))(/\w{1,3})?/files/(?P\w+)' @@ -22,18 +22,18 @@ class DepositfilesCom(SimpleHoster): ("Walter Purcaro", "vuolter@gmail.com")] - NAME_PATTERN = r'