diff options
65 files changed, 1628 insertions, 884 deletions
diff --git a/module/Api.py b/module/Api.py index 11072b2cd..ba79a31ef 100644 --- a/module/Api.py +++ b/module/Api.py @@ -22,14 +22,18 @@ from os.path import join from time import time import re -from remote.thriftbackend.thriftgen.pyload.ttypes import * -from remote.thriftbackend.thriftgen.pyload.Pyload import Iface - from PyFile import PyFile from utils import freeSpace, compare_time from common.packagetools import parseNames from network.RequestFactory import getURL - +from remote import activated + +if activated: + from remote.thriftbackend.thriftgen.pyload.ttypes import * + from remote.thriftbackend.thriftgen.pyload.Pyload import Iface + BaseObject = TBase +else: + from remote.socketbackend.ttypes import * # contains function names mapped to their permissions # unlisted functions are for admins only diff --git a/module/ConfigParser.py b/module/ConfigParser.py index bcf5bcd2a..78b612f13 100644 --- a/module/ConfigParser.py +++ b/module/ConfigParser.py @@ -2,15 +2,17 @@ from __future__ import with_statement from time import sleep -from os.path import exists -from os.path import join +from os.path import exists, join from shutil import copy from traceback import print_exc from utils import chmod -IGNORE = ("FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx") # ignore these plugin configs, mainly because plugins were wiped out +IGNORE = ( + "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('hooks', 'UnRar'), + 'EasyShareCom', 'FlyshareCz' + ) CONF_VERSION = 1 @@ -35,7 +37,7 @@ class ConfigParser: """ - #---------------------------------------------------------------------- + def __init__(self): """Constructor""" self.config = {} # the config values @@ -50,7 +52,7 @@ class ConfigParser: self.deleteOldPlugins() - #---------------------------------------------------------------------- + def checkVersion(self, n=0): """determines if config need to be copied""" try: @@ -88,7 +90,6 @@ class ConfigParser: else: raise - #---------------------------------------------------------------------- def readConfig(self): """reads the config file""" @@ -109,7 +110,7 @@ class ConfigParser: print "Config Warning" print_exc() - #---------------------------------------------------------------------- + def parseConfig(self, config): """parses a given configfile""" @@ -127,8 +128,7 @@ class ConfigParser: for line in config: comment = line.rfind("#") - if line.find(":", comment) < 0 and line.find("=", comment) < 0 and comment > 0 and line[ - comment - 1].isspace(): + if line.find(":", comment) < 0 > line.find("=", comment) and comment > 0 and line[comment - 1].isspace(): line = line.rpartition("#") # removes comments if line[1]: line = line[0] @@ -194,7 +194,6 @@ class ConfigParser: return conf - #---------------------------------------------------------------------- def updateValues(self, config, dest): """sets the config values from a parsed config file to values in destination""" @@ -213,7 +212,6 @@ class ConfigParser: #else: # dest[section] = config[section] - #---------------------------------------------------------------------- def saveConfig(self, config, filename): """saves config to filename""" with open(filename, "wb") as f: @@ -261,19 +259,19 @@ class ConfigParser: else: return value - #---------------------------------------------------------------------- + def save(self): """saves the configs to disk""" self.saveConfig(self.config, "pyload.conf") self.saveConfig(self.plugin, "plugin.conf") - #---------------------------------------------------------------------- + def __getitem__(self, section): """provides dictonary like access: c['section']['option']""" return Section(self, section) - #---------------------------------------------------------------------- + def get(self, section, option): """get value""" val = self.config[section][option]["value"] @@ -318,7 +316,6 @@ class ConfigParser: """ get all config data for an option """ return self.config[section][option] - #---------------------------------------------------------------------- def addPluginConfig(self, name, config, outline=""): """adds config options with tuples (name, type, desc, default)""" if name not in self.plugin: diff --git a/module/PluginThread.py b/module/PluginThread.py index c4bdd59c8..56c36c778 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -459,7 +459,7 @@ class HookThread(PluginThread): self.f(*self.args, **self.kwargs) except TypeError, e: #dirty method to filter out exceptions - if "unexpected keyword argument 'thread'" not in e.message: + if "unexpected keyword argument 'thread'" not in e.args[0]: raise del self.kwargs["thread"] diff --git a/module/PyPackage.py b/module/PyPackage.py index 991b18e54..f3be6c886 100644 --- a/module/PyPackage.py +++ b/module/PyPackage.py @@ -18,6 +18,7 @@ """ from module.PullEvents import UpdateEvent +from module.utils import save_path class PyPackage(): """ @@ -29,15 +30,17 @@ class PyPackage(): self.id = int(id) self.name = name - self.folder = folder + self._folder = folder self.site = site self.password = password self.queue = queue self.order = order - - self.setFinished = False + @property + def folder(self): + return save_path(self._folder) + def toDict(self): """ Returns a dictionary representation of the data. diff --git a/module/Utils.py b/module/Utils.py index 939baad96..3919b5ff0 100644 --- a/module/Utils.py +++ b/module/Utils.py @@ -32,6 +32,12 @@ def removeChars(string, repl): elif type(string) == unicode: return string.translate(dict([(ord(s), None) for s in repl])) +def save_path(name): + #remove some chars + if os.name == 'nt': + return removeChars(name, '/\\?%*:|"<>') + else: + return removeChars(name, '/\\"') def save_join(*args): """ joins a path, encoding aware """ @@ -76,9 +82,9 @@ def compare_time(start, end): if start == end: return True now = list(time.localtime()[3:5]) - if start < now and end > now: return True + if start < now < end: return True elif start > end and (now > start or now < end): return True - elif start < now and end < now and start > end: return True + elif start < now > end < start: return True else: return False diff --git a/module/common/ImportDebugger.py b/module/common/ImportDebugger.py new file mode 100644 index 000000000..a997f7b0c --- /dev/null +++ b/module/common/ImportDebugger.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +import sys + +class ImportDebugger(object): + + def __init__(self): + self.imported = {} + + def find_module(self, name, path=None): + + if name not in self.imported: + self.imported[name] = 0 + + self.imported[name] += 1 + + print name, path + +sys.meta_path.append(ImportDebugger())
\ No newline at end of file diff --git a/module/config/default.conf b/module/config/default.conf index 88ccc70e0..b63a06b4c 100644 --- a/module/config/default.conf +++ b/module/config/default.conf @@ -4,6 +4,7 @@ remote - "Remote": int port : "Port" = 7227
ip listenaddr : "Adress" = 0.0.0.0
bool nolocalauth : "No authentication on local connections" = True
+ bool activated : "Activated" = True
ssl - "SSL":
bool activated : "Activated"= False
file cert : "SSL Certificate" = ssl.crt
diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index 7e30f4817..fc521d36c 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -35,18 +35,19 @@ class AccountManager(): """Constructor""" self.core = core + self.lock = Lock() + + self.initPlugins() + self.saveAccounts() # save to add categories to conf + def initPlugins(self): self.accounts = {} # key = ( plugin ) self.plugins = {} - self.lock = Lock() self.initAccountPlugins() - self.loadAccounts() - self.saveAccounts() # save to add categories to conf - #---------------------------------------------------------------------- def getAccountPlugin(self, plugin): """get account instance for plugin or None if anonymous""" if plugin in self.accounts: diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 7a7ecd974..1abf02bbe 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -17,17 +17,11 @@ @author: RaNaN, spoob, mkaay """ -from time import time -from time import sleep - +from time import time, sleep from random import randint import os -from os import remove -from os import makedirs -from os import chmod -from os import stat -from os import name as os_name +from os import remove, makedirs, chmod, stat from os.path import exists, join if os.name != "nt": @@ -37,7 +31,7 @@ if os.name != "nt": from itertools import islice -from module.utils import save_join, fs_encode, removeChars +from module.utils import save_join, save_path, fs_encode def chunks(iterable, size): it = iter(iterable) @@ -495,12 +489,7 @@ class Plugin(Base): except Exception, e: self.log.warning(_("Setting User and Group failed: %s") % str(e)) - name = self.pyfile.name - if os_name == 'nt': - #delete illegal characters - name = removeChars(name, '/\\?%*:|"<>') - else: - name = removeChars(name, '/\\"') + name = save_path(self.pyfile.name) filename = join(location, fs_encode(name)) diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index e6ad14b66..09d5f58e7 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -46,16 +46,9 @@ class PluginManager: #self.config = self.core.config self.log = core.log + self.plugins = {} self.createIndex() - self.plugins = {"crypter": self.crypterPlugins, - "container": self.containerPlugins, - "hoster": self.hosterPlugins, - "captcha": self.captchaPlugins, - "accounts": self.accountPlugins, - "hooks": self.hookPlugins, - "internal": self.internalPlugins} - #register for import hook sys.meta_path.append(self) @@ -71,14 +64,14 @@ class PluginManager: f = open(join("userplugins", "__init__.py"), "wb") f.close() - self.crypterPlugins = self.parse("crypter", pattern=True) - self.containerPlugins = self.parse("container", pattern=True) - self.hosterPlugins = self.parse("hoster", pattern=True) + self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) + self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) + self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - self.captchaPlugins = self.parse("captcha") - self.accountPlugins = self.parse("accounts") - self.hookPlugins = self.parse("hooks") - self.internalPlugins = self.parse("internal") + self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") + self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") + self.plugins["hooks"] = self.hookPlugins = self.parse("hooks") + self.plugins["internal"] = self.internalPlugins = self.parse("internal") self.log.debug("created index of plugins") @@ -132,21 +125,16 @@ class PluginManager: if home[name]["v"] >= version: continue + if name in IGNORE or (folder, name) in IGNORE: + continue + plugins[name] = {} plugins[name]["v"] = version module = f.replace(".pyc", "").replace(".py", "") - if home: - if name in IGNORE: - del plugins[name] - continue - - user = True - else: - user = False # the plugin is loaded from user directory - plugins[name]["user"] = user + plugins[name]["user"] = True if home else False plugins[name]["name"] = module if pattern: @@ -334,9 +322,43 @@ class PluginManager: return sys.modules[name] - def reloadPlugins(self): + def reloadPlugins(self, type_plugins): """ reloads and reindexes plugins """ - pass + if not type_plugins: return False + + self.log.debug("Reload plugins: %s" % type_plugins) + + as_dict = {} + for t,n in type_plugins: + if t in as_dict: + as_dict[t].append(n) + else: + as_dict[t] = [n] + + # we do not reload hooks or internals, would cause to much side effects + if "hooks" in as_dict or "internal" in as_dict: + return False + + for type in as_dict.iterkeys(): + for plugin in as_dict[type]: + if plugin in self.plugins[type]: + if "module" in self.plugins[type][plugin]: + self.log.debug("Reloading %s" % plugin) + reload(self.plugins[type][plugin]["module"]) + + #index creation + self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) + self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) + self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) + self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") + self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") + + if "accounts" in as_dict: #accounts needs to be reloaded + self.core.accountManager.initPlugins() + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + + return True + if __name__ == "__main__": diff --git a/module/plugins/accounts/FshareVn.py b/module/plugins/accounts/FshareVn.py new file mode 100644 index 000000000..46d2d5166 --- /dev/null +++ b/module/plugins/accounts/FshareVn.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: zoidberg +""" + +from module.plugins.Account import Account +from time import mktime, strptime +from pycurl import REFERER +import re + +class FshareVn(Account): + __name__ = "FshareVn" + __version__ = "0.01" + __type__ = "account" + __description__ = """fshare.vn account plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + VALID_UNTIL_PATTERN = ur'<dt>Thời hạn dùng:</dt>\s*<dd>([^<]+)</dd>' + TRAFFIC_LEFT_PATTERN = ur'<dt>Bandwidth Còn Lại</dt>\s*<dd[^>]*>([0-9.]+) ([kKMG])B</dd>' + DIRECT_DOWNLOAD_PATTERN = ur'<input type="checkbox"\s*([^=>]*)[^>]*/>Kích hoạt download trực tiếp</dt>' + + def loadAccountInfo(self, user, req): + #self.relogin(user) + html = req.load("http://www.fshare.vn/account_info.php", decode = True) + + found = re.search(self.VALID_UNTIL_PATTERN, html) + validuntil = mktime(strptime(found.group(1), '%I:%M:%S %p %d-%m-%Y')) if found else 0 + + found = re.search(self.TRAFFIC_LEFT_PATTERN, html) + trafficleft = float(found.group(1)) * 1024 ** {'k': 0, 'K': 0, 'M': 1, 'G': 2}[found.group(2)] if found else 0 + + return {"validuntil": validuntil, "trafficleft": trafficleft} + + def login(self, user, data, req): + req.http.c.setopt(REFERER, "http://www.fshare.vn/login.php") + + html = req.load('http://www.fshare.vn/login.php', post = { + "login_password" : data['password'], + "login_useremail" : user, + "url_refe" : "http://www.fshare.vn/login.php" + }, referer = True, decode = True) + + if not '<img alt="VIP"' in html: + self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/accounts/HellspyCz.py b/module/plugins/accounts/HellspyCz.py index c07fd748a..5f14a093e 100644 --- a/module/plugins/accounts/HellspyCz.py +++ b/module/plugins/accounts/HellspyCz.py @@ -29,8 +29,7 @@ class HellspyCz(Account): __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - ACTION_PATTERN = r'<div id="snippet--loginBoxSn"><form[^>]*action="([^"]+user_hash=([^"]+))">' - CREDIT_LEFT_PATTERN = r'<strong class="text-credits">(\d+)</strong>' + CREDIT_LEFT_PATTERN = r'<strong>Credits: </strong>\s*(\d+)' WRONG_PASSWORD_PATTERN = r'<p class="block-error-3 marg-tb-050">\s*Wrong user or password was entered<br />' phpsessid = '' @@ -50,16 +49,13 @@ class HellspyCz(Account): return {"validuntil": -1, "trafficleft": credits} def login(self, user, data,req): - html = req.load('http://www.hellspy.com/') - found = re.search(self.ACTION_PATTERN, html) - if found is None: - self.logError('Parse error (FORM)') - action, self.phpsessid = found.group(1).replace('&','&'), found.group(2) - + header = req.load('http://www.hellspy.com/', just_header = True) + self.phpsessid = re.search(r'PHPSESSID=(\w+)', header).group(1) self.logDebug("PHPSESSID:" + self.phpsessid) + html = req.load("http://www.hellspy.com/--%s-" % self.phpsessid) - html = req.load(action, post={ + html = req.load("http://www.hell-share.com/user/login/?do=apiLoginForm-submit&api_hash=hellspy_iq&user_hash=%s" % self.phpsessid, post={ "login": "1", "password": data["password"], "username": user, @@ -70,10 +66,5 @@ class HellspyCz(Account): cj = self.getAccountCookies(user) cj.setCookie(".hellspy.com", "PHPSESSID", self.phpsessid) - self.logDebug(req.lastURL) - self.logDebug(req.lastEffectiveURL) - - html = req.load("http://www.hellspy.com/", get = {"do":"loginBox-login"}) - if not re.search(self.CREDIT_LEFT_PATTERN, html): self.wrongPassword()
\ No newline at end of file diff --git a/module/plugins/crypter/FileserveComFolder.py b/module/plugins/crypter/FileserveComFolder.py new file mode 100644 index 000000000..9fe806971 --- /dev/null +++ b/module/plugins/crypter/FileserveComFolder.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.Crypter import Crypter + +class FileserveComFolder(Crypter): + __name__ = "FileserveComFolder" + __type__ = "crypter" + __pattern__ = r"http://(?:www\.)?fileserve.com/list/\w+" + __version__ = "0.11" + __description__ = """FileServeCom.com Folder Plugin""" + __author_name__ = ("fionnc") + __author_mail__ = ("fionnc@gmail.com") + + FOLDER_PATTERN = r'<table class="file_list">(.*?)</table>' + LINK_PATTERN = r'<a href="([^"]+)" class="sheet_icon wbold">' + + def decrypt(self, pyfile): + html = self.load(self.pyfile.url) + + new_links = [] + + folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) + if folder is None: self.fail("Parse error (FOLDER)") + + new_links.extend(re.findall(self.LINK_PATTERN, folder.group(1))) + + if new_links: + self.core.files.addLinks(map(lambda s:"http://fileserve.com%s" % s, new_links), self.pyfile.package().id) + else: + self.fail('Could not extract any links')
\ No newline at end of file diff --git a/module/plugins/crypter/FilesonicComFolder.py b/module/plugins/crypter/FilesonicComFolder.py index 7bf1df381..b967a74a1 100644 --- a/module/plugins/crypter/FilesonicComFolder.py +++ b/module/plugins/crypter/FilesonicComFolder.py @@ -6,8 +6,8 @@ from module.plugins.Crypter import Crypter class FilesonicComFolder(Crypter): __name__ = "FilesonicComFolder" __type__ = "crypter" - __pattern__ = r"http://(\w*\.)?(sharingmatrix|filesonic|wupload)\.[^/]*/folder/\d+/?" - __version__ = "0.10" + __pattern__ = r"http://(\w*\.)?(sharingmatrix|filesonic|wupload)\.[^/]*/folder/\w+/?" + __version__ = "0.11" __description__ = """Filesonic.com/Wupload.com Folder Plugin""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") diff --git a/module/plugins/crypter/MediafireComFolder.py b/module/plugins/crypter/MediafireComFolder.py new file mode 100644 index 000000000..49a72ca76 --- /dev/null +++ b/module/plugins/crypter/MediafireComFolder.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +import re +from module.plugins.Crypter import Crypter +from module.common.json_layer import json_loads + +class MediafireComFolder(Crypter): + __name__ = "MediafireComFolder" + __type__ = "crypter" + __pattern__ = r"http://(\w*\.)*mediafire\.com/(folder/|\?).*" + __version__ = "0.10" + __description__ = """Mediafire.com Folder Plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FOLDER_KEY_PATTERN = r"var afI= '(\w+)';" + FILE_URL_PATTERN = '<meta property="og:url" content="http://www.mediafire.com/\?(\w+)"/>' + + def decrypt(self, pyfile): + new_links = [] + + html = self.load(pyfile.url) + found = re.search(self.FILE_URL_PATTERN, html) + if found: + new_links.append("http://www.mediafire.com/download.php?" + found.group(1)) + else: + found = re.search(self.FOLDER_KEY_PATTERN, html) + if not found: self.fail('Parse error: Folder Key') + folder_key = found.group(1) + self.logDebug("FOLDER KEY: %s" % folder_key) + + json_resp = json_loads(self.load("http://www.mediafire.com/api/folder/get_info.php?folder_key=%s&response_format=json&version=1" % folder_key)) + #self.logInfo(json_resp) + if json_resp['response']['result'] == "Success": + for link in json_resp['response']['folder_info']['files']: + new_links.append("http://www.mediafire.com/download.php?%s" % link['quickkey']) + else: + self.fail(json_resp['response']['message']) + + if new_links: + self.core.files.addLinks(new_links, self.pyfile.package().id) + else: + self.fail('Could not extract any links')
\ No newline at end of file diff --git a/module/plugins/crypter/SerienjunkiesOrg.py b/module/plugins/crypter/SerienjunkiesOrg.py index 5b6295fe7..2178f5300 100644 --- a/module/plugins/crypter/SerienjunkiesOrg.py +++ b/module/plugins/crypter/SerienjunkiesOrg.py @@ -12,12 +12,14 @@ class SerienjunkiesOrg(Crypter): __type__ = "container" __pattern__ = r"http://.*?serienjunkies.org/.*?" __version__ = "0.31" - __config__ = [ ("preferredHoster", "str", "preferred hoster" , "RapidshareCom,UploadedTo,NetloadIn,FilefactoryCom,FreakshareNet,FilebaseTo,MegauploadCom,HotfileCom,DepositfilesCom,EasyshareCom,KickloadCom"), - ("changeName", "bool", "Take SJ.org episode name", "True") ] + __config__ = [("preferredHoster", "str", "preferred hoster", + "RapidshareCom,UploadedTo,NetloadIn,FilefactoryCom,FreakshareNet,FilebaseTo,MegauploadCom,HotfileCom,DepositfilesCom,EasyshareCom,KickloadCom") + , + ("changeName", "bool", "Take SJ.org episode name", "True")] __description__ = """serienjunkies.org Container Plugin""" __author_name__ = ("mkaay") __author_mail__ = ("mkaay@mkaay.de") - + def setup(self): self.hosterMap = { "rc": "RapidshareCom", @@ -33,26 +35,26 @@ class SerienjunkiesOrg(Crypter): "es": "EasyshareCom", "kl": "KickloadCom", "fc": "FilesonicCom", - } - self.hosterMapReverse = dict((v,k) for k, v in self.hosterMap.iteritems()) - + } + self.hosterMapReverse = dict((v, k) for k, v in self.hosterMap.iteritems()) + self.multiDL = False self.limitDL = 4 - + def getSJSrc(self, url): src = self.req.load(str(url)) if not src.find("Enter Serienjunkies") == -1: sleep(1) src = self.req.load(str(url)) return src - + def handleShow(self, url): src = self.getSJSrc(url) soup = BeautifulSoup(src) nav = soup.find("div", attrs={"id": "scb"}) for a in nav.findAll("a"): self.packages.append((unescape(a.text), [a["href"]], unescape(a.text))) - + def handleSeason(self, url): src = self.getSJSrc(url) soup = BeautifulSoup(src) @@ -63,7 +65,7 @@ class SerienjunkiesOrg(Crypter): self.log.debug("Preferred hoster: %s" % ", ".join(preferredHoster)) groups = {} gid = -1 - seasonName = unescape(soup.find("a", attrs={"rel":"bookmark"}).string) + seasonName = unescape(soup.find("a", attrs={"rel": "bookmark"}).string) for p in ps: if re.search("<strong>Dauer|<strong>Sprache|<strong>Format", str(p)): var = p.findAll("strong") @@ -115,37 +117,40 @@ class SerienjunkiesOrg(Crypter): if hmatch: break self.packages.append((package, links, package)) - + def handleEpisode(self, url): src = self.getSJSrc(url) - if not src.find("Du hast das Download-Limit überschritten! Bitte versuche es später nocheinmal.") == -1: + if not src.find( + "Du hast das Download-Limit überschritten! Bitte versuche es später nocheinmal.") == -1: self.fail(_("Downloadlimit reached")) else: soup = BeautifulSoup(src) form = soup.find("form") - packageName = soup.find("h1", attrs={"class":"wrap"}).text - captchaTag = soup.find(attrs={"src":re.compile("^/secure/")}) - if not captchaTag: - sleep(1) - self.retry() - - captchaUrl = "http://download.serienjunkies.org"+captchaTag["src"] - result = self.decryptCaptcha(str(captchaUrl), imgtype="png") - sinp = form.find(attrs={"name":"s"}) - - self.req.lastURL = str(url) - sj = self.load(str(url), post={'s': sinp["value"], 'c': result, 'action': "Download"}) - - soup = BeautifulSoup(sj) + h1 = soup.find("h1") + packageName = h1.text + if h1.get("class") == "wrap": + captchaTag = soup.find(attrs={"src": re.compile("^/secure/")}) + if not captchaTag: + sleep(1) + self.retry() + + captchaUrl = "http://download.serienjunkies.org" + captchaTag["src"] + result = self.decryptCaptcha(str(captchaUrl), imgtype="png") + sinp = form.find(attrs={"name": "s"}) + + self.req.lastURL = str(url) + sj = self.load(str(url), post={'s': sinp["value"], 'c': result, 'action': "Download"}) + + soup = BeautifulSoup(sj) rawLinks = soup.findAll(attrs={"action": re.compile("^http://download.serienjunkies.org/")}) - + if not len(rawLinks) > 0: sleep(1) self.retry() return - + self.correctCaptcha() - + links = [] for link in rawLinks: frameUrl = link["action"].replace("/go-", "/frame/go-") @@ -156,27 +161,28 @@ class SerienjunkiesOrg(Crypter): packageName = self.pyfile.package().name self.packages.append((packageName, links, packageName)) - + def handleOldStyleLink(self, url): sj = self.req.load(str(url)) soup = BeautifulSoup(sj) - form = soup.find("form", attrs={"action":re.compile("^http://serienjunkies.org")}) - captchaTag = form.find(attrs={"src":re.compile("^/safe/secure/")}) - captchaUrl = "http://serienjunkies.org"+captchaTag["src"] + form = soup.find("form", attrs={"action": re.compile("^http://serienjunkies.org")}) + captchaTag = form.find(attrs={"src": re.compile("^/safe/secure/")}) + captchaUrl = "http://serienjunkies.org" + captchaTag["src"] result = self.decryptCaptcha(str(captchaUrl)) url = form["action"] - sinp = form.find(attrs={"name":"s"}) - - self.req.load(str(url), post={'s': sinp["value"], 'c': result, 'dl.start': "Download"}, cookies=False, just_header=True) + sinp = form.find(attrs={"name": "s"}) + + self.req.load(str(url), post={'s': sinp["value"], 'c': result, 'dl.start': "Download"}, cookies=False, + just_header=True) decrypted = self.req.lastEffectiveURL if decrypted == str(url): self.retry() self.packages.append((self.pyfile.package().name, [decrypted], self.pyfile.package().folder)) - + def handleFrame(self, url): self.req.load(str(url)) return self.req.lastEffectiveURL - + def decrypt(self, pyfile): showPattern = re.compile("^http://serienjunkies.org/serie/(.*)/$") seasonPattern = re.compile("^http://serienjunkies.org/.*?/(.*)/$") diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py index 2a85e505f..8212ddb65 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/hooks/UpdateManager.py @@ -17,8 +17,12 @@ @author: RaNaN @interface-version: 0.1 """ + +import sys import re +from os import stat from os.path import join +from time import time from module.network.RequestFactory import getURL from module.plugins.Hook import threaded, Expose, Hook @@ -28,12 +32,27 @@ class UpdateManager(Hook): __version__ = "0.1" __description__ = """checks for updates""" __config__ = [("activated", "bool", "Activated", "True"), - ("interval", "int", "Check interval in minutes", "360")] + ("interval", "int", "Check interval in minutes", "360"), + ("debug", "bool", "Check for plugin changes when in debug mode", False)] __author_name__ = ("RaNaN") __author_mail__ = ("ranan@pyload.org") + @property + def debug(self): + return self.core.debug and self.getConfig("debug") + + def setup(self): - self.interval = self.getConfig("interval") * 60 + if self.debug: + self.logDebug("Monitoring file changes") + self.interval = 4 + self.last_check = 0 #timestamp of updatecheck + self.old_periodical = self.periodical + self.periodical = self.checkChanges + self.mtimes = {} #recordes times + else: + self.interval = self.getConfig("interval") * 60 + self.updated = False self.reloaded = True @@ -47,12 +66,13 @@ class UpdateManager(Hook): else: self.log.info(_("No Updates for pyLoad")) self.checkPlugins() - + if self.updated and not self.reloaded: self.info["plugins"] = True self.log.info(_("*** Plugins have been updated, please restart pyLoad ***")) elif self.updated and self.reloaded: self.log.info(_("Plugins updated and reloaded")) + self.updated = False else: self.log.info(_("No plugin updates available")) @@ -90,6 +110,7 @@ class UpdateManager(Hook): return False updates = updates.splitlines() + reloads = [] vre = re.compile(r'__version__.*=.*("|\')([0-9.]+)') @@ -135,5 +156,32 @@ class UpdateManager(Hook): f.close() self.updated = True - self.reloaded = False - self.core.pluginManager.reloadPlugins() + reloads.append((type, name)) + + self.reloaded = self.core.pluginManager.reloadPlugins(reloads) + + def checkChanges(self): + + if self.last_check + self.getConfig("interval") * 60 < time(): + self.old_periodical() + self.last_check = time() + + modules = filter( + lambda m: m and (m.__name__.startswith("module.plugins.") or m.__name__.startswith("userplugins.")), + sys.modules.itervalues()) + + reloads = [] + + for m in modules: + root, type, name = m.__name__.rsplit(".", 2) + id = (type, name) + if type in self.core.pluginManager.plugins: + mtime = stat(m.__file__.replace(".pyc", ".py")).st_mtime + + if id not in self.mtimes: + self.mtimes[id] = mtime + elif self.mtimes[id] < mtime: + reloads.append(id) + self.mtimes[id] = mtime + + self.core.pluginManager.reloadPlugins(reloads)
\ No newline at end of file diff --git a/module/plugins/hoster/BezvadataCz.py b/module/plugins/hoster/BezvadataCz.py index 4c198d95f..f061fa2b5 100644 --- a/module/plugins/hoster/BezvadataCz.py +++ b/module/plugins/hoster/BezvadataCz.py @@ -17,29 +17,19 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(BezvadataCz, url, getURL(url, decode=True)) - result.append(file_info) - - yield result +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class BezvadataCz(SimpleHoster): __name__ = "BezvadataCz" __type__ = "hoster" __pattern__ = r"http://(\w*\.)*bezvadata.cz/stahnout/.*" - __version__ = "0.21" + __version__ = "0.22" __description__ = """BezvaData.cz""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - FILE_NAME_PATTERN = r'<p><b>Soubor: ([^<]+)</b></p>' - FILE_SIZE_PATTERN = r'<li><strong>Velikost:</strong> ([0-9.]+) ([kKMG]i?B)</li>' + FILE_NAME_PATTERN = r'<p><b>Soubor: (?P<N>[^<]+)</b></p>' + FILE_SIZE_PATTERN = r'<li><strong>Velikost:</strong> (?P<S>[0-9.]+) (?P<U>[kKMG])i?)</li>' FILE_OFFLINE_PATTERN = r'<title>BezvaData \| Soubor nenalezen</title>' DOWNLOAD_FORM_PATTERN = r'<form class="download" action="([^"]+)" method="post" id="frm-stahnoutForm">' @@ -50,4 +40,6 @@ class BezvadataCz(SimpleHoster): self.logDebug("Download form: %s" % url) self.download(url, post = {"stahnoutSoubor": "St%C3%A1hnout"}, cookies = True) + +getInfo = create_getInfo(BezvadataCz)
\ No newline at end of file diff --git a/module/plugins/hoster/CrockoCom.py b/module/plugins/hoster/CrockoCom.py new file mode 100644 index 000000000..7eafa67ed --- /dev/null +++ b/module/plugins/hoster/CrockoCom.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from module.plugins.ReCaptcha import ReCaptcha +import re + +class CrockoCom(SimpleHoster): + __name__ = "CrockoCom" + __type__ = "hoster" + __pattern__ = r"http://(www\.)?(crocko|easy-share).com/.*" + __version__ = "0.10" + __description__ = """Crocko Download Hoster""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FILE_INFO_PATTERN = r'<strong>(?P<N>.*)\s*<span class="tip1"><span class="inner">(?P<S>[0-9,.]+) (?P<U>[kKMG])i?B</span></span>' + FILE_OFFLINE_PATTERN = r"<h1>Sorry,<br />the page you're looking for <br />isn't here.</h1>" + DOWNLOAD_URL_PATTERN = r"window.location ='([^']+)';" + CAPTCHA_URL_PATTERN = re.compile(r"u='(/file_contents/captcha/\w+)';\s*w='(\d+)';") + CAPTCHA_KEY_PATTERN = re.compile(r'Recaptcha.create\("([^"]+)"') + + FORM_PATTERN = r'<form method="post" action="([^"]+)">(.*?)</form>' + FORM_INPUT_PATTERN = r'<input[^>]* name="?([^" ]+)"? value="?([^" ]+)"?[^>]*>' + + NAME_REPLACEMENTS = [(r'<[^>]*>', '')] + + def handleFree(self): + if "You need Premium membership to download this file." in self.html: + self.fail("You need Premium membership to download this file.") + + url = False + for i in range(5): + found = re.search(self.CAPTCHA_URL_PATTERN, self.html) + if found: + url, wait_time = 'http://crocko.com' + found.group(1), found.group(2) + self.setWait(wait_time) + self.wait() + self.html = self.load(url) + else: + break + + found = re.search(self.CAPTCHA_KEY_PATTERN, self.html) + if not found: self.parseError('Captcha KEY') + captcha_key = found.group(1) + + found = re.search(self.FORM_PATTERN, self.html, re.DOTALL) + if not found: self.parseError('ACTION') + action, form = found.groups() + inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) + + recaptcha = ReCaptcha(self) + + for i in range(5): + inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) + self.download(action, post = inputs) + + check = self.checkDownload({ + "captcha_err": self.CAPTCHA_KEY_PATTERN + }) + + if check == "captcha_err": + self.invalidCaptcha() + else: + break + else: + self.fail('No valid captcha solution received') + +getInfo = create_getInfo(CrockoCom) +
\ No newline at end of file diff --git a/module/plugins/hoster/CzshareCom.py b/module/plugins/hoster/CzshareCom.py index 140aa9569..1a705e302 100644 --- a/module/plugins/hoster/CzshareCom.py +++ b/module/plugins/hoster/CzshareCom.py @@ -17,7 +17,7 @@ """ import re -from module.plugins.Hoster import Hoster +from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo from module.network.RequestFactory import getURL def toInfoPage(url): @@ -36,93 +36,55 @@ def getInfo(urls): for url in urls: info_url = toInfoPage(url) if info_url: - html = getURL(info_url, decode=True) - if re.search(CzshareCom.FILE_OFFLINE_PATTERN, html): - # File offline - result.append((url, 0, 1, url)) - else: - # Get file info - name, size = url, 0 - - found = re.search(CzshareCom.FILE_SIZE_PATTERN, html) - if found is not None: - size = float(found.group(1).replace(',','.').replace(' ','')) - units = found.group(2) - pow = {'KiB': 1, 'MiB': 2, 'GiB': 3}[units] - size = int(size * 1024 ** pow) - - found = re.search(CzshareCom.FILE_NAME_PATTERN, html) - if found is not None: - name = found.group(1) - - if found or size > 0: - result.append((name, size, 2, url)) + file_info = parseFileInfo(CzshareCom, url, getURL(info_url, decode=True)) + result.append(file_info) + yield result -class CzshareCom(Hoster): +class CzshareCom(SimpleHoster): __name__ = "CzshareCom" __type__ = "hoster" __pattern__ = r"http://(\w*\.)*czshare\.(com|cz)/(\d+/|download.php\?).*" - __version__ = "0.8a" + __version__ = "0.84" __description__ = """CZshare.com""" __author_name__ = ("zoidberg") + SIZE_REPLACEMENTS = {',': '.', ' ': ''} FREE_URL_PATTERN = r'<a href="([^"]+)" class="page-download">[^>]*alt="([^"]+)" /></a>' FREE_FORM_PATTERN = r'<form action="download.php" method="post">\s*<img src="captcha.php" id="captcha" />(.*?)</form>' PREMIUM_FORM_PATTERN = r'<form action="/profi_down.php" method="post">(.*?)</form>' FORM_INPUT_PATTERN = r'<input[^>]* name="([^"]+)" value="([^"]+)"[^>]*/>' - FILE_OFFLINE_PATTERN = r'<h2 class="red">[^<]*[Ss]oubor (nenalezen|expiroval|je po.kozen|byl smaz.n)[^<]*<span> </span></h2>' + #FILE_OFFLINE_PATTERN = r'<h2 class="red">[^<]*[Ss]oubor (nenalezen|expiroval|je po.kozen)[^<]*<span> </span></h2>' + FILE_OFFLINE_PATTERN = r'<div class="header clearfix">\s*<h2 class="red">' MULTIDL_PATTERN = r"<p><font color='red'>Z[^<]*PROFI.</font></p>" #FILE_NAME_PATTERN = r'<h1>([^<]+)<span> </span></h1>' - FILE_NAME_PATTERN = r'<div class="tab" id="parameters">\s*<p>\s*Cel. n.zev: <a href=[^>]*>([^<]+)</a>' - FILE_SIZE_PATTERN = r'<div class="tab" id="category">(?:\s*<p>[^\n]*</p>)*\s*Velikost:\s*([0-9., ]+)(KiB|MiB|GiB)\s*</div>' - USER_CREDIT_PATTERN = r'<div class="credit">\s*kredit: <strong>([0-9., ]+)(KB|MB|GB)</strong>\s*</div><!-- .credit -->' + FILE_NAME_PATTERN = r'<div class="tab" id="parameters">\s*<p>\s*Cel. n.zev: <a href=[^>]*>(?P<N>[^<]+)</a>' + FILE_SIZE_PATTERN = r'<div class="tab" id="category">(?:\s*<p>[^\n]*</p>)*\s*Velikost:\s*(?P<S>[0-9., ]+)(?P<U>[kKMG])i?B\s*</div>' + USER_CREDIT_PATTERN = r'<div class="credit">\s*kredit: <strong>([0-9., ]+)([kKMG]i?B)</strong>\s*</div><!-- .credit -->' def setup(self): self.resumeDownload = self.multiDL = True if self.premium else False self.chunkLimit = 1 def process(self, pyfile): - self.getFileInfo(pyfile) - - if not self.account or not self.handlePremium(pyfile): - self.handleFree(pyfile) - self.checkDownloadedFile() - - def getFileInfo(self, pyfile): url = toInfoPage(pyfile.url) if not url: self.logError(e) self.fail("Invalid URL") self.html = self.load(url, cookies=True, decode=True) + self.getFileInfo() - #marks the file as "offline" when the pattern was found on the html-page - if re.search(self.FILE_OFFLINE_PATTERN, self.html) is not None: - self.offline() - - # parse the name from the site and set attribute in pyfile - found = re.search(self.FILE_NAME_PATTERN, self.html) - if found is None: - self.fail("Parse error (NAME)") - pyfile.name = found.group(1) - self.logDebug("NAME:" + pyfile.name) - - found = re.search(self.FILE_SIZE_PATTERN, self.html) - if found is None: - self.logError("Parse error (SIZE)") - else: - size = float(found.group(1).replace(',','.').replace(' ','')) - pyfile.size = size * 1024 ** {'KiB': 1, 'MiB': 2, 'GiB': 3}[found.group(2)] - - pyfile.url = url + if not self.account or not self.handlePremium(): + self.handleFree() + self.checkDownloadedFile() - def handlePremium(self, pyfile): + def handlePremium(self): # check if user logged in found = re.search(self.USER_CREDIT_PATTERN, self.html) if not found: self.account.relogin(self.user) - self.html = self.load(pyfile.url, cookies=True, decode=True) + self.html = self.load(self.pyfile.url, cookies=True, decode=True) found = re.search(self.USER_CREDIT_PATTERN, self.html) if not found: return False @@ -130,10 +92,10 @@ class CzshareCom(Hoster): try: credit = float(found.group(1).replace(',','.').replace(' ','')) credit = credit * 1024 ** {'KB': 0, 'MB': 1, 'GB': 2}[found.group(2)] - self.logInfo("Premium download for %i KiB of Credit" % (pyfile.size / 1024)) + self.logInfo("Premium download for %i KiB of Credit" % (self.pyfile.size / 1024)) self.logInfo("User %s has %i KiB left" % (self.user, credit)) - if credit * 1024 < pyfile.size: - self.logInfo("Not enough credit to download file %s" % pyfile.name) + if credit * 1024 < self.pyfile.size: + self.logInfo("Not enough credit to download file %s" % self.pyfile.name) self.resetAccount() except Exception, e: # let's continue and see what happens... @@ -151,11 +113,11 @@ class CzshareCom(Hoster): self.download("http://czshare.com/profi_down.php", cookies=True, post=inputs) return True - def handleFree(self, pyfile): + def handleFree(self): # get free url found = re.search(self.FREE_URL_PATTERN, self.html) if found is None: - self.fail("Parse error (URL)") + raise PluginParseError('Free URL') parsed_url = "http://czshare.com" + found.group(1) self.logDebug("PARSED_URL:" + parsed_url) @@ -169,10 +131,10 @@ class CzshareCom(Hoster): try: form = re.search(self.FREE_FORM_PATTERN, self.html, re.DOTALL).group(1) inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) - pyfile.size = int(inputs['size']) + self.pyfile.size = int(inputs['size']) except Exception, e: self.logError(e) - self.fail("Parse error (FORM)") + raise PluginParseError('Form') # get and decrypt captcha captcha_url = 'http://czshare.com/captcha.php' diff --git a/module/plugins/hoster/DataportCz.py b/module/plugins/hoster/DataportCz.py index ed4ffa07c..487766201 100644 --- a/module/plugins/hoster/DataportCz.py +++ b/module/plugins/hoster/DataportCz.py @@ -17,28 +17,18 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(DataportCz, url, getURL(url, decode=True)) - result.append(file_info) - - yield result +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class DataportCz(SimpleHoster): __name__ = "DataportCz" __type__ = "hoster" __pattern__ = r"http://.*dataport.cz/file/.*" - __version__ = "0.32" + __version__ = "0.33" __description__ = """Dataport.cz plugin - free only""" __author_name__ = ("zoidberg") - FILE_NAME_PATTERN = r'<h2 style="color: red;">([^<]+)</h2>' - FILE_SIZE_PATTERN = r'<td>Velikost souboru:</td>\s*<td>([0-9.]+)([kKMG]i?B)</td>' + FILE_NAME_PATTERN = r'<h2 style="color: red;">(?P<N>[^<]+)</h2>' + FILE_SIZE_PATTERN = r'<td>Velikost souboru:</td>\s*<td>(?P<S>[0-9.]+)(?P<U>[kKMG])i?B</td>' URL_PATTERN = r'<td><a href="([^"]+)"[^>]*class="ui-state-default button hover ui-corner-all "><strong>' NO_SLOTS_PATTERN = r'<td><a href="http://dataport.cz/kredit/"[^>]*class="ui-state-default button hover ui-corner-all ui-state-disabled">' FILE_OFFLINE_PATTERN = r'<h2>Soubor nebyl nalezen</h2>' @@ -54,4 +44,6 @@ class DataportCz(SimpleHoster): self.fail("Parse error (URL)") download_url = found.group(1) - self.download(download_url)
\ No newline at end of file + self.download(download_url) + +create_getInfo(DataportCz)
\ No newline at end of file diff --git a/module/plugins/hoster/DepositfilesCom.py b/module/plugins/hoster/DepositfilesCom.py index 13a8abe03..87e5e7254 100644 --- a/module/plugins/hoster/DepositfilesCom.py +++ b/module/plugins/hoster/DepositfilesCom.py @@ -20,12 +20,12 @@ class DepositfilesCom(SimpleHoster): __name__ = "DepositfilesCom" __type__ = "hoster" __pattern__ = r"http://[\w\.]*?depositfiles\.com(/\w{1,3})?/files/[\w]+" - __version__ = "0.35" + __version__ = "0.36" __description__ = """Depositfiles.com Download Hoster""" __author_name__ = ("spoob", "zoidberg") __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz") - FILE_INFO_PATTERN = r'File name: <b title="([^"]+)">.*\s*<span class="nowrap">File size: <b>([0-9.]+) ([kKMG]i?B)</b>' + FILE_INFO_PATTERN = r'File name: <b title="(?P<N>[^"]+)">.*\s*<span class="nowrap">File size: <b>(?P<S>[0-9.]+) (?P<U>[kKMG])i?B</b>' FILE_OFFLINE_PATTERN = r'<span class="html_download_api-not_exists"></span>' RECAPTCHA_PATTERN = r"Recaptcha.create\('([^']+)', this\);" DOWNLOAD_LINK_PATTERN = r'<form action="(http://.+?\.depositfiles.com/.+?)" method="get"' diff --git a/module/plugins/hoster/EasyShareCom.py b/module/plugins/hoster/EasyShareCom.py deleted file mode 100644 index 542b4367f..000000000 --- a/module/plugins/hoster/EasyShareCom.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import re
-from module.plugins.Hoster import Hoster
-from module.plugins.ReCaptcha import ReCaptcha
-
-class EasyShareCom(Hoster):
- __name__ = "EasyShareCom"
- __type__ = "hoster"
- __pattern__ = r"http://[\w\d\.]*?easy-share\.com/(\d{6}).*"
- __version__ = "0.1"
- __description__ = """easy-share.com One-Klick Hoster"""
- __author_name__ = ("jeix")
- __author_mail__ = ("jeix@hasnomail.de")
-
-
- def setup(self):
- self.multiDL = False
- self.html = None
-
- def process(self, pyfile):
- self.pyfile = pyfile
-
- self.html = self.load(self.pyfile.url)
- if re.search("Die von ihnen angeforderte Datei wurde gel\xc3\xb6scht.", self.html):
- self.offline()
-
- self.pyfile.name = self.getFileName()
-
- self.download(self.getFileUrl())
-
-
- def getFileName(self):
- return re.search(r'requesting:</span>\s*(.*?)<', self.html).group(1)
-
-
- def getFileUrl(self):
-
- if "There is another download in progress from your IP" in self.html:
- self.log.info("%s: IP blocked, retry in 5 minutes." % self.__name__)
- self.setWait(5 * 60)
- self.wait()
- self.retry()
-
- if "You need a premium membership to download this file" in self.html:
- self.fail("You need a premium membership to download this file.")
-
-
- wait = re.search(r"w='(\d+)'", self.html)
- if wait:
- wait = int( wait.group(1).strip() )
- self.log.info("%s: Waiting %d seconds." % (self.__name__, wait))
- self.setWait(wait)
- self.wait()
-
- tempurl = self.pyfile.url
- if not tempurl.endswith("/"):
- tempurl += "/"
- id = re.search(r'http://[\w\d\.]*?easy-share\.com/(\d+)/', tempurl).group(1)
- self.html = self.load("http://www.easy-share.com/file_contents/captcha/" + id)
-
- challenge = re.search(r'Recaptcha\.create\("(.*?)"', self.html).group(1)
- re_captcha = ReCaptcha(self)
- challenge, result = re_captcha.challenge(challenge)
-
- link = re.search(r'<form\s+method="post"\s+action="(http://[\w\d\.]*?easy-share.com/file_contents/.*?)">', self.html).group(1)
- id = re.search(r'file/id/(\d+)/', link)
- self.download( link, post={"id" : id,
- "recaptcha_challenge_field" : challenge,
- "recaptcha_response_field": result} )
-
diff --git a/module/plugins/hoster/EdiskCz.py b/module/plugins/hoster/EdiskCz.py index f5291547f..a253be0d9 100644 --- a/module/plugins/hoster/EdiskCz.py +++ b/module/plugins/hoster/EdiskCz.py @@ -17,36 +17,18 @@ """ import re -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - html = getURL(url, decode=True) - if re.search(EdiskCz.FILE_OFFLINE_PATTERN, html): - # File offline - result.append((url, 0, 1, url)) - else: - # Get file info - found = re.search(EdiskCz.FILE_NAME_PATTERN, html) - if found is not None: - name = found.group(1) - result.append((name, 0, 2, url)) - yield result - - -class EdiskCz(Hoster): +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo + +class EdiskCz(SimpleHoster): __name__ = "EdiskCz" __type__ = "hoster" __pattern__ = r"http://(\w*\.)?edisk.(cz|sk|eu)/(stahni|sk/stahni|en/download)/.*" - __version__ = "0.2" + __version__ = "0.21" __description__ = """Edisk.cz""" __author_name__ = ("zoidberg") URL_PATTERN = r'<form name = "formular" action = "([^"]+)" method = "post">' - FILE_NAME_PATTERN = r'<span class="fl" title="([^"]+)">' + FILE_INFO_PATTERN = r'<span class="fl" title="(?P<N>[^"]+)">\s*.*?\((?P<S>[0-9.]*) (?P<U>[kKMG])i?B\)</h1></span>' ACTION_PATTERN = r'/en/download/(\d+/.*\.html)' DLLINK_PATTERN = r'http://.*edisk.cz.*\.html' FILE_OFFLINE_PATTERN = r'<h3>This file does not exist due to one of the following:</h3><ul><li>' @@ -60,21 +42,11 @@ class EdiskCz(Hoster): self.logDebug('URL:' + url) found = re.search(self.ACTION_PATTERN, url) - if found is None: - self.fail("Parse error (ACTION)") + if found is None: self.parseError("ACTION") action = found.group(1) self.html = self.load(url, decode=True) - - if re.search(self.FILE_OFFLINE_PATTERN, self.html) is not None: - self.offline() - - found = re.search(self.FILE_NAME_PATTERN, self.html) - if found is None: - self.fail("Parse error (FILENAME)") - pyfile.name = found.group(1) - - self.logDebug('NAME:' + pyfile.name) + self.getFileInfo() self.html = self.load(re.sub("/en/download/", "/en/download-slow/", url)) @@ -85,4 +57,6 @@ class EdiskCz(Hoster): if not re.match(self.DLLINK_PATTERN, url): self.fail("Unexpected server response") - self.download(url)
\ No newline at end of file + self.download(url) + +getInfo = create_getInfo(EdiskCz)
\ No newline at end of file diff --git a/module/plugins/hoster/FilejungleCom.py b/module/plugins/hoster/FilejungleCom.py index 597d1aa3f..b880086a6 100644 --- a/module/plugins/hoster/FilejungleCom.py +++ b/module/plugins/hoster/FilejungleCom.py @@ -17,29 +17,20 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo from module.network.RequestFactory import getURL from module.plugins.ReCaptcha import ReCaptcha -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(FilejungleCom, url, getURL(url, decode=True)) - result.append(file_info) - - yield result - class FilejungleCom(SimpleHoster): __name__ = "FilejungleCom" __type__ = "hoster" __pattern__ = r"http://(?:www\.)?filejungle\.com/f/([^/]+).*" - __version__ = "0.22" + __version__ = "0.23" __description__ = """Filejungle.com plugin - free only""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - FILE_INFO_PATTERN = r'<div id="file_name">([^<]+) <span class="filename_normal">\(([0-9.]+) (KB|MB|GB)\)</span></div>' + FILE_INFO_PATTERN = r'<div id="file_name">(?P<N>[^<]+) <span class="filename_normal">\((?P<S>[0-9.]+) (?P<U>[kKMG])i?B\)</span></div>' FILE_OFFLINE_PATTERN = r'class="error_msg_title"> Invalid or Deleted File. </div>' RECAPTCHA_KEY_PATTERN = r"var reCAPTCHA_publickey='([^']+)'" WAIT_TIME_PATTERN = r'<h1>Please wait for (\d+) seconds to download the next file\.</h1>' @@ -89,4 +80,6 @@ class FilejungleCom(SimpleHoster): self.wait() response = self.load(url, post = {"downloadLink" : "show"}) - self.download(url, post = {"download" : "normal"})
\ No newline at end of file + self.download(url, post = {"download" : "normal"}) + +getInfo = create_getInfo(FilejungleCom)
\ No newline at end of file diff --git a/module/plugins/hoster/FilepostCom.py b/module/plugins/hoster/FilepostCom.py index 779eef1d2..42ec0788b 100644 --- a/module/plugins/hoster/FilepostCom.py +++ b/module/plugins/hoster/FilepostCom.py @@ -17,44 +17,36 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo from module.network.RequestFactory import getURL from module.plugins.ReCaptcha import ReCaptcha from module.common.json_layer import json_loads from time import time -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(FilepostCom, url, getURL(url, decode=True)) - result.append(file_info) - - yield result - class FilepostCom(SimpleHoster): __name__ = "FilepostCom" __type__ = "hoster" __pattern__ = r"https?://(?:www\.)?filepost\.com/files/([^/]+).*" - __version__ = "0.22" + __version__ = "0.23" __description__ = """Filepost.com plugin - free only""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - FILE_INFO_PATTERN = r'<h1>([^<]+)</h1>\s*<div class="ul">\s*<ul>\s*<li><span>Size:</span> ([0-9.]+) (KB|MB|GB)</li>' + FILE_INFO_PATTERN = r'<h1>(?P<N>[^<]+)</h1>\s*<div class="ul">\s*<ul>\s*<li><span>Size:</span> (?P<S>[0-9.]+) (?P<U>[kKMG])i?B</li>' FILE_OFFLINE_PATTERN = r'class="error_msg_title"> Invalid or Deleted File. </div>' RECAPTCHA_KEY_PATTERN = r"Captcha.init\({\s*key:\s*'([^']+)'" - FLP_TOKEN_PATTERN = r"store.set\('flp_token', '([^']+)'\);" + FLP_TOKEN_PATTERN = r"store.set\('(?:flp_)?token', '([^']+)'\);" def handleFree(self): # Find token and captcha key file_id = re.search(self.__pattern__, self.pyfile.url).group(1) + found = re.search(self.FLP_TOKEN_PATTERN, self.html) - if not found: self.fail("Parse error (token)") + if not found: self.parseError("Token") flp_token = found.group(1) found = re.search(self.RECAPTCHA_KEY_PATTERN, self.html) - if not found: self.fail("Parse error (captcha key)") + if not found: self.parseError("Captcha key") captcha_key = found.group(1) url = 'https://filepost.com/files/get/' @@ -68,7 +60,7 @@ class FilepostCom(SimpleHoster): self.setWait(int(json_response['js']['answer']['wait_time'])) except Exception, e: self.logError(e) - self.fail("Parse error (wait time)") + self.self.parseError("Wait time") self.wait() # Solve recaptcha @@ -93,4 +85,6 @@ class FilepostCom(SimpleHoster): else: self.fail("Invalid captcha") # Download - self.download(download_url)
\ No newline at end of file + self.download(download_url) + +getInfo = create_getInfo(FilepostCom)
\ No newline at end of file diff --git a/module/plugins/hoster/FlyshareCz.py b/module/plugins/hoster/FlyshareCz.py deleted file mode 100644 index fc7e9f13b..000000000 --- a/module/plugins/hoster/FlyshareCz.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: zoidberg -""" - -import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(FlyshareCz, url, getURL(url, decode=True)) - result.append(file_info) - - yield result - -class FlyshareCz(SimpleHoster): - __name__ = "FlyshareCz" - __type__ = "hoster" - __pattern__ = r"http://.*flyshare.cz/stahni/.*" - __version__ = "0.31" - __description__ = """flyshare.cz""" - __author_name__ = ("zoidberg") - - FILE_NAME_PATTERN = r'<p><span class="filename">([^<]+)</span>' - ERR_PATTERN = r'<p class="errorreport_error">Chyba: ([^<]+)</p>' - FILE_OFFLINE_PATTERN = r'<p class="errorreport_error">Chyba: File is not available on the server</p>' - - def process(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - - found = re.search(self.ERR_PATTERN, self.html) - if found is not None: - err_dsc = found.group(1) - if err_dsc == "Too many simultaneous downloads, try again later": - self.waitForFreeSlot() - elif err_dsc == "File is not available on the server": - self.offline() - else: - self.fail(err_dsc) - - self.getFileInfo() - self.handleFree() - - def handleFree(self): - self.download(self.pyfile.url, post={ - "wmod_command": "wmod_fileshare3:startDownload", - "method": "free" - }) - - check = self.checkDownload({ - "sim_dl": '<p class="errorreport_error">Chyba: Too many simultaneous downloads, try again later</p>' - }) - - if check == "sim_dl": - self.waitForFreeSlot() - - def waitForFreeSlot(self): - self.setWait(600, True) - self.wait() - self.retry() - - diff --git a/module/plugins/hoster/FourSharedCom.py b/module/plugins/hoster/FourSharedCom.py index 1c12a2d34..5d10204a7 100644 --- a/module/plugins/hoster/FourSharedCom.py +++ b/module/plugins/hoster/FourSharedCom.py @@ -1,55 +1,41 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from module.plugins.internal.SimpleHoster import SimpleHoster -from module.network.RequestFactory import getURL +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo import re -def getInfo(urls): - result = [] - - for url in urls: - name, size, status, url = parseFileInfo(FourSharedCom, url, getURL(url, decode=True)) - if status == 2: - name = re.sub(r"&#(\d+).", lambda m: unichr(int(m.group(1))), name) - result.append(name, size, status, url) - - yield result - class FourSharedCom(SimpleHoster): __name__ = "FourSharedCom" __type__ = "hoster" __pattern__ = r"http://[\w\.]*?4shared(-china)?\.com/(account/)?(download|get|file|document|photo|video|audio)/.+?/.*" - __version__ = "0.21" + __version__ = "0.23" __description__ = """4Shared Download Hoster""" __author_name__ = ("jeix", "zoidberg") __author_mail__ = ("jeix@hasnomail.de", "zoidberg@mujmail.cz") - FILE_NAME_PATTERN = '<meta name="title" content="([^"]+)" />' - FILE_SIZE_PATTERN = '<span title="Size: ([0-9,.]+) ([kKMG]i?B)">' + FILE_NAME_PATTERN = '<meta name="title" content="(?P<N>[^"]+)" />' + FILE_SIZE_PATTERN = '<span title="Size: (?P<S>[0-9,.]+) (?P<U>[kKMG])i?B">' FILE_OFFLINE_PATTERN = 'The file link that you requested is not valid\.|This file was deleted.' - FREE_LINK_PATTERN = '<a href="([^"]+)" class="dbtn"' + DOWNLOAD_BUTTON_PATTERN = '<a href="([^"]+)"\s*class="dbtn' DOWNLOAD_URL_PATTERN = "<div class=\"(?:dl|xxlarge bold)\">\s*<a href='([^']+)'" - - def process(self, pyfile): - self.html = self.load(pyfile.url, decode=True) - self.getFileInfo() - pyfile.name = re.sub(r"&#(\d+).", lambda m: unichr(int(m.group(1))), pyfile.name) - self.handleFree() + + NAME_REPLACEMENTS = [(r"&#(\d+).", lambda m: unichr(int(m.group(1))))] def handleFree(self): - found = re.search(self.FREE_LINK_PATTERN, self.html) - if not found: raise PluginParseError('Free download button') - link = found.group(1) - + found = re.search(self.DOWNLOAD_BUTTON_PATTERN, self.html) + if found: + link = found.group(1) + else: + link = re.sub(r'/(download|get|file|document|photo|video|audio)/', r'/get/', self.pyfile.url) + self.html = self.load(link) found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) - if not found: raise PluginParseError('Download link') + if not found: self.parseError('Download link') link = found.group(1) self.setWait(20) self.wait() self.download(link) -
\ No newline at end of file +getInfo = create_getInfo(FourSharedCom)
\ No newline at end of file diff --git a/module/plugins/hoster/FshareVn.py b/module/plugins/hoster/FshareVn.py new file mode 100644 index 000000000..91cc167b1 --- /dev/null +++ b/module/plugins/hoster/FshareVn.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo +from module.network.RequestFactory import getURL +import re + +def getInfo(urls): + for url in urls: + html = getURL('http://www.fshare.vn/check_link.php', post = { + "action" : "check_link", + "arrlinks" : url + }, decode = True) + + file_info = parseFileInfo(FshareVn, url, html) + + yield file_info + +class FshareVn(SimpleHoster): + __name__ = "FshareVn" + __type__ = "hoster" + __pattern__ = r"http://(www\.)?fshare.vn/file/.*" + __version__ = "0.11" + __description__ = """FshareVn Download Hoster""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FILE_INFO_PATTERN = r'<p>(?P<N>[^<]+)<\\/p>\\r\\n\s*<p>(?P<S>[0-9,.]+)\s*(?P<U>[kKMG])i?B<\\/p>' + FILE_OFFLINE_PATTERN = r'<div class=\\"f_left file_(enable|w)\\">' + + DOWNLOAD_URL_PATTERN = r"<a class=\"bt_down\" id=\"down\".*window.location='([^']+)'\">" + FORM_PATTERN = r'<form action="" method="post" name="frm_download">(.*?)</form>' + FORM_INPUT_PATTERN = r'<input[^>]* name="?([^" ]+)"? value="?([^" ]+)"?[^>]*>' + VIP_URL_PATTERN = r'<form action="([^>]+)" method="get" name="frm_download">' + + def process(self, pyfile): + self.html = self.load('http://www.fshare.vn/check_link.php', post = { + "action": "check_link", + "arrlinks": pyfile.url + }, decode = True) + self.getFileInfo() + if self.account: + self.handlePremium() + else: + self.handleFree() + + def handleFree(self): + self.html = self.load(self.pyfile.url, decode = True) + found = re.search(self.FORM_PATTERN, self.html, re.DOTALL) + if not found: self.parseError('FORM') + form = found.group(1) + inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) + + self.html = self.load(self.pyfile.url, post = inputs) + + found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) + if not found: self.parseError('Free URL') + url = found.group(1) + + found = re.search(r'var count = (\d+)', self.html) + self.setWait(int(found.group(1)) if found else 30) + self.wait() + + self.download(url) + + def handlePremium(self): + header = self.load(self.pyfile.url, just_header = True) + if 'location' in header and header['location'].startswith('http://download'): + self.logDebug('Direct download') + self.download(self.pyfile.url) + else: + self.html = self.load(self.pyfile.url) + found = re.search(self.VIP_URL_PATTERN, self.html) + if not found: self.parseError('VIP URL') + url = found.group(1) + self.logDebug('VIP URL: ' + url) + self.download(url)
\ No newline at end of file diff --git a/module/plugins/hoster/HellshareCz.py b/module/plugins/hoster/HellshareCz.py index d2f5c8e40..cc8341f8e 100644 --- a/module/plugins/hoster/HellshareCz.py +++ b/module/plugins/hoster/HellshareCz.py @@ -18,35 +18,28 @@ import re import datetime -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo +from math import ceil +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo from module.network.RequestFactory import getURL -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(HellshareCz, url, getURL(url, decode=True)) - result.append(file_info) - - yield result - class HellshareCz(SimpleHoster): __name__ = "HellshareCz" __type__ = "hoster" - __pattern__ = r"http://(?:.*\.)*hellshare\.(?:cz|com|sk|hu)/[^?]*/(\d+).*" - __version__ = "0.74" + __pattern__ = r"(http://(?:.*\.)*hellshare\.(?:cz|com|sk|hu)/[^?]*/\d+).*" + __version__ = "0.76" __description__ = """Hellshare.cz""" __author_name__ = ("zoidberg") FREE_URL_PATTERN = r'<form[^>]*action="(http://free\d*\.helldata[^"]*)"' PREMIUM_URL_PATTERN = r"launchFullDownload\('([^']*)'\);" - FILE_NAME_PATTERN = r'<h1 id="filename">([^<]+)</h1>' - FILE_SIZE_PATTERN = r'<td><span>Size</span></td>\s*<th><span>([0-9.]*) (kB|KB|MB|GB)</span></th>' + FILE_NAME_PATTERN = r'<h1 id="filename">(?P<N>[^<]+)</h1>' + FILE_SIZE_PATTERN = r'<td><span>Size</span></td>\s*<th><span>(?P<S>[0-9.]*) (?P<U>[kKMG])i?B</span></th>' FILE_OFFLINE_PATTERN = r'<h1>File not found.</h1>' CAPTCHA_PATTERN = r'<img class="left" id="captcha-img"src="([^"]*)" />' #FILE_CREDITS_PATTERN = r'<strong class="filesize">(\d+) MB</strong>' CREDIT_LEFT_PATTERN = r'<p>After downloading this file you will have (\d+) MB for future downloads.' DOWNLOAD_AGAIN_PATTERN = r'<p>This file you downloaded already and re-download is for free. </p>' + SHOW_WINDOW_PATTERN = r'<a href="([^?]+/(\d+)/\?do=(fileDownloadButton|relatedFileDownloadButton-\2)-showDownloadWindow)"' def setup(self): self.resumeDownload = self.multiDL = True if self.account else False @@ -56,16 +49,15 @@ class HellshareCz(SimpleHoster): if self.account: self.account.relogin(self.user) - pyfile.url = re.search(r'([^?]*)', pyfile.url).group(1) + pyfile.url = re.search(self.__pattern__, pyfile.url).group(1) self.html = self.load(pyfile.url, decode = True) self.getFileInfo() - - if "do=relatedFileDownloadButton" in self.html: - found = re.search(self.__pattern__, self.pyfile.url) - show_window = "relatedFileDownloadButton-%s-showDownloadWindow" % found.group(1) - else: - show_window = "fileDownloadButton-showDownloadWindow" - self.html = self.load(pyfile.url, get = {"do" : show_window}, decode=True) + + found = re.search(self.SHOW_WINDOW_PATTERN, self.html) + if not found: self.parseError('SHOW WINDOW') + url = found.group(1) + self.logDebug("SHOW WINDOW: " + url) + self.html = self.load("http://download.hellshare.com" + url, decode=True) if self.account: self.handlePremium() @@ -128,4 +120,6 @@ class HellshareCz(SimpleHoster): self.download(download_url) info = self.account.getAccountInfo(self.user, True) - self.logInfo("User %s has %i credits left" % (self.user, info["trafficleft"] / 1024))
\ No newline at end of file + self.logInfo("User %s has %i credits left" % (self.user, info["trafficleft"] / 1024)) + +getInfo = create_getInfo(HellshareCz)
\ No newline at end of file diff --git a/module/plugins/hoster/HellspyCz.py b/module/plugins/hoster/HellspyCz.py index e1077a0cb..6d3f84b55 100644 --- a/module/plugins/hoster/HellspyCz.py +++ b/module/plugins/hoster/HellspyCz.py @@ -17,6 +17,7 @@ """ import re +from math import ceil from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo from module.network.RequestFactory import getURL @@ -33,17 +34,15 @@ class HellspyCz(SimpleHoster): __name__ = "HellspyCz" __type__ = "hoster" __pattern__ = r"http://(?:\w*\.)*hellspy\.(?:cz|com|sk|hu)(/\S+/\d+)/?.*" - __version__ = "0.21" + __version__ = "0.22" __description__ = """HellSpy.cz""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - FILE_NAME_PATTERN = r'<h1 class="text-yellow-1 " "><span ><span class="text" title="">([^<]+)</span></span></h1>' + FILE_INFO_PATTERN = '<span class="filesize right">(?P<S>[0-9.]+) <span>(?P<U>[kKMG])i?B</span></span>\s*<h1>(?P<N>[^<]+)</h1>' FILE_OFFLINE_PATTERN = r'<h2>(404 - Page|File) not found</h2>' - FILE_CREDITS_PATTERN = r'<span class="text-credit-taken-1">\s*<span class="text-size"><span class="hidden">Size: </span>(\S+) (kB|MB|GB)</span>\s*<span >\((\d+) credits\)</span>' - CREDIT_LEFT_PATTERN = r'<strong class="text-credits">(\d+)</strong>' - - PREMIUM_URL_PATTERN = r"launchFullDownload\('(http://[^']+)',\s*\d*\);" + CREDIT_LEFT_PATTERN = r'<strong>Credits: </strong>\s*(\d+)' + PREMIUM_URL_PATTERN = r'<a href="([^"]+)" class="ajax button button-blue button-download"' DOWNLOAD_AGAIN_PATTERN = r'<a id="button-download-start"[^>]*title="You can download the file without deducting your credit.">' def setup(self): @@ -51,8 +50,7 @@ class HellspyCz(SimpleHoster): self.chunkLimit = 1 def process(self, pyfile): - if not self.premium: self.fail("Only premium users can download from HellSpy.cz") - if not self.account: self.fail("Not logged in") + if not self.account: self.fail("Only premium users can download from HellSpy.cz") # set PHPSESSID cookie cj = self.account.getAccountCookies(self.user) @@ -65,33 +63,20 @@ class HellspyCz(SimpleHoster): # load html rel_url = re.search(self.__pattern__, pyfile.url).group(1) self.html = self.load("http://www.hellspy.com/--%s-/%s" % (self.account.phpsessid, rel_url), decode = True) + + self.getFileInfo() - # get premium download URL - download_url = self.getPremiumURL() - if download_url is None: - self.checkFile(pyfile) - self.html = self.load("http://www.hellspy.com/%s?download=1" % rel_url) - download_url = self.getPremiumURL() - - # download - if download_url is None: self.fail("Parse error (DOWNLOAD URL)") + # get premium download URL and download + found = re.search(self.PREMIUM_URL_PATTERN, self.html) + if not found: self.parseError('Download URL') + download_url = "http://www.hellspy.cz" + found.group(1) self.logDebug("Download URL: " + download_url) self.download(download_url, disposition = True) info = self.account.getAccountInfo(self.user) self.logInfo("User %s has %i credits left" % (self.user, info["trafficleft"]/1024)) - def checkFile(self, pyfile): - # marks the file as "offline" when the pattern was found on the html-page - if re.search(self.FILE_OFFLINE_PATTERN, self.html) is not None: - self.offline() - - # parse the name from the site and set attribute in pyfile - found = re.search(self.FILE_NAME_PATTERN, self.html) - if found is None: - self.fail("Parse error (FILENAME)") - pyfile.name = found.group(1) - + """ # parse credits left info found = re.search(self.CREDIT_LEFT_PATTERN, self.html) if found is None: @@ -111,8 +96,4 @@ class HellspyCz(SimpleHoster): file_credits = int(found.group(3)) if file_credits > credits_left: self.fail("Not enough credits left to download file") self.logInfo("Premium download for %i credits" % file_credits) - - - def getPremiumURL(self): - found = re.search(self.PREMIUM_URL_PATTERN, self.html) - return found.group(1) if found else None
\ No newline at end of file + """ diff --git a/module/plugins/hoster/IfileIt.py b/module/plugins/hoster/IfileIt.py index 3bc60220a..ec830f3b2 100644 --- a/module/plugins/hoster/IfileIt.py +++ b/module/plugins/hoster/IfileIt.py @@ -17,25 +17,16 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo from module.common.json_layer import json_loads from module.plugins.ReCaptcha import ReCaptcha from module.network.RequestFactory import getURL -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(IfileIt, url, getURL(url, decode=True)) - result.append(file_info) - - yield result - class IfileIt(SimpleHoster): __name__ = "IfileIt" __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*ifile\.it/(\w+).*" - __version__ = "0.22" + __pattern__ = r"http://(?:\w*\.)*(?:ifile\.it|mihd\.net)/(\w+).*" + __version__ = "0.24" __description__ = """Ifile.it""" __author_name__ = ("zoidberg") @@ -43,7 +34,7 @@ class IfileIt(SimpleHoster): #DEC_PATTERN = r"requestBtn_clickEvent[^}]*url:\s*([^,]+)" DOWNLOAD_LINK_PATTERN = r'</span> If it doesn\'t, <a target="_blank" href="([^"]+)">' RECAPTCHA_KEY_PATTERN = r"var __recaptcha_public\s*=\s*'([^']+)';" - FILE_INFO_PATTERN = r'<span style="cursor: default;[^>]*>\s*(.*?)\s* \s*<strong>\s*([0-9.]+)\s*([kKMG]i?B)\s*</strong>\s*</span>' + FILE_INFO_PATTERN = r'<span style="cursor: default;[^>]*>\s*(?P<N>.*?)\s* \s*<strong>\s*(?P<S>[0-9.]+)\s*(?P<U>[kKMG])i?B\s*</strong>\s*</span>' FILE_OFFLINE_PATTERN = r'$\("#errorPnl"\)\.empty\(\)\.append\( "no such file" \);' def handleFree(self): @@ -79,4 +70,6 @@ class IfileIt(SimpleHoster): self.html = self.load(self.pyfile.url) download_url = re.search(self.DOWNLOAD_LINK_PATTERN, self.html).group(1) - self.download(download_url)
\ No newline at end of file + self.download(download_url) + +getInfo = create_getInfo(IfileIt)
\ No newline at end of file diff --git a/module/plugins/hoster/IfolderRu.py b/module/plugins/hoster/IfolderRu.py index 8a8e18282..3177271c4 100644 --- a/module/plugins/hoster/IfolderRu.py +++ b/module/plugins/hoster/IfolderRu.py @@ -18,35 +18,26 @@ import re from urllib import quote -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo from module.network.RequestFactory import getURL -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(IfolderRu, url, getURL(url, decode=True)) - result.append(file_info) - - yield result - class IfolderRu(SimpleHoster): __name__ = "IfolderRu" __type__ = "hoster" __pattern__ = r"http://(?:\w*\.)?ifolder.ru/(\d+).*" - __version__ = "0.32" + __version__ = "0.33" __description__ = """ifolder.ru""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") SIZE_UNITS = {u'Кб': 1, u'Мб': 2, u'Гб': 3} - FILE_NAME_PATTERN = ur'(?:<div><span>)?Название:(?:</span>)? <b>([^<]+)</b><(?:/div|br)>' - FILE_SIZE_PATTERN = ur'(?:<div><span>)?Размер:(?:</span>)? <b>([0-9.]+) ([^<]+)</b><(?:/div|br)>' + FILE_NAME_PATTERN = ur'(?:<div><span>)?Название:(?:</span>)? <b>(?P<N>[^<]+)</b><(?:/div|br)>' + FILE_SIZE_PATTERN = ur'(?:<div><span>)?Размер:(?:</span>)? <b>(?P<S>[0-9.]+) (?P<U>[^<]+)</b><(?:/div|br)>' SESSION_ID_PATTERN = r'<a href=(http://ints.ifolder.ru/ints/sponsor/\?bi=\d*&session=([^&]+)&u=[^>]+)>' FORM1_PATTERN = r'<form method=post name="form1" ID="Form1" style="margin-bottom:200px">(.*?)</form>' FORM_INPUT_PATTERN = r'<input[^>]* name="?([^" ]+)"? value="?([^" ]+)"?[^>]*>' INTS_SESSION_PATTERN = r'\(\'ints_session\'\);\s*if\(tag\)\{tag.value = "([^"]+)";\}' - HIDDEN_INPUT_PATTERN = r"var s= 'hh([^']*)';" + HIDDEN_INPUT_PATTERN = r"var v = .*?name='([^']+)' value='1'" DOWNLOAD_LINK_PATTERN = r'<a id="download_file_href" href="([^"]+)"' WRONG_CAPTCHA_PATTERN = ur'<font color=Red>неверный код,<br>введите еще раз</font><br>' FILE_OFFLINE_PATTERN = ur'<p>Файл номер <b>[^<]*</b> не найден !!!</p>' @@ -81,7 +72,7 @@ class IfolderRu(SimpleHoster): inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) inputs['ints_session'] = re.search(self.INTS_SESSION_PATTERN, form).group(1) inputs['Submit1'] = u"Подтвердить".encode("utf-8") - inputs[re.search(self.HIDDEN_INPUT_PATTERN, form).group(1)] = '1' + inputs[re.search(self.HIDDEN_INPUT_PATTERN, self.html).group(1)] = '1' inputs['confirmed_number'] = self.decryptCaptcha(captcha_url, cookies = True) self.logDebug(inputs) @@ -98,4 +89,6 @@ class IfolderRu(SimpleHoster): download_url = re.search(self.DOWNLOAD_LINK_PATTERN, self.html).group(1) self.correctCaptcha() self.logDebug("Download URL: %s" % download_url) - self.download(download_url)
\ No newline at end of file + self.download(download_url) + +getInfo = create_getInfo(IfolderRu)
\ No newline at end of file diff --git a/module/plugins/hoster/LetitbitNet.py b/module/plugins/hoster/LetitbitNet.py index dd65d7e7a..01d796330 100644 --- a/module/plugins/hoster/LetitbitNet.py +++ b/module/plugins/hoster/LetitbitNet.py @@ -17,45 +17,22 @@ """ import re -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -def getInfo(urls): - result = [] - - for url in urls: - - html = getURL(url, decode=True) - if re.search(LetitbitNet.FILE_OFFLINE_PATTERN, html): - # File offline - result.append((url, 0, 1, url)) - else: - # Get file info - found = re.search(r'<input[^>]* name="name" value="([^"]+)" />', html) - if found is not None: - name = found.group(1) - found = re.search(r'<input[^>]* name="sssize" value="([^"]+)" />', html) - if found is not None: - size = float(found.group(1))/1024 - result.append((name, size, 2, url)) - yield result - - -class LetitbitNet(Hoster): +class LetitbitNet(SimpleHoster): __name__ = "LetitbitNet" __type__ = "hoster" __pattern__ = r"http://(?:\w*\.)*letitbit.net/download/.*" - __version__ = "0.1" + __version__ = "0.12" __description__ = """letitbit.net""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - IFREE_FORM_PATTERN = r'<form id="ifree_form" action="([^"]+)" method="post">(.*?)</form>' - DVIFREE_FORM_PATTERN = r'<form action="([^"]+)" method="post" id="dvifree">(.*?)</form>' + FORM_PATTERN = r'<form%s action="([^"]+)" method="post"%s>(.*?)</form>' FORM_INPUT_PATTERN = r'<input[^>]* name="([^"]+)" value="([^"]+)" />' JS_SCRIPT_PATTERN = r'<title>[^<]*</title>\s*<script language="JavaScript">(.*?)</script>' JS_VARS_PATTERN = r"(\S+) = '?([^';]+)'?;" - + FILE_INFO_PATTERN = r'<h1[^>]*>File: <a[^>]*><span>(?P<N>[^<]+)</span></a> [<span>(?P<S>[0-9.]+)\s*(?P<U>[kKMG])i?[Bb]</span>]</h1>' FILE_OFFLINE_PATTERN = r'<div id="download_content" class="hide-block">[^<]*<br>File not found<br /></div>' def setup(self): @@ -67,21 +44,24 @@ class LetitbitNet(Hoster): if re.search(self.FILE_OFFLINE_PATTERN, self.html): self.offline() try: - action, form = re.search(self.IFREE_FORM_PATTERN, self.html, re.DOTALL).groups() + action, form = re.search(self.FORM_PATTERN % (' id="ifree_form"', ''), self.html, re.DOTALL).groups() inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) pyfile.name = inputs['name'] pyfile.size = float(inputs['sssize'])/1024 except Exception, e: self.logError(e) - self.fail("Parse error on page 1") + self.parseError("page 1 / ifree_form") + + #self.logDebug(inputs) + inputs['desc'] = "" + self.html = self.load("http://letitbit.net" + action, post = inputs) - self.html = self.load("http://letitbit.net"+action, post = inputs) try: - action, form = re.search(self.DVIFREE_FORM_PATTERN, self.html, re.DOTALL).groups() + action, form = re.search(self.FORM_PATTERN % ('', ' id="d3_form"'), self.html, re.DOTALL).groups() inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) except Exception, e: self.logError(e) - self.fail("Parse error on page 2") + self.parseError("page 2 / d3_form") self.html = self.load(action, post = inputs) try: @@ -92,7 +72,9 @@ class LetitbitNet(Hoster): self.wait() except Exception, e: self.logError(e) - self.fail("Parse error on page 3") + self.parseError("page 3 / js") download_url = self.load(ajax_check_url, post = inputs) - self.download(download_url)
\ No newline at end of file + self.download(download_url) + +getInfo = create_getInfo(LetitbitNet)
\ No newline at end of file diff --git a/module/plugins/hoster/MediafireCom.py b/module/plugins/hoster/MediafireCom.py index e499a406f..f014c58e8 100644 --- a/module/plugins/hoster/MediafireCom.py +++ b/module/plugins/hoster/MediafireCom.py @@ -26,8 +26,8 @@ def replace_eval(js_expr): class MediafireCom(SimpleHoster): __name__ = "MediafireCom" __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)*mediafire\.com/.*" - __version__ = "0.64" + __pattern__ = r"http://(?:\w*\.)*mediafire\.com/download.php\?.*" + __version__ = "0.66" __description__ = """Mediafire.com plugin - free only""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") @@ -37,7 +37,7 @@ class MediafireCom(SimpleHoster): PAGE1_RESULT_PATTERN = r"(\w+)\('(?P<qk>[^']+)','(?P<pk1>[^']+)'\)" PAGE1_DIV_PATTERN = r'getElementById\("(\w{32})"\)' PAGE1_PKR_PATTERN = r"pKr='([^']+)';" - RECAPTCHA_PATTERN = r'src="http://api.recaptcha.net/challenge?k=([^"]+)">' + RECAPTCHA_PATTERN = r'src="http://(?:api.recaptcha.net|www.google.com/recaptcha/api)/challenge\?k=([^"]+)">' PAGE1_ACTION_PATTERN = r'<link rel="canonical" href="([^"]+)"/>' PAGE2_VARS_PATTERN = r'<script language="Javascript"><!--\s*(var.*?unescape.*?)eval\(' @@ -45,22 +45,20 @@ class MediafireCom(SimpleHoster): PAGE2_LINK_PATTERN = r"(\w+='';\w+=unescape.*?)eval\(\w+\);(.{0,10}href=[^>]*>)?" FINAL_LINK_PATTERN = r'parent.document.getElementById\(\'(\w{32})\'\).*(http://[^"]+)" \+(\w+)\+ "([^"]+)">' - FILE_NAME_PATTERN = r'<META NAME="description" CONTENT="([^"]+)"/>' - FILE_SIZE_PATTERN = r'<input type="hidden" id="sharedtabsfileinfo1-fs" value="([0-9.]+) ([kKMG]i?B)">' + FILE_NAME_PATTERN = r'<META NAME="description" CONTENT="(?P<N>[^"]+)"/>' + FILE_SIZE_PATTERN = r'<input type="hidden" id="sharedtabsfileinfo1-fs" value="(?P<S>[0-9.]+) (?P<U>[kKMG])i?B">' FILE_OFFLINE_PATTERN = r'class="error_msg_title"> Invalid or Deleted File. </div>' + def process(self, pyfile): + self.html = self.load(pyfile.url, decode = True) + self.checkCaptcha() + self.getFileInfo() + if self.account: + self.handlePremium() + else: + self.handleFree() + def handleFree(self): - found = re.search(self.RECAPTCHA_PATTERN, self.html) - if found: - captcha_action = re.search(self.PAGE1_ACTION_PATTERN, self.html).group(1) - captcha_key = found.group(1) - recaptcha = ReCaptcha(self) - captcha_challenge, captcha_response = recaptcha.challenge(captcha_key) - self.html = self.load(captcha_action, post = { - "recaptcha_challenge_field": captcha_challenge, - "recaptcha_response_field": captcha_response - }) - found = re.search(self.PAGE1_KEY_PATTERN, self.html) if found: result = self.js.eval(found.group(1)) @@ -116,6 +114,22 @@ class MediafireCom(SimpleHoster): self.logDebug("FINAL LINK: %s" % final_link) self.download(final_link) - + + def checkCaptcha(self): + for i in range(5): + found = re.search(self.RECAPTCHA_PATTERN, self.html) + if found: + captcha_action = re.search(self.PAGE1_ACTION_PATTERN, self.html).group(1) + captcha_key = found.group(1) + recaptcha = ReCaptcha(self) + captcha_challenge, captcha_response = recaptcha.challenge(captcha_key) + self.html = self.load(captcha_action, post = { + "recaptcha_challenge_field": captcha_challenge, + "recaptcha_response_field": captcha_response + }) + else: + break + else: + self.fail("No valid recaptcha solution received") getInfo = create_getInfo(MediafireCom)
\ No newline at end of file diff --git a/module/plugins/hoster/MegasharesCom.py b/module/plugins/hoster/MegasharesCom.py index 3c17eb979..a1dd6e694 100644 --- a/module/plugins/hoster/MegasharesCom.py +++ b/module/plugins/hoster/MegasharesCom.py @@ -18,29 +18,20 @@ import re from time import time -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo from module.network.RequestFactory import getURL -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(MegasharesCom, url, getURL(url, decode=True)) - result.append(file_info) - - yield result - class MegasharesCom(SimpleHoster): __name__ = "MegasharesCom" __type__ = "hoster" __pattern__ = r"http://(\w+\.)?megashares.com/.*" - __version__ = "0.12" + __version__ = "0.13" __description__ = """megashares.com plugin - free only""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - FILE_NAME_PATTERN = '<h1 class="black xxl"[^>]*title="([^"]+)">' - FILE_SIZE_PATTERN = '<strong><span class="black">Filesize:</span></strong> ([0-9.]+) (KB|MB|GB)<br />' + FILE_NAME_PATTERN = '<h1 class="black xxl"[^>]*title="(?P<N>[^"]+)">' + FILE_SIZE_PATTERN = '<strong><span class="black">Filesize:</span></strong> (?P<S>[0-9.]+) (?P<U>[kKMG])i?B<br />' DOWNLOAD_URL_PATTERN = '<div id="show_download_button_2" style="display:none">\s*<a href="([^"]+)">' PASSPORT_LEFT_PATTERN = 'Your Download Passport is: <[^>]*>(\w+).*\s*You have\s*<[^>]*>\s*([0-9.]+) ([kKMG]i?B)' PASSPORT_RENEW_PATTERN = 'Your download passport will renew in\s*<strong>(\d+)</strong>:<strong>(\d+)</strong>:<strong>(\d+)</strong>' @@ -104,4 +95,6 @@ class MegasharesCom(SimpleHoster): # Download self.logDebug("Download URL: %s" % download_url) - self.download(download_url)
\ No newline at end of file + self.download(download_url) + +getInfo = create_getInfo(MegasharesCom)
\ No newline at end of file diff --git a/module/plugins/hoster/MultishareCz.py b/module/plugins/hoster/MultishareCz.py index ac35d93e1..bf1d9f3bf 100644 --- a/module/plugins/hoster/MultishareCz.py +++ b/module/plugins/hoster/MultishareCz.py @@ -17,28 +17,18 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(MultishareCz, url, getURL(url, decode=True)) - result.append(file_info) - - yield result +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class MultishareCz(SimpleHoster): __name__ = "MultishareCz" __type__ = "hoster" __pattern__ = r"http://(\w*\.)?multishare.cz/stahnout/.*" - __version__ = "0.31" + __version__ = "0.32" __description__ = """MultiShare.cz""" __author_name__ = ("zoidberg") FILE_ID_PATTERN = r'/stahnout/(\d+)/' - FILE_INFO_PATTERN = ur'<ul class="no-padding"><li>Název: <strong>([^<]+)</strong></li><li>Velikost: <strong>([^&]+) ([^<]+)</strong>' + FILE_INFO_PATTERN = ur'<ul class="no-padding"><li>Název: <strong>(?P<N>[^<]+)</strong></li><li>Velikost: <strong>(?P<S>[^&]+) (?P<U>[^<]+)</strong>' FILE_OFFLINE_PATTERN = ur'<h1>Stáhnout soubor</h1><p><strong>Požadovaný soubor neexistuje.</strong></p>' def handleFree(self): @@ -49,4 +39,6 @@ class MultishareCz(SimpleHoster): self.download("http://www.multishare.cz/html/download_free.php", get={ "ID": file_id - })
\ No newline at end of file + }) + +getInfo = create_getInfo(MultishareCz)
\ No newline at end of file diff --git a/module/plugins/hoster/QuickshareCz.py b/module/plugins/hoster/QuickshareCz.py index 4c82ff1a9..ef33d3263 100644 --- a/module/plugins/hoster/QuickshareCz.py +++ b/module/plugins/hoster/QuickshareCz.py @@ -17,44 +17,31 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(QuickshareCz, url, getURL(url, decode=True)) - result.append(file_info) - - yield result +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class QuickshareCz(SimpleHoster): __name__ = "QuickshareCz" __type__ = "hoster" __pattern__ = r"http://.*quickshare.cz/stahnout-soubor/.*" - __version__ = "0.51" + __version__ = "0.52" __description__ = """Quickshare.cz""" __author_name__ = ("zoidberg") VAR_PATTERN = r"var ID1 = '(?P<ID1>[^']+)';var ID2 = '(?P<ID2>[^']+)';var ID3 = '(?P<ID3>[^']+)';var ID4 = '(?P<ID4>[^']+)';var ID5 = '[^']*';var UU_prihlasen = '[^']*';var UU_kredit = [^;]*;var velikost = [^;]*;var kredit_odecet = [^;]*;var CaptchaText = '(?P<CaptchaText>[^']+)';var server = '(?P<Server>[^']+)';" FILE_OFFLINE_PATTERN = r'<script type="text/javascript">location.href=\'/chyba\';</script>' - FILE_SIZE_PATTERN = r'<br>Velikost: <strong>([0-9.]+)</strong>\s*(KB|MB|GB)<br>' + FILE_NAME_PATTERN = r'<th width="145px">Název:</th>\s*<td style="word-wrap:break-word;">(?P<N>[^<]+)</td>' + FILE_SIZE_PATTERN = r'<th>Velikost:</th>\s*<td>(?P<S>[0-9.]+) (?P<U>[kKMG])i?B</td>' def setup(self): self.multiDL = False def process(self, pyfile): self.html = self.load(pyfile.url, decode=True) - - # marks the file as "offline" when the pattern was found on the html-page - if re.search(self.FILE_OFFLINE_PATTERN, self.html) is not None: - self.offline() + self.getFileInfo() # parse the name from the site and set attribute in pyfile parsed_vars = re.search(self.VAR_PATTERN, self.html) - if parsed_vars is None: - self.fail("Parser error") + if parsed_vars is None: self.parseError("VARs") pyfile.name = parsed_vars.group('ID3') @@ -77,3 +64,5 @@ class QuickshareCz(SimpleHoster): if check == "no_slots": self.retry(5, 600, "No free slots") + +create_getInfo(QuickshareCz)
\ No newline at end of file diff --git a/module/plugins/hoster/RealdebridCom.py b/module/plugins/hoster/RealdebridCom.py index c23e19b4d..62c09aaa8 100644 --- a/module/plugins/hoster/RealdebridCom.py +++ b/module/plugins/hoster/RealdebridCom.py @@ -34,7 +34,7 @@ class RealdebridCom(Hoster): def process(self, pyfile):
if not self.account:
- self.log.error(_("Please enter your Real-debrid account or deactivate this plugin"))
+ self.logError(_("Please enter your Real-debrid account or deactivate this plugin"))
self.fail("No Real-debrid account provided")
self.log.debug("Real-Debrid: Old URL: %s" % pyfile.url)
@@ -52,11 +52,13 @@ class RealdebridCom(Hoster): if error:
msg = error.group(1).strip()
- self.log.debug(page)
+ self.logDebug(page)
if msg == "Your file is unavailable on the hoster.":
self.offline()
else:
self.fail(msg)
+ elif url == 'error':
+ self.fail("Your IP is most likely blocked. Please contact RealDebrid support")
else:
new_url = page
diff --git a/module/plugins/hoster/SendspaceCom.py b/module/plugins/hoster/SendspaceCom.py index b28df29f9..22abaff56 100644 --- a/module/plugins/hoster/SendspaceCom.py +++ b/module/plugins/hoster/SendspaceCom.py @@ -17,29 +17,19 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(SendspaceCom, url, getURL(url, decode=True)) - result.append(file_info) - - yield result +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class SendspaceCom(SimpleHoster): __name__ = "SendspaceCom" __type__ = "hoster" __pattern__ = r"http://(www\.)?sendspace.com/file/.*" - __version__ = "0.12" + __version__ = "0.13" __description__ = """sendspace.com plugin - free only""" __author_name__ = ("zoidberg") DOWNLOAD_URL_PATTERN = r'<a id="download_button" href="([^"]+)"' - FILE_NAME_PATTERN = r'<h2 class="bgray">\s*<(?:b|strong)>([^<]+)</' - FILE_SIZE_PATTERN = r'<div class="file_description reverse margin_center">\s*<b>File Size:</b>\s*([0-9.]+)([kKMG]i?B)\s*</div>' + FILE_NAME_PATTERN = r'<h2 class="bgray">\s*<(?:b|strong)>(?P<N>[^<]+)</' + FILE_SIZE_PATTERN = r'<div class="file_description reverse margin_center">\s*<b>File Size:</b>\s*(?P<S>[0-9.]+)(?P<U>[kKMG])i?B\s*</div>' FILE_OFFLINE_PATTERN = r'<div class="msg error" style="cursor: default">Sorry, the file you requested is not available.</div>' CAPTCHA_PATTERN = r'<td><img src="(/captchas/captcha.php?captcha=([^"]+))"></td>' USER_CAPTCHA_PATTERN = r'<td><img src="/captchas/captcha.php?user=([^"]+))"></td>' @@ -73,3 +63,5 @@ class SendspaceCom(SimpleHoster): self.logDebug("Download URL: %s" % download_url) self.download(download_url) + +create_getInfo(SendspaceCom)
\ No newline at end of file diff --git a/module/plugins/hoster/ShareRapidCom.py b/module/plugins/hoster/ShareRapidCom.py index 17c981b61..21512046e 100644 --- a/module/plugins/hoster/ShareRapidCom.py +++ b/module/plugins/hoster/ShareRapidCom.py @@ -27,14 +27,13 @@ class ShareRapidCom(SimpleHoster): __name__ = "ShareRapidCom" __type__ = "hoster" __pattern__ = r"http://(?:www\.)?((share(-?rapid\.(biz|com|cz|info|eu|net|org|pl|sk)|-(central|credit|free|net)\.cz|-ms\.net)|(s-?rapid|rapids)\.(cz|sk))|(e-stahuj|mediatack|premium-rapidshare|rapidshare-premium|qiuck)\.cz|kadzet\.com|stahuj-zdarma\.eu|strelci\.net|universal-share\.com)/(stahuj/.+)" - __version__ = "0.44" + __version__ = "0.46" __description__ = """Share-rapid.com plugin - premium only""" __author_name__ = ("MikyWoW", "zoidberg") __author_mail__ = ("MikyWoW@seznam.cz", "zoidberg@mujmail.cz") - FILE_NAME_PATTERN = r'<h3>([^<]+)</h3>' - FILE_NAME_INFO_PATTERN = r'<h1[^>]*><span[^>]*>([^<]+)</ br> </h1>' - FILE_SIZE_PATTERN = r'<td class="i">Velikost:</td>\s*<td class="h"><strong>\s*([0-9.]+) (kB|MB|GB)</strong></td>' + FILE_NAME_PATTERN = r'(?:title="Stahnout"|<h3>)(?P<N>[^<]+)</(?:a|h3)>' + FILE_SIZE_PATTERN = r'<td class="i">Velikost:</td>\s*<td class="h"><strong>\s*(?P<S>[0-9.]+) (?P<U>[kKMG])i?B</strong></td>' DOWNLOAD_URL_PATTERN = r'<a href="([^"]+)" title="Stahnout">([^<]+)</a>' ERR_LOGIN_PATTERN = ur'<div class="error_div"><strong>Stahování je přístupné pouze přihlášeným uživatelům' ERR_CREDIT_PATTERN = ur'<div class="error_div"><strong>Stahování zdarma je možné jen přes náš' diff --git a/module/plugins/hoster/SpeedfileCz.py b/module/plugins/hoster/SpeedfileCz.py index 0c5010b15..bfd316dfa 100644 --- a/module/plugins/hoster/SpeedfileCz.py +++ b/module/plugins/hoster/SpeedfileCz.py @@ -17,35 +17,18 @@ """ import re -from module.plugins.Hoster import Hoster -from module.network.RequestFactory import getURL +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -def getInfo(urls): - result = [] - - for url in urls: - html = getURL(url, decode=True) - if re.search(SpeedfileCz.FILE_OFFLINE_PATTERN, html): - # File offline - result.append((url, 0, 1, url)) - else: - # Get file info - found = re.search(SpeedfileCz.FILE_NAME_PATTERN, html) - if found is not None: - name = found.group(1) - result.append((name, 0, 2, url)) - yield result - - -class SpeedfileCz(Hoster): +class SpeedfileCz(SimpleHoster): __name__ = "SpeedFileCz" __type__ = "hoster" __pattern__ = r"http://speedfile.cz/.*" - __version__ = "0.3" + __version__ = "0.31" __description__ = """speedfile.cz""" __author_name__ = ("zoidberg") - FILE_NAME_PATTERN = r'<meta property="og:title" content="([^"]+)" />' + FILE_NAME_PATTERN = r'<meta property="og:title" content="(?P<N>[^"]+)" />' + FILE_SIZE_PATTERN = r'<strong><big>(?P<S>[0-9.]+) (?P<U>[kKMG])i?B' URL_PATTERN = r'<a id="request" class="caps" href="([^"]+)" rel="nofollow">' FILE_OFFLINE_PATTERN = r'<title>Speedfile \| 404' WAIT_PATTERN = r'"requestedAt":(\d+),"allowedAt":(\d+),"adUri"' @@ -78,3 +61,5 @@ class SpeedfileCz(Hoster): self.wait() self.download(download_url) + +create_getInfo(SpeedfileCz)
\ No newline at end of file diff --git a/module/plugins/hoster/StahnuTo.py b/module/plugins/hoster/StahnuTo.py index fb17fad7c..a78615dba 100644 --- a/module/plugins/hoster/StahnuTo.py +++ b/module/plugins/hoster/StahnuTo.py @@ -33,12 +33,12 @@ class StahnuTo(SimpleHoster): __name__ = "StahnuTo" __type__ = "hoster" __pattern__ = r"http://(\w*\.)?stahnu.to/(files/get/|.*\?file=)([^/]+).*" - __version__ = "0.11" + __version__ = "0.12" __description__ = """stahnu.to""" __author_name__ = ("zoidberg") - FILE_NAME_PATTERN = r"<div class='nadpis-01'><h2>([^<]+)</h2></div>" - FILE_SIZE_PATTERN = r'<td>Velikost souboru<br /><span>([^<]+)\s*(kb|Mb|Gb)</span></td>' + FILE_NAME_PATTERN = r"<div class='nadpis-01'><h2>(?<N>[^<]+)</h2></div>" + FILE_SIZE_PATTERN = r'<td>Velikost souboru<br /><span>(?<S>[^<]+)\s*(?<U>[kKMG])i?[Bb]</span></td>' FILE_OFFLINE_PATTERN = r'<!-- Obsah - start -->\s*<!-- Obsah - end -->' #FILE_OFFLINE_PATTERN = r'<h2 align="center">Tento soubor neexistuje nebo byl odstraněn! </h2>' CAPTCHA_PATTERN = r'<img src="captcha/captcha.php" id="captcha" /></td>' diff --git a/module/plugins/hoster/TwoSharedCom.py b/module/plugins/hoster/TwoSharedCom.py new file mode 100644 index 000000000..89c4b7781 --- /dev/null +++ b/module/plugins/hoster/TwoSharedCom.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +import re + +class TwoSharedCom(SimpleHoster): + __name__ = "TwoSharedCom" + __type__ = "hoster" + __pattern__ = r"http://[\w\.]*?2shared.com/(account/)?(download|get|file|document|photo|video|audio)/.*" + __version__ = "0.10" + __description__ = """2Shared Download Hoster""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FILE_NAME_PATTERN = r'<meta name="Description" content="(?P<N>.*) download free at 2shared' + FILE_SIZE_PATTERN = r'<span class="dtitle">File size:</span>\s*(?P<S>[0-9,.]+) (?P<U>[kKMG])i?B' + FILE_OFFLINE_PATTERN = r'The file link that you requested is not valid\.|This file was deleted\.' + DOWNLOAD_URL_PATTERN = r"window.location ='([^']+)';" + + def handleFree(self): + found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) + if not found: self.parseError('Download link') + link = found.group(1) + self.logDebug("Download URL %s" % link) + + self.download(link) + +getInfo = create_getInfo(TwoSharedCom) +
\ No newline at end of file diff --git a/module/plugins/hoster/UlozTo.py b/module/plugins/hoster/UlozTo.py index ffb09a655..53c5cd81f 100644 --- a/module/plugins/hoster/UlozTo.py +++ b/module/plugins/hoster/UlozTo.py @@ -17,24 +17,14 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo from module.network.RequestFactory import getURL -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(UlozTo, url, getURL(url, decode=True)) - print file_info - result.append(file_info) - - yield result - class UlozTo(SimpleHoster): __name__ = "UlozTo" __type__ = "hoster" __pattern__ = r"http://(\w*\.)?(uloz\.to|ulozto\.(cz|sk|net)|bagruj.cz|zachowajto.pl)/.*" - __version__ = "0.73" + __version__ = "0.74" __description__ = """uloz.to""" __config__ = [("reuseCaptcha", "bool", "Reuse captcha", "True"), ("captchaUser", "str", "captcha_user", ""), @@ -42,14 +32,14 @@ class UlozTo(SimpleHoster): __author_name__ = ("zoidberg") FILE_URL_PATTERN = r'<form name="dwn" action="([^"]+)"' - FILE_NAME_PATTERN = r'<h2 class="nadpis" style="margin-left:196px;"><a href="[^"]+">([^<]+)</a></h2>' + FILE_NAME_PATTERN = r'<h2 class="nadpis" style="margin-left:196px;"><a href="[^"]+">(?P<N>[^<]+)</a></h2>' CAPTCHA_PATTERN = r'<img style=".*src="([^"]+)" alt="Captcha" class="captcha"' CAPTCHA_NB_PATTERN = r'<input class="captcha_nb" type="hidden" name="captcha_nb" value="([0-9]+)" >' - FILE_OFFLINE_PATTERN = r'href="http://www.ulozto.net/(neexistujici|smazano)/\?lg=en&' + FILE_OFFLINE_PATTERN = r'http://www.uloz.to/(neexistujici|smazano|nenalezeno)' PASSWD_PATTERN = r'<input type="password" class="text" name="file_password" id="frmfilepasswordForm-file_password" />' LIVE_URL_PATTERN = r'<div id="flashplayer"[^>]*>\s*<a href="([^"]+)"' LIVE_NAME_PATTERN = r'<a share_url="[^&]*&t=([^"]+)"' - FILE_SIZE_PATTERN = r'<div class="info_velikost" style="top:-55px;">\s*<div>[^<]*\s+([0-9.]+)\s([kKMG]i?B)\s*</div>\s*</div>' + FILE_SIZE_PATTERN = r'<div class="info_velikost" style="top:-55px;">\s*<div>[^<]*\s+(?P<S>[0-9.]+)\s(?P<U>[kKMG])i?B\s*</div>\s*</div>' VIPLINK_PATTERN = r'<a href="[^"]*\?disclaimer=1" class="linkVip">' def setup(self): @@ -57,8 +47,10 @@ class UlozTo(SimpleHoster): def process(self, pyfile): header = self.load(pyfile.url, just_header=True) - if "location" in header and "utm_source=old" in header['location']: - self.offline() + if "location" in header: + self.logDebug('LOCATION: ' + header['location']) + if "utm_source=old" in header['location'] or re.search(self.FILE_OFFLINE_PATTERN, header['location']): + self.offline() self.html = self.load(pyfile.url, decode=True) @@ -140,4 +132,7 @@ class UlozTo(SimpleHoster): if reuse_captcha: self.setConfig("captchaUser", captcha) self.setConfig("captchaNb", captcha_nb) + +getInfo = create_getInfo(UlozTo) +
\ No newline at end of file diff --git a/module/plugins/hoster/UloziskoSk.py b/module/plugins/hoster/UloziskoSk.py index 1fd15502d..8d950f0a6 100644 --- a/module/plugins/hoster/UloziskoSk.py +++ b/module/plugins/hoster/UloziskoSk.py @@ -17,17 +17,7 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(UloziskoSk, url, getURL(url, decode=True)) - result.append(file_info) - - yield result +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class UloziskoSk(SimpleHoster): __name__ = "UloziskoSk" @@ -39,8 +29,8 @@ class UloziskoSk(SimpleHoster): URL_PATTERN = r'<form name = "formular" action = "([^"]+)" method = "post">' ID_PATTERN = r'<input type = "hidden" name = "id" value = "([^"]+)" />' - FILE_NAME_PATTERN = r'<div class="down1">([^<]+)</div>' - FILE_SIZE_PATTERN = ur'Veľkosť súboru: <strong>([0-9.]+) ([kKMG]i?B)</strong><br />' + FILE_NAME_PATTERN = r'<div class="down1">(?P<N>[^<]+)</div>' + FILE_SIZE_PATTERN = ur'Veľkosť súboru: <strong>(?P<S>[0-9.]+) (?P<U>[kKMG])i?B</strong><br />' CAPTCHA_PATTERN = r'<img src="(/obrazky/obrazky.php\?fid=[^"]+)" alt="" />' FILE_OFFLINE_PATTERN = ur'<span class = "red">Zadaný súbor neexistuje z jedného z nasledujúcich dôvodov:</span>' IMG_PATTERN = ur'<strong>PRE ZVÄČŠENIE KLIKNITE NA OBRÁZOK</strong><br /><a href = "([^"]+)">' @@ -80,4 +70,6 @@ class UloziskoSk(SimpleHoster): "id": id, "name": self.pyfile.name, "but": "++++STIAHNI+S%DABOR++++" - })
\ No newline at end of file + }) + +getInfo = create_getInfo(UloziskoSk)
\ No newline at end of file diff --git a/module/plugins/hoster/UploadboxCom.py b/module/plugins/hoster/UploadboxCom.py new file mode 100644 index 000000000..584c64e77 --- /dev/null +++ b/module/plugins/hoster/UploadboxCom.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: zoidberg +""" + +import re +from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo +from module.network.RequestFactory import getURL + +def getInfo(urls): + for url in urls: + file_id = re.search(UploadboxCom.__pattern__, url).group(1) + html = getURL('http://uploadbox.com/files/%s/?ac=lang&lang_new=en' % file_id, decode = False) + file_info = parseFileInfo(UploadboxCom, url, html) + yield file_info + +class UploadboxCom(SimpleHoster): + __name__ = "Uploadbox" + __type__ = "hoster" + __pattern__ = r"http://(?:www\.)?uploadbox\.com/files/([^/]+).*" + __version__ = "0.03" + __description__ = """UploadBox.com plugin - free only""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + FILE_NAME_PATTERN = r'<p><span>File name:</span>\s*(?P<N>[^<]+)</p>' + FILE_SIZE_PATTERN = r'<span>Size:</span>\s*(?P<S>[0-9.]+) (?P<U>[kKMG])i?B <span>' + FILE_OFFLINE_PATTERN = r'<strong>File deleted from service</strong>' + NAME_REPLACEMENTS = [(r"(.*)", lambda m: unicode(m.group(1), 'koi8_r'))] + + FREE_FORM_PATTERN = r'<form action="([^"]+)" method="post" id="free" name="free">(.*?)</form>' + FORM_INPUT_PATTERN = r'<input[^>]* name="([^"]+)" value="([^"]+)" />' + LIMIT_EXCEEDED_PATTERN = r'<strong>The limit of traffic for you is exceeded. Please </strong> <br />wait <strong>(\d+)</strong> minutes' + DOWNLOAD_URL_PATTERN = r'<iframe id="file_download"[^>]*src="([^"]+)"></iframe>' + + def process(self, pyfile): + self.file_id = re.search(self.__pattern__, pyfile.url).group(1) + self.html = self.load('http://uploadbox.com/files/%s/?ac=lang&lang_new=en' % self.file_id, decode = False) + self.getFileInfo() + self.handleFree() + + def handleFree(self): + # Solve captcha + url = 'http://uploadbox.com/files/' + self.file_id + + for i in range(5): + self.html = self.load(url, post = {"free": "yes"}) + found = re.search(self.LIMIT_EXCEEDED_PATTERN, self.html) + if found: + self.setWait(int(found.group(1))*60, True) + self.wait() + else: + break + else: + self.fail("Unable to get free download slot") + + for i in range(5): + try: + action, form = re.search(self.FREE_FORM_PATTERN, self.html, re.DOTALL).groups() + inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) + except Exception, e: + self.logError(e) + self.fail("Parse error on page 2") + + captcha_url = 'http://uploadbox.com/?ac=captcha&code=' + inputs['code'] + inputs['enter'] = self.decryptCaptcha(captcha_url) + self.html = self.load('http://uploadbox.com/' + action, post = inputs) + found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) + if found: + self.correctCaptcha() + download_url = found.group(1) + break + else: + self.invalidCaptcha() + else: + self.fail("Incorrect captcha entered 5 times") + + # Download + self.download(download_url)
\ No newline at end of file diff --git a/module/plugins/hoster/UploadhereCom.py b/module/plugins/hoster/UploadhereCom.py index 35ddf25c8..385e77dc7 100644 --- a/module/plugins/hoster/UploadhereCom.py +++ b/module/plugins/hoster/UploadhereCom.py @@ -16,15 +16,18 @@ @author: zoidberg """ -from UploadkingCom import UploadkingCom, getInfo +from module.plugins.internal.SimpleHoster import create_getInfo +from UploadkingCom import UploadkingCom class UploadhereCom(UploadkingCom): __name__ = "UploadhereCom" __type__ = "hoster" __pattern__ = r"http://(?:www\.)?uploadhere\.com/\w{10}" - __version__ = "0.1" + __version__ = "0.11" __description__ = """Uploadhere.com plugin - free only""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - # shares code with UploadkingCom
\ No newline at end of file + # shares code with UploadkingCom + +create_getInfo(UploadhereCom)
\ No newline at end of file diff --git a/module/plugins/hoster/UploadkingCom.py b/module/plugins/hoster/UploadkingCom.py index a706e95bc..a2e246d29 100644 --- a/module/plugins/hoster/UploadkingCom.py +++ b/module/plugins/hoster/UploadkingCom.py @@ -17,28 +17,19 @@ """ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, parseFileInfo -from module.network.RequestFactory import getURL - -def getInfo(urls): - result = [] - - for url in urls: - file_info = parseFileInfo(UploadkingCom, url, getURL(url, decode=False)) - result.append(file_info) - - yield result +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class UploadkingCom(SimpleHoster): __name__ = "UploadkingCom" __type__ = "hoster" __pattern__ = r"http://(?:www\.)?uploadking\.com/\w{10}" - __version__ = "0.12" + __version__ = "0.13" __description__ = """UploadKing.com plugin - free only""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - FILE_INFO_PATTERN = r'<font style="font-size:\d*px;">File(?:name)?:\s*<(?:b|/font><font[^>]*)>([^<]+)(?:</b>)?</div></TD></TR><TR><TD[^>]*><font style="font-size:\d*px;">(?:Files|S)ize:\s*<(?:b|/font><font[^>]*)>([0-9.]+) ([kKMG]i?B)</(?:b|font)>' + FILE_NAME_PATTERN = r'<font style="font-size:\d*px;">File(?:name)?:\s*<(?:b|/font><font[^>]*)>(?P<N>[^<]+)' + FILE_SIZE_PATTERN = r'<font style="font-size:\d*px;">(?:Files|S)ize:\s*<(?:b|/font><font[^>]*)>(?P<S>[0-9.]+) (?P<U>[kKMG])i?B' FILE_OFFLINE_PATTERN = r'<center><font[^>]*>Unfortunately, this file is unavailable</font></center>' FILE_URL_PATTERN = r'id="dlbutton"><a href="([^"]+)"' @@ -47,4 +38,6 @@ class UploadkingCom(SimpleHoster): if not found: self.fail("Download URL not found") url = found.group(1) self.logDebug("DOWNLOAD URL: " + url) - self.download(url)
\ No newline at end of file + self.download(url) + +create_getInfo(UploadkingCom)
\ No newline at end of file diff --git a/module/plugins/hoster/ZippyshareCom.py b/module/plugins/hoster/ZippyshareCom.py index 335f218d4..5b32b4068 100644 --- a/module/plugins/hoster/ZippyshareCom.py +++ b/module/plugins/hoster/ZippyshareCom.py @@ -2,81 +2,50 @@ # -*- coding: utf-8 -*- import re -from module.plugins.Hoster import Hoster +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo -class ZippyshareCom(Hoster): +class ZippyshareCom(SimpleHoster): __name__ = "ZippyshareCom" __type__ = "hoster" - __pattern__ = r"(http://)?www?\d{0,2}\.zippyshare.com/v/" - __version__ = "0.3" + __pattern__ = r"(http://www\d{0,2}\.zippyshare.com)/v(?:/|iew.jsp.*key=)(\d+)" + __version__ = "0.31" __description__ = """Zippyshare.com Download Hoster""" - __author_name__ = ("spoob") - __author_mail__ = ("spoob@pyload.org") + __author_name__ = ("spoob", "zoidberg") + __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz") + + FILE_NAME_PATTERN = r'>Name:</font>\s*<font [^>]*>(?P<N>[^<]+)</font><br />' + FILE_SIZE_PATTERN = r'>Size:</font>\s*<font [^>]*>(?P<S>[0-9.,]+) (?P<U>[kKMG]+)i?B</font><br />' + FILE_OFFLINE_PATTERN = r'>File does not exist on this server</div>' + + DOWNLOAD_URL_PATTERN = r"document\.getElementById\('dlbutton'\).href = ([^;]+);" + SEED_PATTERN = r"seed: (\d*)" def setup(self): self.html = None self.wantReconnect = False self.multiDL = True - - def process(self, pyfile): - self.pyfile = pyfile - self.download_html() - if not self.file_exists(): - self.offline() - - pyfile.name = self.get_file_name() - pyfile.url = self.get_file_url() - if pyfile.url: - self.download(pyfile.url) - else: - self.fail("URL could not be extracted") - - def download_html(self): - url = self.pyfile.url - self.html = self.load(url) - - + def handleFree(self): + url = self.get_file_url() + self.logDebug("Download URL %s" % url) + self.download(url, cookies = True) + def get_file_url(self): """ returns the absolute downloadable filepath """ + file_host, file_key = re.search(self.__pattern__, self.pyfile.url).groups() - file_host_pattern = r"http://([^/]*)/v/(\d*)/.*" - file_host_search = re.search(file_host_pattern, self.pyfile.url) - if file_host_search is None: - return False - - file_host = "http://" + file_host_search.group(1) - file_key = file_host_search.group(2) - - seed_pattern = r"seed: (\d*)" - seed_search = re.search(seed_pattern, self.html) - if seed_search is None: - self.fail("Problem downloading file.. offline?") - - file_seed = int(seed_search.group(1)) - time = str((file_seed * 24) % 6743256) - - file_url = file_host + "/download?key=" + str(file_key) + "&time=" + str(time) - return file_url - - - def get_file_name(self): - if self.html is None: - self.download_html() - if not self.wantReconnect: - file_name = re.search(r'<meta property="og:title" content="([^"]*)" />', self.html).group(1) - return file_name + found = re.search(self.DOWNLOAD_URL_PATTERN, self.html) + if found: + url = self.js.eval(found.group(1)) else: - return self.pyfile.url - + seed_search = re.search(self.SEED_PATTERN, self.html) + if seed_search is None: self.parseError('SEED') + + file_seed = int(seed_search.group(1)) + time = str((file_seed * 24) % 6743256) + url = "/download?key=" + str(file_key) + "&time=" + str(time) + + return file_host + url - def file_exists(self): - """ returns True or False - """ - if self.html is None: - self.download_html() - if re.search(r'File does not exist on this server', self.html) is not None: - return False - else: - return True +getInfo = create_getInfo(ZippyshareCom)
\ No newline at end of file diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e0963fd91..05ef03d58 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -16,12 +16,12 @@ @author: zoidberg """ +from urlparse import urlparse +from re import search, sub from module.plugins.Hoster import Hoster from module.utils import html_unescape from module.network.RequestFactory import getURL -from re import search - def parseFileInfo(self, url = '', html = ''): if not html and hasattr(self, "html"): html = self.html @@ -33,17 +33,17 @@ def parseFileInfo(self, url = '', html = ''): elif hasattr(self, "FILE_INFO_PATTERN"): found = search(self.FILE_INFO_PATTERN, html) if found: - name, size, units = found.groups() + name, size, units = found.group('N'), found.group('S'), found.group('U') else: if hasattr(self, "FILE_NAME_PATTERN"): found = search(self.FILE_NAME_PATTERN, html) if found: - name = found.group(1) + name = found.group('N') if hasattr(self, "FILE_SIZE_PATTERN"): found = search(self.FILE_SIZE_PATTERN, html) if found: - size, units = found.groups() + size, units = found.group('S'), found.group('U') if size: # File online, return name and size @@ -52,7 +52,12 @@ def parseFileInfo(self, url = '', html = ''): size = float(size) * 1024 ** self.SIZE_UNITS[units] status = 2 - if not name: name = url + if name: + for r in self.NAME_REPLACEMENTS: + rf, rt = r + name = sub(rf, rt, name) + else: + name = url return name, size, status, url @@ -66,33 +71,37 @@ def create_getInfo(plugin): class PluginParseError(Exception): def __init__(self, msg): + Exception.__init__ self.value = 'Parse error (%s) - plugin may be out of date' % msg def __str__(self): return repr(self.value) class SimpleHoster(Hoster): __name__ = "SimpleHoster" - __version__ = "0.1" + __version__ = "0.12" __pattern__ = None __type__ = "hoster" __description__ = """Base hoster plugin""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") - - SIZE_UNITS = {'kB': 1, 'KB': 1, 'KiB': 1, 'MB': 2, 'MiB': 2, 'GB': 3, 'GiB': 3} + + #TODO: could be replaced when using utils.parseFileSize ? + #some plugins need to override these + SIZE_UNITS = {'k': 1, 'K': 1, 'M': 2, 'G': 3} SIZE_REPLACEMENTS = {',': '', ' ': ''} - + NAME_REPLACEMENTS = [] + def setup(self): self.resumeDownload = self.multiDL = True if self.account else False def process(self, pyfile): - self.html = self.load(pyfile.url, decode = True) + self.html = self.load(pyfile.url, decode = True, cookies = True) self.getFileInfo() if self.account: self.handlePremium() else: self.handleFree() - + def getFileInfo(self): self.logDebug("URL: %s" % self.pyfile.url) name, size, status, url = parseFileInfo(self) @@ -102,7 +111,7 @@ class SimpleHoster(Hoster): self.parseError('File info') if not name: - name = html_unescape(urlparse(pyfile.url).path.split("/")[-1]) + name = html_unescape(urlparse(self.pyfile.url).path.split("/")[-1]) self.logDebug("FILE NAME: %s FILE SIZE: %s" % (name, size)) self.pyfile.name, self.pyfile.size = name, size diff --git a/module/remote/ClickAndLoadBackend.py b/module/remote/ClickAndLoadBackend.py new file mode 100644 index 000000000..ad8031587 --- /dev/null +++ b/module/remote/ClickAndLoadBackend.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" +import re +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from cgi import FieldStorage +from urllib import unquote +from base64 import standard_b64decode +from binascii import unhexlify + +try: + from Crypto.Cipher import AES +except: + pass + +from RemoteManager import BackendBase + +core = None +js = None + +class ClickAndLoadBackend(BackendBase): + def setup(self, host, port): + self.httpd = HTTPServer((host, port), CNLHandler) + global core, js + core = self.m.core + js = core.js + + def serve(self): + while self.enabled: + self.httpd.handle_request() + +class CNLHandler(BaseHTTPRequestHandler): + + def add_package(self, name, urls, queue=0): + print "name", name + print "urls", urls + print "queue", queue + + def get_post(self, name, default=""): + if name in self.post: + return self.post[name] + else: + return default + + def start_response(self, string): + + self.send_response(200) + + self.send_header("Content-Length", len(string)) + self.send_header("Content-Language", "de") + self.send_header("Vary", "Accept-Language, Cookie") + self.send_header("Cache-Control", "no-cache, must-revalidate") + self.send_header("Content-type", "text/html") + self.end_headers() + + def do_GET(self): + path = self.path.strip("/").lower() + #self.wfile.write(path+"\n") + + self.map = [ (r"add$", self.add), + (r"addcrypted$", self.addcrypted), + (r"addcrypted2$", self.addcrypted2), + (r"flashgot", self.flashgot), + (r"crossdomain\.xml", self.crossdomain), + (r"checkSupportForUrl", self.checksupport), + (r"jdcheck.js", self.jdcheck), + (r"", self.flash) ] + + func = None + for r, f in self.map: + if re.match(r"(flash(got)?/?)?"+r, path): + func = f + break + + if func: + try: + resp = func() + if not resp: resp = "success" + resp += "\r\n" + self.start_response(resp) + self.wfile.write(resp) + except Exception,e : + self.send_error(500, str(e)) + else: + self.send_error(404, "Not Found") + + def do_POST(self): + form = FieldStorage( + fp=self.rfile, + headers=self.headers, + environ={'REQUEST_METHOD':'POST', + 'CONTENT_TYPE':self.headers['Content-Type'], + }) + + self.post = {} + for name in form.keys(): + self.post[name] = form[name].value + + return self.do_GET() + + def flash(self): + return "JDownloader" + + def add(self): + package = self.get_post('referer', 'ClickAndLoad Package') + urls = filter(lambda x: x != "", self.get_post('urls').split("\n")) + + self.add_package(package, urls, 0) + + def addcrypted(self): + package = self.get_post('referer', 'ClickAndLoad Package') + dlc = self.get_post('crypted').replace(" ", "+") + + core.upload_container(package, dlc) + + def addcrypted2(self): + package = self.get_post("source", "ClickAndLoad Package") + crypted = self.get_post("crypted") + jk = self.get_post("jk") + + crypted = standard_b64decode(unquote(crypted.replace(" ", "+"))) + jk = "%s f()" % jk + jk = js.eval(jk) + Key = unhexlify(jk) + IV = Key + + obj = AES.new(Key, AES.MODE_CBC, IV) + result = obj.decrypt(crypted).replace("\x00", "").replace("\r","").split("\n") + + result = filter(lambda x: x != "", result) + + self.add_package(package, result, 0) + + + def flashgot(self): + autostart = int(self.get_post('autostart', 0)) + package = self.get_post('package', "FlashGot") + urls = filter(lambda x: x != "", self.get_post('urls').split("\n")) + + self.add_package(package, urls, autostart) + + def crossdomain(self): + rep = "<?xml version=\"1.0\"?>\n" + rep += "<!DOCTYPE cross-domain-policy SYSTEM \"http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd\">\n" + rep += "<cross-domain-policy>\n" + rep += "<allow-access-from domain=\"*\" />\n" + rep += "</cross-domain-policy>" + return rep + + def checksupport(self): + pass + + def jdcheck(self): + rep = "jdownloader=true;\n" + rep += "var version='10629';\n" + return rep diff --git a/module/remote/RemoteManager.py b/module/remote/RemoteManager.py index 2ac26a677..6caedad90 100644 --- a/module/remote/RemoteManager.py +++ b/module/remote/RemoteManager.py @@ -24,14 +24,19 @@ class BackendBase(Thread): Thread.__init__(self) self.m = manager self.core = manager.core + self.enabled = True + self.running = False def run(self): + self.running = True try: self.serve() except Exception, e: self.core.log.error(_("Remote backend error: %s") % e) if self.core.debug: print_exc() + finally: + self.running = False def setup(self, host, port): pass @@ -42,14 +47,27 @@ class BackendBase(Thread): def serve(self): pass + def shutdown(self): + pass + + def stop(self): + self.enabled = False# set flag and call shutdowm message, so thread can react + self.shutdown() + class RemoteManager(): - available = ["ThriftBackend"] + available = [] def __init__(self, core): self.core = core self.backends = [] + if self.core.remote: + self.available.append("ThriftBackend") + else: + self.available.append("SocketBackend") + + def startBackends(self): host = self.core.config["remote"]["listenaddr"] port = self.core.config["remote"]["port"] diff --git a/module/remote/SocketBackend.py b/module/remote/SocketBackend.py new file mode 100644 index 000000000..1a157cf1d --- /dev/null +++ b/module/remote/SocketBackend.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +import SocketServer + +from RemoteManager import BackendBase + +class RequestHandler(SocketServer.BaseRequestHandler): + + def setup(self): + pass + + def handle(self): + + print self.request.recv(1024) + + + +class SocketBackend(BackendBase): + + def setup(self, host, port): + #local only + self.server = SocketServer.ThreadingTCPServer(("localhost", port), RequestHandler) + + def serve(self): + self.server.serve_forever() diff --git a/module/remote/XMLRPCBackend.py b/module/remote/XMLRPCBackend.py deleted file mode 100644 index 003dc29ea..000000000 --- a/module/remote/XMLRPCBackend.py +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay, RaNaN -""" -from os.path import exists - -import module.lib.SecureXMLRPCServer as Server -from module.remote.RemoteManager import BackendBase - -class XMLRPCBackend(BackendBase): - def setup(self, host, port): - server_addr = (host, port) - if self.core.config['ssl']['activated']: - if exists(self.core.config['ssl']['cert']) and exists(self.core.config['ssl']['key']): - self.core.log.info(_("Using SSL XMLRPCBackend")) - self.server = Server.SecureXMLRPCServer(server_addr, self.core.config['ssl']['cert'], - self.core.config['ssl']['key'], self.checkAuth) - else: - self.core.log.warning(_("SSL Certificates not found, fallback to auth XMLRPC server")) - self.server = Server.AuthXMLRPCServer(server_addr, self.checkAuth) - else: - self.server = Server.AuthXMLRPCServer(server_addr, self.checkAuth) - - self.server.register_instance(self.core.api) - - def serve(self): - self.server.serve_forever() diff --git a/module/remote/__init__.py b/module/remote/__init__.py index 8d1c8b69c..9298f5337 100644 --- a/module/remote/__init__.py +++ b/module/remote/__init__.py @@ -1 +1,2 @@ - +# -*- coding: utf-8 -*- +activated = True diff --git a/module/remote/socketbackend/__init__.py b/module/remote/socketbackend/__init__.py new file mode 100644 index 000000000..de6d13128 --- /dev/null +++ b/module/remote/socketbackend/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'christian' +
\ No newline at end of file diff --git a/module/remote/socketbackend/create_ttypes.py b/module/remote/socketbackend/create_ttypes.py new file mode 100644 index 000000000..0d6192a5a --- /dev/null +++ b/module/remote/socketbackend/create_ttypes.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +from os.path import abspath, dirname, join + +path = dirname(abspath(__file__)) +module = join(path, "..", "..") + +sys.path.append(join(module, "lib")) +sys.path.append(join(module, "remote")) + +from thriftbackend.thriftgen.pyload import ttypes +from thriftbackend.thriftgen.pyload.Pyload import Iface + + +def main(): + + enums = [] + classes = [] + + print "generating lightweight ttypes.py" + + for name in dir(ttypes): + klass = getattr(ttypes, name) + + if name in ("TBase", "TExceptionBase") or name.startswith("_") or not (issubclass(klass, ttypes.TBase) or issubclass(klass, ttypes.TExceptionBase)): + continue + + if hasattr(klass, "thrift_spec"): + classes.append(klass) + else: + enums.append(klass) + + + f = open(join(path, "ttypes.py"), "wb") + + f.write( + """#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Autogenerated by pyload +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + +class BaseObject(object): +\t__slots__ = [] + +""") + + ## generate enums + for enum in enums: + name = enum.__name__ + f.write("class %s:\n" % name) + + for attr in dir(enum): + if attr.startswith("_") or attr in ("read", "write"): continue + + f.write("\t%s = %s\n" % (attr, getattr(enum, attr))) + + f.write("\n") + + for klass in classes: + name = klass.__name__ + base = "Exception" if issubclass(klass, ttypes.TExceptionBase) else "BaseObject" + f.write("class %s(%s):\n" % (name, base)) + f.write("\t__slots__ = %s\n\n" % klass.__slots__) + + #create init + args = ["self"] + ["%s=None" % x for x in klass.__slots__] + + f.write("\tdef init(%s):\n" % ", ".join(args)) + for attr in klass.__slots__: + f.write("\t\tself.%s = %s\n" % (attr, attr)) + + f.write("\n") + + f.write("class Iface:\n") + + for name in dir(Iface): + if name.startswith("_"): continue + f.write("\tdef %s(self):\n\t\tpass\n" % name) + + f.write("\n") + + f.close() + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/module/remote/socketbackend/ttypes.py b/module/remote/socketbackend/ttypes.py new file mode 100644 index 000000000..527dff767 --- /dev/null +++ b/module/remote/socketbackend/ttypes.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Autogenerated by pyload +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + +class BaseObject(object): + __slots__ = [] + +class Destination: + Collector = 0 + Queue = 1 + +class DownloadStatus: + Aborted = 9 + Custom = 11 + Decrypting = 10 + Downloading = 12 + Failed = 8 + Finished = 0 + Offline = 1 + Online = 2 + Processing = 13 + Queued = 3 + Skipped = 4 + Starting = 7 + TempOffline = 6 + Unknown = 14 + Waiting = 5 + +class ElementType: + File = 1 + Package = 0 + +class AccountInfo(BaseObject): + __slots__ = ['validuntil', 'login', 'options', 'valid', 'trafficleft', 'maxtraffic', 'premium', 'type'] + + def init(self, validuntil=None, login=None, options=None, valid=None, trafficleft=None, maxtraffic=None, premium=None, type=None): + self.validuntil = validuntil + self.login = login + self.options = options + self.valid = valid + self.trafficleft = trafficleft + self.maxtraffic = maxtraffic + self.premium = premium + self.type = type + +class CaptchaTask(BaseObject): + __slots__ = ['tid', 'data', 'type', 'resultType'] + + def init(self, tid=None, data=None, type=None, resultType=None): + self.tid = tid + self.data = data + self.type = type + self.resultType = resultType + +class ConfigItem(BaseObject): + __slots__ = ['name', 'description', 'value', 'type'] + + def init(self, name=None, description=None, value=None, type=None): + self.name = name + self.description = description + self.value = value + self.type = type + +class ConfigSection(BaseObject): + __slots__ = ['name', 'description', 'items', 'outline'] + + def init(self, name=None, description=None, items=None, outline=None): + self.name = name + self.description = description + self.items = items + self.outline = outline + +class DownloadInfo(BaseObject): + __slots__ = ['fid', 'name', 'speed', 'eta', 'format_eta', 'bleft', 'size', 'format_size', 'percent', 'status', 'statusmsg', 'format_wait', 'wait_until', 'packageID', 'packageName', 'plugin'] + + def init(self, fid=None, name=None, speed=None, eta=None, format_eta=None, bleft=None, size=None, format_size=None, percent=None, status=None, statusmsg=None, format_wait=None, wait_until=None, packageID=None, packageName=None, plugin=None): + self.fid = fid + self.name = name + self.speed = speed + self.eta = eta + self.format_eta = format_eta + self.bleft = bleft + self.size = size + self.format_size = format_size + self.percent = percent + self.status = status + self.statusmsg = statusmsg + self.format_wait = format_wait + self.wait_until = wait_until + self.packageID = packageID + self.packageName = packageName + self.plugin = plugin + +class EventInfo(BaseObject): + __slots__ = ['eventname', 'id', 'type', 'destination'] + + def init(self, eventname=None, id=None, type=None, destination=None): + self.eventname = eventname + self.id = id + self.type = type + self.destination = destination + +class FileData(BaseObject): + __slots__ = ['fid', 'url', 'name', 'plugin', 'size', 'format_size', 'status', 'statusmsg', 'packageID', 'error', 'order'] + + def init(self, fid=None, url=None, name=None, plugin=None, size=None, format_size=None, status=None, statusmsg=None, packageID=None, error=None, order=None): + self.fid = fid + self.url = url + self.name = name + self.plugin = plugin + self.size = size + self.format_size = format_size + self.status = status + self.statusmsg = statusmsg + self.packageID = packageID + self.error = error + self.order = order + +class FileDoesNotExists(Exception): + __slots__ = ['fid'] + + def init(self, fid=None): + self.fid = fid + +class OnlineCheck(BaseObject): + __slots__ = ['rid', 'data'] + + def init(self, rid=None, data=None): + self.rid = rid + self.data = data + +class OnlineStatus(BaseObject): + __slots__ = ['name', 'plugin', 'packagename', 'status', 'size'] + + def init(self, name=None, plugin=None, packagename=None, status=None, size=None): + self.name = name + self.plugin = plugin + self.packagename = packagename + self.status = status + self.size = size + +class PackageData(BaseObject): + __slots__ = ['pid', 'name', 'folder', 'site', 'password', 'dest', 'order', 'linksdone', 'sizedone', 'sizetotal', 'linkstotal', 'links', 'fids'] + + def init(self, pid=None, name=None, folder=None, site=None, password=None, dest=None, order=None, linksdone=None, sizedone=None, sizetotal=None, linkstotal=None, links=None, fids=None): + self.pid = pid + self.name = name + self.folder = folder + self.site = site + self.password = password + self.dest = dest + self.order = order + self.linksdone = linksdone + self.sizedone = sizedone + self.sizetotal = sizetotal + self.linkstotal = linkstotal + self.links = links + self.fids = fids + +class PackageDoesNotExists(Exception): + __slots__ = ['pid'] + + def init(self, pid=None): + self.pid = pid + +class ServerStatus(BaseObject): + __slots__ = ['pause', 'active', 'queue', 'total', 'speed', 'download', 'reconnect'] + + def init(self, pause=None, active=None, queue=None, total=None, speed=None, download=None, reconnect=None): + self.pause = pause + self.active = active + self.queue = queue + self.total = total + self.speed = speed + self.download = download + self.reconnect = reconnect + +class ServiceCall(BaseObject): + __slots__ = ['plugin', 'func', 'arguments', 'parseArguments'] + + def init(self, plugin=None, func=None, arguments=None, parseArguments=None): + self.plugin = plugin + self.func = func + self.arguments = arguments + self.parseArguments = parseArguments + +class ServiceDoesNotExists(Exception): + __slots__ = ['plugin', 'func'] + + def init(self, plugin=None, func=None): + self.plugin = plugin + self.func = func + +class ServiceException(Exception): + __slots__ = ['msg'] + + def init(self, msg=None): + self.msg = msg + +class UserData(BaseObject): + __slots__ = ['name', 'email', 'role', 'permission', 'templateName'] + + def init(self, name=None, email=None, role=None, permission=None, templateName=None): + self.name = name + self.email = email + self.role = role + self.permission = permission + self.templateName = templateName + +class Iface: + def addFiles(self): + pass + def addPackage(self): + pass + def call(self): + pass + def checkOnlineStatus(self): + pass + def checkOnlineStatusContainer(self): + pass + def checkURLs(self): + pass + def deleteFiles(self): + pass + def deleteFinished(self): + pass + def deletePackages(self): + pass + def freeSpace(self): + pass + def generateAndAddPackages(self): + pass + def generatePackages(self): + pass + def getAccountTypes(self): + pass + def getAccounts(self): + pass + def getAllInfo(self): + pass + def getAllUserData(self): + pass + def getCaptchaTask(self): + pass + def getCaptchaTaskStatus(self): + pass + def getCollector(self): + pass + def getCollectorData(self): + pass + def getConfig(self): + pass + def getConfigValue(self): + pass + def getEvents(self): + pass + def getFileData(self): + pass + def getFileOrder(self): + pass + def getInfoByPlugin(self): + pass + def getLog(self): + pass + def getPackageData(self): + pass + def getPackageInfo(self): + pass + def getPackageOrder(self): + pass + def getPluginConfig(self): + pass + def getQueue(self): + pass + def getQueueData(self): + pass + def getServerVersion(self): + pass + def getServices(self): + pass + def getUserData(self): + pass + def hasService(self): + pass + def isCaptchaWaiting(self): + pass + def isTimeDownload(self): + pass + def isTimeReconnect(self): + pass + def kill(self): + pass + def login(self): + pass + def moveFiles(self): + pass + def movePackage(self): + pass + def orderFile(self): + pass + def orderPackage(self): + pass + def parseURLs(self): + pass + def pauseServer(self): + pass + def pollResults(self): + pass + def pullFromQueue(self): + pass + def pushToQueue(self): + pass + def recheckPackage(self): + pass + def removeAccount(self): + pass + def restart(self): + pass + def restartFailed(self): + pass + def restartFile(self): + pass + def restartPackage(self): + pass + def setCaptchaResult(self): + pass + def setConfigValue(self): + pass + def setPackageData(self): + pass + def setPackageName(self): + pass + def statusDownloads(self): + pass + def statusServer(self): + pass + def stopAllDownloads(self): + pass + def stopDownloads(self): + pass + def togglePause(self): + pass + def toggleReconnect(self): + pass + def unpauseServer(self): + pass + def updateAccount(self): + pass + def uploadContainer(self): + pass + diff --git a/module/setup.py b/module/setup.py index 8677fb65a..4a1c59da6 100644 --- a/module/setup.py +++ b/module/setup.py @@ -238,12 +238,14 @@ class Setup(): try: import jinja2 - if jinja2.__version__ and "unknown" not in jinja2.__version__ and not jinja2.__version__.startswith("2.5"): #@TODO: could be to new aswell - print _("Your installed jinja2 version %s seems too old.") % jinja2.__version__ - print _("You can safely continue but if the webinterface is not working,") - print _("please upgrade or deinstall it, pyLoad includes a sufficient jinja2 libary.") - print - jinja = False + v = jinja2.__version__ + if v and "unknown" not in v: + if not v.startswith("2.5") and not v.startswith("2.6"): + print _("Your installed jinja2 version %s seems too old.") % jinja2.__version__ + print _("You can safely continue but if the webinterface is not working,") + print _("please upgrade or deinstall it, pyLoad includes a sufficient jinja2 libary.") + print + jinja = False except : pass @@ -278,6 +280,12 @@ class Setup(): db.shutdown() print "" + print _("External clients (GUI, CLI or other) need remote access to work over the network.") + print _("However, if you only want to use the webinterface you may disable it to save ram.") + self.config["remote"]["activated"] = self.ask(_("Enable remote access"), "y", bool=True) + + + print "" langs = self.config.getMetaData("general", "language") self.config["general"]["language"] = self.ask(_("Language"), "en", langs["type"].split(";")) diff --git a/module/web/api_app.py b/module/web/api_app.py index db735a5b9..1629c1677 100644 --- a/module/web/api_app.py +++ b/module/web/api_app.py @@ -7,19 +7,18 @@ from traceback import format_exc, print_exc from bottle import route, request, response, HTTPError -from thrift.protocol.TBase import TBase - from utils import toDict, set_session from webinterface import PYLOAD from module.common.json_layer import json from module.lib.SafeEval import const_eval as literal_eval +from module.Api import BaseObject # json encoder that accepts TBase objects class TBaseEncoder(json.JSONEncoder): def default(self, o): - if isinstance(o, TBase): + if isinstance(o, BaseObject): return toDict(o) return json.JSONEncoder.default(self, o) diff --git a/module/web/templates/default/queue.html b/module/web/templates/default/queue.html index b11910bac..c88fa3568 100644 --- a/module/web/templates/default/queue.html +++ b/module/web/templates/default/queue.html @@ -28,6 +28,7 @@ document.addEvent("domready", function(){ {% endblock %}
{% block content %}
+{% autoescape true %}
<ul id="package-list" style="list-style: none; padding-left: 0; margin-top: -10px;">
{% for package in content %}
@@ -69,6 +70,7 @@ document.addEvent("domready", function(){ </li>
{% endfor %}
</ul>
+{% endautoescape %}
{% endblock %}
{% block hidden %}
diff --git a/module/web/webinterface.py b/module/web/webinterface.py index b531d0583..68724e3f6 100644 --- a/module/web/webinterface.py +++ b/module/web/webinterface.py @@ -79,7 +79,7 @@ loader = PrefixLoader({ 'js': FileSystemLoader(join(PROJECT_DIR, 'media', 'js')) }) -env = Environment(loader=loader, extensions=['jinja2.ext.i18n'], trim_blocks=True, auto_reload=False, +env = Environment(loader=loader, extensions=['jinja2.ext.i18n', 'jinja2.ext.autoescape'], trim_blocks=True, auto_reload=False, bytecode_cache=bcc) from filters import quotepath, path_make_relative, path_make_absolute, truncate, date @@ -118,8 +118,6 @@ web = GZipMiddleWare(web) if PREFIX: web = PrefixMiddleware(web, prefix=PREFIX) -#TODO: compress plugin, install(otfcompress) - import pyload_app import json_app import cnl_app diff --git a/pavement.py b/pavement.py index 92fbef573..c534b4877 100644 --- a/pavement.py +++ b/pavement.py @@ -173,6 +173,10 @@ def thrift(options): (outdir / "gen-py").move(outdir / "thriftgen") + #create light ttypes + from module.remote.socketbackend.create_ttypes import main + main() + @task def compile_js(): """ Compile .coffee files to javascript""" diff --git a/pyLoadCore.py b/pyLoadCore.py index 56f32c9c5..97c9ec64b 100755 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -43,14 +43,13 @@ from module import InitHomeDir from module.plugins.AccountManager import AccountManager from module.CaptchaManager import CaptchaManager from module.ConfigParser import ConfigParser -from module.HookManager import HookManager from module.plugins.PluginManager import PluginManager from module.PullEvents import PullManager from module.network.RequestFactory import RequestFactory -from module.ThreadManager import ThreadManager from module.web.ServerThread import WebServer from module.Scheduler import Scheduler from module.common.JsEngine import JsEngine +from module import remote from module.remote.RemoteManager import RemoteManager from module.database import DatabaseBackend, FileHandler @@ -77,20 +76,24 @@ class Core(object): self.startedInGui = False self.running = False self.daemon = False + self.remote = True self.arg_links = [] self.pidfile = "pyload.pid" self.deleteLinks = False # will delete links on startup if len(argv) > 1: try: - options, args = getopt(argv[1:], 'vchdusq', - ["version", "clear", "clean", "help", "debug", "user", "setup", "configdir=", "changedir", "daemon", - "quit", "status"]) + options, args = getopt(argv[1:], 'vchdusqp:', + ["version", "clear", "clean", "help", "debug", "user", + "setup", "configdir=", "changedir", "daemon", + "quit", "status", "no-remote","pidfile="]) for option, argument in options: if option in ("-v", "--version"): print "pyLoad", CURRENT_VERSION exit() + elif option in ("-p", "--pidfile"): + self.pidfile = argument elif option == "--daemon": self.daemon = True elif option in ("-c", "--clear"): @@ -134,6 +137,8 @@ class Core(object): elif option == "--clean": self.cleanTree() exit() + elif option == "--no-remote": + self.remote = False except GetoptError: print 'Unknown Argument(s) "%s"' % " ".join(argv[1:]) @@ -157,8 +162,10 @@ class Core(object): print " -d, --debug", " " * 12, "Enable debug mode" print " -s, --setup", " " * 12, "Run Setup Assistent" print " --configdir=<dir>", " " * 6, "Run with <dir> as config directory" + print " -p, --pidfile=<file>", " " * 3, "Set pidfile to <file>" print " --changedir", " " * 12, "Change config dir permanently" print " --daemon", " " * 15, "Daemonmize after start" + print " --no-remote", " " * 12, "Disable remote access (saves RAM)" print " --status", " " * 15, "Display pid if running or False" print " --clean", " " * 16, "Remove .pyc/.pyo files" print " -q, --quit", " " * 13, "Quit running pyLoad instance" @@ -286,6 +293,7 @@ class Core(object): translation.install(True) self.debug = self.doDebug or self.config['general']['debug_mode'] + self.remote &= self.config['remote']['activated'] pid = self.isAlreadyRunning() if pid: @@ -333,6 +341,9 @@ class Core(object): #@TODO refractor + remote.activated = self.remote + self.log.debug("Remote activated: %s" % self.remote) + self.check_install("Crypto", _("pycrypto to decode container files")) #img = self.check_install("Image", _("Python Image Libary (PIL) for captcha reading")) #self.check_install("pycurl", _("pycurl to download any files"), True, True) @@ -362,9 +373,15 @@ class Core(object): self.lastClientConnected = 0 - from module.Api import Api + # later imported because they would trigger api import, and remote value not set correctly + from module.HookManager import HookManager + from module.ThreadManager import ThreadManager + from module import Api + + if Api.activated != self.remote: + self.log.warning("Import error: API remote status not correct.") - self.api = Api(self) + self.api = Api.Api(self) self.scheduler = Scheduler(self) @@ -387,8 +404,6 @@ class Core(object): if web: self.init_webserver() - #linkFile = self.config['general']['link_file'] - spaceLeft = freeSpace(self.config["general"]["download_folder"]) self.log.info(_("Free space: %s") % formatSize(spaceLeft)) |