From acddd89ad9e0ce9107ac9a99443cfbecc58a1ff9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 07:46:34 +0100 Subject: [SimpleCrypter][SimpleHoster] Fix info attr init --- module/plugins/internal/SimpleCrypter.py | 6 +++--- module/plugins/internal/SimpleHoster.py | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index d1938ec66..75c687c16 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -71,9 +71,6 @@ class SimpleCrypter(Crypter): #@TODO: remove in 0.4.10 def init(self): - self.info = {} #@TODO: Remove in 0.4.10 - self.links = [] - account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") account = self.core.accountManager.getAccountPlugin(account_name) @@ -92,6 +89,9 @@ class SimpleCrypter(Crypter): if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) + self.info = {} + self.links = [] + self.req.setOption("timeout", 120) if isinstance(self.COOKIES, list): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 9cf8eb541..7ab562779 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -237,16 +237,14 @@ class SimpleHoster(Hoster): return info - def init(self): - self.info = {} #@TODO: Remove in 0.4.10 - self.link = "" #@TODO: Move to hoster class in 0.4.10 - - def setup(self): self.resumeDownload = self.multiDL = self.premium def prepare(self): + self.info = {} + self.link = "" #@TODO: Move to hoster class in 0.4.10 + if self.CHECK_DIRECT_LINK is None: self.CHECK_DIRECT_LINK = bool(self.account) -- cgit v1.2.3 From e9c00be0d343ad30bca51147160b5067d7f9c1a8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:18:13 +0100 Subject: [XFSHoster] Update info patterns --- module/plugins/internal/XFSHoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index ea5889190..de9fbc831 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.17" + __version__ = "0.18" __pattern__ = r'^unmatchable$' @@ -35,8 +35,8 @@ class XFSHoster(SimpleHoster): COOKIES = [(HOSTER_DOMAIN, "lang", "english")] INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' - NAME_PATTERN = r'[\d.,]+) ?(?P[\w^_]+)?\)' + NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' + SIZE_PATTERN = r'(>Size:|>File:.*>|)(?P[\d.,]+)\s*(?P[\w^_]+)' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' -- cgit v1.2.3 From adf154d358abfed43f49873cb08ba71379416471 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:19:00 +0100 Subject: [EasybytezCom] Use XFSHoster INFO_PATTERN --- module/plugins/hoster/EasybytezCom.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/module/plugins/hoster/EasybytezCom.py b/module/plugins/hoster/EasybytezCom.py index 3103d403f..cd54bdc70 100644 --- a/module/plugins/hoster/EasybytezCom.py +++ b/module/plugins/hoster/EasybytezCom.py @@ -6,7 +6,7 @@ from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo class EasybytezCom(XFSHoster): __name__ = "EasybytezCom" __type__ = "hoster" - __version__ = "0.22" + __version__ = "0.23" __pattern__ = r'http://(?:www\.)?easybytez\.com/\w{12}' @@ -18,7 +18,6 @@ class EasybytezCom(XFSHoster): HOSTER_DOMAIN = "easybytez.com" - INFO_PATTERN = r'(?P.+)
\s*(?P[^<]+)' OFFLINE_PATTERN = r'>File not available' LINK_PATTERN = r'(http://(\w+\.(easybytez|easyload|ezbytez|zingload)\.(com|to)|\d+\.\d+\.\d+\.\d+)/files/\d+/\w+/.+?)["\'<]' -- cgit v1.2.3 From 48d785908bafd2615189c6792cbfec36db7e97df Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:20:17 +0100 Subject: [SimpleCrypter] Extend SimpleHoster --- module/plugins/internal/SimpleCrypter.py | 47 +++++++------------------------- 1 file changed, 10 insertions(+), 37 deletions(-) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 75c687c16..8c7f0bef3 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -6,14 +6,14 @@ from urlparse import urlparse from module.plugins.Crypter import Crypter from module.plugins.Plugin import Fail -from module.plugins.internal.SimpleHoster import _error, _wait, parseFileInfo, replace_patterns, set_cookies -from module.utils import fixup, html_unescape +from module.plugins.internal.SimpleHoster import SimpleHoster, _error, _wait, replace_patterns, set_cookies +from module.utils import fixup -class SimpleCrypter(Crypter): +class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.30" + __version__ = "0.31" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -69,7 +69,7 @@ class SimpleCrypter(Crypter): LOGIN_PREMIUM = False - #@TODO: remove in 0.4.10 + #@TODO: Remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") account = self.core.accountManager.getAccountPlugin(account_name) @@ -100,13 +100,6 @@ class SimpleCrypter(Crypter): self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - def preload(self): - self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING) - - if isinstance(self.TEXT_ENCODING, basestring): - self.html = unicode(self.html, self.TEXT_ENCODING) - - def decrypt(self, pyfile): self.prepare() @@ -138,30 +131,19 @@ class SimpleCrypter(Crypter): self.tempOffline() - def checkName(self): + def checkNameSize(self): name = self.info['name'] url = self.info['url'] if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split('/')[-1] - - self.info['folder'] = self.pyfile.name - - self.logDebug("File name: %s" % self.pyfile.name) - + self.pyfile.name = self.info['name'] = urlparse(name).path.split('/')[-1] - def checkInfo(self): - self.logDebug(_("File info (previous): %s") % self.info) + folder = self.info['folder'] = self.pyfile.name - info = parseFileInfo(self.pyfile.url, self.html or "") - self.info.update(info) - - self.logDebug(_("File info (current): %s") % self.info) - - self.checkName() - self.checkStatus() + self.logDebug("File name: %s" % self.pyfile.name, + "File folder: %s" % folder) def getLinks(self): @@ -182,12 +164,3 @@ class SimpleCrypter(Crypter): for p in xrange(2, pages + 1): self.html = self.loadPage(p) self.links += self.getLinks() - - - #@TODO: Remove in 0.4.10 - def wait(self, seconds=0, reconnect=None): - return _wait(self, seconds, reconnect) - - - def error(self, reason="", type="parse"): - return _error(self, reason, type) -- cgit v1.2.3 From e08fb5d3361627ad866aed86be9597b32c8c50f9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:23:33 +0100 Subject: [SimpleHoster] Code cosmetics --- module/plugins/Plugin.py | 2 +- module/plugins/internal/SimpleHoster.py | 39 ++++++++++++++++++++------------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 55d93e357..550169034 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -16,7 +16,7 @@ from itertools import islice from traceback import print_exc from urlparse import urlparse -from module.utils import fs_decode, fs_encode, html_unescape, save_join, save_path +from module.utils import fs_decode, fs_encode, save_join, save_path def chunks(iterable, size): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 7ab562779..1de872261 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -7,12 +7,16 @@ from urlparse import urlparse from pycurl import FOLLOWLOCATION -from module.PyFile import statusMap +from module.PyFile import statusMap as _statusMap from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster from module.plugins.Plugin import Fail -from module.utils import fixup, html_unescape, parseFileSize +from module.utils import fixup, parseFileSize + + +#@TODO: Adapt and move to PyFile in 0.4.10 +statusMap = {v: k for k, v in _statusMap.iteritems()} #@TODO: Remove in 0.4.10 and redirect to self.error instead @@ -21,7 +25,7 @@ def _error(self, reason, type): type = "unknown" msg = _("%s error") % type.strip().capitalize() if type else _("Error") - msg += ": " + reason.strip() if reason else "" + msg += ": %s" % reason.strip() if reason else "" msg += _(" | Plugin may be out of date") raise Fail(msg) @@ -89,11 +93,13 @@ def parseHtmlForm(attr_str, html, input_names=None): return {}, None #: no matching form found +#: Deprecated def parseFileInfo(plugin, url="", html=""): info = plugin.getInfo(url, html) return info['name'], info['size'], info['status'], info['url'] +#@TODO: Remove in 0.4.10 def create_getInfo(plugin): return lambda urls: list(plugin.parseInfo(urls)) @@ -120,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.56" + __version__ = "0.57" __pattern__ = r'^unmatchable$' @@ -279,12 +285,13 @@ class SimpleHoster(Hoster): self.fail(_("No html retrieved")) info = self.getInfo(pyfile.url, self.html) - self._updateInfo(info) + premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) + self._updateInfo(info) self.checkNameSize() - premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only: #: Usually premium only pages doesn't show any file information + #: Usually premium only pages doesn't show any file information + if not premium_only: self.checkStatus() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): @@ -312,8 +319,9 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is not 2: - self.error(_("File status: %s") % filter(lambda key, val: val == status, statusMap.iteritems())[0], - _("File info: %s") % self.info) + self.logDebug("File status: %s" % statusMap[status], + "File info: %s" % self.info) + self.error(_("No file info")) def checkNameSize(self): @@ -324,14 +332,15 @@ class SimpleHoster(Hoster): if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split('/')[-1] + self.pyfile.name = name = self.info['name'] = urlparse(name).path.split('/')[-1] if size > 0: self.pyfile.size = size else: - self.logError(_("File size not found")) + size = "Unknown" - self.logDebug("File name: %s" % self.pyfile.name, "File size: %s" % self.pyfile.size or _("Unknown")) + self.logDebug("File name: %s" % name, + "File size: %s" % size) def checkInfo(self): @@ -340,10 +349,10 @@ class SimpleHoster(Hoster): self.checkStatus() - def _updateInfo(self, info) - self.logDebug(_("File info (previous): %s") % self.info) + def _updateInfo(self, info): + self.logDebug(_("File info (before update): %s") % self.info) self.info.update(info) - self.logDebug(_("File info (current): %s") % self.info) + self.logDebug(_("File info (after update): %s") % self.info) def handleDirect(self): -- cgit v1.2.3 From 911ef70f5e39444d4b885b79ba825855b54e7606 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:59:47 +0100 Subject: [SimpleHoster] Fix create_getInfo --- module/PluginThread.py | 11 ++++++----- module/plugins/internal/SimpleHoster.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/module/PluginThread.py b/module/PluginThread.py index 59d2cabc9..051236c3e 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -209,7 +209,7 @@ class DownloadThread(PluginThread): if self.m.core.debug: print_exc() - + self.clean(pyfile) continue @@ -244,7 +244,7 @@ class DownloadThread(PluginThread): if self.m.core.debug: print_exc() - + self.m.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) continue @@ -393,7 +393,7 @@ class DecrypterThread(PluginThread): except Abort: self.m.log.info(_("Download aborted: %s") % pyfile.name) pyfile.setStatus("aborted") - + if self.m.core.debug: print_exc() return @@ -630,9 +630,10 @@ class InfoThread(PluginThread): self.m.log.debug("Run Info Fetching for %s" % pluginname) for result in plugin.getInfo(process): #result = [ .. (name, size, status, url) .. ] - if not type(result) == list: result = [result] + if not type(result) == list: + result = [result] - for res in result: + for res in result: #: why don't assign res dict directly? self.m.infoCache[res[3]] = res cb(pluginname, result) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1de872261..c5875a8da 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -101,7 +101,7 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 def create_getInfo(plugin): - return lambda urls: list(plugin.parseInfo(urls)) + return lambda urls: [info['name'], info['size'], info['status'], info['url'] for info in plugin.parseInfo(urls)] def timestamp(): -- cgit v1.2.3 From 70759ae9fd8561436bef08322d45ac41637c66d6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 20:01:24 +0100 Subject: [XFSHoster] Fix getPostParameters --- module/plugins/internal/XFSHoster.py | 102 +++++++++++++++++------------------ 1 file changed, 50 insertions(+), 52 deletions(-) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index de9fbc831..d2a771507 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.18" + __version__ = "0.19" __pattern__ = r'^unmatchable$' @@ -32,7 +32,9 @@ class XFSHoster(SimpleHoster): URL_REPLACEMENTS = [(r'/(?:embed-)?(\w{12}).*', r'/\1')] #: plus support embedded files - COOKIES = [(HOSTER_DOMAIN, "lang", "english")] + TEXT_ENCODING = False + COOKIES = [(HOSTER_DOMAIN, "lang", "english")] + CHECK_DIRECT_LINK = None INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' @@ -128,19 +130,19 @@ class XFSHoster(SimpleHoster): self.req.http.c.setopt(FOLLOWLOCATION, 0) self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True) - self.header = self.req.http.header self.req.http.c.setopt(FOLLOWLOCATION, 1) - m = re.search(r'Location\s*:\s*(.+)', self.header, re.I) - if m: + m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) + if m and not "op=" in m.group(1): break m = re.search(self.LINK_PATTERN, self.html, re.S) if m: break else: - return + self.logError(data['op'] if 'op' in data else _("UNKNOWN")) + return "" self.errmsg = None @@ -197,6 +199,7 @@ class XFSHoster(SimpleHoster): self.error(_("OVR_LINK_PATTERN not found")) header = self.load(m.group(1).strip(), just_header=True, decode=True) #@TODO: Remove .strip() in 0.4.10 + if 'location' in header: #: Direct download link self.download(header['location'], ref=True, cookies=True, disposition=True) else: @@ -251,58 +254,53 @@ class XFSHoster(SimpleHoster): def getPostParameters(self): - for _i in xrange(3): - if hasattr(self, "FORM_PATTERN"): - action, inputs = self.parseHtmlForm(self.FORM_PATTERN) - else: - action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + if hasattr(self, "FORM_PATTERN"): + action, inputs = self.parseHtmlForm(self.FORM_PATTERN) + else: + action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + if not inputs: + action, inputs = self.parseHtmlForm('F1') if not inputs: - action, inputs = self.parseHtmlForm('F1') - if not inputs: - if self.errmsg: - self.retry(reason=self.errmsg) - else: - self.error(_("TEXTAREA F1 not found")) - - self.logDebug(inputs) - - if 'op' in inputs and inputs['op'] in ("download1", "download2", "download3"): - if "password" in inputs: - if self.passwords: - inputs['password'] = self.passwords.pop(0) - else: - self.fail(_("No or invalid passport")) - - if not self.premium: - m = re.search(self.WAIT_PATTERN, self.html) - if m: - wait_time = int(m.group(1)) - self.setWait(wait_time, False) - else: - wait_time = 0 - - self.captcha = self.handleCaptcha(inputs) - - if wait_time: - self.wait() - - return inputs - else: - inputs['referer'] = self.pyfile.url + if self.errmsg: + self.retry(reason=self.errmsg) + else: + self.error(_("TEXTAREA F1 not found")) - if self.premium: - inputs['method_premium'] = "Premium Download" - if 'method_free' in inputs: - del inputs['method_free'] + self.logDebug(inputs) + + if 'op' in inputs: + if "password" in inputs: + if self.passwords: + inputs['password'] = self.passwords.pop(0) + else: + self.fail(_("No or invalid passport")) + + if not self.premium: + m = re.search(self.WAIT_PATTERN, self.html) + if m: + wait_time = int(m.group(1)) + self.setWait(wait_time, False) else: - inputs['method_free'] = "Free Download" - if 'method_premium' in inputs: - del inputs['method_premium'] + wait_time = 0 + + self.captcha = self.handleCaptcha(inputs) - self.html = self.load(self.pyfile.url, post=inputs, ref=True) + if wait_time: + self.wait() else: - self.error(_("FORM: %s") % (inputs['op'] if 'op' in inputs else _("UNKNOWN"))) + inputs['referer'] = self.pyfile.url + + if self.premium: + inputs['method_premium'] = "Premium Download" + if 'method_free' in inputs: + del inputs['method_free'] + else: + inputs['method_free'] = "Free Download" + if 'method_premium' in inputs: + del inputs['method_premium'] + + return inputs def handleCaptcha(self, inputs): -- cgit v1.2.3 From 770464d0ab27ef948c1b4d97025bc07a4676ab46 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 01:34:52 +0100 Subject: [RarefileNet] Remove old handleCaptcha --- module/plugins/hoster/RarefileNet.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/module/plugins/hoster/RarefileNet.py b/module/plugins/hoster/RarefileNet.py index f6f9143c5..8c6ec43a2 100644 --- a/module/plugins/hoster/RarefileNet.py +++ b/module/plugins/hoster/RarefileNet.py @@ -3,13 +3,12 @@ import re from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo -from module.utils import html_unescape class RarefileNet(XFSHoster): __name__ = "RarefileNet" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.07" __pattern__ = r'http://(?:www\.)?rarefile\.net/\w{12}' @@ -20,19 +19,7 @@ class RarefileNet(XFSHoster): HOSTER_DOMAIN = "rarefile.net" - NAME_PATTERN = r'(?P.*?)' - SIZE_PATTERN = r'Size : (?P.+?) ' - LINK_PATTERN = r'(?P=link)' - def handleCaptcha(self, inputs): - captcha_div = re.search(r'Enter code.*?(.*?)', self.html, re.S).group(1) - self.logDebug(captcha_div) - numerals = re.findall('(\d)', html_unescape(captcha_div)) - inputs['code'] = "".join([a[1] for a in sorted(numerals, key=lambda num: int(num[0]))]) - self.logDebug("CAPTCHA", inputs['code'], numerals) - return 3 - - getInfo = create_getInfo(RarefileNet) -- cgit v1.2.3 From 4dfd0fe2f76e67218158d76644a48f58a95bcf8d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 01:35:32 +0100 Subject: [MultiHoster] Fix __name__ --- module/plugins/internal/MultiHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index f0d3993a7..c86f9b286 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -7,7 +7,7 @@ from module.utils import remove_chars class MultiHoster(Hook): - __name__ = "AbtractExtractor" + __name__ = "MultiHoster" __type__ = "hook" __version__ = "0.19" -- cgit v1.2.3 From e6c7bc85b5a18bef90ea0f3a3c235d3849c28b89 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 01:37:33 +0100 Subject: [XFSHoster] Improvements --- module/plugins/internal/XFSHoster.py | 71 +++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index d2a771507..0c428c032 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.19" + __version__ = "0.20" __pattern__ = r'^unmatchable$' @@ -35,6 +35,7 @@ class XFSHoster(SimpleHoster): TEXT_ENCODING = False COOKIES = [(HOSTER_DOMAIN, "lang", "english")] CHECK_DIRECT_LINK = None + MULTI_HOSTER = False INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' @@ -43,18 +44,18 @@ class XFSHoster(SimpleHoster): OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' - WAIT_PATTERN = r'.*?>(\d+)|id="countdown" value=".*?(\d+).*?"' + WAIT_PATTERN = r'.*?>(\d+)|id="countdown" value=".*?(\d+).*?"' + PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' + ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter>|>Error\s*\s*(?:<.+?>)?)(.+?)(?:["\']|\(ERROR:(.+?)\)' OVR_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' LINK_PATTERN = None #: final download url pattern - CAPTCHA_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' + CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' CAPTCHA_DIV_PATTERN = r'>Enter code.*?(.+?)' RECAPTCHA_PATTERN = None SOLVEMEDIA_PATTERN = None - ERROR_PATTERN = r'(?:class=["\']err["\'][^>]*>|<[Cc]enter>)(.+?)(?:["\']|\(ERROR:(.+?)\)' - def setup(self): self.chunkLimit = 1 @@ -73,23 +74,14 @@ class XFSHoster(SimpleHoster): pattern = r'(https?://(www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') - if self.CHECK_DIRECT_LINK is None: - self.CHECK_DIRECT_LINK = bool(self.premium) - - self.captcha = None - self.errmsg = None + self.captcha = None + self.errmsg = None self.passwords = self.getPassword().splitlines() - if (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] - and re.match(self.__pattern__, self.pyfile.url) is None): - self.logInfo(_("Multi hoster detected")) - if self.premium: - self.logDebug(_("Looking for download link...")) - self.handleOverriden() - else: - self.fail(_("Only premium users can use url leech feature")) + super(XFSHoster, self).prepare() - return super(XFSHoster, self).prepare() + if self.CHECK_DIRECT_LINK is None: + self.directDL = bool(self.premium) def handleFree(self): @@ -134,8 +126,11 @@ class XFSHoster(SimpleHoster): self.req.http.c.setopt(FOLLOWLOCATION, 1) m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) - if m and not "op=" in m.group(1): - break + if m: + if 'down_direct' in data and data['down_direct']: + break + else: + self.error(_("Direct download link detected but not found")) m = re.search(self.LINK_PATTERN, self.html, re.S) if m: @@ -149,7 +144,7 @@ class XFSHoster(SimpleHoster): return m.group(1).strip() #@TODO: Remove .strip() in 0.4.10 - def handleOverriden(self): + def handleMulti(self): #only tested with easybytez.com self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) @@ -201,12 +196,17 @@ class XFSHoster(SimpleHoster): header = self.load(m.group(1).strip(), just_header=True, decode=True) #@TODO: Remove .strip() in 0.4.10 if 'location' in header: #: Direct download link - self.download(header['location'], ref=True, cookies=True, disposition=True) + self.link = header['location'] else: self.fail(_("Download link not found")) def checkErrors(self): + m = re.search(self.PREMIUM_ONLY_PATTERN, self.html) + if m: + self.info['error'] = "premium-only" + return + m = re.search(self.ERROR_PATTERN, self.html) if m is None: @@ -217,10 +217,13 @@ class XFSHoster(SimpleHoster): self.logWarning(re.sub(r"<.*?>", " ", self.errmsg)) if 'wait' in self.errmsg: - wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in - re.findall(r'(\d+)\s*(hour|minute|second)', self.errmsg)]) + wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec)', self.errmsg, re.I)]) self.wait(wait_time, True) + elif 'country' in self.errmsg: + self.fail(_("Downloads are disabled for your country")) + elif 'captcha' in self.errmsg: self.invalidCaptcha() @@ -250,6 +253,11 @@ class XFSHoster(SimpleHoster): else: self.retry(wait_time=60, reason=self.errmsg) + if self.errmsg: + self.info['error'] = self.errmsg + else: + self.info.pop('error', None) + return self.errmsg @@ -274,31 +282,26 @@ class XFSHoster(SimpleHoster): if self.passwords: inputs['password'] = self.passwords.pop(0) else: - self.fail(_("No or invalid passport")) + self.fail(_("Missing password")) if not self.premium: m = re.search(self.WAIT_PATTERN, self.html) if m: wait_time = int(m.group(1)) self.setWait(wait_time, False) - else: - wait_time = 0 self.captcha = self.handleCaptcha(inputs) - if wait_time: - self.wait() + self.wait() else: inputs['referer'] = self.pyfile.url if self.premium: inputs['method_premium'] = "Premium Download" - if 'method_free' in inputs: - del inputs['method_free'] + inputs.pop('method_free', None) else: inputs['method_free'] = "Free Download" - if 'method_premium' in inputs: - del inputs['method_premium'] + inputs.pop('method_premium', None) return inputs -- cgit v1.2.3 From d85b5694472c4bf3983729a077a76dd58da25ac4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 01:46:38 +0100 Subject: [SimpleHoster] Multi-hoster and error handling support + fixes (thx MaPePeR) --- module/plugins/internal/SimpleCrypter.py | 3 +- module/plugins/internal/SimpleHoster.py | 95 ++++++++++++++++++++++++++------ 2 files changed, 79 insertions(+), 19 deletions(-) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 8c7f0bef3..5d00daa3c 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -5,8 +5,7 @@ import re from urlparse import urlparse from module.plugins.Crypter import Crypter -from module.plugins.Plugin import Fail -from module.plugins.internal.SimpleHoster import SimpleHoster, _error, _wait, replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies from module.utils import fixup diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c5875a8da..9b74c7cef 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -101,7 +101,7 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 def create_getInfo(plugin): - return lambda urls: [info['name'], info['size'], info['status'], info['url'] for info in plugin.parseInfo(urls)] + return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfo(urls)] def timestamp(): @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.57" + __version__ = "0.58" __pattern__ = r'^unmatchable$' @@ -138,7 +138,7 @@ class SimpleHoster(Hoster): """ - Following patterns should be defined by each hoster: + Info patterns should be defined by each hoster: INFO_PATTERN: (optional) Name and Size of the file example: INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' @@ -148,17 +148,26 @@ class SimpleHoster(Hoster): SIZE_PATTERN: (optional) Size that will be checked for the file example: SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' - OFFLINE_PATTERN: (optional) Checks if the file is yet available online + OFFLINE_PATTERN: (optional) Check if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN: (optional) Checks if the file is temporarily offline + TEMP_OFFLINE_PATTERN: (optional) Check if the file is temporarily offline example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' - PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account + + Error handling patterns are all optional: + + WAIT_PATTERN: (optional) Detect waiting time + example: WAIT_PATTERN = r'' + + PREMIUM_ONLY_PATTERN: (optional) Check if the file can be downloaded only with a premium account example: PREMIUM_ONLY_PATTERN = r'Premium account required' + ERROR_PATTERN: (optional) Detect any error preventing download + example: ERROR_PATTERN = r'' + - Instead overriding handleFree and handlePremium methods now you can define patterns for direct download: + Instead overriding handleFree and handlePremium methods you can define the following patterns for direct download: LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download example: LINK_FREE_PATTERN = r'
Date: Sat, 22 Nov 2014 05:15:25 +0100 Subject: [SimpleHoster] Fixup --- module/plugins/internal/SimpleHoster.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 9b74c7cef..a39531b82 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.58" + __version__ = "0.59" __pattern__ = r'^unmatchable$' @@ -192,7 +192,7 @@ class SimpleHoster(Hoster): def parseInfo(cls, urls): for url in urls: url = replace_patterns(url, cls.FILE_URL_REPLACEMENTS if hasattr(cls, "FILE_URL_REPLACEMENTS") else cls.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 - yield cls.getInfo(cls, url) + yield cls.getInfo(url) @classmethod @@ -272,7 +272,7 @@ class SimpleHoster(Hoster): and self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] and re.match(self.__pattern__, self.pyfile.url) is None): - self.logDebug("Multi hoster detected") + self.logInfo("Multi hoster detected") if self.account: self.multihost = True @@ -373,9 +373,9 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is not 2: - self.logDebug("File status: %s" % statusMap[status], - "File info: %s" % self.info) - self.error(_("No file info")) + self.logInfo(_("File status: %s") % statusMap[status], + _("File info: %s") % self.info) + self.error(_("No file info retrieved")) def checkNameSize(self): @@ -406,6 +406,11 @@ class SimpleHoster(Hoster): self.checkStatus() + #: Deprecated + def getFileInfo(self): + return self.checkInfo() + + def _updateInfo(self, info): self.logDebug(_("File info (before update): %s") % self.info) self.info.update(info) -- cgit v1.2.3 From ed4dd7c759a48f5fa8a4411292cd12a06f662d68 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 05:31:56 +0100 Subject: [RarefileNet] Restore NAME_PATTERN and SIZE_PATTERN --- module/plugins/hoster/RarefileNet.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/module/plugins/hoster/RarefileNet.py b/module/plugins/hoster/RarefileNet.py index 8c6ec43a2..987028c8f 100644 --- a/module/plugins/hoster/RarefileNet.py +++ b/module/plugins/hoster/RarefileNet.py @@ -8,7 +8,7 @@ from module.plugins.internal.XFSHoster import XFSHoster, create_getInfo class RarefileNet(XFSHoster): __name__ = "RarefileNet" __type__ = "hoster" - __version__ = "0.07" + __version__ = "0.08" __pattern__ = r'http://(?:www\.)?rarefile\.net/\w{12}' @@ -19,6 +19,9 @@ class RarefileNet(XFSHoster): HOSTER_DOMAIN = "rarefile.net" + NAME_PATTERN = r'(?P.+?)<' + SIZE_PATTERN = r'>Size : (?P[\d.,]+) (?P[\w^_]+)' + LINK_PATTERN = r'(?P=link)' -- cgit v1.2.3 From cc9cb5c44e9cce4a855f76e2ada0795de00b4412 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 06:14:41 +0100 Subject: [UptoboxCom] Account fixup (thx ajarc) --- module/plugins/accounts/UptoboxCom.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/module/plugins/accounts/UptoboxCom.py b/module/plugins/accounts/UptoboxCom.py index d3fb4e559..299a0acc2 100644 --- a/module/plugins/accounts/UptoboxCom.py +++ b/module/plugins/accounts/UptoboxCom.py @@ -6,7 +6,7 @@ from module.plugins.internal.XFSAccount import XFSAccount class UptoboxCom(XFSAccount): __name__ = "UptoboxCom" __type__ = "account" - __version__ = "0.06" + __version__ = "0.07" __description__ = """DDLStorage.com account plugin""" __license__ = "GPLv3" @@ -14,3 +14,4 @@ class UptoboxCom(XFSAccount): HOSTER_DOMAIN = "uptobox.com" + HOSTER_URL = "https://uptobox.com/" -- cgit v1.2.3 From 9f10e758325f4ce80c479bc16149e677b9e1885f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 06:38:17 +0100 Subject: [XFSAccount][XFSHoster] Tiny pattern improvement --- module/plugins/internal/XFSAccount.py | 5 ++++- module/plugins/internal/XFSHoster.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index ba2c00ec9..627dc75c4 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.25" + __version__ = "0.26" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -97,6 +97,9 @@ class XFSAccount(Account): except Exception, e: self.logError(e) + else: + if premium: + trafficleft = -1 return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 0c428c032..a6a3f3260 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.20" + __version__ = "0.21" __pattern__ = r'^unmatchable$' @@ -38,7 +38,7 @@ class XFSHoster(SimpleHoster): MULTI_HOSTER = False INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' - NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' + NAME_PATTERN = r'(>Filename:|name="fname" value="||<[Tt]itle>.*?Download )(?P.+?)(\s*<|")' SIZE_PATTERN = r'(>Size:|>File:.*>|)(?P[\d.,]+)\s*(?P[\w^_]+)' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' -- cgit v1.2.3 From 561357c1e1fdb86a4ad89f146f8a1d1241d21a3b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 17:09:17 +0100 Subject: [XFSHoster] Improve ERROR_PATTERN + revert direct link detection in getDownloadLink --- module/plugins/internal/XFSHoster.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index a6a3f3260..d5bbf5c63 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.21" + __version__ = "0.22" __pattern__ = r'^unmatchable$' @@ -46,7 +46,7 @@ class XFSHoster(SimpleHoster): WAIT_PATTERN = r'.*?>(\d+)|id="countdown" value=".*?(\d+).*?"' PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' - ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter>|>Error\s*\s*(?:<.+?>)?)(.+?)(?:["\']|\(ERROR:(.+?)\)' + ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter>|>Error|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))' OVR_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' LINK_PATTERN = None #: final download url pattern @@ -126,11 +126,8 @@ class XFSHoster(SimpleHoster): self.req.http.c.setopt(FOLLOWLOCATION, 1) m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) - if m: - if 'down_direct' in data and data['down_direct']: - break - else: - self.error(_("Direct download link detected but not found")) + if m and not "op=" in m.group(1): + break m = re.search(self.LINK_PATTERN, self.html, re.S) if m: @@ -212,7 +209,7 @@ class XFSHoster(SimpleHoster): if m is None: self.errmsg = None else: - self.errmsg = m.group(1) + self.errmsg = m.group(1).strip() self.logWarning(re.sub(r"<.*?>", " ", self.errmsg)) -- cgit v1.2.3