From 2d893a24d4bb8b5a7a00807ed6320358454577f8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 29 Jun 2014 21:55:29 +0200 Subject: [SimpleHoster] Improve documentation of own public attributes --- module/plugins/internal/SimpleHoster.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index f10433e78..8c2e3daae 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -169,15 +169,19 @@ class SimpleHoster(Hoster): __description__ = """Simple hoster plugin""" __author_name__ = ("zoidberg", "stickell") __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") + """ - These patterns should be defined by each hoster: - FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Punits)' - or FILE_NAME_PATTERN = r'(?Pfile_name)' - and FILE_SIZE_PATTERN = r'(?Pfile_size) (?Punits)' + Following patterns should be defined by each hoster: + FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' + or + FILE_NAME_PATTERN = r'(?Pfile_name)' + FILE_SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' + FILE_OFFLINE_PATTERN = r'File (deleted|not found)' TEMP_OFFLINE_PATTERN = r'Server maintainance' - You can also define a PREMIUM_ONLY_PATTERN to detect links that can be downloaded only with a premium account. + You can also define: + PREMIUM_ONLY_PATTERN = r'Premium account required' """ FILE_SIZE_REPLACEMENTS = [] -- cgit v1.2.3 From 93592862b520a862c01f80c019e5c4bc43746c19 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 5 Jul 2014 16:54:20 +0200 Subject: [SimpleHoster] Better inline docs + changed "FILE_OFFLINE_PATTERN" to "OFFLINE_PATTERN" --- module/plugins/internal/SimpleHoster.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 8c2e3daae..b5066437e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -98,7 +98,8 @@ def parseFileInfo(self, url='', html=''): if hasattr(self, "html"): self.html = html - if hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html): + if (hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html)) or \ + (hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html)): # File offline info['status'] = 1 else: @@ -163,7 +164,7 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" - __version__ = "0.33" + __version__ = "0.34" __pattern__ = None __type__ = "hoster" __description__ = """Simple hoster plugin""" @@ -172,16 +173,25 @@ class SimpleHoster(Hoster): """ Following patterns should be defined by each hoster: - FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' - or - FILE_NAME_PATTERN = r'(?Pfile_name)' - FILE_SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' - FILE_OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN = r'Server maintainance' + FILE_INFO_PATTERN: Name and Size of the file + example: FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' + or + FILE_NAME_PATTERN: Name that will be set for the file + example: FILE_NAME_PATTERN = r'(?Pfile_name)' + FILE_SIZE_PATTERN: Size that will be checked for the file + example: FILE_SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' - You can also define: - PREMIUM_ONLY_PATTERN = r'Premium account required' + OFFLINE_PATTERN: Checks if the file is yet available online + example: OFFLINE_PATTERN = r'File (deleted|not found)' + or: + FILE_OFFLINE_PATTERN: Deprecated + + TEMP_OFFLINE_PATTERN: Checks if the file is temporarily offline + example: TEMP_OFFLINE_PATTERN = r'Server maintainance' + + PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account + example: PREMIUM_ONLY_PATTERN = r'Premium account required' """ FILE_SIZE_REPLACEMENTS = [] -- cgit v1.2.3 From 04038a2cf0c4c2d9cc9a0c8e8bf9beb6426afae8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 10 Jul 2014 03:02:26 +0200 Subject: Use parseError instead PluginParseError + unified all download pattern attributes as LINK_PATTERN + removed some old patterns (not used anymore) + other code cosmetics --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index b5066437e..8e6b8b42f 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -194,8 +194,8 @@ class SimpleHoster(Hoster): example: PREMIUM_ONLY_PATTERN = r'Premium account required' """ - FILE_SIZE_REPLACEMENTS = [] FILE_NAME_REPLACEMENTS = [("&#?\w+;", fixup)] + FILE_SIZE_REPLACEMENTS = [] FILE_URL_REPLACEMENTS = [] SH_BROKEN_ENCODING = False # Set to True or encoding name if encoding in http header is not correct -- cgit v1.2.3 From 48c0c42fd6faffc56432d5f037cd575979f180cc Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 14 Jul 2014 02:23:37 +0200 Subject: Removed all @author flags + key attributes cleanup for internal & hooks plugins --- module/plugins/internal/SimpleHoster.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 8e6b8b42f..e9ba59c0d 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -13,8 +13,6 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - - @author: zoidberg """ from urlparse import urlparse import re @@ -165,8 +163,10 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __version__ = "0.34" - __pattern__ = None __type__ = "hoster" + + __pattern__ = None + __description__ = """Simple hoster plugin""" __author_name__ = ("zoidberg", "stickell") __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") @@ -202,6 +202,7 @@ class SimpleHoster(Hoster): SH_COOKIES = True # or False or list of tuples [(domain, name, value)] SH_CHECK_TRAFFIC = False # True = force check traffic left for a premium account + def init(self): self.file_info = {} -- cgit v1.2.3 From 5060e4c6374a5116d0d8b02528f910f8c5f8bcf9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 15 Jul 2014 16:25:41 +0200 Subject: Fix code indentation, some bad whitespaces and missing authors + use 'not' instead 'is None' + replace __pattern__'s r" with r' + other minor cosmetics --- module/plugins/internal/SimpleHoster.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e9ba59c0d..ba143f4d8 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -53,7 +53,7 @@ def parseHtmlForm(attr_str, html, input_names=None): name = parseHtmlTagAttrValue("name", inputtag.group(1)) if name: value = parseHtmlTagAttrValue("value", inputtag.group(1)) - if value is None: + if not value: inputs[name] = inputtag.group(3) or '' else: inputs[name] = value @@ -135,6 +135,7 @@ def parseFileInfo(self, url='', html=''): def create_getInfo(plugin): + def getInfo(urls): for url in urls: cj = CookieJar(plugin.__name__) @@ -152,6 +153,7 @@ def timestamp(): class PluginParseError(Exception): + def __init__(self, msg): Exception.__init__(self) self.value = 'Parse error (%s) - plugin may be out of date' % msg -- cgit v1.2.3 From 7b8c458cca7d21a029620f98e453f746fce69cd1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 14 Jul 2014 16:10:01 +0200 Subject: Prefer single quote for dict key name --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ba143f4d8..0dfecf0fb 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -291,7 +291,7 @@ class SimpleHoster(Hoster): return parseHtmlForm(attr_str, self.html, input_names) def checkTrafficLeft(self): - traffic = self.account.getAccountInfo(self.user, True)["trafficleft"] + traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] if traffic == -1: return True size = self.pyfile.size / 1024 -- cgit v1.2.3 From ba916633f2bedb04c7358000b91aed69f52e8e43 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 1 Aug 2014 19:35:59 +0200 Subject: Remove trailing whitespaces + remove license headers + import urllib methods directly + sort and fix key attributes + use save_join instead join + sort some import declarations + other minor code cosmetics --- module/plugins/internal/SimpleHoster.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 0dfecf0fb..23b8a5292 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -1,27 +1,14 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . -""" -from urlparse import urlparse import re + from time import time +from urlparse import urlparse -from module.plugins.Hoster import Hoster -from module.utils import html_unescape, fixup, parseFileSize -from module.network.RequestFactory import getURL from module.network.CookieJar import CookieJar +from module.network.RequestFactory import getURL +from module.plugins.Hoster import Hoster +from module.utils import fixup, html_unescape, parseFileSize def replace_patterns(string, ruleslist): @@ -164,8 +151,8 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" - __version__ = "0.34" __type__ = "hoster" + __version__ = "0.34" __pattern__ = None -- cgit v1.2.3 From a2b1a203a7666052d4857620b4712e82160747c2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 4 Sep 2014 22:29:59 +0200 Subject: [SimpleHoster] Improve parseFileInfo --- module/plugins/internal/SimpleHoster.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 23b8a5292..ec9cf1b70 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -83,10 +83,12 @@ def parseFileInfo(self, url='', html=''): if hasattr(self, "html"): self.html = html - if (hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html)) or \ - (hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html)): - # File offline + if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html): info['status'] = 1 + elif hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html): #@TODO: Remove in 0.4.10 + info['status'] = 1 + elif hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, html): + info['status'] = 6 else: online = False try: @@ -152,7 +154,7 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.34" + __version__ = "0.35" __pattern__ = None @@ -173,8 +175,6 @@ class SimpleHoster(Hoster): OFFLINE_PATTERN: Checks if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' - or: - FILE_OFFLINE_PATTERN: Deprecated TEMP_OFFLINE_PATTERN: Checks if the file is temporarily offline example: TEMP_OFFLINE_PATTERN = r'Server maintainance' @@ -227,13 +227,13 @@ class SimpleHoster(Hoster): def getFileInfo(self): self.logDebug("URL: %s" % self.pyfile.url) - if hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, self.html): - self.tempOffline() name, size, status = parseFileInfo(self)[:3] if status == 1: self.offline() + elif status == 6: + self.tempOffline() elif status != 2: self.logDebug(self.file_info) self.parseError('File info') -- cgit v1.2.3 From 76c67b22c0b2dbd275fae77e8a3121acb2d4301b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Sep 2014 15:39:33 +0200 Subject: Fix cookie deletion + Account support for SimpleCrypter Fix: https://github.com/pyload/pyload/issues/749 --- module/plugins/internal/SimpleHoster.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ec9cf1b70..b2db56f3f 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -187,24 +187,26 @@ class SimpleHoster(Hoster): FILE_SIZE_REPLACEMENTS = [] FILE_URL_REPLACEMENTS = [] - SH_BROKEN_ENCODING = False # Set to True or encoding name if encoding in http header is not correct - SH_COOKIES = True # or False or list of tuples [(domain, name, value)] - SH_CHECK_TRAFFIC = False # True = force check traffic left for a premium account + SH_BROKEN_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + SH_COOKIES = True #: or False or list of tuples [(domain, name, value)] + SH_CHECK_TRAFFIC = False #: True = force check traffic left for a premium account def init(self): self.file_info = {} + def setup(self): self.resumeDownload = self.multiDL = self.premium if isinstance(self.SH_COOKIES, list): set_cookies(self.req.cj, self.SH_COOKIES) + def process(self, pyfile): pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) self.req.setOption("timeout", 120) # Due to a 0.4.9 core bug self.load would keep previous cookies even if overridden by cookies parameter. - # Workaround using getURL. Can be reverted in 0.5 as the cookies bug has been fixed. + # Workaround using getURL. Can be reverted in 0.4.10 as the cookies bug has been fixed. self.html = getURL(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) if not premium_only: # Usually premium only pages doesn't show the file information @@ -215,16 +217,18 @@ class SimpleHoster(Hoster): elif premium_only: self.fail("This link require a premium account") else: - # This line is required due to the getURL workaround. Can be removed in 0.5 + # This line is required due to the getURL workaround. Can be removed in 0.4.10 self.html = self.load(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) self.handleFree() + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): if type(url) == unicode: url = url.encode('utf8') return Hoster.load(self, url=url, get=get, post=post, ref=ref, cookies=cookies, just_header=just_header, decode=decode) + def getFileInfo(self): self.logDebug("URL: %s" % self.pyfile.url) @@ -251,15 +255,19 @@ class SimpleHoster(Hoster): self.logDebug("FILE NAME: %s FILE SIZE: %s" % (self.pyfile.name, self.pyfile.size)) return self.file_info + def handleFree(self): self.fail("Free download not implemented") + def handlePremium(self): self.fail("Premium download not implemented") + def parseError(self, msg): raise PluginParseError(msg) + def longWait(self, wait_time=None, max_tries=3): if wait_time and isinstance(wait_time, (int, long, float)): time_str = "%dh %dm" % divmod(wait_time / 60, 60) @@ -274,9 +282,11 @@ class SimpleHoster(Hoster): self.wait() self.retry(max_tries=max_tries, reason="Download limit reached") + def parseHtmlForm(self, attr_str='', input_names=None): return parseHtmlForm(attr_str, self.html, input_names) + def checkTrafficLeft(self): traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] if traffic == -1: @@ -285,7 +295,8 @@ class SimpleHoster(Hoster): self.logInfo("Filesize: %i KiB, Traffic left for user %s: %i KiB" % (size, self.user, traffic)) return size <= traffic - # TODO: Remove in 0.5 + + #@TODO: Remove in 0.4.10 def wait(self, seconds=False, reconnect=False): if seconds: self.setWait(seconds, reconnect) -- cgit v1.2.3 From 4709a41c176dd2947709b7f35ccc84213d574624 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Sep 2014 16:59:26 +0200 Subject: Rename SH flags --- module/plugins/internal/SimpleHoster.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index b2db56f3f..6ecb2d9f2 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -78,8 +78,8 @@ def parseFileInfo(self, url='', html=''): else: if not html and hasattr(self, "html"): html = self.html - if isinstance(self.SH_BROKEN_ENCODING, (str, unicode)): - html = unicode(html, self.SH_BROKEN_ENCODING) + if isinstance(self.TEXT_ENCODING, basestring): + html = unicode(html, self.TEXT_ENCODING) if hasattr(self, "html"): self.html = html @@ -112,7 +112,7 @@ def parseFileInfo(self, url='', html=''): size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], self.FILE_SIZE_REPLACEMENTS) info['size'] = parseFileSize(size) - elif isinstance(info['size'], (str, unicode)): + elif isinstance(info['size'], basestring): if 'units' in info: info['size'] += info['units'] info['size'] = parseFileSize(info['size']) @@ -128,10 +128,10 @@ def create_getInfo(plugin): def getInfo(urls): for url in urls: cj = CookieJar(plugin.__name__) - if isinstance(plugin.SH_COOKIES, list): - set_cookies(cj, plugin.SH_COOKIES) + if isinstance(plugin.COOKIES, list): + set_cookies(cj, plugin.COOKIES) file_info = parseFileInfo(plugin, url, getURL(replace_patterns(url, plugin.FILE_URL_REPLACEMENTS), - decode=not plugin.SH_BROKEN_ENCODING, cookies=cj)) + decode=not plugin.TEXT_ENCODING, cookies=cj)) yield file_info return getInfo @@ -187,9 +187,9 @@ class SimpleHoster(Hoster): FILE_SIZE_REPLACEMENTS = [] FILE_URL_REPLACEMENTS = [] - SH_BROKEN_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - SH_COOKIES = True #: or False or list of tuples [(domain, name, value)] - SH_CHECK_TRAFFIC = False #: True = force check traffic left for a premium account + TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + COOKIES = True #: or False or list of tuples [(domain, name, value)] + FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account def init(self): @@ -198,8 +198,8 @@ class SimpleHoster(Hoster): def setup(self): self.resumeDownload = self.multiDL = self.premium - if isinstance(self.SH_COOKIES, list): - set_cookies(self.req.cj, self.SH_COOKIES) + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) def process(self, pyfile): @@ -207,18 +207,18 @@ class SimpleHoster(Hoster): self.req.setOption("timeout", 120) # Due to a 0.4.9 core bug self.load would keep previous cookies even if overridden by cookies parameter. # Workaround using getURL. Can be reverted in 0.4.10 as the cookies bug has been fixed. - self.html = getURL(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) + self.html = getURL(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) if not premium_only: # Usually premium only pages doesn't show the file information self.getFileInfo() - if self.premium and (not self.SH_CHECK_TRAFFIC or self.checkTrafficLeft()): + if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): self.handlePremium() elif premium_only: self.fail("This link require a premium account") else: # This line is required due to the getURL workaround. Can be removed in 0.4.10 - self.html = self.load(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) + self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) self.handleFree() -- cgit v1.2.3 From 0d220d634e512d89bda540f91c643b361c82ea8a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 27 Sep 2014 01:38:32 +0200 Subject: Logging string cosmetics --- module/plugins/internal/SimpleHoster.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6ecb2d9f2..c18bfff15 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -15,7 +15,6 @@ def replace_patterns(string, ruleslist): for r in ruleslist: rf, rt = r string = re.sub(rf, rt, string) - #self.logDebug(rf, rt, string) return string @@ -230,7 +229,7 @@ class SimpleHoster(Hoster): def getFileInfo(self): - self.logDebug("URL: %s" % self.pyfile.url) + self.logDebug("URL", self.pyfile.url) name, size, status = parseFileInfo(self)[:3] @@ -250,7 +249,7 @@ class SimpleHoster(Hoster): if size: self.pyfile.size = size else: - self.logError("File size not parsed") + self.logError(_("File size not parsed")) self.logDebug("FILE NAME: %s FILE SIZE: %s" % (self.pyfile.name, self.pyfile.size)) return self.file_info @@ -276,7 +275,7 @@ class SimpleHoster(Hoster): time_str = "(unknown time)" max_tries = 100 - self.logInfo("Download limit reached, reconnect or wait %s" % time_str) + self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str) self.setWait(wait_time, True) self.wait() @@ -292,7 +291,7 @@ class SimpleHoster(Hoster): if traffic == -1: return True size = self.pyfile.size / 1024 - self.logInfo("Filesize: %i KiB, Traffic left for user %s: %i KiB" % (size, self.user, traffic)) + self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) return size <= traffic -- cgit v1.2.3 From 8add4efc7dc3eeaa873030930e5bd31d6bdf8126 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 28 Sep 2014 14:57:02 +0200 Subject: [XFileSharingPro] COOKIES preset to english + improved setup routine --- module/plugins/internal/SimpleHoster.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c18bfff15..ecbeec979 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -217,17 +217,10 @@ class SimpleHoster(Hoster): self.fail("This link require a premium account") else: # This line is required due to the getURL workaround. Can be removed in 0.4.10 - self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) + self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING) self.handleFree() - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): - if type(url) == unicode: - url = url.encode('utf8') - return Hoster.load(self, url=url, get=get, post=post, ref=ref, cookies=cookies, - just_header=just_header, decode=decode) - - def getFileInfo(self): self.logDebug("URL", self.pyfile.url) -- cgit v1.2.3 From 2e4101c6b373867d556a410901efb8c85c56593f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 28 Sep 2014 16:27:29 +0200 Subject: Prepare method improves setup one --- module/plugins/internal/SimpleHoster.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ecbeec979..1c0916541 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -153,13 +153,13 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.35" + __version__ = "0.36" __pattern__ = None __description__ = """Simple hoster plugin""" - __author_name__ = ("zoidberg", "stickell") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") + __author_name__ = ("zoidberg", "stickell", "Walter Purcaro") + __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it", "vuolter@gmail.com") """ Following patterns should be defined by each hoster: @@ -197,13 +197,19 @@ class SimpleHoster(Hoster): def setup(self): self.resumeDownload = self.multiDL = self.premium + + + def prepare(self): if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) + self.req.setOption("timeout", 120) def process(self, pyfile): + self.prepare() + pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) - self.req.setOption("timeout", 120) + # Due to a 0.4.9 core bug self.load would keep previous cookies even if overridden by cookies parameter. # Workaround using getURL. Can be reverted in 0.4.10 as the cookies bug has been fixed. self.html = getURL(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) -- cgit v1.2.3 From 23ae563604dca1dae262fbc598154b99b2f1eae8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 3 Oct 2014 09:06:05 +0200 Subject: Update plugins after CaptchaService and XFileSharingPro changes --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1c0916541..3625f171e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -176,7 +176,7 @@ class SimpleHoster(Hoster): example: OFFLINE_PATTERN = r'File (deleted|not found)' TEMP_OFFLINE_PATTERN: Checks if the file is temporarily offline - example: TEMP_OFFLINE_PATTERN = r'Server maintainance' + example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account example: PREMIUM_ONLY_PATTERN = r'Premium account required' -- cgit v1.2.3 From b994d8fa6a7dcc902787bb2450acdbed535d72fc Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 5 Oct 2014 22:11:49 +0200 Subject: [SimpleHoster] Added default handle patterns LINK_FREE_PATTERN and LINK_PREMIUM_PATTERN --- module/plugins/internal/SimpleHoster.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3625f171e..160125cdf 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -153,7 +153,7 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.36" + __version__ = "0.37" __pattern__ = None @@ -255,11 +255,35 @@ class SimpleHoster(Hoster): def handleFree(self): - self.fail("Free download not implemented") + if not hasattr(self, 'LINK_FREE_PATTERN'): + self.fail("Free download not implemented") + + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m is None: + self.parseError("Free download link not found") + + try: + link = m.group(1) + except Exception, e: + self.logError(str(e)) + else: + self.download(link, ref=True, cookies=True, disposition=True) def handlePremium(self): - self.fail("Premium download not implemented") + if not hasattr(self, 'LINK_PREMIUM_PATTERN'): + self.fail("Premium download not implemented") + + m = re.search(self.LINK_PREMIUM_PATTERN, self.html) + if m is None: + self.parseError("Premium download link not found") + + try: + link = m.group(1) + except Exception, e: + self.logError(str(e)) + else: + self.download(link, ref=True, cookies=True, disposition=True) def parseError(self, msg): -- cgit v1.2.3 From 82facc20b829d182bd0ae5787d29ed21f0f93fbc Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 6 Oct 2014 14:28:32 +0200 Subject: [SimpleHoster] Better filename processing + info about new pattern attributes --- module/plugins/internal/SimpleHoster.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 160125cdf..6ad8398bb 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -106,7 +106,7 @@ def parseFileInfo(self, url='', html=''): # File online, return name and size info['status'] = 2 if 'N' in info: - info['name'] = replace_patterns(info['N'], self.FILE_NAME_REPLACEMENTS) + info['name'] = replace_patterns(info['N'].strip(), self.FILE_NAME_REPLACEMENTS) if 'S' in info: size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], self.FILE_SIZE_REPLACEMENTS) @@ -153,7 +153,7 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.37" + __version__ = "0.38" __pattern__ = None @@ -180,6 +180,12 @@ class SimpleHoster(Hoster): PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account example: PREMIUM_ONLY_PATTERN = r'Premium account required' + + + Instead overriding handleFree and handlePremium methods now you can define patterns for direct download: + + LINK_FREE_PATTERN: (optional) Get direct link for free download + LINK_PREMIUM_PATTERN: (optional) Get direct link for premium download """ FILE_NAME_REPLACEMENTS = [("&#?\w+;", fixup)] -- cgit v1.2.3 From dd24571c5015e3fea1877671b464104557463f6f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 7 Oct 2014 13:52:47 +0200 Subject: [SimpleCrypter][SimpleHoster] Better exception handling --- module/plugins/internal/SimpleHoster.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6ad8398bb..238c98b05 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -112,9 +112,8 @@ def parseFileInfo(self, url='', html=''): self.FILE_SIZE_REPLACEMENTS) info['size'] = parseFileSize(size) elif isinstance(info['size'], basestring): - if 'units' in info: - info['size'] += info['units'] - info['size'] = parseFileSize(info['size']) + unit = info['units'] if 'units' in info else None + info['size'] = parseFileSize(info['size'], unit) if hasattr(self, "file_info"): self.file_info = info @@ -161,6 +160,7 @@ class SimpleHoster(Hoster): __author_name__ = ("zoidberg", "stickell", "Walter Purcaro") __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it", "vuolter@gmail.com") + """ Following patterns should be defined by each hoster: @@ -184,8 +184,11 @@ class SimpleHoster(Hoster): Instead overriding handleFree and handlePremium methods now you can define patterns for direct download: - LINK_FREE_PATTERN: (optional) Get direct link for free download - LINK_PREMIUM_PATTERN: (optional) Get direct link for premium download + LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download + example: LINK_FREE_PATTERN = r'
Date: Tue, 7 Oct 2014 18:57:59 +0200 Subject: New __authors__ key replaces __author_name__ and __author_mail__ + Whitespaces and EOF fixup --- module/plugins/internal/SimpleHoster.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 238c98b05..3887f1579 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -157,8 +157,9 @@ class SimpleHoster(Hoster): __pattern__ = None __description__ = """Simple hoster plugin""" - __author_name__ = ("zoidberg", "stickell", "Walter Purcaro") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it", "vuolter@gmail.com") + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] """ -- cgit v1.2.3 From ae7a7e66981456e5bbe2b54006d79b6f907be7a4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 8 Oct 2014 20:18:13 +0200 Subject: Add __license__ key attribute to plugins --- module/plugins/internal/SimpleHoster.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3887f1579..895584af6 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -157,6 +157,7 @@ class SimpleHoster(Hoster): __pattern__ = None __description__ = """Simple hoster plugin""" + __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), ("stickell", "l.stickell@yahoo.it"), ("Walter Purcaro", "vuolter@gmail.com")] -- cgit v1.2.3 From d0aa0ece5e506469f631b388f6c38eaaa63b63c9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Oct 2014 13:52:01 +0200 Subject: Tiny code cosmetics --- module/plugins/internal/SimpleHoster.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 895584af6..c2bc4909e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -166,18 +166,18 @@ class SimpleHoster(Hoster): """ Following patterns should be defined by each hoster: - FILE_INFO_PATTERN: Name and Size of the file + FILE_INFO_PATTERN: (optional) Name and Size of the file example: FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' or - FILE_NAME_PATTERN: Name that will be set for the file + FILE_NAME_PATTERN: (optional) Name that will be set for the file example: FILE_NAME_PATTERN = r'(?Pfile_name)' - FILE_SIZE_PATTERN: Size that will be checked for the file + FILE_SIZE_PATTERN: (optional) Size that will be checked for the file example: FILE_SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' - OFFLINE_PATTERN: Checks if the file is yet available online + OFFLINE_PATTERN: (optional) Checks if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN: Checks if the file is temporarily offline + TEMP_OFFLINE_PATTERN: (optional) Checks if the file is temporarily offline example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account -- cgit v1.2.3 From bb1f6b3cbfc34914a1a336f8f5a2e15b57e2b5f5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Oct 2014 14:35:53 +0200 Subject: parseError -> error now calls Fail instead Exception --- module/plugins/internal/SimpleHoster.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c2bc4909e..bbd37e978 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -8,7 +8,8 @@ from urlparse import urlparse from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster -from module.utils import fixup, html_unescape, parseFileSize +from module.plugins.Plugin import Fail +reasonfrom module.utils import fixup, html_unescape, parseFileSize def replace_patterns(string, ruleslist): @@ -139,20 +140,10 @@ def timestamp(): return int(time() * 1000) -class PluginParseError(Exception): - - def __init__(self, msg): - Exception.__init__(self) - self.value = 'Parse error (%s) - plugin may be out of date' % msg - - def __str__(self): - return repr(self.value) - - class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.38" + __version__ = "0.39" __pattern__ = None @@ -249,7 +240,7 @@ class SimpleHoster(Hoster): self.tempOffline() elif status != 2: self.logDebug(self.file_info) - self.parseError('File info') + self.error('File info') if name: self.pyfile.name = name @@ -272,7 +263,7 @@ class SimpleHoster(Hoster): try: m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: - self.parseError("Free download link not found") + self.error("Free download link not found") link = m.group(1) except Exception, e: @@ -288,7 +279,7 @@ class SimpleHoster(Hoster): try: m = re.search(self.LINK_PREMIUM_PATTERN, self.html) if m is None: - self.parseError("Premium download link not found") + self.error("Premium download link not found") link = m.group(1) except Exception, e: @@ -297,10 +288,6 @@ class SimpleHoster(Hoster): self.download(link, ref=True, cookies=True, disposition=True) - def parseError(self, msg): - raise PluginParseError(msg) - - def longWait(self, wait_time=None, max_tries=3): if wait_time and isinstance(wait_time, (int, long, float)): time_str = "%dh %dm" % divmod(wait_time / 60, 60) @@ -334,3 +321,7 @@ class SimpleHoster(Hoster): if seconds: self.setWait(seconds, reconnect) super(SimpleHoster, self).wait() + + + def error(self, reason): + raise Fail("Parse error (%s) - hoster plugin may be out of date" % reason) -- cgit v1.2.3 From 4bfdf03e804fcafabab0ff08b2b6ef1daec2cf91 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Oct 2014 15:11:18 +0200 Subject: Improve error method --- module/plugins/internal/SimpleHoster.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index bbd37e978..fe9b2d523 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -323,5 +323,8 @@ class SimpleHoster(Hoster): super(SimpleHoster, self).wait() - def error(self, reason): - raise Fail("Parse error (%s) - hoster plugin may be out of date" % reason) + def error(self, reason=None, type="parse"): + if reason: + raise Fail("%s error: %s | Plugin may be out of date" % (type.capitalize(), reason)) + else: + raise Fail("%s error | Plugin out of date" % type.capitalize()) -- cgit v1.2.3 From 39b304c6495e6a77562e9ce2c9cdf921851c668a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Oct 2014 16:28:10 +0200 Subject: Fixes --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index fe9b2d523..c62ef53b4 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -9,7 +9,7 @@ from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster from module.plugins.Plugin import Fail -reasonfrom module.utils import fixup, html_unescape, parseFileSize +from module.utils import fixup, html_unescape, parseFileSize def replace_patterns(string, ruleslist): @@ -143,7 +143,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.39" + __version__ = "0.40" __pattern__ = None -- cgit v1.2.3 From 4e0e93fe8a9c9a98f3f299afacf07fbc855ef88b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 20 Oct 2014 12:59:27 +0200 Subject: Preload self.html --- module/plugins/internal/SimpleHoster.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c62ef53b4..6fc61460a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -204,17 +204,16 @@ class SimpleHoster(Hoster): def prepare(self): if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) + self.req.setOption("timeout", 120) + url = self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) + self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) + def process(self, pyfile): self.prepare() - pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) - - # Due to a 0.4.9 core bug self.load would keep previous cookies even if overridden by cookies parameter. - # Workaround using getURL. Can be reverted in 0.4.10 as the cookies bug has been fixed. - self.html = getURL(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) if not premium_only: # Usually premium only pages doesn't show the file information self.getFileInfo() @@ -224,8 +223,6 @@ class SimpleHoster(Hoster): elif premium_only: self.fail("This link require a premium account") else: - # This line is required due to the getURL workaround. Can be removed in 0.4.10 - self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING) self.handleFree() -- cgit v1.2.3 From b409e7782d48830489f47cc2364b4728d98f363a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 02:06:31 +0200 Subject: Improve COOKIES parsing --- module/plugins/internal/SimpleHoster.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6fc61460a..74910692f 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -67,7 +67,7 @@ def parseHtmlForm(attr_str, html, input_names=None): return {}, None # no matching form found -def parseFileInfo(self, url='', html=''): +def parseFileInfo(self, url="", html=""): info = {"name": url, "size": 0, "status": 3} if hasattr(self, "pyfile"): @@ -129,6 +129,7 @@ def create_getInfo(plugin): cj = CookieJar(plugin.__name__) if isinstance(plugin.COOKIES, list): set_cookies(cj, plugin.COOKIES) + file_info = parseFileInfo(plugin, url, getURL(replace_patterns(url, plugin.FILE_URL_REPLACEMENTS), decode=not plugin.TEXT_ENCODING, cookies=cj)) yield file_info @@ -208,7 +209,7 @@ class SimpleHoster(Hoster): self.req.setOption("timeout", 120) url = self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) - self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) + self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) def process(self, pyfile): -- cgit v1.2.3 From 6badd4f1f210d2e9385c4d73c5b41ee5a6f04e65 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 09:59:16 +0200 Subject: Improve error method + fix TEXT_ENCODING in SimpleCrypter --- module/plugins/internal/SimpleHoster.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 74910692f..17543ba8d 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -3,6 +3,7 @@ import re from time import time +from traceback import print_exc from urlparse import urlparse from module.network.CookieJar import CookieJar @@ -321,8 +322,8 @@ class SimpleHoster(Hoster): super(SimpleHoster, self).wait() + #@TODO: remove in 0.4.10 def error(self, reason=None, type="parse"): - if reason: - raise Fail("%s error: %s | Plugin may be out of date" % (type.capitalize(), reason)) - else: - raise Fail("%s error | Plugin out of date" % type.capitalize()) + raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ':' + str(reason) if reason else "")) + if self.core.debug: + print_exc() -- cgit v1.2.3 From 1b096b2eb2634e8dea80b06ab9ecde206b198b35 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 19:47:00 +0200 Subject: Spare code cosmetics --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 17543ba8d..a8a44d953 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -324,6 +324,6 @@ class SimpleHoster(Hoster): #@TODO: remove in 0.4.10 def error(self, reason=None, type="parse"): - raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ':' + str(reason) if reason else "")) + raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ': ' + str(reason) if reason else "")) if self.core.debug: print_exc() -- cgit v1.2.3 From 8e89cb7781ec3b135feeb750f6321bbc85e89d67 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 23 Oct 2014 22:54:40 +0200 Subject: Fix error method --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index a8a44d953..be3e3c8fc 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -323,7 +323,7 @@ class SimpleHoster(Hoster): #@TODO: remove in 0.4.10 - def error(self, reason=None, type="parse"): - raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ': ' + str(reason) if reason else "")) + def error(self, reason="", type="parse"): if self.core.debug: print_exc() + raise Fail("%s error%s | Plugin may be out of date" % (type.strip().capitalize(), ': ' + reason.strip() if reason else "")) -- cgit v1.2.3 From c41d4ec4c5935ee702a44e919730971028bf5724 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Oct 2014 01:16:52 +0200 Subject: Fix and improve plugins logging --- module/plugins/internal/SimpleHoster.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index be3e3c8fc..51010d569 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -221,10 +221,12 @@ class SimpleHoster(Hoster): self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handle as premium download") self.handlePremium() elif premium_only: self.fail("This link require a premium account") else: + self.logDebug("Handle as free download") self.handleFree() -- cgit v1.2.3 From e3f5280529921100f48bb8a79853bf480c7611e4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Oct 2014 02:53:05 +0200 Subject: Replace single quotes with doubles in self.error and self.fail msg --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 51010d569..951a05783 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -241,7 +241,7 @@ class SimpleHoster(Hoster): self.tempOffline() elif status != 2: self.logDebug(self.file_info) - self.error('File info') + self.error("File info") if name: self.pyfile.name = name -- cgit v1.2.3 From 9f2ebe486a3e155fb6a60e07cccb77ab6a772eb2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 02:31:54 +0200 Subject: Extend translation support in plugins + a lot of code cosmetics and typo fixes --- module/plugins/internal/SimpleHoster.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 951a05783..135d3c9d9 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -224,7 +224,7 @@ class SimpleHoster(Hoster): self.logDebug("Handle as premium download") self.handlePremium() elif premium_only: - self.fail("This link require a premium account") + self.fail(_("This link require a premium account")) else: self.logDebug("Handle as free download") self.handleFree() @@ -241,7 +241,7 @@ class SimpleHoster(Hoster): self.tempOffline() elif status != 2: self.logDebug(self.file_info) - self.error("File info") + self.error(_("File info")) if name: self.pyfile.name = name @@ -259,12 +259,12 @@ class SimpleHoster(Hoster): def handleFree(self): if not hasattr(self, 'LINK_FREE_PATTERN'): - self.fail("Free download not implemented") + self.fail(_("Free download not implemented")) try: m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: - self.error("Free download link not found") + self.error(_("Free download link not found")) link = m.group(1) except Exception, e: @@ -275,12 +275,12 @@ class SimpleHoster(Hoster): def handlePremium(self): if not hasattr(self, 'LINK_PREMIUM_PATTERN'): - self.fail("Premium download not implemented") + self.fail(_("Premium download not implemented")) try: m = re.search(self.LINK_PREMIUM_PATTERN, self.html) if m is None: - self.error("Premium download link not found") + self.error(_("Premium download link not found")) link = m.group(1) except Exception, e: -- cgit v1.2.3 From 8f0303277fce7471539508f7a8fad033e5e87076 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 19:36:46 +0100 Subject: [SimpleHoster][SimpleCrypter] Clean error method --- module/plugins/internal/SimpleHoster.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 135d3c9d9..84a77a312 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -13,6 +13,20 @@ from module.plugins.Plugin import Fail from module.utils import fixup, html_unescape, parseFileSize +#@TODO: Remove in 0.4.10 and redirect to self.error instead +def _error(self, reason="", type=""): + if not reason and not type: + type = "unknown" + + msg = _("%s error") % type.strip().capitalize() if type else _("Error") + msg += ": " + reason.strip() if reason else "" + msg += _(" | Plugin may be out of date") + + if self.core.debug: + print_exc() + raise Fail(msg) + + def replace_patterns(string, ruleslist): for r in ruleslist: rf, rt = r @@ -324,8 +338,5 @@ class SimpleHoster(Hoster): super(SimpleHoster, self).wait() - #@TODO: remove in 0.4.10 def error(self, reason="", type="parse"): - if self.core.debug: - print_exc() - raise Fail("%s error%s | Plugin may be out of date" % (type.strip().capitalize(), ': ' + reason.strip() if reason else "")) + return _error(reason, type) -- cgit v1.2.3 From f6aed08062c3b6a4dad09d238a32ef5ab58479a4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 20:49:31 +0100 Subject: [SimpleHoster][SimpleCrypter] Bump up __version__ --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 84a77a312..87ebfc533 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -159,7 +159,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.40" + __version__ = "0.41" __pattern__ = None -- cgit v1.2.3 From f10f74fb695afbaa5539f6dae7d96be0ab667249 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Oct 2014 01:16:51 +0100 Subject: [SimpleHoster] Direct download link support --- module/plugins/internal/SimpleHoster.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 87ebfc533..ca84dc33e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -159,7 +159,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.41" + __version__ = "0.42" __pattern__ = None @@ -208,6 +208,8 @@ class SimpleHoster(Hoster): COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + CHECK_DIRECT_DOWNLOAD = True #: Search for a direct download link in the page header (premium only) + def init(self): self.file_info = {} @@ -223,15 +225,22 @@ class SimpleHoster(Hoster): self.req.setOption("timeout", 120) - url = self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) - self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) + self.html = getURL(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) def process(self, pyfile): self.prepare() + if self.CHECK_DIRECT_DOWNLOAD and self.premium: + header = self.load(self.pyfile.url, just_header=True) + if 'location' in header: + self.logDebug("Direct download link detected") + self.download(header['location'], ref=True, cookies=True, disposition=True) + return + premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only: # Usually premium only pages doesn't show the file information + if not premium_only: #: Usually premium only pages doesn't show the file information self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): @@ -282,7 +291,7 @@ class SimpleHoster(Hoster): link = m.group(1) except Exception, e: - self.logError(str(e)) + self.fail(str(e)) else: self.download(link, ref=True, cookies=True, disposition=True) @@ -298,7 +307,7 @@ class SimpleHoster(Hoster): link = m.group(1) except Exception, e: - self.logError(str(e)) + self.fail(str(e)) else: self.download(link, ref=True, cookies=True, disposition=True) @@ -308,14 +317,14 @@ class SimpleHoster(Hoster): time_str = "%dh %dm" % divmod(wait_time / 60, 60) else: wait_time = 900 - time_str = "(unknown time)" + time_str = _("(unknown time)") max_tries = 100 self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str) self.setWait(wait_time, True) self.wait() - self.retry(max_tries=max_tries, reason="Download limit reached") + self.retry(max_tries=max_tries, reason=_("Download limit reached")) def parseHtmlForm(self, attr_str='', input_names=None): -- cgit v1.2.3 From aa0751bcfd995e308bcd586a6965c75e68b1274b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Oct 2014 23:05:13 +0100 Subject: Code cosmetics --- module/plugins/internal/SimpleHoster.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ca84dc33e..782655007 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -157,17 +157,18 @@ def timestamp(): class SimpleHoster(Hoster): - __name__ = "SimpleHoster" - __type__ = "hoster" + __name__ = "SimpleHoster" + __type__ = "hoster" __version__ = "0.42" __pattern__ = None + __config__ = [] __description__ = """Simple hoster plugin""" - __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("stickell", "l.stickell@yahoo.it"), - ("Walter Purcaro", "vuolter@gmail.com")] + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] """ -- cgit v1.2.3 From 885f8ed782e64d9e73367905e642a84d0a8999f1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 04:01:38 +0100 Subject: Update __config__ --- module/plugins/internal/SimpleHoster.py | 1 - 1 file changed, 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 782655007..3d2bcda86 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -162,7 +162,6 @@ class SimpleHoster(Hoster): __version__ = "0.42" __pattern__ = None - __config__ = [] __description__ = """Simple hoster plugin""" __license__ = "GPLv3" -- cgit v1.2.3 From 6a5b13502648cf51007c15948b778e5a9d54ad20 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 04:52:39 +0100 Subject: [SimpleCrypter][SimpleHoster] Fix typo in _error method call --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3d2bcda86..e0b31416a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -159,7 +159,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.42" + __version__ = "0.43" __pattern__ = None @@ -348,4 +348,4 @@ class SimpleHoster(Hoster): def error(self, reason="", type="parse"): - return _error(reason, type) + return _error(self, reason, type) -- cgit v1.2.3 From 302616d94f76ab8794b58d3d54b780cc58e86b06 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 04:58:47 +0100 Subject: Code cosmetics: items() -> iteritems() --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e0b31416a..9e270ff3d 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -62,7 +62,7 @@ def parseHtmlForm(attr_str, html, input_names=None): if isinstance(input_names, dict): # check input attributes - for key, val in input_names.items(): + for key, val in input_names.iteritems(): if key in inputs: if isinstance(val, basestring) and inputs[key] == val: continue -- cgit v1.2.3 From 8447df36b147bab1c3caa70aaa67732115ad735b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 16:03:14 +0100 Subject: [SimpleHoster] Improve wait method --- module/plugins/internal/SimpleHoster.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 9e270ff3d..c47a715f0 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -14,7 +14,7 @@ from module.utils import fixup, html_unescape, parseFileSize #@TODO: Remove in 0.4.10 and redirect to self.error instead -def _error(self, reason="", type=""): +def _error(self, reason, type): if not reason and not type: type = "unknown" @@ -27,6 +27,13 @@ def _error(self, reason="", type=""): raise Fail(msg) +#@TODO: Remove in 0.4.10 +def _wait(self, seconds, reconnect): + if seconds: + self.setWait(seconds, reconnect) + super(SimpleHoster, self).wait() + + def replace_patterns(string, ruleslist): for r in ruleslist: rf, rt = r @@ -159,7 +166,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.43" + __version__ = "0.44" __pattern__ = None @@ -341,10 +348,8 @@ class SimpleHoster(Hoster): #@TODO: Remove in 0.4.10 - def wait(self, seconds=False, reconnect=False): - if seconds: - self.setWait(seconds, reconnect) - super(SimpleHoster, self).wait() + def wait(self, seconds=0, reconnect=None): + return _wait(self, seconds, reconnect) def error(self, reason="", type="parse"): -- cgit v1.2.3 From 56ff5aff808975456163c027c670844baeebc2a8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 30 Oct 2014 17:16:09 +0100 Subject: [SimpleHoster] Fix checkTrafficLeft, ddl routine and cookies support --- module/plugins/internal/SimpleHoster.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c47a715f0..de21f4d3c 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -166,7 +166,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.44" + __version__ = "0.45" __pattern__ = None @@ -233,19 +233,20 @@ class SimpleHoster(Hoster): self.req.setOption("timeout", 120) self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) - self.html = getURL(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) - - - def process(self, pyfile): - self.prepare() if self.CHECK_DIRECT_DOWNLOAD and self.premium: - header = self.load(self.pyfile.url, just_header=True) + header = self.load(self.pyfile.url, just_header=True, decode=True) if 'location' in header: self.logDebug("Direct download link detected") self.download(header['location'], ref=True, cookies=True, disposition=True) return + self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + + + def process(self, pyfile): + self.prepare() + premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) if not premium_only: #: Usually premium only pages doesn't show the file information self.getFileInfo() @@ -340,11 +341,15 @@ class SimpleHoster(Hoster): def checkTrafficLeft(self): traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] - if traffic == -1: + + if traffic is None: + return False + elif traffic == -1: return True - size = self.pyfile.size / 1024 - self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) - return size <= traffic + else: + size = self.pyfile.size / 1024 + self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) + return size <= traffic #@TODO: Remove in 0.4.10 -- cgit v1.2.3 From 011c308fdce25df396824ea97c2bf634b976c8d8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 1 Nov 2014 18:51:19 +0100 Subject: [SimpleHoster] Fix and improve ddl detection, parseFileInfo and getInfo --- module/plugins/internal/SimpleHoster.py | 122 +++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 42 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index de21f4d3c..e601759cf 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -90,27 +90,30 @@ def parseHtmlForm(attr_str, html, input_names=None): def parseFileInfo(self, url="", html=""): - info = {"name": url, "size": 0, "status": 3} - - if hasattr(self, "pyfile"): + if not url and hasattr(self, "pyfile"): url = self.pyfile.url - if hasattr(self, "req") and self.req.http.code == '404': - info['status'] = 1 - else: - if not html and hasattr(self, "html"): - html = self.html - if isinstance(self.TEXT_ENCODING, basestring): - html = unicode(html, self.TEXT_ENCODING) - if hasattr(self, "html"): - self.html = html + info = {"name": url, "size": 0, "status": 3} + if not html: + if url: + return tuple(create_getInfo(self)([url])) + elif hasattr(self, "html"): + if hasattr(self, "req") and self.req.http.code == '404': + info['status'] = 1 + else: + html = self.html + + if html: if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html): info['status'] = 1 + elif hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html): #@TODO: Remove in 0.4.10 info['status'] = 1 + elif hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, html): info['status'] = 6 + else: online = False try: @@ -128,18 +131,23 @@ def parseFileInfo(self, url="", html=""): if online: # File online, return name and size info['status'] = 2 + if 'N' in info: info['name'] = replace_patterns(info['N'].strip(), self.FILE_NAME_REPLACEMENTS) + if 'S' in info: size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], self.FILE_SIZE_REPLACEMENTS) info['size'] = parseFileSize(size) + elif isinstance(info['size'], basestring): unit = info['units'] if 'units' in info else None info['size'] = parseFileSize(info['size'], unit) - if hasattr(self, "file_info"): - self.file_info = info + if not hasattr(self, "file_info"): + self.file_info = {} + + self.file_info.update(info) return info['name'], info['size'], info['status'], url @@ -148,13 +156,23 @@ def create_getInfo(plugin): def getInfo(urls): for url in urls: - cj = CookieJar(plugin.__name__) - if isinstance(plugin.COOKIES, list): + if hasattr(plugin, "COOKIES") and isinstance(plugin.COOKIES, list): + cj = CookieJar(plugin.__name__) set_cookies(cj, plugin.COOKIES) + else: + cj = None - file_info = parseFileInfo(plugin, url, getURL(replace_patterns(url, plugin.FILE_URL_REPLACEMENTS), - decode=not plugin.TEXT_ENCODING, cookies=cj)) - yield file_info + if hasattr(plugin, "FILE_URL_REPLACEMENTS"): + url = replace_patterns(url, plugin.FILE_URL_REPLACEMENTS) + + if hasattr(plugin, "TEXT_ENCODING"): + html = getURL(url, cookies=bool(cj), decode=not plugin.TEXT_ENCODING) + if isinstance(plugin.TEXT_ENCODING, basestring): + html = unicode(html, plugin.TEXT_ENCODING) + else: + html = getURL(url, cookies=bool(cj), decode=True) + + yield parseFileInfo(plugin, url, html) return getInfo @@ -166,7 +184,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.45" + __version__ = "0.46" __pattern__ = None @@ -215,11 +233,10 @@ class SimpleHoster(Hoster): COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - CHECK_DIRECT_DOWNLOAD = True #: Search for a direct download link in the page header (premium only) - def init(self): self.file_info = {} + self.html = "" #@TODO: Remove in 0.4.10 def setup(self): @@ -234,31 +251,53 @@ class SimpleHoster(Hoster): self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) - if self.CHECK_DIRECT_DOWNLOAD and self.premium: - header = self.load(self.pyfile.url, just_header=True, decode=True) - if 'location' in header: - self.logDebug("Direct download link detected") - self.download(header['location'], ref=True, cookies=True, disposition=True) - return + if self.premium: + direct_link = self.getDirectLink(self.pyfile.url) + if direct_link: + return direct_link self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + if isinstance(self.TEXT_ENCODING, basestring): + self.html = unicode(self.html, self.TEXT_ENCODING) + def process(self, pyfile): - self.prepare() + direct_link = self.prepare() - premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only: #: Usually premium only pages doesn't show the file information - self.getFileInfo() + if isinstance(direct_link, basestring): + self.download(direct_link, ref=True, cookies=True, disposition=True) + + elif self.html is None: + self.fail(_("Attribute html should never be set to None")) - if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handle as premium download") - self.handlePremium() - elif premium_only: - self.fail(_("This link require a premium account")) else: - self.logDebug("Handle as free download") - self.handleFree() + premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) + if not premium_only and 'status' not in self.file_info: #: Usually premium only pages doesn't show the file information + self.getFileInfo() + + if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as premium download") + self.handlePremium() + elif premium_only: + self.fail(_("Link require a premium account to be handled")) + else: + self.logDebug("Handled as free download") + self.handleFree() + + + def getDirectLink(self, url): + self.req.http.c.setopt(FOLLOWLOCATION, 0) + + html = self.load(url, ref=True, decode=True) + + self.req.http.c.setopt(FOLLOWLOCATION, 1) + + if parseFileInfo(self, url, html)[2] != 2: + try + return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1) + except: + pass def getFileInfo(self): @@ -271,8 +310,7 @@ class SimpleHoster(Hoster): elif status == 6: self.tempOffline() elif status != 2: - self.logDebug(self.file_info) - self.error(_("File info")) + self.error(_("File info: %s") % self.file_info) if name: self.pyfile.name = name @@ -284,7 +322,7 @@ class SimpleHoster(Hoster): else: self.logError(_("File size not parsed")) - self.logDebug("FILE NAME: %s FILE SIZE: %s" % (self.pyfile.name, self.pyfile.size)) + self.logDebug("FILE NAME: %s" % self.pyfile.name, "FILE SIZE: %d" % self.pyfile.size) return self.file_info -- cgit v1.2.3 From 11fcb4ebb1d3f38e8a49f316308520ebcf8cf658 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 1 Nov 2014 22:20:06 +0100 Subject: [SimpleHoster] Fix typo --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e601759cf..1d85dc01b 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -294,7 +294,7 @@ class SimpleHoster(Hoster): self.req.http.c.setopt(FOLLOWLOCATION, 1) if parseFileInfo(self, url, html)[2] != 2: - try + try: return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1) except: pass -- cgit v1.2.3 From 4ec11afb741149ea86e6ba647862b461d54c747c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Nov 2014 22:37:19 +0100 Subject: Update internal Simple plugins --- module/plugins/internal/SimpleHoster.py | 67 +++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 28 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1d85dc01b..749cdf557 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -90,19 +90,20 @@ def parseHtmlForm(attr_str, html, input_names=None): def parseFileInfo(self, url="", html=""): + info = {'name': url, 'size': 0, 'status': 3} + if not url and hasattr(self, "pyfile"): url = self.pyfile.url - info = {"name": url, "size": 0, "status": 3} - if not html: if url: - return tuple(create_getInfo(self)([url])) + return next(create_getInfo(self)([url])) + + elif hasattr(self, "req") and self.req.http.code == '404': + info['status'] = 1 + elif hasattr(self, "html"): - if hasattr(self, "req") and self.req.http.code == '404': - info['status'] = 1 - else: - html = self.html + html = self.html if html: if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html): @@ -121,7 +122,8 @@ def parseFileInfo(self, url="", html=""): except: pass - for pattern in ("FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): + for pattern in ("INFO_PATTERN", "NAME_PATTERN", "SIZE_PATTERN", + "FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): #@TODO: Remove in 0.4.10 try: info.update(re.search(getattr(self, pattern), html).groupdict()) online = True @@ -133,17 +135,21 @@ def parseFileInfo(self, url="", html=""): info['status'] = 2 if 'N' in info: - info['name'] = replace_patterns(info['N'].strip(), self.FILE_NAME_REPLACEMENTS) + info['name'] = replace_patterns(info['N'].strip(), + self.FILE_NAME_REPLACEMENTS if hasattr(self, "FILE_NAME_REPLACEMENTS") else self.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 if 'S' in info: size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], - self.FILE_SIZE_REPLACEMENTS) + self.FILE_SIZE_REPLACEMENTS if hasattr(self, "FILE_SIZE_REPLACEMENTS") else self.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 info['size'] = parseFileSize(size) elif isinstance(info['size'], basestring): unit = info['units'] if 'units' in info else None info['size'] = parseFileSize(info['size'], unit) + if not hasattr(self, "html") or self.html is None: + self.html = html + if not hasattr(self, "file_info"): self.file_info = {} @@ -162,7 +168,10 @@ def create_getInfo(plugin): else: cj = None - if hasattr(plugin, "FILE_URL_REPLACEMENTS"): + if hasattr(plugin, "URL_REPLACEMENTS"): + url = replace_patterns(url, plugin.URL_REPLACEMENTS) + + elif hasattr(plugin, "FILE_URL_REPLACEMENTS"): #@TODO: Remove in 0.4.10 url = replace_patterns(url, plugin.FILE_URL_REPLACEMENTS) if hasattr(plugin, "TEXT_ENCODING"): @@ -198,13 +207,13 @@ class SimpleHoster(Hoster): """ Following patterns should be defined by each hoster: - FILE_INFO_PATTERN: (optional) Name and Size of the file - example: FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' + INFO_PATTERN: (optional) Name and Size of the file + example: INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' or - FILE_NAME_PATTERN: (optional) Name that will be set for the file - example: FILE_NAME_PATTERN = r'(?Pfile_name)' - FILE_SIZE_PATTERN: (optional) Size that will be checked for the file - example: FILE_SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' + NAME_PATTERN: (optional) Name that will be set for the file + example: NAME_PATTERN = r'(?Pfile_name)' + SIZE_PATTERN: (optional) Size that will be checked for the file + example: SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' OFFLINE_PATTERN: (optional) Checks if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' @@ -225,9 +234,9 @@ class SimpleHoster(Hoster): example: LINK_PREMIUM_PATTERN = r'
Date: Mon, 3 Nov 2014 08:18:32 +0100 Subject: [SimpleHoster] Fix missing FOLLOWLOCATION import --- module/plugins/internal/SimpleHoster.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 749cdf557..feb373541 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -6,6 +6,8 @@ from time import time from traceback import print_exc from urlparse import urlparse +from pycurl import FOLLOWLOCATION + from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster @@ -193,7 +195,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.46" + __version__ = "0.47" __pattern__ = None -- cgit v1.2.3 From 4666843d7e7b90e409019b471bcf35c67c4df645 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Nov 2014 12:12:09 +0100 Subject: [SimpleHoster] Fix parseFileInfo and improve getFileInfo --- module/plugins/internal/SimpleHoster.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index feb373541..b9ac84783 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -92,14 +92,14 @@ def parseHtmlForm(attr_str, html, input_names=None): def parseFileInfo(self, url="", html=""): - info = {'name': url, 'size': 0, 'status': 3} - if not url and hasattr(self, "pyfile"): url = self.pyfile.url + info = {'name': url, 'size': 0, 'status': 3} + if not html: if url: - return next(create_getInfo(self)([url])) + return create_getInfo(self)([url]).next() elif hasattr(self, "req") and self.req.http.code == '404': info['status'] = 1 @@ -155,8 +155,12 @@ def parseFileInfo(self, url="", html=""): if not hasattr(self, "file_info"): self.file_info = {} + self.logDebug(_("File info (before update): %s") % self.file_info) + self.file_info.update(info) + self.logDebug(_("File info (after update): %s") % self.file_info) + return info['name'], info['size'], info['status'], url @@ -195,7 +199,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.47" + __version__ = "0.48" __pattern__ = None @@ -245,10 +249,6 @@ class SimpleHoster(Hoster): FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - def init(self): - self.file_info = {} - - def setup(self): self.resumeDownload = self.multiDL = self.premium @@ -313,29 +313,26 @@ class SimpleHoster(Hoster): def getFileInfo(self): - name, size, status = parseFileInfo(self)[:3] + name, size, status, url = parseFileInfo(self) - msg = _("File info: %s") % self.file_info - self.logDebug(msg) + if name and name != url: + self.pyfile.name = name + else: + self.pyfile.name = self.file_info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) if status == 1: self.offline() elif status == 6: self.tempOffline() elif status != 2: - self.error(msg) - - if name: - self.pyfile.name = name - else: - self.pyfile.name = html_unescape(urlparse(self.pyfile.url).path.split("/")[-1]) + self.error(_("File info: %s") % self.file_info) if size: self.pyfile.size = size else: self.logError(_("File size not parsed")) - self.logDebug("FILE NAME: %s" % self.pyfile.name, "FILE SIZE: %d" % self.pyfile.size) + self.logDebug("FILE NAME: %s" % self.pyfile.name, "FILE SIZE: %d" % self.pyfile.size or _("Unknown")) return self.file_info -- cgit v1.2.3 From d7729f9020bc5959f295ca5efb75f3ce0dd605bb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Nov 2014 15:46:50 +0100 Subject: [SimpleHoster] Fix file_info stuff --- module/plugins/internal/SimpleHoster.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index b9ac84783..1555151a5 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -152,14 +152,20 @@ def parseFileInfo(self, url="", html=""): if not hasattr(self, "html") or self.html is None: self.html = html - if not hasattr(self, "file_info"): - self.file_info = {} + if not hasattr(self, "info"): + self.info = {} - self.logDebug(_("File info (before update): %s") % self.file_info) + try: #: Remove try-except statement in 0.4.10 + self.logDebug(_("File info (before update): %s") % self.info) + except: + pass - self.file_info.update(info) + self.info.update(info) - self.logDebug(_("File info (after update): %s") % self.file_info) + try: #: Remove try-except statement in 0.4.10 + self.logDebug(_("File info (after update): %s") % self.info) + except: + pass return info['name'], info['size'], info['status'], url @@ -199,9 +205,9 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.48" + __version__ = "0.49" - __pattern__ = None + __pattern__ = r'^unmatchable$' __description__ = """Simple hoster plugin""" __license__ = "GPLv3" @@ -249,6 +255,10 @@ class SimpleHoster(Hoster): FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + def init(): + self.info = {} + + def setup(self): self.resumeDownload = self.multiDL = self.premium @@ -285,7 +295,7 @@ class SimpleHoster(Hoster): else: premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only and 'status' not in self.file_info: #: Usually premium only pages doesn't show any file information + if not premium_only and 'status' not in self.info: #: Usually premium only pages doesn't show any file information self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): @@ -318,14 +328,14 @@ class SimpleHoster(Hoster): if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.file_info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) + self.pyfile.name = self.info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) if status == 1: self.offline() elif status == 6: self.tempOffline() elif status != 2: - self.error(_("File info: %s") % self.file_info) + self.error(_("File info: %s") % self.info) if size: self.pyfile.size = size @@ -333,7 +343,7 @@ class SimpleHoster(Hoster): self.logError(_("File size not parsed")) self.logDebug("FILE NAME: %s" % self.pyfile.name, "FILE SIZE: %d" % self.pyfile.size or _("Unknown")) - return self.file_info + return self.info def handleFree(self): -- cgit v1.2.3 From 03f3b86f500c495932fd118b54569d92f700847c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Nov 2014 16:57:55 +0100 Subject: Code cosmetics about file_info and other stuff --- module/plugins/internal/SimpleHoster.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1555151a5..1f673a5c2 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -155,14 +155,14 @@ def parseFileInfo(self, url="", html=""): if not hasattr(self, "info"): self.info = {} - try: #: Remove try-except statement in 0.4.10 + try: self.logDebug(_("File info (before update): %s") % self.info) except: pass self.info.update(info) - try: #: Remove try-except statement in 0.4.10 + try: self.logDebug(_("File info (after update): %s") % self.info) except: pass @@ -255,7 +255,7 @@ class SimpleHoster(Hoster): FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - def init(): + def init(self): self.info = {} @@ -330,11 +330,11 @@ class SimpleHoster(Hoster): else: self.pyfile.name = self.info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) - if status == 1: + if status is 1: self.offline() - elif status == 6: + elif status is 6: self.tempOffline() - elif status != 2: + elif status is not 2: self.error(_("File info: %s") % self.info) if size: -- cgit v1.2.3 From 2f4d7d614171054c3c2b0b6185fa4ea465f5312f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 01:09:43 +0100 Subject: Use more print_exc + code cosmetics --- module/plugins/internal/SimpleHoster.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1f673a5c2..ac90e193a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -24,8 +24,6 @@ def _error(self, reason, type): msg += ": " + reason.strip() if reason else "" msg += _(" | Plugin may be out of date") - if self.core.debug: - print_exc() raise Fail(msg) @@ -315,7 +313,7 @@ class SimpleHoster(Hoster): self.req.http.c.setopt(FOLLOWLOCATION, 1) - if parseFileInfo(self, url, html)[2] != 2: + if parseFileInfo(self, url, html)[2] is not 2: try: return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1) except: -- cgit v1.2.3 From e9376045f48255d3cbfdbaf5b5c6fbfd9112f17d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 03:23:06 +0100 Subject: [SimpleCrypter][SimpleHoster] Fix info data retrieving --- module/plugins/internal/SimpleHoster.py | 42 ++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 19 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ac90e193a..3de49bef9 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -147,23 +147,21 @@ def parseFileInfo(self, url="", html=""): unit = info['units'] if 'units' in info else None info['size'] = parseFileSize(info['size'], unit) - if not hasattr(self, "html") or self.html is None: + if hasattr(self, "html") and self.html is None: self.html = html - if not hasattr(self, "info"): - self.info = {} - - try: - self.logDebug(_("File info (before update): %s") % self.info) - except: - pass + if hasattr(self, "info"): + try: + self.logDebug(_("File info (before update): %s") % self.info) + except: + pass - self.info.update(info) + self.info.update(info) - try: - self.logDebug(_("File info (after update): %s") % self.info) - except: - pass + try: + self.logDebug(_("File info (after update): %s") % self.info) + except: + pass return info['name'], info['size'], info['status'], url @@ -203,7 +201,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.49" + __version__ = "0.50" __pattern__ = r'^unmatchable$' @@ -246,10 +244,10 @@ class SimpleHoster(Hoster): NAME_REPLACEMENTS = [("&#?\w+;", fixup)] SIZE_REPLACEMENTS = [] - URL_REPLACEMENTS = [] + URL_REPLACEMENTS = [] - TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] + TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account @@ -274,8 +272,11 @@ class SimpleHoster(Hoster): direct_link = self.getDirectLink(self.pyfile.url) if direct_link: return direct_link + else: + self.html = None + self.info = {} - if not self.html: + if self.html is None: self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) if isinstance(self.TEXT_ENCODING, basestring): @@ -286,6 +287,7 @@ class SimpleHoster(Hoster): direct_link = self.prepare() if isinstance(direct_link, basestring): + self.logInfo(_("Direct download link detected")) self.download(direct_link, ref=True, cookies=True, disposition=True) elif self.html is None: @@ -293,7 +295,7 @@ class SimpleHoster(Hoster): else: premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only and 'status' not in self.info: #: Usually premium only pages doesn't show any file information + if not premium_only and not self.info: #: Usually premium only pages doesn't show any file information self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): @@ -330,8 +332,10 @@ class SimpleHoster(Hoster): if status is 1: self.offline() + elif status is 6: self.tempOffline() + elif status is not 2: self.error(_("File info: %s") % self.info) -- cgit v1.2.3 From 6412e51af626ee89ce6d9dac99a6c53e1878d142 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 14:19:32 +0100 Subject: logDebug when detects ddl or multi hoster link --- module/plugins/internal/SimpleHoster.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3de49bef9..2b729dae4 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -269,10 +269,12 @@ class SimpleHoster(Hoster): self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 if self.premium: + self.logDebug(_("Checking for direct download link...")) direct_link = self.getDirectLink(self.pyfile.url) if direct_link: return direct_link else: + self.logDebug(_("No direct download link found")) self.html = None self.info = {} -- cgit v1.2.3 From 0d1fe176a7963a71f9eb6735afc4132c26550b75 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 19:59:21 +0100 Subject: Fix missing urlparse import --- module/plugins/internal/SimpleHoster.py | 1 - 1 file changed, 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2b729dae4..bd6c69db6 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -3,7 +3,6 @@ import re from time import time -from traceback import print_exc from urlparse import urlparse from pycurl import FOLLOWLOCATION -- cgit v1.2.3 From 6cb8c84d19066db3ad32f84dff864f058b6a2be9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 6 Nov 2014 15:13:01 +0100 Subject: [SimpleHoster] Fix getDirectLink --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index bd6c69db6..2c95c8b38 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -200,7 +200,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.50" + __version__ = "0.51" __pattern__ = r'^unmatchable$' @@ -318,7 +318,7 @@ class SimpleHoster(Hoster): if parseFileInfo(self, url, html)[2] is not 2: try: - return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1) + return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1).strip() #@TODO: Remove .strip() in 0.4.10 except: pass -- cgit v1.2.3 From 6bcfdef32e7e62f8f749ceb6618ee7550bce28de Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 6 Nov 2014 22:42:59 +0100 Subject: Lowercase url to load --- module/plugins/internal/SimpleHoster.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2c95c8b38..87a348367 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -200,7 +200,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.51" + __version__ = "0.52" __pattern__ = r'^unmatchable$' @@ -268,7 +268,7 @@ class SimpleHoster(Hoster): self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 if self.premium: - self.logDebug(_("Checking for direct download link...")) + self.logDebug(_("Looking for direct download link...")) direct_link = self.getDirectLink(self.pyfile.url) if direct_link: return direct_link @@ -300,12 +300,12 @@ class SimpleHoster(Hoster): self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as premium download") + self.logDebug("Handle as premium download") self.handlePremium() elif premium_only: self.fail(_("Link require a premium account to be handled")) else: - self.logDebug("Handled as free download") + self.logDebug("Handle as free download") self.handleFree() @@ -318,7 +318,7 @@ class SimpleHoster(Hoster): if parseFileInfo(self, url, html)[2] is not 2: try: - return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1).strip() #@TODO: Remove .strip() in 0.4.10 + return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1).strip().lower() #@TODO: Remove .strip().lower() in 0.4.10 except: pass -- cgit v1.2.3 From 58c9c8d78417ec6ccac5b6b117ea540033a09574 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 7 Nov 2014 17:08:21 +0100 Subject: Fix download link formatting in some plugins --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 87a348367..6a8a6ff6f 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -200,7 +200,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.52" + __version__ = "0.53" __pattern__ = r'^unmatchable$' @@ -318,7 +318,7 @@ class SimpleHoster(Hoster): if parseFileInfo(self, url, html)[2] is not 2: try: - return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1).strip().lower() #@TODO: Remove .strip().lower() in 0.4.10 + return re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I).group(1).rstrip() #@TODO: Remove .rstrip() in 0.4.10 except: pass -- cgit v1.2.3 From 395ac7930ef4e1598186a7a0636d3f245b7e5134 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 11 Nov 2014 00:36:45 +0100 Subject: Fix url to name conversion in api and simple plugins --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6a8a6ff6f..6726d4fa5 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -329,7 +329,7 @@ class SimpleHoster(Hoster): if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) + self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split("/")[-1] if status is 1: self.offline() -- cgit v1.2.3 From d7d2b0c56e16866313b7f746f94569f085acef01 Mon Sep 17 00:00:00 2001 From: Radek Senfeld Date: Fri, 14 Nov 2014 01:47:39 +0100 Subject: [UlozTo] Fixed hoster and account --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6726d4fa5..1f7753003 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -324,7 +324,7 @@ class SimpleHoster(Hoster): def getFileInfo(self): - name, size, status, url = parseFileInfo(self) + name, size, status, url = parseFileInfo(self, html=self.html) if name and name != url: self.pyfile.name = name -- cgit v1.2.3 From 0521acfc4e62096d8f78e6f5d1966835c7acbcef Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 14 Nov 2014 01:53:24 +0100 Subject: [SimpleCrypter][SimpleHoster] Temp workaround --- module/plugins/internal/SimpleHoster.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1f7753003..2562f0af3 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -200,7 +200,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.53" + __version__ = "0.54" __pattern__ = r'^unmatchable$' @@ -274,11 +274,8 @@ class SimpleHoster(Hoster): return direct_link else: self.logDebug(_("No direct download link found")) - self.html = None - self.info = {} - if self.html is None: - self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) if isinstance(self.TEXT_ENCODING, basestring): self.html = unicode(self.html, self.TEXT_ENCODING) @@ -296,7 +293,7 @@ class SimpleHoster(Hoster): else: premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only and not self.info: #: Usually premium only pages doesn't show any file information + if not premium_only: #: Usually premium only pages doesn't show any file information self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): -- cgit v1.2.3 From 7d8d0991e4d9848325692f8bbb81df00afe4e065 Mon Sep 17 00:00:00 2001 From: Thierry Merle Date: Sat, 15 Nov 2014 07:14:06 +0100 Subject: [SimpleHoster][OneFichierCom] Fix issue #836 ('status': 3) --- module/plugins/internal/SimpleHoster.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2562f0af3..18df2de92 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -170,10 +170,9 @@ def create_getInfo(plugin): def getInfo(urls): for url in urls: if hasattr(plugin, "COOKIES") and isinstance(plugin.COOKIES, list): - cj = CookieJar(plugin.__name__) - set_cookies(cj, plugin.COOKIES) + set_cookies(plugin.req.cj, plugin.COOKIES) else: - cj = None + plugin.req.cj = None if hasattr(plugin, "URL_REPLACEMENTS"): url = replace_patterns(url, plugin.URL_REPLACEMENTS) @@ -182,11 +181,11 @@ def create_getInfo(plugin): url = replace_patterns(url, plugin.FILE_URL_REPLACEMENTS) if hasattr(plugin, "TEXT_ENCODING"): - html = getURL(url, cookies=bool(cj), decode=not plugin.TEXT_ENCODING) + html = plugin.load(url, decode=not plugin.TEXT_ENCODING, cookies=bool(plugin.COOKIES)) if isinstance(plugin.TEXT_ENCODING, basestring): html = unicode(html, plugin.TEXT_ENCODING) else: - html = getURL(url, cookies=bool(cj), decode=True) + html = plugin.load(url, decode=True, cookies=bool(plugin.COOKIES)) yield parseFileInfo(plugin, url, html) @@ -200,7 +199,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.54" + __version__ = "0.55" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 64afb5db7af0acdc74011b4e1a95ea83bd68bf39 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 06:30:16 +0100 Subject: [SimpleHoster] Updated --- module/plugins/internal/SimpleHoster.py | 319 +++++++++++++++++--------------- 1 file changed, 166 insertions(+), 153 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 18df2de92..e052a79ae 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -7,6 +7,7 @@ from urlparse import urlparse from pycurl import FOLLOWLOCATION +from module.PyFile import statusMap from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster @@ -76,130 +77,50 @@ def parseHtmlForm(attr_str, html, input_names=None): continue elif hasattr(val, "search") and re.match(val, inputs[key]): continue - break # attibute value does not match + break #: attibute value does not match else: - break # attibute name does not match + break #: attibute name does not match else: - return action, inputs # passed attribute check + return action, inputs #: passed attribute check else: # no attribute check return action, inputs - return {}, None # no matching form found + return {}, None #: no matching form found -def parseFileInfo(self, url="", html=""): - if not url and hasattr(self, "pyfile"): - url = self.pyfile.url +def parseFileInfo(plugin, url="", html=""): + info = plugin.getInfo(url, html) + return info['name'], info['size'], info['status'], info['url'] - info = {'name': url, 'size': 0, 'status': 3} - if not html: - if url: - return create_getInfo(self)([url]).next() - - elif hasattr(self, "req") and self.req.http.code == '404': - info['status'] = 1 - - elif hasattr(self, "html"): - html = self.html - - if html: - if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html): - info['status'] = 1 - - elif hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html): #@TODO: Remove in 0.4.10 - info['status'] = 1 - - elif hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, html): - info['status'] = 6 - - else: - online = False - try: - info.update(re.match(self.__pattern__, url).groupdict()) - except: - pass - - for pattern in ("INFO_PATTERN", "NAME_PATTERN", "SIZE_PATTERN", - "FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): #@TODO: Remove in 0.4.10 - try: - info.update(re.search(getattr(self, pattern), html).groupdict()) - online = True - except AttributeError: - continue - - if online: - # File online, return name and size - info['status'] = 2 +def create_getInfo(plugin): + return lambda urls: list(plugin.parseInfo(urls)) - if 'N' in info: - info['name'] = replace_patterns(info['N'].strip(), - self.FILE_NAME_REPLACEMENTS if hasattr(self, "FILE_NAME_REPLACEMENTS") else self.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 - if 'S' in info: - size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], - self.FILE_SIZE_REPLACEMENTS if hasattr(self, "FILE_SIZE_REPLACEMENTS") else self.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 - info['size'] = parseFileSize(size) +def timestamp(): + return int(time() * 1000) - elif isinstance(info['size'], basestring): - unit = info['units'] if 'units' in info else None - info['size'] = parseFileSize(info['size'], unit) - if hasattr(self, "html") and self.html is None: - self.html = html +#@TODO: Move to hoster class in 0.4.10 +def _getDirectLink(self, url): + self.req.http.c.setopt(FOLLOWLOCATION, 0) - if hasattr(self, "info"): - try: - self.logDebug(_("File info (before update): %s") % self.info) - except: - pass + html = self.load(url, ref=True, decode=True) - self.info.update(info) + self.req.http.c.setopt(FOLLOWLOCATION, 1) + if self.getInfo(url, html)['status'] is not 2: try: - self.logDebug(_("File info (after update): %s") % self.info) + return re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I).group(1).rstrip() #@TODO: Remove .rstrip() in 0.4.10 except: pass - return info['name'], info['size'], info['status'], url - - -def create_getInfo(plugin): - - def getInfo(urls): - for url in urls: - if hasattr(plugin, "COOKIES") and isinstance(plugin.COOKIES, list): - set_cookies(plugin.req.cj, plugin.COOKIES) - else: - plugin.req.cj = None - - if hasattr(plugin, "URL_REPLACEMENTS"): - url = replace_patterns(url, plugin.URL_REPLACEMENTS) - - elif hasattr(plugin, "FILE_URL_REPLACEMENTS"): #@TODO: Remove in 0.4.10 - url = replace_patterns(url, plugin.FILE_URL_REPLACEMENTS) - - if hasattr(plugin, "TEXT_ENCODING"): - html = plugin.load(url, decode=not plugin.TEXT_ENCODING, cookies=bool(plugin.COOKIES)) - if isinstance(plugin.TEXT_ENCODING, basestring): - html = unicode(html, plugin.TEXT_ENCODING) - else: - html = plugin.load(url, decode=True, cookies=bool(plugin.COOKIES)) - - yield parseFileInfo(plugin, url, html) - - return getInfo - - -def timestamp(): - return int(time() * 1000) - class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.55" + __version__ = "0.56" __pattern__ = r'^unmatchable$' @@ -245,12 +166,79 @@ class SimpleHoster(Hoster): URL_REPLACEMENTS = [] TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] + COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + CHECK_DIRECT_LINK = None #: Set to None to set True if self.account else False + + + @classmethod + def parseInfo(cls, urls): + for url in urls: + url = replace_patterns(url, cls.FILE_URL_REPLACEMENTS if hasattr(cls, "FILE_URL_REPLACEMENTS") else cls.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 + yield cls.getInfo(cls, url) + + + @classmethod + def getInfo(cls, url="", html=""): + info = {'name': url or _("Unknown"), 'size': 0, 'status': 3, 'url': url} + + if not html: + if url: + html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING) + if isinstance(cls.TEXT_ENCODING, basestring): + html = unicode(html, cls.TEXT_ENCODING) + else: + return info + + online = False + + if hasattr(cls, "OFFLINE_PATTERN") and re.search(cls.OFFLINE_PATTERN, html): + info['status'] = 1 + + elif hasattr(cls, "FILE_OFFLINE_PATTERN") and re.search(cls.FILE_OFFLINE_PATTERN, html): #@TODO: Remove in 0.4.10 + info['status'] = 1 + + elif hasattr(cls, "TEMP_OFFLINE_PATTERN") and re.search(cls.TEMP_OFFLINE_PATTERN, html): + info['status'] = 6 + + else: + try: + info.update(re.match(cls.__pattern__, url).groupdict()) + except: + pass + + for pattern in ("INFO_PATTERN", "NAME_PATTERN", "SIZE_PATTERN", + "FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): #@TODO: Remove in 0.4.10 + try: + attr = getattr(cls, pattern) + info.update(re.search(attr, html).groupdict()) + except AttributeError: + continue + else: + online = True + + if online: + info['status'] = 2 + + if 'N' in info: + info['name'] = replace_patterns(info['N'].strip(), + cls.FILE_NAME_REPLACEMENTS if hasattr(cls, "FILE_NAME_REPLACEMENTS") else cls.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 + + if 'S' in info: + size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], + cls.FILE_SIZE_REPLACEMENTS if hasattr(cls, "FILE_SIZE_REPLACEMENTS") else cls.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 + info['size'] = parseFileSize(size) + + elif isinstance(info['size'], basestring): + unit = info['units'] if 'units' in info else None + info['size'] = parseFileSize(info['size'], unit) + + return info def init(self): - self.info = {} + self.info = {} #@TODO: Remove in 0.4.10 + self.link = None #@TODO: Move to hoster class in 0.4.10 def setup(self): @@ -258,91 +246,118 @@ class SimpleHoster(Hoster): def prepare(self): - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) + if self.CHECK_DIRECT_LINK is None: + self.CHECK_DIRECT_LINK = bool(self.account) self.req.setOption("timeout", 120) + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) + self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 - if self.premium: - self.logDebug(_("Looking for direct download link...")) - direct_link = self.getDirectLink(self.pyfile.url) - if direct_link: - return direct_link - else: - self.logDebug(_("No direct download link found")) - self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + def preload(self): + self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING) if isinstance(self.TEXT_ENCODING, basestring): self.html = unicode(self.html, self.TEXT_ENCODING) def process(self, pyfile): - direct_link = self.prepare() + self.prepare() - if isinstance(direct_link, basestring): - self.logInfo(_("Direct download link detected")) - self.download(direct_link, ref=True, cookies=True, disposition=True) + if self.CHECK_DIRECT_LINK: + self.logDebug("Looking for direct download link...") + self.handleDirect() - elif self.html is None: - self.fail(_("No html retrieved")) + if not self.link: + self.preload() + + #@TODO: Remove in 0.4.10 + if self.html is None: + self.fail(_("No html retrieved")) + + info = self.getInfo(pyfile.url, self.html) + self._updateInfo(info) + + self.checkNameSize() - else: premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) if not premium_only: #: Usually premium only pages doesn't show any file information - self.getFileInfo() + self.checkStatus() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handle as premium download") + self.logDebug("Handled as premium download") self.handlePremium() + elif premium_only: self.fail(_("Link require a premium account to be handled")) + else: - self.logDebug("Handle as free download") + self.logDebug("Handled as free download") self.handleFree() + if self.link: + self.download(self.link) - def getDirectLink(self, url): - self.req.http.c.setopt(FOLLOWLOCATION, 0) - html = self.load(url, ref=True, decode=True) + def checkStatus(self): + status = self.info['status'] - self.req.http.c.setopt(FOLLOWLOCATION, 1) + if status is 1: + self.offline() - if parseFileInfo(self, url, html)[2] is not 2: - try: - return re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I).group(1).rstrip() #@TODO: Remove .rstrip() in 0.4.10 - except: - pass + elif status is 6: + self.tempOffline() + + elif status is not 2: + self.error(_("File status: %s") % filter(lambda key, val: val == status, statusMap.iteritems())[0], + _("File info: %s") % self.info) - def getFileInfo(self): - name, size, status, url = parseFileInfo(self, html=self.html) + def checkNameSize(self): + name = self.info['name'] + size = self.info['size'] + url = self.info['url'] if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split("/")[-1] + self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split('/')[-1] - if status is 1: - self.offline() + if size > 0: + self.pyfile.size = size + else: + self.logError(_("File size not found")) - elif status is 6: - self.tempOffline() + self.logDebug("File name: %s" % self.pyfile.name, "File size: %s" % self.pyfile.size or _("Unknown")) - elif status is not 2: - self.error(_("File info: %s") % self.info) - if size: - self.pyfile.size = size - else: - self.logError(_("File size not parsed")) + def checkInfo(self): + self._updateInfo(self.getInfo(self.pyfile.url, self.html or "")) + self.checkNameSize() + self.checkStatus() + + + def _updateInfo(self, info) + self.logDebug(_("File info (previous): %s") % self.info) + self.info.update(info) + self.logDebug(_("File info (current): %s") % self.info) + - self.logDebug("FILE NAME: %s" % self.pyfile.name, "FILE SIZE: %d" % self.pyfile.size or _("Unknown")) - return self.info + def handleDirect(self): + self.link = _getDirectLink(self, self.pyfile.url) + + if self.link: + self.logInfo(_("Direct download link detected")) + + self._updateInfo(self.getInfo(self.pyfile.url)) + self.checkNameSize() + + else: + self.logDebug(_("Direct download link not found")) def handleFree(self): @@ -354,11 +369,10 @@ class SimpleHoster(Hoster): if m is None: self.error(_("Free download link not found")) - link = m.group(1) + self.link = m.group(1) + except Exception, e: self.fail(str(e)) - else: - self.download(link, ref=True, cookies=True, disposition=True) def handlePremium(self): @@ -370,19 +384,18 @@ class SimpleHoster(Hoster): if m is None: self.error(_("Premium download link not found")) - link = m.group(1) + self.link = m.group(1) + except Exception, e: self.fail(str(e)) - else: - self.download(link, ref=True, cookies=True, disposition=True) def longWait(self, wait_time=None, max_tries=3): if wait_time and isinstance(wait_time, (int, long, float)): - time_str = "%dh %dm" % divmod(wait_time / 60, 60) + time_str = "%dh %dm" % divmod(wait_time / 60, 60) else: wait_time = 900 - time_str = _("(unknown time)") + time_str = _("(unknown time)") max_tries = 100 self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str) -- cgit v1.2.3 From 8bcb70d042bf93fefe6f6b3737ce0bee396bcf75 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 07:10:34 +0100 Subject: [SimpleHoster] CONTENT_DISPOSITION feature --- module/plugins/internal/SimpleHoster.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e052a79ae..9cf8eb541 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -165,10 +165,11 @@ class SimpleHoster(Hoster): SIZE_REPLACEMENTS = [] URL_REPLACEMENTS = [] - TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - CHECK_DIRECT_LINK = None #: Set to None to set True if self.account else False + CHECK_DIRECT_LINK = None #: Set to None self-set to True if self.account else to False + CONTENT_DISPOSITION = False #: Set to True to replace file name with content-disposition value in http header @classmethod @@ -237,8 +238,8 @@ class SimpleHoster(Hoster): def init(self): - self.info = {} #@TODO: Remove in 0.4.10 - self.link = None #@TODO: Move to hoster class in 0.4.10 + self.info = {} #@TODO: Remove in 0.4.10 + self.link = "" #@TODO: Move to hoster class in 0.4.10 def setup(self): @@ -300,7 +301,7 @@ class SimpleHoster(Hoster): self.handleFree() if self.link: - self.download(self.link) + self.download(self.link, disposition=self.CONTENT_DISPOSITION) def checkStatus(self): -- cgit v1.2.3 From acddd89ad9e0ce9107ac9a99443cfbecc58a1ff9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 07:46:34 +0100 Subject: [SimpleCrypter][SimpleHoster] Fix info attr init --- module/plugins/internal/SimpleHoster.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 9cf8eb541..7ab562779 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -237,16 +237,14 @@ class SimpleHoster(Hoster): return info - def init(self): - self.info = {} #@TODO: Remove in 0.4.10 - self.link = "" #@TODO: Move to hoster class in 0.4.10 - - def setup(self): self.resumeDownload = self.multiDL = self.premium def prepare(self): + self.info = {} + self.link = "" #@TODO: Move to hoster class in 0.4.10 + if self.CHECK_DIRECT_LINK is None: self.CHECK_DIRECT_LINK = bool(self.account) -- cgit v1.2.3 From e08fb5d3361627ad866aed86be9597b32c8c50f9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:23:33 +0100 Subject: [SimpleHoster] Code cosmetics --- module/plugins/internal/SimpleHoster.py | 39 ++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 15 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 7ab562779..1de872261 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -7,12 +7,16 @@ from urlparse import urlparse from pycurl import FOLLOWLOCATION -from module.PyFile import statusMap +from module.PyFile import statusMap as _statusMap from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster from module.plugins.Plugin import Fail -from module.utils import fixup, html_unescape, parseFileSize +from module.utils import fixup, parseFileSize + + +#@TODO: Adapt and move to PyFile in 0.4.10 +statusMap = {v: k for k, v in _statusMap.iteritems()} #@TODO: Remove in 0.4.10 and redirect to self.error instead @@ -21,7 +25,7 @@ def _error(self, reason, type): type = "unknown" msg = _("%s error") % type.strip().capitalize() if type else _("Error") - msg += ": " + reason.strip() if reason else "" + msg += ": %s" % reason.strip() if reason else "" msg += _(" | Plugin may be out of date") raise Fail(msg) @@ -89,11 +93,13 @@ def parseHtmlForm(attr_str, html, input_names=None): return {}, None #: no matching form found +#: Deprecated def parseFileInfo(plugin, url="", html=""): info = plugin.getInfo(url, html) return info['name'], info['size'], info['status'], info['url'] +#@TODO: Remove in 0.4.10 def create_getInfo(plugin): return lambda urls: list(plugin.parseInfo(urls)) @@ -120,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.56" + __version__ = "0.57" __pattern__ = r'^unmatchable$' @@ -279,12 +285,13 @@ class SimpleHoster(Hoster): self.fail(_("No html retrieved")) info = self.getInfo(pyfile.url, self.html) - self._updateInfo(info) + premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) + self._updateInfo(info) self.checkNameSize() - premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only: #: Usually premium only pages doesn't show any file information + #: Usually premium only pages doesn't show any file information + if not premium_only: self.checkStatus() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): @@ -312,8 +319,9 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is not 2: - self.error(_("File status: %s") % filter(lambda key, val: val == status, statusMap.iteritems())[0], - _("File info: %s") % self.info) + self.logDebug("File status: %s" % statusMap[status], + "File info: %s" % self.info) + self.error(_("No file info")) def checkNameSize(self): @@ -324,14 +332,15 @@ class SimpleHoster(Hoster): if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split('/')[-1] + self.pyfile.name = name = self.info['name'] = urlparse(name).path.split('/')[-1] if size > 0: self.pyfile.size = size else: - self.logError(_("File size not found")) + size = "Unknown" - self.logDebug("File name: %s" % self.pyfile.name, "File size: %s" % self.pyfile.size or _("Unknown")) + self.logDebug("File name: %s" % name, + "File size: %s" % size) def checkInfo(self): @@ -340,10 +349,10 @@ class SimpleHoster(Hoster): self.checkStatus() - def _updateInfo(self, info) - self.logDebug(_("File info (previous): %s") % self.info) + def _updateInfo(self, info): + self.logDebug(_("File info (before update): %s") % self.info) self.info.update(info) - self.logDebug(_("File info (current): %s") % self.info) + self.logDebug(_("File info (after update): %s") % self.info) def handleDirect(self): -- cgit v1.2.3 From 911ef70f5e39444d4b885b79ba825855b54e7606 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:59:47 +0100 Subject: [SimpleHoster] Fix create_getInfo --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1de872261..c5875a8da 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -101,7 +101,7 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 def create_getInfo(plugin): - return lambda urls: list(plugin.parseInfo(urls)) + return lambda urls: [info['name'], info['size'], info['status'], info['url'] for info in plugin.parseInfo(urls)] def timestamp(): -- cgit v1.2.3 From d85b5694472c4bf3983729a077a76dd58da25ac4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 01:46:38 +0100 Subject: [SimpleHoster] Multi-hoster and error handling support + fixes (thx MaPePeR) --- module/plugins/internal/SimpleHoster.py | 95 +++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 17 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c5875a8da..9b74c7cef 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -101,7 +101,7 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 def create_getInfo(plugin): - return lambda urls: [info['name'], info['size'], info['status'], info['url'] for info in plugin.parseInfo(urls)] + return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfo(urls)] def timestamp(): @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.57" + __version__ = "0.58" __pattern__ = r'^unmatchable$' @@ -138,7 +138,7 @@ class SimpleHoster(Hoster): """ - Following patterns should be defined by each hoster: + Info patterns should be defined by each hoster: INFO_PATTERN: (optional) Name and Size of the file example: INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' @@ -148,17 +148,26 @@ class SimpleHoster(Hoster): SIZE_PATTERN: (optional) Size that will be checked for the file example: SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' - OFFLINE_PATTERN: (optional) Checks if the file is yet available online + OFFLINE_PATTERN: (optional) Check if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN: (optional) Checks if the file is temporarily offline + TEMP_OFFLINE_PATTERN: (optional) Check if the file is temporarily offline example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' - PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account + + Error handling patterns are all optional: + + WAIT_PATTERN: (optional) Detect waiting time + example: WAIT_PATTERN = r'' + + PREMIUM_ONLY_PATTERN: (optional) Check if the file can be downloaded only with a premium account example: PREMIUM_ONLY_PATTERN = r'Premium account required' + ERROR_PATTERN: (optional) Detect any error preventing download + example: ERROR_PATTERN = r'' + - Instead overriding handleFree and handlePremium methods now you can define patterns for direct download: + Instead overriding handleFree and handlePremium methods you can define the following patterns for direct download: LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download example: LINK_FREE_PATTERN = r'
Date: Sat, 22 Nov 2014 05:15:25 +0100 Subject: [SimpleHoster] Fixup --- module/plugins/internal/SimpleHoster.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 9b74c7cef..a39531b82 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.58" + __version__ = "0.59" __pattern__ = r'^unmatchable$' @@ -192,7 +192,7 @@ class SimpleHoster(Hoster): def parseInfo(cls, urls): for url in urls: url = replace_patterns(url, cls.FILE_URL_REPLACEMENTS if hasattr(cls, "FILE_URL_REPLACEMENTS") else cls.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 - yield cls.getInfo(cls, url) + yield cls.getInfo(url) @classmethod @@ -272,7 +272,7 @@ class SimpleHoster(Hoster): and self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] and re.match(self.__pattern__, self.pyfile.url) is None): - self.logDebug("Multi hoster detected") + self.logInfo("Multi hoster detected") if self.account: self.multihost = True @@ -373,9 +373,9 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is not 2: - self.logDebug("File status: %s" % statusMap[status], - "File info: %s" % self.info) - self.error(_("No file info")) + self.logInfo(_("File status: %s") % statusMap[status], + _("File info: %s") % self.info) + self.error(_("No file info retrieved")) def checkNameSize(self): @@ -406,6 +406,11 @@ class SimpleHoster(Hoster): self.checkStatus() + #: Deprecated + def getFileInfo(self): + return self.checkInfo() + + def _updateInfo(self, info): self.logDebug(_("File info (before update): %s") % self.info) self.info.update(info) -- cgit v1.2.3 From 2e9b625a215835841bc283cf1f7275a88ea8161b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 18:52:42 +0100 Subject: [SimpleHoster] Fix statusMap generator for python 2.5 --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index a39531b82..ea298cc39 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -16,7 +16,7 @@ from module.utils import fixup, parseFileSize #@TODO: Adapt and move to PyFile in 0.4.10 -statusMap = {v: k for k, v in _statusMap.iteritems()} +statusMap = dict((v, k) for k, v in _statusMap.iteritems()) #@TODO: Remove in 0.4.10 and redirect to self.error instead @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.59" + __version__ = "0.60" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 9804b57e02d663328af048a96e60a330ac27b60d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 01:30:45 +0100 Subject: Fixes about create_getInfo --- module/plugins/internal/SimpleHoster.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ea298cc39..01702d423 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -100,6 +100,7 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 +#@NOTE: Every plugin must have own parseInfo classmethod to work with 0.4.10 def create_getInfo(plugin): return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfo(urls)] @@ -126,7 +127,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.60" + __version__ = "0.61" __pattern__ = r'^unmatchable$' @@ -197,7 +198,7 @@ class SimpleHoster(Hoster): @classmethod def getInfo(cls, url="", html=""): - info = {'name': url or _("Unknown"), 'size': 0, 'status': 3, 'url': url} + info = {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url or ""} if not html: if url: -- cgit v1.2.3 From 65d290c21e5426f515684f624b8872dec7c0bec0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 23:20:53 +0100 Subject: [SimpleHoster] Fix _getDirectLink routine --- module/plugins/internal/SimpleHoster.py | 37 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 19 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 01702d423..80ec2669b 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -19,7 +19,6 @@ from module.utils import fixup, parseFileSize statusMap = dict((v, k) for k, v in _statusMap.iteritems()) -#@TODO: Remove in 0.4.10 and redirect to self.error instead def _error(self, reason, type): if not reason and not type: type = "unknown" @@ -57,12 +56,13 @@ def parseHtmlTagAttrValue(attr_name, tag): return m.group(2) if m else None -def parseHtmlForm(attr_str, html, input_names=None): - for form in re.finditer(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, +def parseHtmlForm(attr_str, html, input_names={}): + for form in re.finditer(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, html, re.S | re.I): inputs = {} - action = parseHtmlTagAttrValue("action", form.group('tag')) - for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=]*>)([^<]*(?= Date: Mon, 24 Nov 2014 01:12:54 +0100 Subject: [XFSHoster] Code improvements --- module/plugins/internal/SimpleHoster.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 80ec2669b..95b52a017 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -19,6 +19,7 @@ from module.utils import fixup, parseFileSize statusMap = dict((v, k) for k, v in _statusMap.iteritems()) +#@TODO: Remove in 0.4.10 and redirect to self.error instead def _error(self, reason, type): if not reason and not type: type = "unknown" @@ -479,7 +480,7 @@ class SimpleHoster(Hoster): self.retry(max_tries=max_tries, reason=_("Download limit reached")) - def parseHtmlForm(self, attr_str='', input_names={}): + def parseHtmlForm(self, attr_str="", input_names={}): return parseHtmlForm(attr_str, self.html, input_names) -- cgit v1.2.3 From 8319b2fa98ee0408b4886ff474604e8199901f5c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 28 Nov 2014 21:16:49 +0100 Subject: [SimpleHoster] Fix deprecated method getFileInfo --- module/plugins/internal/SimpleHoster.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 95b52a017..d74da1301 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.62" + __version__ = "0.63" __pattern__ = r'^unmatchable$' @@ -315,8 +315,7 @@ class SimpleHoster(Hoster): premium_only = 'error' in self.info and self.info['error'] == "premium-only" - info = self.getInfo(pyfile.url, self.html) - self._updateInfo(info) + self._updateInfo(self.getInfo(pyfile.url, self.html)) self.checkNameSize() @@ -408,7 +407,9 @@ class SimpleHoster(Hoster): #: Deprecated def getFileInfo(self): - return self.checkInfo() + self.info = {} + self.checkInfo() + return self.info def _updateInfo(self, info): -- cgit v1.2.3 From a85526b5a3b1683b928ecfe9d93ec953dc29dae7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 28 Nov 2014 22:29:40 +0100 Subject: [SimpleHoster] Improve getInfo routine --- module/plugins/internal/SimpleHoster.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index d74da1301..529abce9a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.63" + __version__ = "0.64" __pattern__ = r'^unmatchable$' @@ -224,13 +224,19 @@ class SimpleHoster(Hoster): except: pass - for pattern in ("INFO_PATTERN", "NAME_PATTERN", "SIZE_PATTERN", - "FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): #@TODO: Remove in 0.4.10 + for pattern in ("FILE_INFO_PATTERN", "INFO_PATTERN", + "FILE_NAME_PATTERN", "NAME_PATTERN", + "FILE_SIZE_PATTERN", "SIZE_PATTERN"): #@TODO: Remove old patterns starting with "FILE_" in 0.4.10 try: attr = getattr(cls, pattern) - info.update(re.search(attr, html).groupdict()) + dict = re.search(attr, html).groupdict() + + if all(True for k in dict if k not in info): + info.update(dict) + except AttributeError: continue + else: online = True -- cgit v1.2.3 From 42bb9542bb2dfeafcf98630cf4a13a661a060398 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 29 Nov 2014 00:29:15 +0100 Subject: [SimpleHoster] Fix _getDirectLink --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 529abce9a..bdd84b74b 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -117,7 +117,7 @@ def _getDirectLink(self, url): if not 'location' in header or not header['location']: return "" - if header['code'] != 302 or 'content-type' not in header or header['content-type'] != "text/plain": + if header['code'] != 302 or 'content-type' in header and header['content-type'] != "text/plain": return "" return header['location'] @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.64" + __version__ = "0.65" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 00277a73a3754a58d1e09b44348a75e0dfaa5f61 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 29 Nov 2014 00:47:04 +0100 Subject: [SimpleHoster] Fix _getDirectLink (2) --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index bdd84b74b..274d95154 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -117,7 +117,7 @@ def _getDirectLink(self, url): if not 'location' in header or not header['location']: return "" - if header['code'] != 302 or 'content-type' in header and header['content-type'] != "text/plain": + if header['code'] != 302 or 'content-type' in header and "text/plain" not in header['content-type']: return "" return header['location'] @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.65" + __version__ = "0.66" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From d4719096cad1127540c32bb22345ebcba8617fa9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 29 Nov 2014 23:02:24 +0100 Subject: [SimpleHoster] Fix _getDirectLink (3) --- module/plugins/internal/SimpleHoster.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 274d95154..24a2fa6b0 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -5,8 +5,6 @@ import re from time import time from urlparse import urlparse -from pycurl import FOLLOWLOCATION - from module.PyFile import statusMap as _statusMap from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL @@ -34,7 +32,11 @@ def _error(self, reason, type): #@TODO: Remove in 0.4.10 def _wait(self, seconds, reconnect): if seconds: - self.setWait(seconds, reconnect) + self.setWait(seconds) + + if reconnect is not None: + self.wantReconnect = reconnect + super(SimpleHoster, self).wait() @@ -114,19 +116,22 @@ def timestamp(): def _getDirectLink(self, url): header = self.load(url, ref=True, just_header=True, decode=True) - if not 'location' in header or not header['location']: + if not 'code' in header or header['code'] != 302: return "" - if header['code'] != 302 or 'content-type' in header and "text/plain" not in header['content-type']: + if not 'location' in header or not header['location']: return "" + # if 'content-type' in header and "text/plain" not in header['content-type']: + # return "" + return header['location'] class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.66" + __version__ = "0.67" __pattern__ = r'^unmatchable$' @@ -197,7 +202,7 @@ class SimpleHoster(Hoster): @classmethod def getInfo(cls, url="", html=""): - info = {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url or ""} + info = {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3 if url else 1, 'url': url or ""} if not html: if url: -- cgit v1.2.3 From bc3f5f4903444e438a8a07688c82f0c2995dafa9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 30 Nov 2014 21:15:54 +0100 Subject: [SimpleHoster] Unquote file name --- module/plugins/internal/SimpleHoster.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 24a2fa6b0..f741cd691 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -3,6 +3,7 @@ import re from time import time +from urllib import unquote from urlparse import urlparse from module.PyFile import statusMap as _statusMap @@ -131,7 +132,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.67" + __version__ = "0.68" __pattern__ = r'^unmatchable$' @@ -249,7 +250,7 @@ class SimpleHoster(Hoster): info['status'] = 2 if 'N' in info: - info['name'] = replace_patterns(info['N'].strip(), + info['name'] = replace_patterns(unquote(info['N'].strip()), cls.FILE_NAME_REPLACEMENTS if hasattr(cls, "FILE_NAME_REPLACEMENTS") else cls.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 if 'S' in info: -- cgit v1.2.3 From 7ef266b8c5681236111fe3e71acbb7eaeefd543f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 1 Dec 2014 18:10:59 +0100 Subject: [SimpleHoster] Fix some stuff, improve others --- module/plugins/internal/SimpleHoster.py | 123 +++++++++++++++++++++----------- 1 file changed, 83 insertions(+), 40 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index f741cd691..736d15f3a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -4,7 +4,7 @@ import re from time import time from urllib import unquote -from urlparse import urlparse +from urlparse import urljoin, urlparse from module.PyFile import statusMap as _statusMap from module.network.CookieJar import CookieJar @@ -33,7 +33,7 @@ def _error(self, reason, type): #@TODO: Remove in 0.4.10 def _wait(self, seconds, reconnect): if seconds: - self.setWait(seconds) + self.setWait(seconds + 1) if reconnect is not None: self.wantReconnect = reconnect @@ -114,25 +114,35 @@ def timestamp(): #@TODO: Move to hoster class in 0.4.10 -def _getDirectLink(self, url): +def _isDirectLink(self, url, resumable=True): header = self.load(url, ref=True, just_header=True, decode=True) - if not 'code' in header or header['code'] != 302: - return "" - if not 'location' in header or not header['location']: return "" - # if 'content-type' in header and "text/plain" not in header['content-type']: - # return "" + location = header['location'] + + if resumable: #: sometimes http code may be wrong... + if self.load(location, ref=True, cookies=True, just_header=True, decode=True)['location']: + return "" + else: + if not 'code' in header or header['code'] != 302: + return "" - return header['location'] + if urlparse(location).scheme: + link = location + else: + p = urlparse(url) + base = "%s://%s" % (p.scheme, p.netloc) + link = urljoin(base, location) + + return link class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.68" + __version__ = "0.69" __pattern__ = r'^unmatchable$' @@ -154,6 +164,9 @@ class SimpleHoster(Hoster): SIZE_PATTERN: (optional) Size that will be checked for the file example: SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' + HASHSUM_PATTERN: (optional) Hash code and type of the file + example: HASHSUM_PATTERN = r'(?Phash_code) (?PMD5)' + OFFLINE_PATTERN: (optional) Check if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' @@ -189,9 +202,9 @@ class SimpleHoster(Hoster): TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - CHECK_DIRECT_LINK = None #: when None self-set to True if self.account else False - MULTI_HOSTER = False #: Set to True to leech other hoster link - CONTENT_DISPOSITION = False #: Set to True to replace file name with content-disposition value in http header + CHECK_DIRECT_LINK = None #: Set to True to check for direct link, set to None to do it only if self.account is True + MULTI_HOSTER = False #: Set to True to leech other hoster link (according its multihoster hook if available) + CONTENT_DISPOSITION = False #: Set to True to replace file name with content-disposition value from http header @classmethod @@ -203,14 +216,32 @@ class SimpleHoster(Hoster): @classmethod def getInfo(cls, url="", html=""): - info = {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3 if url else 1, 'url': url or ""} + info = {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url} if not html: - if url: - html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING) - if isinstance(cls.TEXT_ENCODING, basestring): - html = unicode(html, cls.TEXT_ENCODING) - else: + try: + if not url: + info['error'] = "missing url" + info['status'] = 1 + raise + + try: + html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING) + + if isinstance(cls.TEXT_ENCODING, basestring): + html = unicode(html, cls.TEXT_ENCODING) + + except BadHeader, e: + info['error'] = "%d: %s" % (e.code, e.content) + + if e.code is 404: + info['status'] = 1 + raise + + if e.code is 503: + info['status'] = 6 + raise + except: return info online = False @@ -226,19 +257,20 @@ class SimpleHoster(Hoster): else: try: - info.update(re.match(cls.__pattern__, url).groupdict()) + info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here, please save api stuff to info['api'] except: pass for pattern in ("FILE_INFO_PATTERN", "INFO_PATTERN", "FILE_NAME_PATTERN", "NAME_PATTERN", - "FILE_SIZE_PATTERN", "SIZE_PATTERN"): #@TODO: Remove old patterns starting with "FILE_" in 0.4.10 + "FILE_SIZE_PATTERN", "SIZE_PATTERN", + "HASHSUM_PATTERN"): #@TODO: Remove old patterns starting with "FILE_" in 0.4.10 try: attr = getattr(cls, pattern) dict = re.search(attr, html).groupdict() - if all(True for k in dict if k not in info): - info.update(dict) + if all(True for k in dict if k not in info['pattern']): + info['pattern'].update(dict) except AttributeError: continue @@ -249,12 +281,12 @@ class SimpleHoster(Hoster): if online: info['status'] = 2 - if 'N' in info: - info['name'] = replace_patterns(unquote(info['N'].strip()), + if 'N' in info['pattern']: + info['name'] = replace_patterns(unquote(info['pattern']['N'].strip()), cls.FILE_NAME_REPLACEMENTS if hasattr(cls, "FILE_NAME_REPLACEMENTS") else cls.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 - if 'S' in info: - size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], + if 'S' in info['pattern']: + size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info else info['pattern']['S'], cls.FILE_SIZE_REPLACEMENTS if hasattr(cls, "FILE_SIZE_REPLACEMENTS") else cls.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 info['size'] = parseFileSize(size) @@ -262,6 +294,10 @@ class SimpleHoster(Hoster): unit = info['units'] if 'units' in info else None info['size'] = parseFileSize(info['size'], unit) + if 'H' in info['pattern']: + hashtype = info['pattern']['T'] if 'T' in info['pattern'] else "hash" + info[hashtype] = info['pattern']['H'] + return info @@ -349,15 +385,20 @@ class SimpleHoster(Hoster): if self.link: self.download(self.link, disposition=self.CONTENT_DISPOSITION) + self.checkFile() + + + def checkFile(self) + if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": #@TODO: Move to hoster in 0.4.10 + self.fail(_("Empty file")) + def checkErrors(self): - if hasattr(self, 'WAIT_PATTERN'): - m = re.search(self.WAIT_PATTERN, self.html) + if hasattr(self, 'ERROR_PATTERN'): + m = re.search(self.ERROR_PATTERN, self.html) if m: - wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec)', m, re.I)]) - self.wait(wait_time, False) - return + e = self.info['error'] = m.group(1) + self.error(e) if hasattr(self, 'PREMIUM_ONLY_PATTERN'): m = re.search(self.PREMIUM_ONLY_PATTERN, self.html) @@ -365,11 +406,13 @@ class SimpleHoster(Hoster): self.info['error'] = "premium-only" return - if hasattr(self, 'ERROR_PATTERN'): - m = re.search(self.ERROR_PATTERN, self.html) + if hasattr(self, 'WAIT_PATTERN'): + m = re.search(self.WAIT_PATTERN, self.html) if m: - e = self.info['error'] = m.group(1) - self.error(e) + wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec)', m, re.I)]) + self.wait(wait_time, False) + return self.info.pop('error', None) @@ -431,7 +474,7 @@ class SimpleHoster(Hoster): def handleDirect(self): - link = _getDirectLink(self, self.pyfile.url) + link = _isDirectLink(self, self.pyfile.url, self.resumeDownload) if link: self.logInfo(_("Direct download link detected")) @@ -460,7 +503,7 @@ class SimpleHoster(Hoster): self.link = m.group(1) except Exception, e: - self.fail(str(e)) + self.fail(e) def handlePremium(self): @@ -475,7 +518,7 @@ class SimpleHoster(Hoster): self.link = m.group(1) except Exception, e: - self.fail(str(e)) + self.fail(e) def longWait(self, wait_time=None, max_tries=3): -- cgit v1.2.3 From 1bcdfb409ded0c369735c413b33f2e344b36bd7a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 1 Dec 2014 20:00:14 +0100 Subject: Fixes --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 736d15f3a..cf03a1242 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -123,7 +123,7 @@ def _isDirectLink(self, url, resumable=True): location = header['location'] if resumable: #: sometimes http code may be wrong... - if self.load(location, ref=True, cookies=True, just_header=True, decode=True)['location']: + if 'location' in self.load(location, ref=True, cookies=True, just_header=True, decode=True): return "" else: if not 'code' in header or header['code'] != 302: @@ -388,7 +388,7 @@ class SimpleHoster(Hoster): self.checkFile() - def checkFile(self) + def checkFile(self): if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": #@TODO: Move to hoster in 0.4.10 self.fail(_("Empty file")) -- cgit v1.2.3 From eb71f308010e35aaec7f7bfa529ada96975c909d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 2 Dec 2014 00:04:41 +0100 Subject: [SimpleHoster] Force _isDirectLink to old style --- module/plugins/internal/SimpleHoster.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index cf03a1242..ba718950d 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -122,6 +122,8 @@ def _isDirectLink(self, url, resumable=True): location = header['location'] + resumable = False #@NOTE: Testing... + if resumable: #: sometimes http code may be wrong... if 'location' in self.load(location, ref=True, cookies=True, just_header=True, decode=True): return "" -- cgit v1.2.3 From be05dce2da77bca4613efbdfb0f6357a983e6e1c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 3 Dec 2014 14:38:22 +0100 Subject: Fix https://github.com/pyload/pyload/issues/943 --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ba718950d..809e89c02 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -33,7 +33,7 @@ def _error(self, reason, type): #@TODO: Remove in 0.4.10 def _wait(self, seconds, reconnect): if seconds: - self.setWait(seconds + 1) + self.setWait(int(seconds) + 1) if reconnect is not None: self.wantReconnect = reconnect @@ -144,7 +144,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.69" + __version__ = "0.70" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 5a23de6f32dc99960cccd634d2fd2cc47c34be2a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 5 Dec 2014 23:08:44 +0100 Subject: Code cosmetics about checkErrors --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 809e89c02..f391445fd 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -399,8 +399,8 @@ class SimpleHoster(Hoster): if hasattr(self, 'ERROR_PATTERN'): m = re.search(self.ERROR_PATTERN, self.html) if m: - e = self.info['error'] = m.group(1) - self.error(e) + errmsg = self.info['error'] = m.group(1) + self.error(errmsg) if hasattr(self, 'PREMIUM_ONLY_PATTERN'): m = re.search(self.PREMIUM_ONLY_PATTERN, self.html) -- cgit v1.2.3 From ef0d8692b04d75f4a39a565d52c2eb381eb43055 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 6 Dec 2014 11:03:57 +0100 Subject: [SimpleHoster] Fix multihost --- module/plugins/internal/SimpleHoster.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index f391445fd..530b67692 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -144,7 +144,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.70" + __version__ = "0.71" __pattern__ = r'^unmatchable$' @@ -319,8 +319,8 @@ class SimpleHoster(Hoster): set_cookies(self.req.cj, self.COOKIES) if (self.MULTI_HOSTER - and self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] - and re.match(self.__pattern__, self.pyfile.url) is None): + and (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] + or re.match(self.__pattern__, self.pyfile.url) is None)): self.logInfo("Multi hoster detected") @@ -384,12 +384,17 @@ class SimpleHoster(Hoster): self.logDebug("Handled as free download") self.handleFree() - if self.link: - self.download(self.link, disposition=self.CONTENT_DISPOSITION) - + self.downloadLink(self.link) self.checkFile() + def downloadLink(self, link): + if not link: + return + + self.download(link, disposition=self.CONTENT_DISPOSITION) + + def checkFile(self): if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": #@TODO: Move to hoster in 0.4.10 self.fail(_("Empty file")) -- cgit v1.2.3 From 1fe525b92affd6d3829af219a9b1b5eeda136ca2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 12 Dec 2014 20:31:37 +0100 Subject: [SimpleHoster] Remove CONTENT_DISPOSITION + Rename parseInfo to parseInfos --- module/plugins/internal/SimpleHoster.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 530b67692..ddaea020a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -104,9 +104,9 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 -#@NOTE: Every plugin must have own parseInfo classmethod to work with 0.4.10 +#@NOTE: Every plugin must have own parseInfos classmethod to work with 0.4.10 def create_getInfo(plugin): - return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfo(urls)] + return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfos(urls)] def timestamp(): @@ -144,7 +144,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.71" + __version__ = "0.72" __pattern__ = r'^unmatchable$' @@ -206,11 +206,10 @@ class SimpleHoster(Hoster): FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account CHECK_DIRECT_LINK = None #: Set to True to check for direct link, set to None to do it only if self.account is True MULTI_HOSTER = False #: Set to True to leech other hoster link (according its multihoster hook if available) - CONTENT_DISPOSITION = False #: Set to True to replace file name with content-disposition value from http header @classmethod - def parseInfo(cls, urls): + def parseInfos(cls, urls): for url in urls: url = replace_patterns(url, cls.FILE_URL_REPLACEMENTS if hasattr(cls, "FILE_URL_REPLACEMENTS") else cls.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 yield cls.getInfo(url) @@ -392,7 +391,7 @@ class SimpleHoster(Hoster): if not link: return - self.download(link, disposition=self.CONTENT_DISPOSITION) + self.download(link, disposition=True) def checkFile(self): -- cgit v1.2.3 From c9b4f3b9a78904d34d7e58593eacdc26b93c2562 Mon Sep 17 00:00:00 2001 From: philou75 Date: Fri, 12 Dec 2014 22:49:23 +0100 Subject: Make the waiting time work again on OneFichier.com --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index f391445fd..47b713173 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -144,7 +144,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.70" + __version__ = "0.71" __pattern__ = r'^unmatchable$' @@ -412,7 +412,7 @@ class SimpleHoster(Hoster): m = re.search(self.WAIT_PATTERN, self.html) if m: wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec)', m, re.I)]) + re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I)]) self.wait(wait_time, False) return -- cgit v1.2.3 From a51d35534fc38650dd7714572f6874ab262bd119 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 16 Dec 2014 23:03:05 +0100 Subject: [SimpleHoster] Improve and fix --- module/plugins/internal/SimpleHoster.py | 58 +++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 24 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 992454451..86d9b4c9a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -99,14 +99,24 @@ def parseHtmlForm(attr_str, html, input_names={}): #: Deprecated def parseFileInfo(plugin, url="", html=""): - info = plugin.getInfo(url, html) - return info['name'], info['size'], info['status'], info['url'] + if hasattr(plugin, "getInfo"): + info = plugin.getInfo(url, html) + res = info['name'], info['size'], info['status'], info['url'] + else: + res = urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 0, 3, url + + return res #@TODO: Remove in 0.4.10 #@NOTE: Every plugin must have own parseInfos classmethod to work with 0.4.10 def create_getInfo(plugin): - return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfos(urls)] + if hasattr(plugin, "parseInfos"): + fn = lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfos(urls)] + else: + fn = lambda urls: [parseFileInfo(url) for url in urls] + + return fn def timestamp(): @@ -144,7 +154,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.73" + __version__ = "0.74" __pattern__ = r'^unmatchable$' @@ -260,18 +270,18 @@ class SimpleHoster(Hoster): try: info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here, please save api stuff to info['api'] except: - pass + info['pattern'] = {} for pattern in ("FILE_INFO_PATTERN", "INFO_PATTERN", "FILE_NAME_PATTERN", "NAME_PATTERN", "FILE_SIZE_PATTERN", "SIZE_PATTERN", "HASHSUM_PATTERN"): #@TODO: Remove old patterns starting with "FILE_" in 0.4.10 try: - attr = getattr(cls, pattern) - dict = re.search(attr, html).groupdict() + attr = getattr(cls, pattern) + pdict = re.search(attr, html).groupdict() - if all(True for k in dict if k not in info['pattern']): - info['pattern'].update(dict) + if all(True for k in pdict if k not in info['pattern']): + info['pattern'].update(pdict) except AttributeError: continue @@ -279,6 +289,9 @@ class SimpleHoster(Hoster): else: online = True + if not info['pattern']: + info.pop('pattern', None) + if online: info['status'] = 2 @@ -360,18 +373,21 @@ class SimpleHoster(Hoster): if self.html is None: self.fail(_("No html retrieved")) - self.checkErrors() - - premium_only = 'error' in self.info and self.info['error'] == "premium-only" - - self._updateInfo(self.getInfo(pyfile.url, self.html)) + self.updateInfo(self.getInfo(pyfile.url, self.html)) self.checkNameSize() + if hasattr(self, 'PREMIUM_ONLY_PATTERN'): + premium_only = re.search(self.PREMIUM_ONLY_PATTERN, self.html) + else: + premium_only = False + #: Usually premium only pages doesn't show any file information if not premium_only: self.checkStatus() + self.checkErrors() + if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() @@ -406,18 +422,12 @@ class SimpleHoster(Hoster): errmsg = self.info['error'] = m.group(1) self.error(errmsg) - if hasattr(self, 'PREMIUM_ONLY_PATTERN'): - m = re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if m: - self.info['error'] = "premium-only" - return - if hasattr(self, 'WAIT_PATTERN'): m = re.search(self.WAIT_PATTERN, self.html) if m: wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I)]) - self.wait(wait_time, True if wait_time > 300 else False) + self.wait(wait_time, wait_time > 300) return self.info.pop('error', None) @@ -460,7 +470,7 @@ class SimpleHoster(Hoster): def checkInfo(self): self.checkErrors() - self._updateInfo(self.getInfo(self.pyfile.url, self.html or "")) + self.updateInfo(self.getInfo(self.pyfile.url, self.html or "")) self.checkNameSize() self.checkStatus() @@ -473,7 +483,7 @@ class SimpleHoster(Hoster): return self.info - def _updateInfo(self, info): + def updateInfo(self, info): self.logDebug(_("File info (before update): %s") % self.info) self.info.update(info) self.logDebug(_("File info (after update): %s") % self.info) @@ -487,7 +497,7 @@ class SimpleHoster(Hoster): self.link = link - self._updateInfo(self.getInfo(self.pyfile.url)) + self.updateInfo(self.getInfo(self.pyfile.url)) self.checkNameSize() else: self.logDebug(_("Direct download link not found")) -- cgit v1.2.3 From 69616099c133fdc783abad44c2a0916acba4b6d1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 17 Dec 2014 22:38:24 +0100 Subject: [SimpleHoster] Fix filesize recognition --- module/plugins/internal/SimpleHoster.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 86d9b4c9a..ad140423c 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -154,7 +154,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.74" + __version__ = "0.75" __pattern__ = r'^unmatchable$' @@ -300,7 +300,7 @@ class SimpleHoster(Hoster): cls.FILE_NAME_REPLACEMENTS if hasattr(cls, "FILE_NAME_REPLACEMENTS") else cls.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 if 'S' in info['pattern']: - size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info else info['pattern']['S'], + size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info['pattern'] else info['pattern']['S'], cls.FILE_SIZE_REPLACEMENTS if hasattr(cls, "FILE_SIZE_REPLACEMENTS") else cls.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 info['size'] = parseFileSize(size) @@ -361,6 +361,7 @@ class SimpleHoster(Hoster): if self.multihost: self.logDebug("Looking for leeched download link...") + self.logDebug("File url: %s" % self.pyfile.url) self.handleMulti() elif self.directDL: @@ -404,10 +405,8 @@ class SimpleHoster(Hoster): def downloadLink(self, link): - if not link: - return - - self.download(link, disposition=True) + if link and isinstance(link, basestring): + self.download(link, disposition=True) def checkFile(self): -- cgit v1.2.3 From 0860e09f5ff16ee3f097f6f9d444f277a38abd72 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 17 Dec 2014 23:03:46 +0100 Subject: Extend SimpleHoster in multi-hoster plugins (3) --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ad140423c..2ff60d6ea 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -361,7 +361,7 @@ class SimpleHoster(Hoster): if self.multihost: self.logDebug("Looking for leeched download link...") - self.logDebug("File url: %s" % self.pyfile.url) + self.logDebug("File url: %s" % pyfile.url) self.handleMulti() elif self.directDL: -- cgit v1.2.3 From 4e9c8f7ab1269966a9eac9e1b6363f5458f9f970 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 18 Dec 2014 16:07:21 +0100 Subject: Update checkFile routine in some hoster plugins --- module/plugins/internal/SimpleHoster.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2ff60d6ea..949a6c66a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -2,6 +2,7 @@ import re +from os.path import exists from time import time from urllib import unquote from urlparse import urljoin, urlparse @@ -11,7 +12,7 @@ from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster from module.plugins.Plugin import Fail -from module.utils import fixup, parseFileSize +from module.utils import fixup, fs_encode, parseFileSize #@TODO: Adapt and move to PyFile in 0.4.10 @@ -154,7 +155,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.75" + __version__ = "0.76" __pattern__ = r'^unmatchable$' @@ -410,8 +411,19 @@ class SimpleHoster(Hoster): def checkFile(self): - if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": #@TODO: Move to hoster in 0.4.10 - self.fail(_("Empty file")) + if not self.lastDownload or not exists(fs_encode(self.lastDownload)): + self.fail(_("No file downloaded")) + + else: + rules = {'empty file': re.compile(r"^$")} + + if hasattr(self, 'ERROR_PATTERN'): + rules['error'] = re.compile(self.ERROR_PATTERN) + + check = self.checkDownload(rules) + if check: #@TODO: Move to hoster in 0.4.10 + errmsg = check.strip().capitalize() + (" | " + self.lastCheck.strip() if self.lastCheck else "") + self.retry(10, 60, errmsg) def checkErrors(self): -- cgit v1.2.3 From 6d9c5ff2e06e06e33fa6674f013dd0e290092bbd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 20 Dec 2014 14:25:57 +0100 Subject: [SimpleHoster] Improve checkFile --- module/plugins/internal/SimpleHoster.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 949a6c66a..784ef8c85 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -155,7 +155,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.76" + __version__ = "0.77" __pattern__ = r'^unmatchable$' @@ -407,11 +407,16 @@ class SimpleHoster(Hoster): def downloadLink(self, link): if link and isinstance(link, basestring): + self.correctCaptcha() self.download(link, disposition=True) def checkFile(self): - if not self.lastDownload or not exists(fs_encode(self.lastDownload)): + if self.cTask and not self.lastDownload: + self.invalidCaptcha() + self.retry(10, reason=_("Wrong captcha")) + + elif not self.lastDownload or not exists(fs_encode(self.lastDownload)): self.fail(_("No file downloaded")) else: -- cgit v1.2.3 From e1d4186c62512d8bb76d35b6f8d1828d8d0aa94e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 23 Dec 2014 13:26:32 +0100 Subject: [SimpleHoster] Improve multi-hoster feature --- module/plugins/internal/SimpleHoster.py | 38 ++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 784ef8c85..ab114c9ec 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -155,7 +155,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.77" + __version__ = "0.78" __pattern__ = r'^unmatchable$' @@ -214,9 +214,9 @@ class SimpleHoster(Hoster): TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct COOKIES = True #: or False or list of tuples [(domain, name, value)] - FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - CHECK_DIRECT_LINK = None #: Set to True to check for direct link, set to None to do it only if self.account is True - MULTI_HOSTER = False #: Set to True to leech other hoster link (according its multihoster hook if available) + CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + DIRECT_LINK = None #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False + MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) @classmethod @@ -334,17 +334,13 @@ class SimpleHoster(Hoster): if (self.MULTI_HOSTER and (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] or re.match(self.__pattern__, self.pyfile.url) is None)): + self.multihost = True + return - self.logInfo("Multi hoster detected") - - if self.account: - self.multihost = True - return - else: - self.fail(_("Only registered or premium users can use url leech feature")) - - if self.CHECK_DIRECT_LINK is None: + if self.DIRECT_LINK is None: self.directDL = bool(self.account) + else: + self.directDL = self.DIRECT_LINK self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 @@ -360,16 +356,20 @@ class SimpleHoster(Hoster): def process(self, pyfile): self.prepare() - if self.multihost: + if self.directDL: + self.logDebug("Looking for direct download link...") + self.handleDirect() + + if self.multihost and not self.link and not self.lastDownload: self.logDebug("Looking for leeched download link...") self.logDebug("File url: %s" % pyfile.url) self.handleMulti() - elif self.directDL: - self.logDebug("Looking for direct download link...") - self.handleDirect() + if not self.link and not self.lastDownload: + self.MULTI_HOSTER = False + self.retry(1, reason="Multi hoster fails") - if not self.link: + if not self.link and not self.lastDownload: self.preload() if self.html is None: @@ -390,7 +390,7 @@ class SimpleHoster(Hoster): self.checkErrors() - if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() -- cgit v1.2.3 From badfd177fb3639869441f4738461c7f576e1d078 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 23 Dec 2014 17:22:47 +0100 Subject: [SimpleHoster] Update --- module/plugins/internal/SimpleHoster.py | 81 +++++++++++++++------------------ 1 file changed, 37 insertions(+), 44 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ab114c9ec..95833d0cf 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -125,29 +125,34 @@ def timestamp(): #@TODO: Move to hoster class in 0.4.10 -def _isDirectLink(self, url, resumable=True): - header = self.load(url, ref=True, just_header=True, decode=True) +def _isDirectLink(self, url, resumable=False): + link = "" - if not 'location' in header or not header['location']: - return "" + for i in xrange(5 if resumable else 1): + header = self.load(url, ref=True, cookies=True, just_header=True, decode=True) - location = header['location'] + if 'content-disposition' in header: + link = url - resumable = False #@NOTE: Testing... + elif 'location' in header and header['location']: + location = header['location'] - if resumable: #: sometimes http code may be wrong... - if 'location' in self.load(location, ref=True, cookies=True, just_header=True, decode=True): - return "" - else: - if not 'code' in header or header['code'] != 302: - return "" + if not urlparse(location).scheme: + p = urlparse(url) + base = "%s://%s" % (p.scheme, p.netloc) + location = urljoin(base, location) + + if 'code' in header and header['code'] == 302: + link = location + + elif resumable: + url = location + self.logDebug("Redirect #%d to: %s" % (++i, location)) + continue - if urlparse(location).scheme: - link = location + break else: - p = urlparse(url) - base = "%s://%s" % (p.scheme, p.netloc) - link = urljoin(base, location) + self.logError(_("Too many redirects")) return link @@ -155,7 +160,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.78" + __version__ = "0.79" __pattern__ = r'^unmatchable$' @@ -355,6 +360,7 @@ class SimpleHoster(Hoster): def process(self, pyfile): self.prepare() + self.checkInfo() if self.directDL: self.logDebug("Looking for direct download link...") @@ -371,32 +377,15 @@ class SimpleHoster(Hoster): if not self.link and not self.lastDownload: self.preload() + self.checkInfo() if self.html is None: self.fail(_("No html retrieved")) - self.updateInfo(self.getInfo(pyfile.url, self.html)) - - self.checkNameSize() - - if hasattr(self, 'PREMIUM_ONLY_PATTERN'): - premium_only = re.search(self.PREMIUM_ONLY_PATTERN, self.html) - else: - premium_only = False - - #: Usually premium only pages doesn't show any file information - if not premium_only: - self.checkStatus() - - self.checkErrors() - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() - elif premium_only: - self.fail(_("Link require a premium account to be handled")) - else: self.logDebug("Handled as free download") self.handleFree() @@ -432,6 +421,9 @@ class SimpleHoster(Hoster): def checkErrors(self): + if hasattr(self, 'PREMIUM_ONLY_PATTERN') and self.premium and re.search(self.PREMIUM_ONLY_PATTERN, self.html): + self.fail(_("Link require a premium account to be handled")) + if hasattr(self, 'ERROR_PATTERN'): m = re.search(self.ERROR_PATTERN, self.html) if m: @@ -459,9 +451,8 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is not 2: - self.logInfo(_("File status: %s") % statusMap[status], - _("File info: %s") % self.info) - self.error(_("No file info retrieved")) + self.logDebug(_("File status: %s") % statusMap[status], + _("File info: %s") % self.info) def checkNameSize(self): @@ -484,9 +475,14 @@ class SimpleHoster(Hoster): def checkInfo(self): - self.checkErrors() + self.updateInfo(self.getInfo(self.pyfile.url, self.html)) - self.updateInfo(self.getInfo(self.pyfile.url, self.html or "")) + self.checkNameSize() + + if self.html: + self.checkErrors() + + self.updateInfo(self.getInfo(self.pyfile.url, self.html)) self.checkNameSize() self.checkStatus() @@ -512,9 +508,6 @@ class SimpleHoster(Hoster): self.logInfo(_("Direct download link detected")) self.link = link - - self.updateInfo(self.getInfo(self.pyfile.url)) - self.checkNameSize() else: self.logDebug(_("Direct download link not found")) -- cgit v1.2.3 From 9c9843e6c7681ef52a882033eaabb44afacc5b8e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 23 Dec 2014 18:51:01 +0100 Subject: [SimpleHoster] Code cosmetics --- module/plugins/internal/SimpleHoster.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 95833d0cf..6726726e1 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -368,7 +368,6 @@ class SimpleHoster(Hoster): if self.multihost and not self.link and not self.lastDownload: self.logDebug("Looking for leeched download link...") - self.logDebug("File url: %s" % pyfile.url) self.handleMulti() if not self.link and not self.lastDownload: @@ -451,8 +450,8 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is not 2: - self.logDebug(_("File status: %s") % statusMap[status], - _("File info: %s") % self.info) + self.logDebug("File status: %s" % statusMap[status], + "File info: %s" % self.info) def checkNameSize(self): @@ -496,9 +495,9 @@ class SimpleHoster(Hoster): def updateInfo(self, info): - self.logDebug(_("File info (before update): %s") % self.info) + self.logDebug(_("File info (BEFORE): %s") % self.info) self.info.update(info) - self.logDebug(_("File info (after update): %s") % self.info) + self.logDebug(_("File info (AFTER): %s") % self.info) def handleDirect(self): -- cgit v1.2.3 From 873f1bc17733148163829e23a456a6e724c486e3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Dec 2014 04:26:20 +0100 Subject: [SimpleHoster] Improve --- module/plugins/internal/SimpleHoster.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6726726e1..56c25e0d9 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -160,7 +160,7 @@ def _isDirectLink(self, url, resumable=False): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.79" + __version__ = "0.80" __pattern__ = r'^unmatchable$' @@ -381,7 +381,7 @@ class SimpleHoster(Hoster): if self.html is None: self.fail(_("No html retrieved")) - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + if self.premium and not self.CHECK_TRAFFIC or self.checkTrafficLeft(): self.logDebug("Handled as premium download") self.handlePremium() @@ -405,7 +405,11 @@ class SimpleHoster(Hoster): self.retry(10, reason=_("Wrong captcha")) elif not self.lastDownload or not exists(fs_encode(self.lastDownload)): - self.fail(_("No file downloaded")) + errmsg = _("No file downloaded") + if 'error' in self.info: + self.fail(errmsg, self.info['error']) + else: + self.fail(errmsg) else: rules = {'empty file': re.compile(r"^$")} -- cgit v1.2.3 From c7187f2142c6e06be961bb3be45ea3b61ce31651 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Dec 2014 17:24:41 +0100 Subject: [SimpleHoster] getInfo handle direct link error --- module/plugins/internal/SimpleHoster.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 56c25e0d9..89d2e629c 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -160,7 +160,7 @@ def _isDirectLink(self, url, resumable=False): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.80" + __version__ = "0.81" __pattern__ = r'^unmatchable$' @@ -233,7 +233,13 @@ class SimpleHoster(Hoster): @classmethod def getInfo(cls, url="", html=""): - info = {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url} + info = {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url} + online = False + + try: + info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here, please save api stuff to info['api'] + except Exception: + pass if not html: try: @@ -242,6 +248,11 @@ class SimpleHoster(Hoster): info['status'] = 1 raise + if _isDirectLink(url): + info['error'] = "direct link" + info['status'] = 2 + raise + try: html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING) @@ -261,8 +272,6 @@ class SimpleHoster(Hoster): except: return info - online = False - if hasattr(cls, "OFFLINE_PATTERN") and re.search(cls.OFFLINE_PATTERN, html): info['status'] = 1 @@ -273,9 +282,7 @@ class SimpleHoster(Hoster): info['status'] = 6 else: - try: - info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here, please save api stuff to info['api'] - except: + if not 'pattern' in info: info['pattern'] = {} for pattern in ("FILE_INFO_PATTERN", "INFO_PATTERN", @@ -381,7 +388,7 @@ class SimpleHoster(Hoster): if self.html is None: self.fail(_("No html retrieved")) - if self.premium and not self.CHECK_TRAFFIC or self.checkTrafficLeft(): + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() -- cgit v1.2.3 From 2260302b71a803da708f3ec6a0cc2200baae527b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 27 Dec 2014 13:44:36 +0100 Subject: [SimpleHoster] Improve _isDirectLink + move here secondsToMidnight routine + spare code cosmetics --- module/plugins/internal/SimpleHoster.py | 55 +++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 16 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 89d2e629c..d9732d063 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -131,7 +131,7 @@ def _isDirectLink(self, url, resumable=False): for i in xrange(5 if resumable else 1): header = self.load(url, ref=True, cookies=True, just_header=True, decode=True) - if 'content-disposition' in header: + if 'content-disposition' in header or 'content-length' in header: link = url elif 'location' in header and header['location']: @@ -150,6 +150,9 @@ def _isDirectLink(self, url, resumable=False): self.logDebug("Redirect #%d to: %s" % (++i, location)) continue + elif 'content-type' in header and header['content-type' ] and "html" not in header['content-type']: + link = url + break else: self.logError(_("Too many redirects")) @@ -157,10 +160,28 @@ def _isDirectLink(self, url, resumable=False): return link +def secondsToMidnight(gmt=0): + now = datetime.utcnow() + timedelta(hours=gmt) + + if now.hour is 0 and now.minute < 10: + midnight = now + else: + midnight = now + timedelta(days=1) + + td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now + + if hasattr(td, 'total_seconds'): + res = td.total_seconds() + else: #@NOTE: work-around for python 2.5 and 2.6 missing timedelta.total_seconds + res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 + + return int(res) + + class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.81" + __version__ = "0.82" __pattern__ = r'^unmatchable$' @@ -185,10 +206,10 @@ class SimpleHoster(Hoster): HASHSUM_PATTERN: (optional) Hash code and type of the file example: HASHSUM_PATTERN = r'(?Phash_code) (?PMD5)' - OFFLINE_PATTERN: (optional) Check if the file is yet available online + OFFLINE_PATTERN: (optional) Check if the page is unreachable example: OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN: (optional) Check if the file is temporarily offline + TEMP_OFFLINE_PATTERN: (optional) Check if the page is temporarily unreachable example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' @@ -217,11 +238,11 @@ class SimpleHoster(Hoster): SIZE_REPLACEMENTS = [] URL_REPLACEMENTS = [] - TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] - CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - DIRECT_LINK = None #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False - MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) + TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct + COOKIES = True #: or False or list of tuples [(domain, name, value)] + CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + DIRECT_LINK = None #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False + MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) @classmethod @@ -451,7 +472,10 @@ class SimpleHoster(Hoster): self.info.pop('error', None) - def checkStatus(self): + def checkStatus(self, getinfo=True): + if getinfo: + self.updateInfo(self.getInfo(self.pyfile.url, self.html)) + status = self.info['status'] if status is 1: @@ -465,7 +489,10 @@ class SimpleHoster(Hoster): "File info: %s" % self.info) - def checkNameSize(self): + def checkNameSize(self, getinfo=True): + if getinfo: + self.updateInfo(self.getInfo(self.pyfile.url, self.html)) + name = self.info['name'] size = self.info['size'] url = self.info['url'] @@ -485,17 +512,13 @@ class SimpleHoster(Hoster): def checkInfo(self): - self.updateInfo(self.getInfo(self.pyfile.url, self.html)) - self.checkNameSize() if self.html: self.checkErrors() - self.updateInfo(self.getInfo(self.pyfile.url, self.html)) - self.checkNameSize() - self.checkStatus() + self.checkStatus(getinfo=False) #: Deprecated -- cgit v1.2.3 From 0c61cbb03493b15970a5ccb9a5e20b02b3884a4e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 28 Dec 2014 01:05:59 +0100 Subject: [SimpleHoster] Improve --- module/plugins/internal/SimpleHoster.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index d9732d063..5c6f8c720 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -181,7 +181,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.82" + __version__ = "0.83" __pattern__ = r'^unmatchable$' @@ -388,7 +388,7 @@ class SimpleHoster(Hoster): def process(self, pyfile): self.prepare() - self.checkInfo() + self.checkNameSize() if self.directDL: self.logDebug("Looking for direct download link...") @@ -406,9 +406,6 @@ class SimpleHoster(Hoster): self.preload() self.checkInfo() - if self.html is None: - self.fail(_("No html retrieved")) - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() -- cgit v1.2.3 From 98ae3d727b60d3f529b39a6c275e31732231bd91 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 28 Dec 2014 11:53:37 +0100 Subject: [SimpleHoster] Fix https://github.com/pyload/pyload/issues/983 --- module/plugins/internal/SimpleHoster.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/SimpleHoster.py') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 5c6f8c720..6075e3f67 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -181,7 +181,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.83" + __version__ = "0.84" __pattern__ = r'^unmatchable$' @@ -444,7 +444,9 @@ class SimpleHoster(Hoster): check = self.checkDownload(rules) if check: #@TODO: Move to hoster in 0.4.10 - errmsg = check.strip().capitalize() + (" | " + self.lastCheck.strip() if self.lastCheck else "") + errmsg = check.strip().capitalize() + if self.lastCheck: + errmsg += " | " + self.lastCheck.group(0).strip() self.retry(10, 60, errmsg) -- cgit v1.2.3