From 28e66d852a66c4db4df7142f641900a0fe996c67 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 9 Aug 2015 03:45:36 +0200 Subject: Fix missing import statement --- module/plugins/internal/Hoster.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index a0cdb1e2e..b397a92a6 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -3,6 +3,7 @@ from __future__ import with_statement import inspect +import mimetypes import os import random import time @@ -43,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.19" + __version__ = "0.20" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From dedce876f1f9cb9ad9f8b135ced26bf19fc1dbc7 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Sat, 29 Aug 2015 16:28:59 +0300 Subject: [Hoster] fix pyfile.name not updated if disposition --- module/plugins/internal/Hoster.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index b397a92a6..a8b8922bb 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.20" + __version__ = "0.21" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -412,7 +412,7 @@ class Hoster(Plugin): if disposition and newname: finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] - if finalname != newname != self.pyfile.name: + if finalname != newname try: os.rename(fs_join(location, newname), fs_join(location, finalname)) @@ -421,8 +421,9 @@ class Hoster(Plugin): finalname = newname self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname)) - self.pyfile.name = finalname - filename = os.path.join(location, finalname) + + self.pyfile.name = finalname + filename = os.path.join(location, finalname) self.set_permissions(fs_encode(filename)) -- cgit v1.2.3 From 2a3405db57862c5644643fd9b8bad0a74b0c1cbe Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Sat, 29 Aug 2015 23:02:36 +0300 Subject: Update Hoster.py --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index a8b8922bb..924672acf 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -412,7 +412,7 @@ class Hoster(Plugin): if disposition and newname: finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] - if finalname != newname + if finalname != newname: try: os.rename(fs_join(location, newname), fs_join(location, finalname)) -- cgit v1.2.3 From 4cd2b7390dd97dc2016ab71f954f191de12f2f46 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 3 Sep 2015 20:37:17 +0200 Subject: Spare fixes (2) --- module/plugins/internal/Hoster.py | 53 +++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 21 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 924672acf..e6bd14329 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.21" + __version__ = "0.22" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -432,9 +432,35 @@ class Hoster(Plugin): return self.last_download - def check_download(self, rules, delete=False, file_size=0, size_tolerance=1024, read_size=1048576): + def check_filesize(self, file_size, size_tolerance=1024): """ - Checks the content of the last downloaded file, re match is saved to `lastCheck` + Checks the file size of the last downloaded file + + :param file_size: expected file size + :param size_tolerance: size check tolerance + """ + if not self.last_download: + return + + download_size = os.stat(fs_encode(self.last_download)).st_size + + if download_size < 1: + self.fail(_("Empty file")) + + elif file_size > 0: + diff = abs(file_size - download_size) + + if diff > size_tolerance: + self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") + % (file_size, download_size)) + + elif diff != 0: + self.log_warning(_("File size is not equal to expected size")) + + + def check_download(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): + """ + Checks the content of the last downloaded file, re match is saved to `last_check` :param rules: dict with names and rules to match (compiled regexp or strings) :param delete: delete if matched @@ -447,26 +473,11 @@ class Hoster(Plugin): last_download = fs_encode(self.last_download) if not self.last_download or not exists(last_download): - self.last_download = "" + self.last_download = "" #@NOTE: Bad place... self.fail(self.pyfile.error or _("No file downloaded")) try: - download_size = os.stat(last_download).st_size - - if download_size < 1: - do_delete = True - self.fail(_("Empty file")) - - elif file_size > 0: - diff = abs(file_size - download_size) - - if diff > size_tolerance: - do_delete = True - self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") - % (file_size, download_size)) - - elif diff != 0: - self.log_warning(_("File size is not equal to expected size")) + self.check_filesize(file_size, size_tolerance) with open(last_download, "rb") as f: content = f.read(read_size) @@ -496,8 +507,8 @@ class Hoster(Plugin): traceback.print_exc() else: + self.log_info(_("File deleted: ") + self.last_download) self.last_download = "" - self.log_info(_("File deleted")) def direct_link(self, url, follow_location=None): -- cgit v1.2.3 From f23c2ed39b842c115751937ebc3d1b1138a9ad88 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 19 Sep 2015 06:48:27 +0200 Subject: [XFSAccount] Update --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index e6bd14329..ac42182c4 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -368,7 +368,7 @@ class Hoster(Plugin): url = self.fixurl(url) if not url or not isinstance(url, basestring): - self.fail(_("No url given")) + self.fail(_("No given url")) if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, -- cgit v1.2.3 From 59d2ad3541bf133ddd69fd3b7c633e7e226e4829 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 21 Sep 2015 01:08:35 +0200 Subject: Spare improvements and fixes --- module/plugins/internal/Hoster.py | 176 +++++++++++++++++++++++++------------- 1 file changed, 116 insertions(+), 60 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index ac42182c4..3449c8128 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.22" + __version__ = "0.23" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -105,6 +105,16 @@ class Hoster(Plugin): self.init() + def _log(self, level, plugintype, pluginname, messages): + log = getattr(self.pyload.log, level) + msg = " | ".join(encode(a).strip() for a in messages if a) + log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" + % {'plugintype': plugintype.upper(), + 'pluginname': pluginname, + 'id' : self.pyfile.id, + 'msg' : msg}) + + @classmethod def get_info(cls, url="", html=""): url = _fixurl(url) @@ -132,6 +142,11 @@ class Hoster(Plugin): def _setup(self): + #@TODO: Remove in 0.4.10 + self.html = "" + self.last_download = "" + self.pyfile.error = "" + if self.account: self.req = self.pyload.requestFactory.getRequest(self.__name__, self.user) self.chunk_limit = -1 #: -1 for unlimited @@ -143,23 +158,10 @@ class Hoster(Plugin): self.resume_download = False self.premium = False - - def load_account(self): - if self.req: - self.req.close() - - if not self.account: - self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - - if self.account: - if not self.user: - self.user = self.account.select()[0] - - if not self.user or not self.account.is_logged(self.user, True): - self.account = False + return self.setup() - def preprocessing(self, thread): + def _process(self, thread): """ Handles important things to do before starting """ @@ -172,19 +174,36 @@ class Hoster(Plugin): self.retry_free = False self._setup() - self.setup() + self.pyfile.setStatus("starting") self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 - if self.pyfile.abort: - self.abort() + self.check_abort() - self.pyfile.setStatus("starting") self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) - return self.process(self.pyfile) + #: Deprecated method, use `_process` instead (Remove in 0.4.10) + def preprocessing(self, *args, **kwargs): + return self._process(*args, **kwargs) + + + def load_account(self): + if self.req: + self.req.close() + + if not self.account: + self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) + + if self.account: + if not self.user: + self.user = self.account.select()[0] + + if not self.user or not self.account.is_logged(self.user, True): + self.account = False + + def process(self, pyfile): """ The 'main' method of every plugin, you **have to** overwrite it @@ -242,15 +261,12 @@ class Hoster(Plugin): self.log_warning("Ignore reconnection due logged account") while pyfile.waitUntil > time.time(): - if pyfile.abort: - self.abort() - + self.check_abort() time.sleep(2) else: while pyfile.waitUntil > time.time(): - if pyfile.abort: - self.abort() + self.check_abort() if self.thread.m.reconnecting.isSet(): self.waiting = False @@ -264,78 +280,94 @@ class Hoster(Plugin): pyfile.status = status #@NOTE: Remove in 0.4.10 - def skip(self, reason=""): + def skip(self, msg=""): """ - Skip and give reason + Skip and give msg """ - raise Skip(encode(reason)) #@TODO: Remove `encode` in 0.4.10 + raise Skip(encode(msg or self.pyfile.error)) #@TODO: Remove `encode` in 0.4.10 - def abort(self, reason=""): + #@TODO: Remove in 0.4.10 + def fail(self, msg): """ - Abort and give reason + Fail and give msg """ - #@TODO: Remove in 0.4.10 - if reason: - self.pyfile.error = encode(reason) + msg = msg.strip() + + if msg: + self.pyfile.error = msg + else: + msg = self.pyfile.error + + raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 + + + def error(self, msg="", type=_("Parse")): + type = _("%s error") % type.strip().capitalize() if type else _("Unknown") + msg = _("%(type)s: %(msg)s | Plugin may be out of date" + % {'type': type, 'msg': msg or self.pyfile.error}) + + self.fail(msg) + + + def abort(self, msg=""): + """ + Abort and give msg + """ + if msg: #@TODO: Remove in 0.4.10 + self.pyfile.error = encode(msg) raise Abort - def offline(self, reason=""): + #@TODO: Recheck in 0.4.10 + def offline(self, msg=""): """ Fail and indicate file is offline """ - #@TODO: Remove in 0.4.10 - if reason: - self.pyfile.error = encode(reason) - - raise Fail("offline") + self.fail("offline") - def temp_offline(self, reason=""): + #@TODO: Recheck in 0.4.10 + def temp_offline(self, msg=""): """ Fail and indicates file ist temporary offline, the core may take consequences """ - #@TODO: Remove in 0.4.10 - if reason: - self.pyfile.error = encode(reason) - - raise Fail("temp. offline") + self.fail("temp. offline") - def retry(self, max_tries=5, wait_time=1, reason=""): + def retry(self, max_tries=5, wait_time=1, msg=""): """ Retries and begin again from the beginning :param max_tries: number of maximum retries :param wait_time: time to wait in seconds - :param reason: reason for retrying, will be passed to fail if max_tries reached + :param msg: msg for retrying, will be passed to fail if max_tries reached """ id = inspect.currentframe().f_back.f_lineno if id not in self.retries: self.retries[id] = 0 if 0 < max_tries <= self.retries[id]: - self.fail(reason or _("Max retries reached")) + self.fail(msg or _("Max retries reached")) self.wait(wait_time, False) self.retries[id] += 1 - raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 + raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def restart(self, reason=None, nopremium=False): - if not reason: - reason = _("Fallback to free download") if nopremium else _("Restart") + def restart(self, msg=None, nopremium=False): + if not msg: + msg = _("Fallback to free download") if nopremium else _("Restart") if nopremium: if self.premium: self.retry_free = True else: - self.fail("%s | %s" % (reason, _("Download was already free"))) + self.fail("%s | %s" % (msg, _("Download was already free"))) - raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 + raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 def fixurl(self, url): @@ -349,6 +381,11 @@ class Hoster(Plugin): return url + def load(self, *args, **kwargs): + self.check_abort() + return super(Hoster, self).load(*args, **kwargs) + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ Downloads the content at url to download folder @@ -362,8 +399,7 @@ class Hoster(Plugin): the filename will be changed if needed :return: The location where the file was saved """ - if self.pyfile.abort: - self.abort() + self.check_abort() url = self.fixurl(url) @@ -398,8 +434,7 @@ class Hoster(Plugin): self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename) - if self.pyfile.abort: - self.abort() + self.check_abort() try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, @@ -432,6 +467,26 @@ class Hoster(Plugin): return self.last_download + def check_abort(self): + if not self.pyfile.abort: + return + + if self.pyfile.hasStatus("failed"): + self.fail() + + elif self.pyfile.hasStatus("skipped"): + self.skip(self.pyfile.statusname) + + elif self.pyfile.hasStatus("offline"): + self.offline() + + elif self.pyfile.hasStatus("temp. offline"): + self.temp_offline() + + else: + self.abort() + + def check_filesize(self, file_size, size_tolerance=1024): """ Checks the file size of the last downloaded file @@ -473,7 +528,6 @@ class Hoster(Plugin): last_download = fs_encode(self.last_download) if not self.last_download or not exists(last_download): - self.last_download = "" #@NOTE: Bad place... self.fail(self.pyfile.error or _("No file downloaded")) try: @@ -609,8 +663,10 @@ class Hoster(Plugin): if traffic is None: return False + elif traffic == -1: return True + else: size = self.pyfile.size / 1024 self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic)) -- cgit v1.2.3 From af896302d7701e2bc63b1523b9ea73f38f5201c8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 21 Sep 2015 14:01:21 +0200 Subject: Fix pyfile.name processing --- module/plugins/internal/Hoster.py | 45 ++++++++++++++------------------------- 1 file changed, 16 insertions(+), 29 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 3449c8128..8457420c1 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -12,16 +12,17 @@ import urlparse from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, encode, exists, fixurl as _fixurl, replace_patterns, - seconds_to_midnight, set_cookie, set_cookies, parse_html_form, - parse_html_tag_attr_value, timestamp) + chunks, encode, exists, parse_html_form, + parse_html_tag_attr_value, parse_name, + replace_patterns, seconds_to_midnight, + set_cookie, set_cookies, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename #@TODO: Remove in 0.4.10 def parse_fileInfo(klass, url="", html=""): info = klass.get_info(url, html) - return info['name'], info['size'], info['status'], info['url'] + return encode(info['name']), info['size'], info['status'], info['url'] #@TODO: Remove in 0.4.10 @@ -44,7 +45,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.23" + __version__ = "0.24" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -117,13 +118,9 @@ class Hoster(Plugin): @classmethod def get_info(cls, url="", html=""): - url = _fixurl(url) - url_p = urlparse.urlparse(url) - return {'name' : (url_p.path.split('/')[-1] or - url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or - url_p.netloc.split('.', 1)[0]), + return {'name' : parse_name(url), 'size' : 0, - 'status': 3 if url else 8, + 'status': 3 if url.strip() else 8, 'url' : url} @@ -370,11 +367,12 @@ class Hoster(Plugin): raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def fixurl(self, url): - url = _fixurl(url) + def fixurl(self, url, baseurl=None): + if not baseurl: + baseurl = self.pyfile.url if not urlparse.urlparse(url).scheme: - url_p = urlparse.urlparse(self.pyfile.url) + url_p = urlparse.urlparse(baseurl) baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) url = urlparse.urljoin(baseurl, url) @@ -401,17 +399,11 @@ class Hoster(Plugin): """ self.check_abort() - url = self.fixurl(url) - - if not url or not isinstance(url, basestring): - self.fail(_("No given url")) - if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) - name = _fixurl(self.pyfile.name) - self.pyfile.name = urlparse.urlparse(name).path.split('/')[-1] or name + self.pyfile.name = parse_name(self.pyfile.name) #: Safe check self.captcha.correct() self.check_for_same_files() @@ -445,7 +437,7 @@ class Hoster(Plugin): #@TODO: Recheck in 0.4.10 if disposition and newname: - finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] + finalname = parse_name(newname).split(' filename*=')[0] if finalname != newname: try: @@ -609,12 +601,7 @@ class Hoster(Plugin): link = url elif 'location' in header and header['location']: - location = header['location'] - - if not urlparse.urlparse(location).scheme: - url_p = urlparse.urlparse(url) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - location = urlparse.urljoin(baseurl, location) + location = self.fixurl(header['location'], url) if 'code' in header and header['code'] == 302: link = location @@ -624,7 +611,7 @@ class Hoster(Plugin): continue else: - extension = os.path.splitext(urlparse.urlparse(url).path.split('/')[-1])[-1] + extension = os.path.splitext(parse_name(url))[-1] if 'content-type' in header and header['content-type']: mimetype = header['content-type'].split(';')[0].strip() -- cgit v1.2.3 From f9fc367427e30b7a3ca2ccad6144cb76b21f0257 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 21 Sep 2015 14:36:22 +0200 Subject: Spare code cosmetics --- module/plugins/internal/Hoster.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 8457420c1..6ac1e95f6 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -416,6 +416,7 @@ class Hoster(Plugin): if not exists(download_location): try: os.makedirs(download_location) + except Exception, e: self.fail(e) @@ -632,6 +633,7 @@ class Hoster(Plugin): else: try: self.log_error(_("Too many redirects")) + except Exception: pass -- cgit v1.2.3 From 34244c7a49b099d30456e0bd80cf26a2be3a3208 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 21 Sep 2015 23:35:58 +0200 Subject: Spare improvements and fixes (2) --- module/plugins/internal/Hoster.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 6ac1e95f6..a77ef05dd 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -7,12 +7,11 @@ import mimetypes import os import random import time -import traceback import urlparse from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, encode, exists, parse_html_form, + chunks, decode, encode, exists, parse_html_form, parse_html_tag_attr_value, parse_name, replace_patterns, seconds_to_midnight, set_cookie, set_cookies, timestamp) @@ -45,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.24" + __version__ = "0.26" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -75,7 +74,6 @@ class Hoster(Plugin): #: Account handler instance, see :py:class:`Account` self.account = None - self.user = None self.req = None #: Browser instance, see `network.Browser` #: Associated pyfile instance, see `PyFile` @@ -108,7 +106,7 @@ class Hoster(Plugin): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) - msg = " | ".join(encode(a).strip() for a in messages if a) + msg = " | ".join(decode(a).strip() for a in messages if a) log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, @@ -145,10 +143,10 @@ class Hoster(Plugin): self.pyfile.error = "" if self.account: - self.req = self.pyload.requestFactory.getRequest(self.__name__, self.user) + self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) self.chunk_limit = -1 #: -1 for unlimited self.resume_download = True - self.premium = self.account.is_premium(self.user) + self.premium = self.account.premium else: self.req = self.pyload.requestFactory.getRequest(self.__name__) self.chunk_limit = 1 @@ -194,10 +192,10 @@ class Hoster(Plugin): self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) if self.account: - if not self.user: - self.user = self.account.select()[0] + if not self.account.user: #@TODO: Move to `Account` in 0.4.10 + self.account.user = self.account.select()[0] - if not self.user or not self.account.is_logged(self.user, True): + if not self.account.logged: self.account = False @@ -333,22 +331,22 @@ class Hoster(Plugin): self.fail("temp. offline") - def retry(self, max_tries=5, wait_time=1, msg=""): + def retry(self, attemps=5, delay=1, msg=""): """ Retries and begin again from the beginning - :param max_tries: number of maximum retries - :param wait_time: time to wait in seconds - :param msg: msg for retrying, will be passed to fail if max_tries reached + :param attemps: number of maximum retries + :param delay: time to wait in seconds + :param msg: msg for retrying, will be passed to fail if attemps value was reached """ id = inspect.currentframe().f_back.f_lineno if id not in self.retries: self.retries[id] = 0 - if 0 < max_tries <= self.retries[id]: + if 0 < attemps <= self.retries[id]: self.fail(msg or _("Max retries reached")) - self.wait(wait_time, False) + self.wait(delay, False) self.retries[id] += 1 raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 @@ -550,8 +548,6 @@ class Hoster(Plugin): except OSError, e: self.log_warning(_("Error removing: %s") % last_download, e) - if self.pyload.debug: - traceback.print_exc() else: self.log_info(_("File deleted: ") + self.last_download) @@ -648,7 +644,7 @@ class Hoster(Plugin): if not self.account: return True - traffic = self.account.get_data(self.user, True)['trafficleft'] + traffic = self.account.get_data(self.account.user, True)['trafficleft'] if traffic is None: return False @@ -658,7 +654,7 @@ class Hoster(Plugin): else: size = self.pyfile.size / 1024 - self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic)) + self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) return size <= traffic -- cgit v1.2.3 From fa5e570bfa4439812b4329f43d6a49e0b5239641 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 22 Sep 2015 03:20:25 +0200 Subject: [Plugin] Fix _log + code cosmetics --- module/plugins/internal/Hoster.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index a77ef05dd..bdb5c4746 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -106,7 +106,7 @@ class Hoster(Plugin): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) - msg = " | ".join(decode(a).strip() for a in messages if a) + msg = u" | ".join(decode(a).strip() for a in messages if a) log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, @@ -207,12 +207,13 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): - reconnect = bool(reconnect) + if reconnect: + self.log_info(_("Requesting line reconnection...") + else: + self.log_debug("Reconnect: %s" % reconnect) - self.log_info(_("RECONNECT ") + ("enabled" if reconnect else "disabled")) self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) - - self.wantReconnect = reconnect + self.wantReconnect = bool(reconnect) def set_wait(self, seconds, reconnect=None): @@ -225,7 +226,7 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_info(_("WAIT %d seconds") % wait_time) + self.log_info(_("Waiting %d seconds...") % wait_time) self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until -- cgit v1.2.3 From 32bfe05611f9cdeb11f8d13d4e57b68c96ad2b58 Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 22 Sep 2015 15:45:39 +0300 Subject: version up --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index bdb5c4746..e655b0544 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.26" + __version__ = "0.27" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 2c61d10fdeeb89199d14ba152490f227f201d67c Mon Sep 17 00:00:00 2001 From: GammaC0de Date: Tue, 22 Sep 2015 18:12:57 +0300 Subject: Update Hoster.py --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index e655b0544..90961d529 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -575,7 +575,7 @@ class Hoster(Plugin): except Exception: #: Bad bad bad... rewrite this part in 0.4.10 res = self.load(url, just_header=True, - req=self.pyload.requestFactory.getRequest()) + req=self.pyload.requestFactory.getRequest(self.__name__)) header = {'code': req.code} for line in res.splitlines(): -- cgit v1.2.3 From 3da545b9be94d5da8d247102538e062ea83b0a56 Mon Sep 17 00:00:00 2001 From: "Marcus \"Chaosblog\"" Date: Wed, 23 Sep 2015 12:53:14 +0200 Subject: Add missing closing bracket --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 90961d529..f610023f4 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -208,7 +208,7 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): if reconnect: - self.log_info(_("Requesting line reconnection...") + self.log_info(_("Requesting line reconnection...")) else: self.log_debug("Reconnect: %s" % reconnect) -- cgit v1.2.3 From 625be1490bee7e8dd44af720eb64864799c06487 Mon Sep 17 00:00:00 2001 From: Stefan Reisich Date: Wed, 23 Sep 2015 18:37:27 +0200 Subject: [FIX] AttributeError: x object has no attribute 'user' --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index f610023f4..b853f2815 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -192,10 +192,10 @@ class Hoster(Plugin): self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) if self.account: - if not self.account.user: #@TODO: Move to `Account` in 0.4.10 + if not hasattr(self.account, 'user'): #@TODO: Move to `Account` in 0.4.10 self.account.user = self.account.select()[0] - if not self.account.logged: + if not hasattr(self.account, 'logged'): self.account = False -- cgit v1.2.3 From 8dd484b68a0be6376d0f38a5abc7bcf0107aa349 Mon Sep 17 00:00:00 2001 From: "Marcus \"Chaosblog\"" Date: Wed, 23 Sep 2015 22:53:40 +0200 Subject: Version up Increase version number --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index b853f2815..5d0a64f1a 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.27" + __version__ = "0.28" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 8af39bfa2c409d4fd07d430485117bf60fdcc935 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 29 Sep 2015 21:43:22 +0200 Subject: Update internal plugins --- module/plugins/internal/Hoster.py | 244 +++++++++++++++++++++----------------- 1 file changed, 134 insertions(+), 110 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 5d0a64f1a..bc340e78f 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -11,8 +11,8 @@ import urlparse from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, decode, encode, exists, parse_html_form, - parse_html_tag_attr_value, parse_name, + chunks, decode, encode, exists, fixurl, + parse_html_form, parse_html_tag_attr_value, parse_name, replace_patterns, seconds_to_midnight, set_cookie, set_cookies, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename @@ -41,21 +41,27 @@ def create_getInfo(klass): return get_info +#@NOTE: `check_abort` decorator +def check_abort(fn): + + def wrapper(self, *args, **kwargs): + self.check_abort() + return fn(self, *args, **kwargs) + + return wrapper + + class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.28" + __version__ = "0.31" __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [] #: [("name", "type", "desc", "default")] __description__ = """Base hoster plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), - ("spoob" , "spoob@pyload.org" ), - ("mkaay" , "mkaay@mkaay.de" ), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] def __init__(self, pyfile): @@ -74,7 +80,7 @@ class Hoster(Plugin): #: Account handler instance, see :py:class:`Account` self.account = None - self.req = None #: Browser instance, see `network.Browser` + self.user = None #@TODO: Remove in 0.4.10 #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile @@ -98,7 +104,7 @@ class Hoster(Plugin): #: Dict of the amount of retries already made self.retries = {} - self.retry_free = False #@TODO: Recheck in 0.4.10 + self.force_free = False #@TODO: Recheck in 0.4.10 self._setup() self.init() @@ -116,9 +122,10 @@ class Hoster(Plugin): @classmethod def get_info(cls, url="", html=""): + url = fixurl(url, unquote=True) return {'name' : parse_name(url), 'size' : 0, - 'status': 3 if url.strip() else 8, + 'status': 3 if url else 8, 'url' : url} @@ -142,6 +149,11 @@ class Hoster(Plugin): self.last_download = "" self.pyfile.error = "" + try: + self.req.close() + except Exception: + pass + if self.account: self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) self.chunk_limit = -1 #: -1 for unlimited @@ -162,16 +174,16 @@ class Hoster(Plugin): """ self.thread = thread - if self.retry_free: + if self.force_free: self.account = False else: self.load_account() #@TODO: Move to PluginThread in 0.4.10 - self.retry_free = False + self.force_free = False self._setup() + # self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 self.pyfile.setStatus("starting") - self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 self.check_abort() @@ -185,17 +197,17 @@ class Hoster(Plugin): def load_account(self): - if self.req: - self.req.close() - if not self.account: self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - if self.account: - if not hasattr(self.account, 'user'): #@TODO: Move to `Account` in 0.4.10 - self.account.user = self.account.select()[0] + if not self.account: + self.account = False + self.user = None #@TODO: Remove in 0.4.10 - if not hasattr(self.account, 'logged'): + else: + self.account.choose() + self.user = self.account.user #@TODO: Remove in 0.4.10 + if self.account.user is None: self.account = False @@ -207,12 +219,8 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): - if reconnect: - self.log_info(_("Requesting line reconnection...")) - else: - self.log_debug("Reconnect: %s" % reconnect) - - self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) + self.log_debug("RECONNECT %s required" % ("" if reconnect else "not"), + "Previous wantReconnect: %s" % self.wantReconnect) self.wantReconnect = bool(reconnect) @@ -226,8 +234,8 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_info(_("Waiting %d seconds...") % wait_time) - self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) + self.log_debug("WAIT set to %d seconds" % wait_time, + "Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until @@ -249,13 +257,17 @@ class Hoster(Plugin): self.waiting = True - status = pyfile.status #@NOTE: Remove in 0.4.10 + status = pyfile.status #@NOTE: Recheck in 0.4.10 pyfile.setStatus("waiting") - if not self.wantReconnect or self.account: + self.log_info(_("Waiting %d seconds...") % pyfile.waitUntil - time.time()) + + if self.wantReconnect: + self.log_info(_("Requiring reconnection...")) if self.account: self.log_warning("Ignore reconnection due logged account") + if not self.wantReconnect or self.account: while pyfile.waitUntil > time.time(): self.check_abort() time.sleep(2) @@ -263,24 +275,24 @@ class Hoster(Plugin): else: while pyfile.waitUntil > time.time(): self.check_abort() + self.thread.m.reconnecting.wait(1) if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False raise Reconnect - self.thread.m.reconnecting.wait(2) time.sleep(2) self.waiting = False - pyfile.status = status #@NOTE: Remove in 0.4.10 + pyfile.status = status #@NOTE: Recheck in 0.4.10 def skip(self, msg=""): """ Skip and give msg """ - raise Skip(encode(msg or self.pyfile.error)) #@TODO: Remove `encode` in 0.4.10 + raise Skip(encode(msg or self.pyfile.error or self.pyfile.pluginname)) #@TODO: Remove `encode` in 0.4.10 #@TODO: Remove in 0.4.10 @@ -293,7 +305,7 @@ class Hoster(Plugin): if msg: self.pyfile.error = msg else: - msg = self.pyfile.error + msg = self.pyfile.error or (self.info['error'] if 'error' in self.info else self.pyfile.getStatusName()) raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 @@ -359,30 +371,33 @@ class Hoster(Plugin): if nopremium: if self.premium: - self.retry_free = True + self.force_free = True else: self.fail("%s | %s" % (msg, _("Download was already free"))) raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def fixurl(self, url, baseurl=None): + def fixurl(self, url, baseurl=None, unquote=None): + url = fixurl(url) + if not baseurl: - baseurl = self.pyfile.url + baseurl = fixurl(self.pyfile.url) if not urlparse.urlparse(url).scheme: url_p = urlparse.urlparse(baseurl) baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) url = urlparse.urljoin(baseurl, url) - return url + return fixurl(url, unquote) + @check_abort def load(self, *args, **kwargs): - self.check_abort() return super(Hoster, self).load(*args, **kwargs) + @check_abort def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ Downloads the content at url to download folder @@ -396,16 +411,18 @@ class Hoster(Plugin): the filename will be changed if needed :return: The location where the file was saved """ - self.check_abort() - if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, - *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) + + url = self.fixurl(url, unquote=True) self.pyfile.name = parse_name(self.pyfile.name) #: Safe check self.captcha.correct() - self.check_for_same_files() + + if self.pyload.config.get("download", "skip_existing"): + self.check_filedupe() self.pyfile.setStatus("downloading") @@ -441,7 +458,9 @@ class Hoster(Plugin): if finalname != newname: try: - os.rename(fs_join(location, newname), fs_join(location, finalname)) + oldname_enc = fs_join(download_location, newname) + newname_enc = fs_join(download_location, finalname) + os.rename(oldname_enc, newname_enc) except OSError, e: self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e) @@ -463,16 +482,16 @@ class Hoster(Plugin): if not self.pyfile.abort: return - if self.pyfile.hasStatus("failed"): + if self.pyfile.status is 8: self.fail() - elif self.pyfile.hasStatus("skipped"): + elif self.pyfile.status is 4: self.skip(self.pyfile.statusname) - elif self.pyfile.hasStatus("offline"): + elif self.pyfile.status is 1: self.offline() - elif self.pyfile.hasStatus("temp. offline"): + elif self.pyfile.status is 6: self.temp_offline() else: @@ -489,7 +508,8 @@ class Hoster(Plugin): if not self.last_download: return - download_size = os.stat(fs_encode(self.last_download)).st_size + download_location = fs_encode(self.last_download) + download_size = os.stat(download_location).st_size if download_size < 1: self.fail(_("Empty file")) @@ -505,7 +525,7 @@ class Hoster(Plugin): self.log_warning(_("File size is not equal to expected size")) - def check_download(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): + def check_file(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): """ Checks the content of the last downloaded file, re match is saved to `last_check` @@ -552,7 +572,66 @@ class Hoster(Plugin): else: self.log_info(_("File deleted: ") + self.last_download) - self.last_download = "" + self.last_download = "" #: Recheck in 0.4.10 + + + def check_traffic(self): + if not self.account: + return True + + traffic = self.account.get_data(refresh=True)['trafficleft'] + + if traffic is None: + return False + + elif traffic is -1: + return True + + else: + size = self.pyfile.size / 1024 #@TODO: Remove in 0.4.10 + self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) #@TODO: Rewrite in 0.4.10 + return size <= traffic + + + def check_filedupe(self): + """ + Checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises Skip: + """ + pack = self.pyfile.package() + + for pyfile in self.pyload.files.cache.values(): + if pyfile is self.pyfile: + continue + + if pyfile.name != self.pyfile.name or pyfile.package().folder != pack.folder: + continue + + if pyfile.status in (0, 5, 7, 12): #: (finished, waiting, starting, downloading) + self.skip(pyfile.pluginname) + + download_folder = self.pyload.config.get("general", "download_folder") + package_folder = pack.folder if self.pyload.config.get("general", "folder_per_package") else "" + download_location = fs_join(download_folder, package_folder, self.pyfile.name) + + if not exists(download_location): + return + + pyfile = self.pyload.db.findDuplicates(self.pyfile.id, package_folder, self.pyfile.name) + if pyfile: + self.skip(pyfile[0]) + + size = os.stat(download_location).st_size + if size >= self.pyfile.size: + self.skip(_("File exists")) + + + #: Deprecated method, use `check_filedupe` instead (Remove in 0.4.10) + def checkForSameFiles(self, *args, **kwargs): + if self.pyload.config.get("download", "skip_existing"): + return self.check_filedupe() def direct_link(self, url, follow_location=None): @@ -598,10 +677,10 @@ class Hoster(Plugin): if 'content-disposition' in header: link = url - elif 'location' in header and header['location']: + elif header.get('location'): location = self.fixurl(header['location'], url) - if 'code' in header and header['code'] == 302: + if header.get('code') == 302: link = location if follow_location: @@ -611,7 +690,7 @@ class Hoster(Plugin): else: extension = os.path.splitext(parse_name(url))[-1] - if 'content-type' in header and header['content-type']: + if header.get('content-type'): mimetype = header['content-type'].split(';')[0].strip() elif extension: @@ -641,63 +720,8 @@ class Hoster(Plugin): return parse_html_form(attr_str, self.html, input_names) - def check_traffic_left(self): - if not self.account: - return True - - traffic = self.account.get_data(self.account.user, True)['trafficleft'] - - if traffic is None: - return False - - elif traffic == -1: - return True - - else: - size = self.pyfile.size / 1024 - self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) - return size <= traffic - - def get_password(self): """ Get the password the user provided in the package """ return self.pyfile.package().password or "" - - - #: Deprecated method, use `check_for_same_files` instead (Remove in 0.4.10) - def checkForSameFiles(self, *args, **kwargs): - return self.check_for_same_files(*args, **kwargs) - - - def check_for_same_files(self, starting=False): - """ - Checks if same file was/is downloaded within same package - - :param starting: indicates that the current download is going to start - :raises Skip: - """ - pack = self.pyfile.package() - - for pyfile in self.pyload.files.cache.values(): - if pyfile != self.pyfile and pyfile.name is self.pyfile.name and pyfile.package().folder is pack.folder: - if pyfile.status in (0, 12): #: Finished or downloading - self.skip(pyfile.pluginname) - elif pyfile.status in (5, 7) and starting: #: A download is waiting/starting and was appenrently started before - self.skip(pyfile.pluginname) - - download_folder = self.pyload.config.get("general", "download_folder") - location = fs_join(download_folder, pack.folder, self.pyfile.name) - - if starting and self.pyload.config.get("download", "skip_existing") and exists(location): - size = os.stat(location).st_size - if size >= self.pyfile.size: - self.skip("File exists") - - pyfile = self.pyload.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) - if pyfile: - if exists(location): - self.skip(pyfile[0]) - - self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name) -- cgit v1.2.3 From 8d85a655d8cc408574f471306613bc05115c8c64 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 1 Oct 2015 18:59:55 +0200 Subject: Improve Hoster and Crypter + general fixup --- module/plugins/internal/Hoster.py | 502 +++++--------------------------------- 1 file changed, 59 insertions(+), 443 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index bc340e78f..d1b894c6f 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -2,19 +2,10 @@ from __future__ import with_statement -import inspect -import mimetypes import os -import random -import time -import urlparse - -from module.plugins.internal.Captcha import Captcha -from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, decode, encode, exists, fixurl, - parse_html_form, parse_html_tag_attr_value, parse_name, - replace_patterns, seconds_to_midnight, - set_cookie, set_cookies, timestamp) + +from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo +from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename @@ -24,40 +15,17 @@ def parse_fileInfo(klass, url="", html=""): return encode(info['name']), info['size'], info['status'], info['url'] -#@TODO: Remove in 0.4.10 -def getInfo(urls): - #: result = [ .. (name, size, status, url) .. ] - pass - - -#@TODO: Remove in 0.4.10 -def create_getInfo(klass): - def get_info(urls): - for url in urls: - if hasattr(klass, "URL_REPLACEMENTS"): - url = replace_patterns(url, klass.URL_REPLACEMENTS) - yield parse_fileInfo(klass, url) - - return get_info - - -#@NOTE: `check_abort` decorator -def check_abort(fn): - - def wrapper(self, *args, **kwargs): - self.check_abort() - return fn(self, *args, **kwargs) - - return wrapper - -class Hoster(Plugin): +class Hoster(Base): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.31" + __version__ = "0.32" __status__ = "testing" __pattern__ = r'^unmatchable$' + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("fallback_premium", "bool", "Fallback to free download if premium fails", True), + ("chk_filesize" , "bool", "Check file size" , True)] __description__ = """Base hoster plugin""" __license__ = "GPLv3" @@ -65,107 +33,39 @@ class Hoster(Plugin): def __init__(self, pyfile): - self._init(pyfile.m.core) - - #: Engage wan reconnection - self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10 + super(Base, self).__init__(pyfile) #: Enable simultaneous processing of multiple downloads - self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10 self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 - #: time.time() + wait in seconds - self.wait_until = 0 - self.waiting = False - - #: Account handler instance, see :py:class:`Account` - self.account = None - self.user = None #@TODO: Remove in 0.4.10 - - #: Associated pyfile instance, see `PyFile` - self.pyfile = pyfile - - self.thread = None #: Holds thread in future - #: Location where the last call to download was saved - self.last_download = "" + self.last_download = None #: Re match of the last call to `checkDownload` self.last_check = None - #: Js engine, see `JsEngine` - self.js = self.pyload.js - - #: Captcha stuff - self.captcha = Captcha(self) - - #: Some plugins store html code here - self.html = None - - #: Dict of the amount of retries already made - self.retries = {} - self.force_free = False #@TODO: Recheck in 0.4.10 + #: Restart flag + self.rst_free = False #@TODO: Recheck in 0.4.10 self._setup() self.init() - def _log(self, level, plugintype, pluginname, messages): - log = getattr(self.pyload.log, level) - msg = u" | ".join(decode(a).strip() for a in messages if a) - log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" - % {'plugintype': plugintype.upper(), - 'pluginname': pluginname, - 'id' : self.pyfile.id, - 'msg' : msg}) - - - @classmethod - def get_info(cls, url="", html=""): - url = fixurl(url, unquote=True) - return {'name' : parse_name(url), - 'size' : 0, - 'status': 3 if url else 8, - 'url' : url} - - - def init(self): - """ - Initialize the plugin (in addition to `__init__`) - """ - pass - - - def setup(self): - """ - Setup for enviroment and other things, called before downloading (possibly more than one time) - """ - pass + def _setup(self): + super(Base, self)._setup() + self.last_download = None + self.last_check = None + self.rst_free = False - def _setup(self): - #@TODO: Remove in 0.4.10 - self.html = "" - self.last_download = "" - self.pyfile.error = "" - try: - self.req.close() - except Exception: - pass - - if self.account: - self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) - self.chunk_limit = -1 #: -1 for unlimited - self.resume_download = True - self.premium = self.account.premium + def load_account(self): + if self.rst_free: + self.account = False + self.user = None #@TODO: Remove in 0.4.10 else: - self.req = self.pyload.requestFactory.getRequest(self.__name__) - self.chunk_limit = 1 - self.resume_download = False - self.premium = False - - return self.setup() + super(Base, self).load_account() + # self.rst_free = False def _process(self, thread): @@ -174,229 +74,44 @@ class Hoster(Plugin): """ self.thread = thread - if self.force_free: - self.account = False - else: - self.load_account() #@TODO: Move to PluginThread in 0.4.10 - self.force_free = False - self._setup() # self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 - self.pyfile.setStatus("starting") - self.check_abort() - self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) - return self.process(self.pyfile) - - - #: Deprecated method, use `_process` instead (Remove in 0.4.10) - def preprocessing(self, *args, **kwargs): - return self._process(*args, **kwargs) - - - def load_account(self): - if not self.account: - self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - - if not self.account: - self.account = False - self.user = None #@TODO: Remove in 0.4.10 - - else: - self.account.choose() - self.user = self.account.user #@TODO: Remove in 0.4.10 - if self.account.user is None: - self.account = False - - - def process(self, pyfile): - """ - The 'main' method of every plugin, you **have to** overwrite it - """ - raise NotImplementedError - - - def set_reconnect(self, reconnect): - self.log_debug("RECONNECT %s required" % ("" if reconnect else "not"), - "Previous wantReconnect: %s" % self.wantReconnect) - self.wantReconnect = bool(reconnect) - - - def set_wait(self, seconds, reconnect=None): - """ - Set a specific wait time later used with `wait` - - :param seconds: wait time in seconds - :param reconnect: True if a reconnect would avoid wait time - """ - wait_time = max(int(seconds), 1) - wait_until = time.time() + wait_time + 1 - - self.log_debug("WAIT set to %d seconds" % wait_time, - "Previous waitUntil: %f" % self.pyfile.waitUntil) - - self.pyfile.waitUntil = wait_until - - if reconnect is not None: - self.set_reconnect(reconnect) - - - def wait(self, seconds=None, reconnect=None): - """ - Waits the time previously set - """ - pyfile = self.pyfile - - if seconds is not None: - self.set_wait(seconds) - - if reconnect is not None: - self.set_reconnect(reconnect) - - self.waiting = True - - status = pyfile.status #@NOTE: Recheck in 0.4.10 - pyfile.setStatus("waiting") - - self.log_info(_("Waiting %d seconds...") % pyfile.waitUntil - time.time()) - - if self.wantReconnect: - self.log_info(_("Requiring reconnection...")) - if self.account: - self.log_warning("Ignore reconnection due logged account") - - if not self.wantReconnect or self.account: - while pyfile.waitUntil > time.time(): - self.check_abort() - time.sleep(2) - - else: - while pyfile.waitUntil > time.time(): - self.check_abort() - self.thread.m.reconnecting.wait(1) - - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - time.sleep(2) - - self.waiting = False - pyfile.status = status #@NOTE: Recheck in 0.4.10 - - - def skip(self, msg=""): - """ - Skip and give msg - """ - raise Skip(encode(msg or self.pyfile.error or self.pyfile.pluginname)) #@TODO: Remove `encode` in 0.4.10 - - - #@TODO: Remove in 0.4.10 - def fail(self, msg): - """ - Fail and give msg - """ - msg = msg.strip() - - if msg: - self.pyfile.error = msg - else: - msg = self.pyfile.error or (self.info['error'] if 'error' in self.info else self.pyfile.getStatusName()) - - raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - - - def error(self, msg="", type=_("Parse")): - type = _("%s error") % type.strip().capitalize() if type else _("Unknown") - msg = _("%(type)s: %(msg)s | Plugin may be out of date" - % {'type': type, 'msg': msg or self.pyfile.error}) - - self.fail(msg) - - - def abort(self, msg=""): - """ - Abort and give msg - """ - if msg: #@TODO: Remove in 0.4.10 - self.pyfile.error = encode(msg) - - raise Abort - - - #@TODO: Recheck in 0.4.10 - def offline(self, msg=""): - """ - Fail and indicate file is offline - """ - self.fail("offline") - - - #@TODO: Recheck in 0.4.10 - def temp_offline(self, msg=""): - """ - Fail and indicates file ist temporary offline, the core may take consequences - """ - self.fail("temp. offline") - + self.pyfile.setStatus("starting") - def retry(self, attemps=5, delay=1, msg=""): - """ - Retries and begin again from the beginning + try: + self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) #@TODO: Remove in 0.4.10 + self.process(self.pyfile) - :param attemps: number of maximum retries - :param delay: time to wait in seconds - :param msg: msg for retrying, will be passed to fail if attemps value was reached - """ - id = inspect.currentframe().f_back.f_lineno - if id not in self.retries: - self.retries[id] = 0 + self.check_abort() - if 0 < attemps <= self.retries[id]: - self.fail(msg or _("Max retries reached")) + self.log_debug("CHECK DOWNLOAD") #@TODO: Recheck in 0.4.10 + self._check_download() - self.wait(delay, False) + except Fail, e: #@TODO: Move to PluginThread in 0.4.10 + if self.get_config('fallback_premium', True) and self.premium: + self.log_warning(_("Premium download failed"), e) + self.restart() - self.retries[id] += 1 - raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 + else: + raise Fail(e) - def restart(self, msg=None, nopremium=False): + def restart(self, msg="", premium=False): if not msg: - msg = _("Fallback to free download") if nopremium else _("Restart") + msg = _("Simple restart") if premium else _("Fallback to free download") - if nopremium: + if not premium: if self.premium: - self.force_free = True + self.rst_free = True else: self.fail("%s | %s" % (msg, _("Download was already free"))) raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def fixurl(self, url, baseurl=None, unquote=None): - url = fixurl(url) - - if not baseurl: - baseurl = fixurl(self.pyfile.url) - - if not urlparse.urlparse(url).scheme: - url_p = urlparse.urlparse(baseurl) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - url = urlparse.urljoin(baseurl, url) - - return fixurl(url, unquote) - - - @check_abort - def load(self, *args, **kwargs): - return super(Hoster, self).load(*args, **kwargs) - - @check_abort def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ @@ -415,7 +130,7 @@ class Hoster(Plugin): self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) - url = self.fixurl(url, unquote=True) + url = self.fixurl(url) self.pyfile.name = parse_name(self.pyfile.name) #: Safe check @@ -475,27 +190,7 @@ class Hoster(Plugin): self.last_download = filename - return self.last_download - - - def check_abort(self): - if not self.pyfile.abort: - return - - if self.pyfile.status is 8: - self.fail() - - elif self.pyfile.status is 4: - self.skip(self.pyfile.statusname) - - elif self.pyfile.status is 1: - self.offline() - - elif self.pyfile.status is 6: - self.temp_offline() - - else: - self.abort() + return filename def check_filesize(self, file_size, size_tolerance=1024): @@ -537,7 +232,7 @@ class Hoster(Plugin): :return: dictionary key of the first rule that matched """ do_delete = False - last_download = fs_encode(self.last_download) + last_download = fs_encode(self.last_download) #@TODO: Recheck in 0.4.10 if not self.last_download or not exists(last_download): self.fail(self.pyfile.error or _("No file downloaded")) @@ -558,7 +253,7 @@ class Hoster(Plugin): elif hasattr(rule, "search"): m = rule.search(content) - if m: + if m is not None: do_delete = True self.last_check = m return name @@ -575,11 +270,25 @@ class Hoster(Plugin): self.last_download = "" #: Recheck in 0.4.10 + def _check_download(self): + if self.captcha.task and not self.last_download: + self.retry_captcha() + + elif self.check_file({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, + delete=True): + self.error(_("Empty file")) + + elif self.get_config('chk_filesize', False) and self.info.get('size'): + # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size + # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB + self.check_filesize(self.info['size'], size_tolerance=10485760) + + def check_traffic(self): if not self.account: return True - traffic = self.account.get_data(refresh=True)['trafficleft'] + traffic = self.account.get_data('trafficleft') if traffic is None: return False @@ -632,96 +341,3 @@ class Hoster(Plugin): def checkForSameFiles(self, *args, **kwargs): if self.pyload.config.get("download", "skip_existing"): return self.check_filedupe() - - - def direct_link(self, url, follow_location=None): - link = "" - - if follow_location is None: - redirect = 1 - - elif type(follow_location) is int: - redirect = max(follow_location, 1) - - else: - redirect = self.get_config("maxredirs", 10, "UserAgentSwitcher") - - for i in xrange(redirect): - try: - self.log_debug("Redirect #%d to: %s" % (i, url)) - header = self.load(url, just_header=True) - - except Exception: #: Bad bad bad... rewrite this part in 0.4.10 - res = self.load(url, - just_header=True, - req=self.pyload.requestFactory.getRequest(self.__name__)) - - header = {'code': req.code} - for line in res.splitlines(): - line = line.strip() - if not line or ":" not in line: - continue - - key, none, value = line.partition(":") - key = key.lower().strip() - value = value.strip() - - if key in header: - if type(header[key]) is list: - header[key].append(value) - else: - header[key] = [header[key], value] - else: - header[key] = value - - if 'content-disposition' in header: - link = url - - elif header.get('location'): - location = self.fixurl(header['location'], url) - - if header.get('code') == 302: - link = location - - if follow_location: - url = location - continue - - else: - extension = os.path.splitext(parse_name(url))[-1] - - if header.get('content-type'): - mimetype = header['content-type'].split(';')[0].strip() - - elif extension: - mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" - - else: - mimetype = "" - - if mimetype and (link or 'html' not in mimetype): - link = url - else: - link = "" - - break - - else: - try: - self.log_error(_("Too many redirects")) - - except Exception: - pass - - return link - - - def parse_html_form(self, attr_str="", input_names={}): - return parse_html_form(attr_str, self.html, input_names) - - - def get_password(self): - """ - Get the password the user provided in the package - """ - return self.pyfile.package().password or "" -- cgit v1.2.3 From ab3c90a7b0d18f47d3efc8259906895c1f50765d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 1 Oct 2015 23:21:14 +0200 Subject: Fixpack (1) --- module/plugins/internal/Hoster.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index d1b894c6f..cb7b215ba 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -4,22 +4,15 @@ from __future__ import with_statement import os -from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo +from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo, parse_fileInfo from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename -#@TODO: Remove in 0.4.10 -def parse_fileInfo(klass, url="", html=""): - info = klass.get_info(url, html) - return encode(info['name']), info['size'], info['status'], info['url'] - - - class Hoster(Base): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.32" + __version__ = "0.33" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -33,7 +26,7 @@ class Hoster(Base): def __init__(self, pyfile): - super(Base, self).__init__(pyfile) + super(Hoster, self).__init__(pyfile) #: Enable simultaneous processing of multiple downloads self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 @@ -52,7 +45,7 @@ class Hoster(Base): def _setup(self): - super(Base, self)._setup() + super(Hoster, self)._setup() self.last_download = None self.last_check = None @@ -64,7 +57,7 @@ class Hoster(Base): self.account = False self.user = None #@TODO: Remove in 0.4.10 else: - super(Base, self).load_account() + super(Hoster, self).load_account() # self.rst_free = False -- cgit v1.2.3 From 7adfcbe536339bb8c51024573cad346cac7e0eda Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 2 Oct 2015 01:37:12 +0200 Subject: [Hoster] Fix https://github.com/pyload/pyload/issues/1918 --- module/plugins/internal/Hoster.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index cb7b215ba..ea5a3cb29 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -2,6 +2,7 @@ from __future__ import with_statement +import re import os from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo, parse_fileInfo @@ -12,7 +13,7 @@ from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path a class Hoster(Base): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.33" + __version__ = "0.34" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 67a21410eeb8a2bd7bc4d71f7fe5cd68312b702b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 2 Oct 2015 02:34:47 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1917 + new function: parse_time --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index ea5a3cb29..26da436a5 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -2,8 +2,8 @@ from __future__ import with_statement -import re import os +import re from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo, parse_fileInfo from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name -- cgit v1.2.3