From 681deb7ff09956be0bee04453e33ac6e9d6dab4a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 05:56:10 +0200 Subject: Move base plugins to internal folder --- module/plugins/internal/Hoster.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 module/plugins/internal/Hoster.py (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py new file mode 100644 index 000000000..814a70949 --- /dev/null +++ b/module/plugins/internal/Hoster.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from module.plugins.Plugin import Plugin + +def getInfo(self): + #result = [ .. (name, size, status, url) .. ] + return + +class Hoster(Plugin): + __name__ = "Hoster" + __version__ = "0.1" + __pattern__ = None + __type__ = "hoster" + __description__ = """Base hoster plugin""" + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") -- cgit v1.2.3 From 0e1ef9bc01579328e17e79416fa3c1c7b77adcc8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Jun 2015 06:08:01 +0200 Subject: Update everything --- module/plugins/internal/Hoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 814a70949..3fed8a7c6 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -13,11 +13,11 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - + @author: mkaay """ -from module.plugins.Plugin import Plugin +from module.plugins.internal.Plugin import Plugin def getInfo(self): #result = [ .. (name, size, status, url) .. ] @@ -25,7 +25,7 @@ def getInfo(self): class Hoster(Plugin): __name__ = "Hoster" - __version__ = "0.1" + __version__ = "0.02" __pattern__ = None __type__ = "hoster" __description__ = """Base hoster plugin""" -- cgit v1.2.3 From c1764e2fea0bb05164c83a876e8cd58b97f58f25 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 16 Jun 2015 17:31:38 +0200 Subject: Update all --- module/plugins/internal/Hoster.py | 704 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 681 insertions(+), 23 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 3fed8a7c6..9db22d1d7 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -1,33 +1,691 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. +from __future__ import with_statement - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. +import inspect +import os +import random +import time +import urlparse - You should have received a copy of the GNU General Public License - along with this program; if not, see . +if os.name != "nt": + import grp + import pwd - @author: mkaay -""" +from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip +from module.utils import fs_decode, fs_encode, save_join as fs_join -from module.plugins.internal.Plugin import Plugin -def getInfo(self): - #result = [ .. (name, size, status, url) .. ] - return +def getInfo(urls): + #result = [ .. (name, size, status, url) .. ] + pass + class Hoster(Plugin): - __name__ = "Hoster" - __version__ = "0.02" - __pattern__ = None - __type__ = "hoster" + __name__ = "Hoster" + __type__ = "hoster" + __version__ = "0.03" + + __pattern__ = r'^unmatchable$' + __config__ = [] #: [("name", "type", "desc", "default")] + __description__ = """Base hoster plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + __license__ = "GPLv3" + __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), + ("spoob" , "spoob@pyload.org" ), + ("mkaay" , "mkaay@mkaay.de" ), + ("Walter Purcaro", "vuolter@gmail.com")] + + + def __init__(self, pyfile): + super(Hoster, self).__init__(pyfile.m.core) + + #: engage wan reconnection + self.wantReconnect = False + + #: enable simultaneous processing of multiple downloads + self.multiDL = True + self.limitDL = 0 + + #: chunk limit + self.chunkLimit = 1 + self.resumeDownload = False + + #: time.time() + wait in seconds + self.waitUntil = 0 + self.waiting = False + + #: captcha reader instance + self.ocr = None + + #: account handler instance, see :py:class:`Account` + self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + + #: premium status + self.premium = False + + #: username/login + self.user = None + + if self.account and not self.account.canUse(): + self.account = None + + if self.account: + self.user, data = self.account.selectAccount() + + #: Browser instance, see `network.Browser` + self.req = self.account.getAccountRequest(self.user) + self.chunkLimit = -1 #: chunk limit, -1 for unlimited + + #: enables resume (will be ignored if server dont accept chunks) + self.resumeDownload = True + + #: premium status + self.premium = self.account.isPremium(self.user) + else: + self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) + + #: associated pyfile instance, see `PyFile` + self.pyfile = pyfile + + self.thread = None #: holds thread in future + + #: location where the last call to download was saved + self.lastDownload = "" + + #: re match of the last call to `checkDownload` + self.lastCheck = None + + #: js engine, see `JsEngine` + self.js = self.core.js + + #: captcha task + self.cTask = None + + #: some plugins store html code here + self.html = None + + #: dict of the amount of retries already made + self.retries = {} + + self.init() + + + def init(self): + """ + Initialize the plugin (in addition to `__init__`) + """ + pass + + + def setup(self): + """ + Setup for enviroment and other things, called before downloading (possibly more than one time) + """ + pass + + + def preprocessing(self, thread): + """ + Handles important things to do before starting + """ + self.thread = thread + + if self.account: + self.account.checkLogin(self.user) + else: + self.req.clearCookies() + + self.setup() + + self.pyfile.setStatus("starting") + + return self.process(self.pyfile) + + + def process(self, pyfile): + """ + The 'main' method of every plugin, you **have to** overwrite it + """ + raise NotImplementedError + + + def getChunkCount(self): + if self.chunkLimit <= 0: + return self.core.config.get("download", "chunks") + return min(self.core.config.get("download", "chunks"), self.chunkLimit) + + + def resetAccount(self): + """ + Don't use account and retry download + """ + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retry() + + + def setReconnect(self, reconnect): + reconnect = bool(reconnect) + self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + self.wantReconnect = reconnect + + + def setWait(self, seconds, reconnect=None): + """ + Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + wait_time = int(seconds) + 1 + wait_until = time.time() + wait_time + + self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), + "Wait: %d seconds" % wait_time) + + self.pyfile.waitUntil = wait_until + + if reconnect is not None: + self.setReconnect(reconnect) + + + def wait(self, seconds=None, reconnect=None): + """ + Waits the time previously set + """ + pyfile = self.pyfile + + if seconds is not None: + self.setWait(seconds) + + if reconnect is not None: + self.setReconnect(reconnect) + + self.waiting = True + + status = pyfile.status + pyfile.setStatus("waiting") + + self.logInfo(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), + _("Reconnect: %s") % self.wantReconnect) + + if self.account: + self.logDebug("Ignore reconnection due account logged") + + while pyfile.waitUntil > time.time(): + if pyfile.abort: + self.abort() + + time.sleep(1) + else: + while pyfile.waitUntil > time.time(): + self.thread.m.reconnecting.wait(2) + + if pyfile.abort: + self.abort() + + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + time.sleep(1) + + self.waiting = False + + pyfile.status = status + + + def skip(self, reason=""): + """ + Skip and give reason + """ + raise Skip(fs_encode(reason)) + + + def abort(self, reason=""): + """ + Abort and give reason + """ + if reason: + self.pyfile.error = fs_encode(reason) + raise Abort + + + def offline(self, reason=""): + """ + Fail and indicate file is offline + """ + if reason: + self.pyfile.error = fs_encode(reason) + raise Fail("offline") + + + def tempOffline(self, reason=""): + """ + Fail and indicates file ist temporary offline, the core may take consequences + """ + if reason: + self.pyfile.error = fs_encode(reason) + raise Fail("temp. offline") + + + def retry(self, max_tries=5, wait_time=1, reason=""): + """ + Retries and begin again from the beginning + + :param max_tries: number of maximum retries + :param wait_time: time to wait in seconds + :param reason: reason for retrying, will be passed to fail if max_tries reached + """ + id = inspect.currentframe().f_back.f_lineno + if id not in self.retries: + self.retries[id] = 0 + + if 0 < max_tries <= self.retries[id]: + self.fail(reason or _("Max retries reached"), "retry") + + self.wait(wait_time, False) + + self.retries[id] += 1 + raise Retry(reason) + + + def invalidCaptcha(self): + self.logError(_("Invalid captcha")) + if self.cTask: + self.cTask.invalid() + + + def correctCaptcha(self): + self.logInfo(_("Correct captcha")) + if self.cTask: + self.cTask.correct() + + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, + imgtype='jpg', result_type='textual'): + """ + Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time.time())[-6:].replace(".", "") + + with open(os.path.join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: + tmpCaptcha.write(img) + + has_plugin = self.__name__ in self.core.pluginManager.ocrPlugins + + if self.core.captcha: + Ocr = self.core.pluginManager.loadClass("ocr", self.__name__) + else: + Ocr = None + + if Ocr and not forceUser: + time.sleep(random.randint(3000, 5000) / 1000.0) + if self.pyfile.abort: + self.abort() + + ocr = Ocr() + result = ocr.get_captcha(tmpCaptcha.name) + else: + captchaManager = self.core.captchaManager + task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) + self.cTask = task + captchaManager.handleCaptcha(task) + + while task.isWaiting(): + if self.pyfile.abort: + captchaManager.removeTask(task) + self.abort() + time.sleep(1) + + captchaManager.removeTask(task) + + if task.error and has_plugin: #: ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) + + result = task.result + self.logDebug("Received captcha result: %s" % result) + + if not self.core.debug: + try: + os.remove(tmpCaptcha.name) + except Exception: + pass + + return result + + + def fixurl(self, url): + url_p = urlparse.urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + + url = super(Hoster, self).fixurl(url) + + if not urlparse.urlparse(url).scheme: + url = urlparse.urljoin(baseurl, url) + + return url + + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + """ + Downloads the content at url to download folder + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param disposition: if True and server provides content-disposition header\ + the filename will be changed if needed + :return: The location where the file was saved + """ + if self.pyfile.abort: + self.abort() + + url = self.fixurl(url) + + if not url or not isinstance(url, basestring): + self.fail(_("No url given")) + + if self.core.debug: + self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + + self.correctCaptcha() + self.checkForSameFiles() + + self.pyfile.setStatus("downloading") + + if disposition: + self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name + + download_folder = self.core.config.get("general", "download_folder") + + location = fs_join(download_folder, self.pyfile.package().folder) + + if not os.path.exists(location): + try: + os.makedirs(location, int(self.core.config.get("permission", "folder"), 8)) + + if self.core.config.get("permission", "change_dl") and os.name != "nt": + uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2] + gid = grp.getgrnam(self.core.config.get("permission", "group"))[2] + os.chown(location, uid, gid) + + except Exception, e: + self.fail(e) + + # convert back to unicode + location = fs_decode(location) + name = safe_filename(self.pyfile.name) + + filename = os.path.join(location, name) + + self.core.addonManager.dispatchEvent("download-start", self.pyfile, url, filename) + + try: + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.setProgress, disposition=disposition) + finally: + self.pyfile.size = self.req.size + + if newname: + newname = urlparse.urlparse(newname).path.split('/')[-1] + + if disposition and newname != name: + self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = os.path.join(location, newname) + + fs_filename = fs_encode(filename) + + if self.core.config.get("permission", "change_file"): + try: + os.chmod(fs_filename, int(self.core.config.get("permission", "file"), 8)) + except Exception, e: + self.logWarning(_("Setting file mode failed"), e) + + if self.core.config.get("permission", "change_dl") and os.name != "nt": + try: + uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2] + gid = grp.getgrnam(self.core.config.get("permission", "group"))[2] + os.chown(fs_filename, uid, gid) + + except Exception, e: + self.logWarning(_("Setting User and Group failed"), e) + + self.lastDownload = filename + return self.lastDownload + + + def checkDownload(self, rules, delete=True, file_size=None, size_tolerance=1000, read_size=100000): + """ + Checks the content of the last downloaded file, re match is saved to `lastCheck` + + :param rules: dict with names and rules to match (compiled regexp or strings) + :param delete: delete if matched + :param file_size: expected file size + :param size_tolerance: size check tolerance + :param read_size: amount of bytes to read from files + :return: dictionary key of the first rule that matched + """ + lastDownload = fs_encode(self.lastDownload) + + if not self.lastDownload or not os.path.exists(lastDownload): + self.lastDownload = "" + self.fail(self.pyfile.error or _("No file downloaded")) + + download_size = os.stat(lastDownload).st_size + + if download_size < 1 or (file_size and abs(file_size - download_size) > size_tolerance): + if delete: + os.remove(lastDownload) + self.fail(_("Empty file")) + + self.logDebug("Download Check triggered") + + with open(lastDownload, "rb") as f: + content = f.read(read_size) + + # produces encoding errors, better log to other file in the future? + # self.logDebug("Content: %s" % content) + for name, rule in rules.iteritems(): + if isinstance(rule, basestring): + if rule in content: + if delete: + os.remove(lastDownload) + return name + + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + if delete: + os.remove(lastDownload) + self.lastCheck = m + return name + + + def directLink(self, url, follow_location=None): + link = "" + + if follow_location is None: + redirect = 1 + + elif type(follow_location) is int: + redirect = max(follow_location, 1) + + else: + redirect = self.getConfig("maxredirs", plugin="UserAgentSwitcher") + + for i in xrange(redirect): + try: + self.logDebug("Redirect #%d to: %s" % (i, url)) + header = self.load(url, just_header=True) + + except Exception: #: Bad bad bad... rewrite this part in 0.4.10 + req = pyreq.getHTTPRequest() + res = self.load(url, just_header=True) + + req.close() + + header = {"code": req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: + continue + + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() + + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value + + if 'content-disposition' in header: + link = url + + elif 'location' in header and header['location']: + location = header['location'] + + if not urlparse.urlparse(location).scheme: + url_p = urlparse.urlparse(url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + location = urlparse.urljoin(baseurl, location) + + if 'code' in header and header['code'] == 302: + link = location + + if follow_location: + url = location + continue + + else: + extension = os.path.splitext(urlparse.urlparse(url).path.split('/')[-1])[-1] + + if 'content-type' in header and header['content-type']: + mimetype = header['content-type'].split(';')[0].strip() + + elif extension: + mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" + + else: + mimetype = "" + + if mimetype and (link or 'html' not in mimetype): + link = url + else: + link = "" + + break + + else: + try: + self.logError(_("Too many redirects")) + except Exception: + pass + + return link + + + def parseHtmlForm(self, attr_str="", input_names={}): + return parseHtmlForm(attr_str, self.html, input_names) + + + def checkTrafficLeft(self): + if not self.account: + return True + + traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] + + if traffic is None: + return False + elif traffic == -1: + return True + else: + size = self.pyfile.size / 1024 + self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) + return size <= traffic + + + def getPassword(self): + """ + Get the password the user provided in the package + """ + return self.pyfile.package().password or "" + + + def checkForSameFiles(self, starting=False): + """ + Checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises Skip: + """ + pack = self.pyfile.package() + + for pyfile in self.core.files.cache.values(): + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #: finished or downloading + self.skip(pyfile.pluginname) + elif pyfile.status in (5, 7) and starting: #: a download is waiting/starting and was appenrently started before + self.skip(pyfile.pluginname) + + download_folder = self.core.config.get("general", "download_folder") + location = fs_join(download_folder, pack.folder, self.pyfile.name) + + if starting and self.core.config.get("download", "skip_existing") and os.path.exists(location): + size = os.stat(location).st_size + if size >= self.pyfile.size: + self.skip("File exists") + + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + if pyfile: + if os.path.exists(location): + self.skip(pyfile[0]) + + self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name) + + + def clean(self): + """ + Clean everything and remove references + """ + if hasattr(self, "pyfile"): + del self.pyfile + + if hasattr(self, "req"): + self.req.close() + del self.req + + if hasattr(self, "thread"): + del self.thread + + if hasattr(self, "html"): + del self.html -- cgit v1.2.3 From fff615f28424c4fbdccd68e13ea329c1220f1c1b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 16 Jun 2015 18:05:15 +0200 Subject: [Hoster] Import fixup --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 9db22d1d7..834a200ac 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -12,7 +12,7 @@ if os.name != "nt": import grp import pwd -from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip +from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip, parseHtmlForm from module.utils import fs_decode, fs_encode, save_join as fs_join -- cgit v1.2.3 From d99d6eddb6af637580bb6fc72013f913077525d6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 17 Jun 2015 11:23:08 +0200 Subject: Spare fixes --- module/plugins/internal/Hoster.py | 78 ++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 33 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 834a200ac..40430c38d 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -178,11 +178,11 @@ class Hoster(Plugin): :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ - wait_time = int(seconds) + 1 - wait_until = time.time() + wait_time + wait_time = max(int(seconds), 1) + wait_until = time.time() + wait_time + 1 self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), - "Wait: %d seconds" % wait_time) + "Wait: %d(+1) seconds" % wait_time) self.pyfile.waitUntil = wait_until @@ -190,13 +190,13 @@ class Hoster(Plugin): self.setReconnect(reconnect) - def wait(self, seconds=None, reconnect=None): + def wait(self, seconds=0, reconnect=None): """ Waits the time previously set """ pyfile = self.pyfile - if seconds is not None: + if seconds > 0: self.setWait(seconds) if reconnect is not None: @@ -284,7 +284,7 @@ class Hoster(Plugin): self.retries[id] = 0 if 0 < max_tries <= self.retries[id]: - self.fail(reason or _("Max retries reached"), "retry") + self.fail(reason or _("Max retries reached"), _("retry")) self.wait(wait_time, False) @@ -480,7 +480,7 @@ class Hoster(Plugin): return self.lastDownload - def checkDownload(self, rules, delete=True, file_size=None, size_tolerance=1000, read_size=100000): + def checkDownload(self, rules, delete=True, file_size=0, size_tolerance=1000, read_size=100000): """ Checks the content of the last downloaded file, re match is saved to `lastCheck` @@ -491,40 +491,52 @@ class Hoster(Plugin): :param read_size: amount of bytes to read from files :return: dictionary key of the first rule that matched """ + do_delete = False lastDownload = fs_encode(self.lastDownload) if not self.lastDownload or not os.path.exists(lastDownload): self.lastDownload = "" self.fail(self.pyfile.error or _("No file downloaded")) - download_size = os.stat(lastDownload).st_size + try: + download_size = os.stat(lastDownload).st_size + + if download_size < 1: + do_delete = True + self.fail(_("Empty file")) + + elif file_size > 0: + diff = abs(file_size - download_size) + + if diff > size_tolerance: + do_delete = True + self.fail(_("File size mismatch")) + + elif diff != 0: + self.logWarning(_("File size is not equal to expected size")) - if download_size < 1 or (file_size and abs(file_size - download_size) > size_tolerance): - if delete: + self.logDebug("Download Check triggered") + + with open(lastDownload, "rb") as f: + content = f.read(read_size) + + # produces encoding errors, better log to other file in the future? + # self.logDebug("Content: %s" % content) + for name, rule in rules.iteritems(): + if isinstance(rule, basestring): + if rule in content: + do_delete = True + return name + + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + do_delete = True + self.lastCheck = m + return name + finally: + if delete and do_delete: os.remove(lastDownload) - self.fail(_("Empty file")) - - self.logDebug("Download Check triggered") - - with open(lastDownload, "rb") as f: - content = f.read(read_size) - - # produces encoding errors, better log to other file in the future? - # self.logDebug("Content: %s" % content) - for name, rule in rules.iteritems(): - if isinstance(rule, basestring): - if rule in content: - if delete: - os.remove(lastDownload) - return name - - elif hasattr(rule, "search"): - m = rule.search(content) - if m: - if delete: - os.remove(lastDownload) - self.lastCheck = m - return name def directLink(self, url, follow_location=None): -- cgit v1.2.3 From 164512b6a74c94a731fcee7435dce1ccfa2f71e7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 17 Jun 2015 18:29:50 +0200 Subject: Spare code cosmetics --- module/plugins/internal/Hoster.py | 1 - 1 file changed, 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 40430c38d..32c3312d0 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -321,7 +321,6 @@ class Hoster(Plugin): :return: result of decrypting """ - img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time.time())[-6:].replace(".", "") -- cgit v1.2.3 From 20b6a2ec022202b0efb6cb69415239fb8f4d1445 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 17 Jun 2015 18:59:20 +0200 Subject: Spare code cosmetics (2) --- module/plugins/internal/Hoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 32c3312d0..e1a13af01 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -435,7 +435,7 @@ class Hoster(Plugin): except Exception, e: self.fail(e) - # convert back to unicode + #: convert back to unicode location = fs_decode(location) name = safe_filename(self.pyfile.name) @@ -519,8 +519,8 @@ class Hoster(Plugin): with open(lastDownload, "rb") as f: content = f.read(read_size) - # produces encoding errors, better log to other file in the future? - # self.logDebug("Content: %s" % content) + #: produces encoding errors, better log to other file in the future? + #: self.logDebug("Content: %s" % content) for name, rule in rules.iteritems(): if isinstance(rule, basestring): if rule in content: -- cgit v1.2.3 From f32fcfcff81b67c8102df3822434f4c2551d7ae6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 5 Jul 2015 13:13:05 +0200 Subject: Fix long int conversion issue --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index e1a13af01..92cb476ea 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -182,7 +182,7 @@ class Hoster(Plugin): wait_until = time.time() + wait_time + 1 self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), - "Wait: %d(+1) seconds" % wait_time) + "Wait: %d+1 seconds" % wait_time) self.pyfile.waitUntil = wait_until @@ -641,7 +641,7 @@ class Hoster(Plugin): return True else: size = self.pyfile.size / 1024 - self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) + self.logInfo(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic)) return size <= traffic -- cgit v1.2.3 From b1759bc440cd6013837697eb8de540914f693ffd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 7 Jul 2015 01:23:55 +0200 Subject: No camelCase style anymore --- module/plugins/internal/Hoster.py | 138 +++++++++++++++++++------------------- 1 file changed, 69 insertions(+), 69 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 92cb476ea..09672f6f0 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -16,15 +16,15 @@ from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry from module.utils import fs_decode, fs_encode, save_join as fs_join -def getInfo(urls): - #result = [ .. (name, size, status, url) .. ] +def get_info(urls): + # result = [ .. (name, size, status, url) .. ] pass class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.03" + __version__ = "0.04" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] @@ -41,18 +41,18 @@ class Hoster(Plugin): super(Hoster, self).__init__(pyfile.m.core) #: engage wan reconnection - self.wantReconnect = False + self.want_reconnect = False #: enable simultaneous processing of multiple downloads - self.multiDL = True - self.limitDL = 0 + self.multi_dl = True + self.limit_dl = 0 #: chunk limit - self.chunkLimit = 1 - self.resumeDownload = False + self.chunk_limit = 1 + self.resume_download = False #: time.time() + wait in seconds - self.waitUntil = 0 + self.wait_until = 0 self.waiting = False #: captcha reader instance @@ -75,10 +75,10 @@ class Hoster(Plugin): #: Browser instance, see `network.Browser` self.req = self.account.getAccountRequest(self.user) - self.chunkLimit = -1 #: chunk limit, -1 for unlimited + self.chunk_limit = -1 #: chunk limit, -1 for unlimited #: enables resume (will be ignored if server dont accept chunks) - self.resumeDownload = True + self.resume_download = True #: premium status self.premium = self.account.isPremium(self.user) @@ -91,16 +91,16 @@ class Hoster(Plugin): self.thread = None #: holds thread in future #: location where the last call to download was saved - self.lastDownload = "" + self.last_download = "" #: re match of the last call to `checkDownload` - self.lastCheck = None + self.last_check = None #: js engine, see `JsEngine` self.js = self.core.js #: captcha task - self.cTask = None + self.c_task = None #: some plugins store html code here self.html = None @@ -150,13 +150,13 @@ class Hoster(Plugin): raise NotImplementedError - def getChunkCount(self): - if self.chunkLimit <= 0: + def get_chunk_count(self): + if self.chunk_limit <= 0: return self.core.config.get("download", "chunks") - return min(self.core.config.get("download", "chunks"), self.chunkLimit) + return min(self.core.config.get("download", "chunks"), self.chunk_limit) - def resetAccount(self): + def reset_account(self): """ Don't use account and retry download """ @@ -165,13 +165,13 @@ class Hoster(Plugin): self.retry() - def setReconnect(self, reconnect): + def set_reconnect(self, reconnect): reconnect = bool(reconnect) - self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) - self.wantReconnect = reconnect + self.log_debug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.want_reconnect)) + self.want_reconnect = reconnect - def setWait(self, seconds, reconnect=None): + def set_wait(self, seconds, reconnect=None): """ Set a specific wait time later used with `wait` @@ -181,13 +181,13 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), + self.log_debug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), "Wait: %d+1 seconds" % wait_time) self.pyfile.waitUntil = wait_until if reconnect is not None: - self.setReconnect(reconnect) + self.set_reconnect(reconnect) def wait(self, seconds=0, reconnect=None): @@ -197,21 +197,21 @@ class Hoster(Plugin): pyfile = self.pyfile if seconds > 0: - self.setWait(seconds) + self.set_wait(seconds) if reconnect is not None: - self.setReconnect(reconnect) + self.set_reconnect(reconnect) self.waiting = True status = pyfile.status pyfile.setStatus("waiting") - self.logInfo(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), - _("Reconnect: %s") % self.wantReconnect) + self.log_info(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), + _("Reconnect: %s") % self.want_reconnect) if self.account: - self.logDebug("Ignore reconnection due account logged") + self.log_debug("Ignore reconnection due account logged") while pyfile.waitUntil > time.time(): if pyfile.abort: @@ -227,7 +227,7 @@ class Hoster(Plugin): if self.thread.m.reconnecting.isSet(): self.waiting = False - self.wantReconnect = False + self.want_reconnect = False raise Reconnect time.sleep(1) @@ -262,7 +262,7 @@ class Hoster(Plugin): raise Fail("offline") - def tempOffline(self, reason=""): + def temp_offline(self, reason=""): """ Fail and indicates file ist temporary offline, the core may take consequences """ @@ -292,19 +292,19 @@ class Hoster(Plugin): raise Retry(reason) - def invalidCaptcha(self): - self.logError(_("Invalid captcha")) - if self.cTask: - self.cTask.invalid() + def invalid_captcha(self): + self.log_error(_("Invalid captcha")) + if self.c_task: + self.c_task.invalid() - def correctCaptcha(self): - self.logInfo(_("Correct captcha")) - if self.cTask: - self.cTask.correct() + def correct_captcha(self): + self.log_info(_("Correct captcha")) + if self.c_task: + self.c_task.correct() - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, + def decrypt_captcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', result_type='textual'): """ Loads a captcha and decrypts it with ocr, plugin, user input @@ -345,7 +345,7 @@ class Hoster(Plugin): else: captchaManager = self.core.captchaManager task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) - self.cTask = task + self.c_task = task captchaManager.handleCaptcha(task) while task.isWaiting(): @@ -364,7 +364,7 @@ class Hoster(Plugin): self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) result = task.result - self.logDebug("Received captcha result: %s" % result) + self.log_debug("Received captcha result: %s" % result) if not self.core.debug: try: @@ -409,10 +409,10 @@ class Hoster(Plugin): self.fail(_("No url given")) if self.core.debug: - self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + self.log_debug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) - self.correctCaptcha() - self.checkForSameFiles() + self.correct_captcha() + self.check_for_same_files() self.pyfile.setStatus("downloading") @@ -445,7 +445,7 @@ class Hoster(Plugin): try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, - chunks=self.getChunkCount(), resume=self.resumeDownload, + chunks=self.get_chunk_count(), resume=self.resume_download, progressNotify=self.pyfile.setProgress, disposition=disposition) finally: self.pyfile.size = self.req.size @@ -454,7 +454,7 @@ class Hoster(Plugin): newname = urlparse.urlparse(newname).path.split('/')[-1] if disposition and newname != name: - self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) + self.log_info(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) self.pyfile.name = newname filename = os.path.join(location, newname) @@ -464,7 +464,7 @@ class Hoster(Plugin): try: os.chmod(fs_filename, int(self.core.config.get("permission", "file"), 8)) except Exception, e: - self.logWarning(_("Setting file mode failed"), e) + self.log_warning(_("Setting file mode failed"), e) if self.core.config.get("permission", "change_dl") and os.name != "nt": try: @@ -473,13 +473,13 @@ class Hoster(Plugin): os.chown(fs_filename, uid, gid) except Exception, e: - self.logWarning(_("Setting User and Group failed"), e) + self.log_warning(_("Setting User and Group failed"), e) - self.lastDownload = filename - return self.lastDownload + self.last_download = filename + return self.last_download - def checkDownload(self, rules, delete=True, file_size=0, size_tolerance=1000, read_size=100000): + def check_download(self, rules, delete=True, file_size=0, size_tolerance=1000, read_size=100000): """ Checks the content of the last downloaded file, re match is saved to `lastCheck` @@ -491,10 +491,10 @@ class Hoster(Plugin): :return: dictionary key of the first rule that matched """ do_delete = False - lastDownload = fs_encode(self.lastDownload) + lastDownload = fs_encode(self.last_download) - if not self.lastDownload or not os.path.exists(lastDownload): - self.lastDownload = "" + if not self.last_download or not os.path.exists(lastDownload): + self.last_download = "" self.fail(self.pyfile.error or _("No file downloaded")) try: @@ -512,15 +512,15 @@ class Hoster(Plugin): self.fail(_("File size mismatch")) elif diff != 0: - self.logWarning(_("File size is not equal to expected size")) + self.log_warning(_("File size is not equal to expected size")) - self.logDebug("Download Check triggered") + self.log_debug("Download Check triggered") with open(lastDownload, "rb") as f: content = f.read(read_size) #: produces encoding errors, better log to other file in the future? - #: self.logDebug("Content: %s" % content) + #: self.log_debug("Content: %s" % content) for name, rule in rules.iteritems(): if isinstance(rule, basestring): if rule in content: @@ -531,14 +531,14 @@ class Hoster(Plugin): m = rule.search(content) if m: do_delete = True - self.lastCheck = m + self.last_check = m return name finally: if delete and do_delete: os.remove(lastDownload) - def directLink(self, url, follow_location=None): + def direct_link(self, url, follow_location=None): link = "" if follow_location is None: @@ -548,11 +548,11 @@ class Hoster(Plugin): redirect = max(follow_location, 1) else: - redirect = self.getConfig("maxredirs", plugin="UserAgentSwitcher") + redirect = self.get_config("maxredirs", plugin="UserAgentSwitcher") for i in xrange(redirect): try: - self.logDebug("Redirect #%d to: %s" % (i, url)) + self.log_debug("Redirect #%d to: %s" % (i, url)) header = self.load(url, just_header=True) except Exception: #: Bad bad bad... rewrite this part in 0.4.10 @@ -618,18 +618,18 @@ class Hoster(Plugin): else: try: - self.logError(_("Too many redirects")) + self.log_error(_("Too many redirects")) except Exception: pass return link - def parseHtmlForm(self, attr_str="", input_names={}): + def parse_html_form(self, attr_str="", input_names={}): return parseHtmlForm(attr_str, self.html, input_names) - def checkTrafficLeft(self): + def check_traffic_left(self): if not self.account: return True @@ -641,18 +641,18 @@ class Hoster(Plugin): return True else: size = self.pyfile.size / 1024 - self.logInfo(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic)) + self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic)) return size <= traffic - def getPassword(self): + def get_password(self): """ Get the password the user provided in the package """ return self.pyfile.package().password or "" - def checkForSameFiles(self, starting=False): + def check_for_same_files(self, starting=False): """ Checks if same file was/is downloaded within same package @@ -681,7 +681,7 @@ class Hoster(Plugin): if os.path.exists(location): self.skip(pyfile[0]) - self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name) + self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name) def clean(self): -- cgit v1.2.3 From d2e2b127651a5a44b56337eb6d9ca246c97a208a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 17 Jul 2015 03:03:26 +0200 Subject: Spare fixes and code cosmetics --- module/plugins/internal/Hoster.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 09672f6f0..722fae2e0 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -12,7 +12,7 @@ if os.name != "nt": import grp import pwd -from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip, parseHtmlForm +from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip, parse_html_form from module.utils import fs_decode, fs_encode, save_join as fs_join @@ -67,21 +67,21 @@ class Hoster(Plugin): #: username/login self.user = None - if self.account and not self.account.canUse(): + if self.account and not self.account.can_use(): self.account = None if self.account: - self.user, data = self.account.selectAccount() + self.user, data = self.account.select_account() #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest(self.user) + self.req = self.account.get_account_request(self.user) self.chunk_limit = -1 #: chunk limit, -1 for unlimited #: enables resume (will be ignored if server dont accept chunks) self.resume_download = True #: premium status - self.premium = self.account.isPremium(self.user) + self.premium = self.account.is_premium(self.user) else: self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) @@ -132,7 +132,7 @@ class Hoster(Plugin): self.thread = thread if self.account: - self.account.checkLogin(self.user) + self.account.check_login(self.user) else: self.req.clearCookies() @@ -626,14 +626,14 @@ class Hoster(Plugin): def parse_html_form(self, attr_str="", input_names={}): - return parseHtmlForm(attr_str, self.html, input_names) + return parse_html_form(attr_str, self.html, input_names) def check_traffic_left(self): if not self.account: return True - traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] + traffic = self.account.get_account_info(self.user, True)['trafficleft'] if traffic is None: return False -- cgit v1.2.3 From ce3788f41d400e486c13126891641ad45740020c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Jul 2015 19:19:51 +0200 Subject: Reorder some functions --- module/plugins/internal/Hoster.py | 40 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 722fae2e0..9a9e61e46 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -6,25 +6,47 @@ import inspect import os import random import time +import urllib import urlparse if os.name != "nt": import grp import pwd -from module.plugins.internal.Plugin import Plugin, Abort, Fail, Reconnect, Retry, Skip, parse_html_form +from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip + chunks, replace_patterns, seconds_to_midnight, + set_cookies, parse_html_form, parse_html_tag_attr_value, + timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join -def get_info(urls): +#@TODO: Remove in 0.4.10 +def parse_fileInfo(klass, url="", html=""): + info = klass.get_info(url, html) + return info['name'], info['size'], info['status'], info['url'] + + +#@TODO: Remove in 0.4.10 +def getInfo(urls): # result = [ .. (name, size, status, url) .. ] pass +#@TODO: Remove in 0.4.10 +def create_getInfo(klass): + def get_info(urls): + for url in urls: + if hasattr(klass, "URL_REPLACEMENTS"): + url = replace_patterns(url, klass.URL_REPLACEMENTS) + yield parse_fileInfo(klass, url) + + return get_info + + class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] @@ -111,6 +133,18 @@ class Hoster(Plugin): self.init() + @classmethod + def get_info(cls, url="", html=""): + url = urllib.unquote(url) + url_p = urlparse.urlparse(url) + return {'name' : (url_p.path.split('/')[-1] + or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] + or url_p.netloc.split('.', 1)[0]), + 'size' : 0, + 'status': 3 if url else 8, + 'url' : url} + + def init(self): """ Initialize the plugin (in addition to `__init__`) -- cgit v1.2.3 From 9e5d813d7721e351ac02ba72bdc473a7d77ba6b7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Jul 2015 20:04:36 +0200 Subject: Code cosmetics --- module/plugins/internal/Hoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 9a9e61e46..15c46101b 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -216,7 +216,7 @@ class Hoster(Plugin): wait_until = time.time() + wait_time + 1 self.log_debug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), - "Wait: %d+1 seconds" % wait_time) + "Wait: %d (+1) seconds" % wait_time) self.pyfile.waitUntil = wait_until @@ -242,7 +242,7 @@ class Hoster(Plugin): pyfile.setStatus("waiting") self.log_info(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), - _("Reconnect: %s") % self.want_reconnect) + _("Reconnect: %s") % self.want_reconnect) if self.account: self.log_debug("Ignore reconnection due account logged") @@ -443,7 +443,7 @@ class Hoster(Plugin): self.fail(_("No url given")) if self.core.debug: - self.log_debug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + self.log_debug("Download url " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) self.correct_captcha() self.check_for_same_files() -- cgit v1.2.3 From 1bc7db9098967a166b17df59065861fbe3cd41d8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Jul 2015 23:26:48 +0200 Subject: [Hoster] Fix the http request issue --- module/plugins/internal/Hoster.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 15c46101b..637d9f3b5 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -165,6 +165,8 @@ class Hoster(Plugin): """ self.thread = thread + self.req.renewHTTPRequest() + if self.account: self.account.check_login(self.user) else: -- cgit v1.2.3 From dad722ac7255640e7e0541c4094a4d2e4de79cd3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 00:05:58 +0200 Subject: Code cosmetics (2) --- module/plugins/internal/Hoster.py | 48 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 637d9f3b5..c0e374a86 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -28,7 +28,7 @@ def parse_fileInfo(klass, url="", html=""): #@TODO: Remove in 0.4.10 def getInfo(urls): - # result = [ .. (name, size, status, url) .. ] + #: result = [ .. (name, size, status, url) .. ] pass @@ -62,14 +62,14 @@ class Hoster(Plugin): def __init__(self, pyfile): super(Hoster, self).__init__(pyfile.m.core) - #: engage wan reconnection + #: Engage wan reconnection self.want_reconnect = False - #: enable simultaneous processing of multiple downloads + #: Enable simultaneous processing of multiple downloads self.multi_dl = True self.limit_dl = 0 - #: chunk limit + #: Chunk limit self.chunk_limit = 1 self.resume_download = False @@ -77,13 +77,13 @@ class Hoster(Plugin): self.wait_until = 0 self.waiting = False - #: captcha reader instance + #: Captcha reader instance self.ocr = None - #: account handler instance, see :py:class:`Account` + #: Account handler instance, see :py:class:`Account` self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) - #: premium status + #: Premium status self.premium = False #: username/login @@ -97,37 +97,37 @@ class Hoster(Plugin): #: Browser instance, see `network.Browser` self.req = self.account.get_account_request(self.user) - self.chunk_limit = -1 #: chunk limit, -1 for unlimited + self.chunk_limit = -1 #: Chunk limit, -1 for unlimited - #: enables resume (will be ignored if server dont accept chunks) + #: Enables resume (will be ignored if server dont accept chunks) self.resume_download = True - #: premium status + #: Premium status self.premium = self.account.is_premium(self.user) else: self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) - #: associated pyfile instance, see `PyFile` + #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile - self.thread = None #: holds thread in future + self.thread = None #: Holds thread in future - #: location where the last call to download was saved + #: Location where the last call to download was saved self.last_download = "" - #: re match of the last call to `checkDownload` + #: Re match of the last call to `checkDownload` self.last_check = None - #: js engine, see `JsEngine` + #: Js engine, see `JsEngine` self.js = self.core.js - #: captcha task + #: Captcha task self.c_task = None - #: some plugins store html code here + #: Some plugins store html code here self.html = None - #: dict of the amount of retries already made + #: Dict of the amount of retries already made self.retries = {} self.init() @@ -392,7 +392,7 @@ class Hoster(Plugin): captchaManager.removeTask(task) - if task.error and has_plugin: #: ignore default error message since the user could use OCR + if task.error and has_plugin: #: Ignore default error message since the user could use OCR self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) elif task.error: self.fail(task.error) @@ -471,7 +471,7 @@ class Hoster(Plugin): except Exception, e: self.fail(e) - #: convert back to unicode + #: Convert back to unicode location = fs_decode(location) name = safe_filename(self.pyfile.name) @@ -555,8 +555,8 @@ class Hoster(Plugin): with open(lastDownload, "rb") as f: content = f.read(read_size) - #: produces encoding errors, better log to other file in the future? - #: self.log_debug("Content: %s" % content) + #: Produces encoding errors, better log to other file in the future? + # self.log_debug("Content: %s" % content) for name, rule in rules.iteritems(): if isinstance(rule, basestring): if rule in content: @@ -699,9 +699,9 @@ class Hoster(Plugin): for pyfile in self.core.files.cache.values(): if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: - if pyfile.status in (0, 12): #: finished or downloading + if pyfile.status in (0, 12): #: Finished or downloading self.skip(pyfile.pluginname) - elif pyfile.status in (5, 7) and starting: #: a download is waiting/starting and was appenrently started before + elif pyfile.status in (5, 7) and starting: #: A download is waiting/starting and was appenrently started before self.skip(pyfile.pluginname) download_folder = self.core.config.get("general", "download_folder") -- cgit v1.2.3 From a0e2459f2d1506912ac4a5d0c330c8dae01a4768 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 00:28:18 +0200 Subject: Improve fixurl method --- module/plugins/internal/Hoster.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index c0e374a86..e7df336b6 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -6,7 +6,6 @@ import inspect import os import random import time -import urllib import urlparse if os.name != "nt": @@ -14,7 +13,7 @@ if os.name != "nt": import pwd from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip - chunks, replace_patterns, seconds_to_midnight, + chunks, fixurl as _fixurl, replace_patterns, seconds_to_midnight, set_cookies, parse_html_form, parse_html_tag_attr_value, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join @@ -135,7 +134,7 @@ class Hoster(Plugin): @classmethod def get_info(cls, url="", html=""): - url = urllib.unquote(url) + url = _fixurl(url) url_p = urlparse.urlparse(url) return {'name' : (url_p.path.split('/')[-1] or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] @@ -412,12 +411,11 @@ class Hoster(Plugin): def fixurl(self, url): - url_p = urlparse.urlparse(self.pyfile.url) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - - url = super(Hoster, self).fixurl(url) + url = _fixurl(url) if not urlparse.urlparse(url).scheme: + url_p = urlparse.urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) url = urlparse.urljoin(baseurl, url) return url -- cgit v1.2.3 From ff9383bfe06d14d23bc0ed6af79aa8967965d078 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 10:59:52 +0200 Subject: Code cosmetics (3) --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index e7df336b6..497df00f3 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -488,7 +488,7 @@ class Hoster(Plugin): newname = urlparse.urlparse(newname).path.split('/')[-1] if disposition and newname != name: - self.log_info(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) + self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) self.pyfile.name = newname filename = os.path.join(location, newname) @@ -595,7 +595,7 @@ class Hoster(Plugin): req.close() - header = {"code": req.code} + header = {'code': req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: -- cgit v1.2.3 From 56389e28ba5d2f5658278bc7f486d73be747f135 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 11:44:49 +0200 Subject: Rename self.core to self.pyload (plugins only) --- module/plugins/internal/Hoster.py | 55 ++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 27 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 497df00f3..f5566fb6e 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -59,7 +59,8 @@ class Hoster(Plugin): def __init__(self, pyfile): - super(Hoster, self).__init__(pyfile.m.core) + self.pyload = pyfile.m.core + self.info = {} #: Provide information in dict here #: Engage wan reconnection self.want_reconnect = False @@ -118,7 +119,7 @@ class Hoster(Plugin): self.last_check = None #: Js engine, see `JsEngine` - self.js = self.core.js + self.js = self.pyload.js #: Captcha task self.c_task = None @@ -187,8 +188,8 @@ class Hoster(Plugin): def get_chunk_count(self): if self.chunk_limit <= 0: - return self.core.config.get("download", "chunks") - return min(self.core.config.get("download", "chunks"), self.chunk_limit) + return self.pyload.config.get("download", "chunks") + return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) def reset_account(self): @@ -196,7 +197,7 @@ class Hoster(Plugin): Don't use account and retry download """ self.account = None - self.req = self.core.requestFactory.getRequest(self.__name__) + self.req = self.pyload.requestFactory.getRequest(self.__name__) self.retry() @@ -363,10 +364,10 @@ class Hoster(Plugin): with open(os.path.join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: tmpCaptcha.write(img) - has_plugin = self.__name__ in self.core.pluginManager.ocrPlugins + has_plugin = self.__name__ in self.pyload.pluginManager.ocrPlugins - if self.core.captcha: - Ocr = self.core.pluginManager.loadClass("ocr", self.__name__) + if self.pyload.captcha: + Ocr = self.pyload.pluginManager.loadClass("ocr", self.__name__) else: Ocr = None @@ -378,7 +379,7 @@ class Hoster(Plugin): ocr = Ocr() result = ocr.get_captcha(tmpCaptcha.name) else: - captchaManager = self.core.captchaManager + captchaManager = self.pyload.captchaManager task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) self.c_task = task captchaManager.handleCaptcha(task) @@ -401,7 +402,7 @@ class Hoster(Plugin): result = task.result self.log_debug("Received captcha result: %s" % result) - if not self.core.debug: + if not self.pyload.debug: try: os.remove(tmpCaptcha.name) except Exception: @@ -442,8 +443,8 @@ class Hoster(Plugin): if not url or not isinstance(url, basestring): self.fail(_("No url given")) - if self.core.debug: - self.log_debug("Download url " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + if self.pyload.debug: + self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) self.correct_captcha() self.check_for_same_files() @@ -453,17 +454,17 @@ class Hoster(Plugin): if disposition: self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name - download_folder = self.core.config.get("general", "download_folder") + download_folder = self.pyload.config.get("general", "download_folder") location = fs_join(download_folder, self.pyfile.package().folder) if not os.path.exists(location): try: - os.makedirs(location, int(self.core.config.get("permission", "folder"), 8)) + os.makedirs(location, int(self.pyload.config.get("permission", "folder"), 8)) - if self.core.config.get("permission", "change_dl") and os.name != "nt": - uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2] - gid = grp.getgrnam(self.core.config.get("permission", "group"))[2] + if self.pyload.config.get("permission", "change_dl") and os.name != "nt": + uid = pwd.getpwnam(self.pyload.config.get("permission", "user"))[2] + gid = grp.getgrnam(self.pyload.config.get("permission", "group"))[2] os.chown(location, uid, gid) except Exception, e: @@ -475,7 +476,7 @@ class Hoster(Plugin): filename = os.path.join(location, name) - self.core.addonManager.dispatchEvent("download-start", self.pyfile, url, filename) + self.pyload.addonManager.dispatchEvent("download-start", self.pyfile, url, filename) try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, @@ -494,16 +495,16 @@ class Hoster(Plugin): fs_filename = fs_encode(filename) - if self.core.config.get("permission", "change_file"): + if self.pyload.config.get("permission", "change_file"): try: - os.chmod(fs_filename, int(self.core.config.get("permission", "file"), 8)) + os.chmod(fs_filename, int(self.pyload.config.get("permission", "file"), 8)) except Exception, e: self.log_warning(_("Setting file mode failed"), e) - if self.core.config.get("permission", "change_dl") and os.name != "nt": + if self.pyload.config.get("permission", "change_dl") and os.name != "nt": try: - uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2] - gid = grp.getgrnam(self.core.config.get("permission", "group"))[2] + uid = pwd.getpwnam(self.pyload.config.get("permission", "user"))[2] + gid = grp.getgrnam(self.pyload.config.get("permission", "group"))[2] os.chown(fs_filename, uid, gid) except Exception, e: @@ -695,22 +696,22 @@ class Hoster(Plugin): """ pack = self.pyfile.package() - for pyfile in self.core.files.cache.values(): + for pyfile in self.pyload.files.cache.values(): if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: if pyfile.status in (0, 12): #: Finished or downloading self.skip(pyfile.pluginname) elif pyfile.status in (5, 7) and starting: #: A download is waiting/starting and was appenrently started before self.skip(pyfile.pluginname) - download_folder = self.core.config.get("general", "download_folder") + download_folder = self.pyload.config.get("general", "download_folder") location = fs_join(download_folder, pack.folder, self.pyfile.name) - if starting and self.core.config.get("download", "skip_existing") and os.path.exists(location): + if starting and self.pyload.config.get("download", "skip_existing") and os.path.exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: self.skip("File exists") - pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + pyfile = self.pyload.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: if os.path.exists(location): self.skip(pyfile[0]) -- cgit v1.2.3 From a5b840079dfa281127c0fc0f5a13708b5ecb5031 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 12:13:02 +0200 Subject: [OCR] Fix __init__ --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index f5566fb6e..e9fc81e7a 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -376,7 +376,7 @@ class Hoster(Plugin): if self.pyfile.abort: self.abort() - ocr = Ocr() + ocr = Ocr(self.pyfile) result = ocr.get_captcha(tmpCaptcha.name) else: captchaManager = self.pyload.captchaManager -- cgit v1.2.3 From d38e830b7c0b3c6561a0072c74bbccb5fcdf4a61 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Jul 2015 14:43:42 +0200 Subject: New __status__ magic key --- module/plugins/internal/Hoster.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index e9fc81e7a..35e6ef23e 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -46,6 +46,7 @@ class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" __version__ = "0.05" + __status__ = "stable" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] -- cgit v1.2.3 From 027cb529d79558de19c47da88a782b31745a65c9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 21 Jul 2015 22:53:37 +0200 Subject: New Captcha skeleton --- module/plugins/internal/Hoster.py | 92 +++------------------------------------ 1 file changed, 5 insertions(+), 87 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 35e6ef23e..af3e80acf 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -12,6 +12,7 @@ if os.name != "nt": import grp import pwd +from module.plugins.internal import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip chunks, fixurl as _fixurl, replace_patterns, seconds_to_midnight, set_cookies, parse_html_form, parse_html_tag_attr_value, @@ -122,8 +123,8 @@ class Hoster(Plugin): #: Js engine, see `JsEngine` self.js = self.pyload.js - #: Captcha task - self.c_task = None + #: Captcha stuff + self.captcha = Captcha(self) #: Some plugins store html code here self.html = None @@ -257,7 +258,7 @@ class Hoster(Plugin): time.sleep(1) else: while pyfile.waitUntil > time.time(): - self.thread.m.reconnecting.wait(2) + self.thread.m.reconnecting.wait(1) if pyfile.abort: self.abort() @@ -329,89 +330,6 @@ class Hoster(Plugin): raise Retry(reason) - def invalid_captcha(self): - self.log_error(_("Invalid captcha")) - if self.c_task: - self.c_task.invalid() - - - def correct_captcha(self): - self.log_info(_("Correct captcha")) - if self.c_task: - self.c_task.correct() - - - def decrypt_captcha(self, url, get={}, post={}, cookies=False, forceUser=False, - imgtype='jpg', result_type='textual'): - """ - Loads a captcha and decrypts it with ocr, plugin, user input - - :param url: url of captcha image - :param get: get part for request - :param post: post part for request - :param cookies: True if cookies should be enabled - :param forceUser: if True, ocr is not used - :param imgtype: Type of the Image - :param result_type: 'textual' if text is written on the captcha\ - or 'positional' for captcha where the user have to click\ - on a specific region on the captcha - - :return: result of decrypting - """ - img = self.load(url, get=get, post=post, cookies=cookies) - - id = ("%.2f" % time.time())[-6:].replace(".", "") - - with open(os.path.join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: - tmpCaptcha.write(img) - - has_plugin = self.__name__ in self.pyload.pluginManager.ocrPlugins - - if self.pyload.captcha: - Ocr = self.pyload.pluginManager.loadClass("ocr", self.__name__) - else: - Ocr = None - - if Ocr and not forceUser: - time.sleep(random.randint(3000, 5000) / 1000.0) - if self.pyfile.abort: - self.abort() - - ocr = Ocr(self.pyfile) - result = ocr.get_captcha(tmpCaptcha.name) - else: - captchaManager = self.pyload.captchaManager - task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) - self.c_task = task - captchaManager.handleCaptcha(task) - - while task.isWaiting(): - if self.pyfile.abort: - captchaManager.removeTask(task) - self.abort() - time.sleep(1) - - captchaManager.removeTask(task) - - if task.error and has_plugin: #: Ignore default error message since the user could use OCR - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) - - result = task.result - self.log_debug("Received captcha result: %s" % result) - - if not self.pyload.debug: - try: - os.remove(tmpCaptcha.name) - except Exception: - pass - - return result - - def fixurl(self, url): url = _fixurl(url) @@ -447,7 +365,7 @@ class Hoster(Plugin): if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) - self.correct_captcha() + self.captcha.correct() self.check_for_same_files() self.pyfile.setStatus("downloading") -- cgit v1.2.3 From 4fc28dc09f9632eb4a15a1ef48778427f9dcae33 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 23 Jul 2015 18:53:06 +0200 Subject: Code cosmetics --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index af3e80acf..ffd7c4ecc 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -83,7 +83,7 @@ class Hoster(Plugin): self.ocr = None #: Account handler instance, see :py:class:`Account` - self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) #: Premium status self.premium = False @@ -107,7 +107,7 @@ class Hoster(Plugin): #: Premium status self.premium = self.account.is_premium(self.user) else: - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) + self.req = self.pyload.requestFactory.getRequest(self.__name__) #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile -- cgit v1.2.3 From f7df6ef48a7c0a8ab6351e046cd12160257c4ef5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 24 Jul 2015 02:15:31 +0200 Subject: Hotfixes --- module/plugins/internal/Hoster.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index ffd7c4ecc..d26592718 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -6,6 +6,7 @@ import inspect import os import random import time +import traceback import urlparse if os.name != "nt": @@ -14,7 +15,7 @@ if os.name != "nt": from module.plugins.internal import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip - chunks, fixurl as _fixurl, replace_patterns, seconds_to_midnight, + chunks, encode, fixurl as _fixurl, replace_patterns, seconds_to_midnight, set_cookies, parse_html_form, parse_html_tag_attr_value, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join @@ -279,7 +280,7 @@ class Hoster(Plugin): """ Skip and give reason """ - raise Skip(fs_encode(reason)) + raise Skip(encode(reason)) def abort(self, reason=""): @@ -287,7 +288,7 @@ class Hoster(Plugin): Abort and give reason """ if reason: - self.pyfile.error = fs_encode(reason) + self.pyfile.error = encode(reason) raise Abort @@ -296,7 +297,7 @@ class Hoster(Plugin): Fail and indicate file is offline """ if reason: - self.pyfile.error = fs_encode(reason) + self.pyfile.error = encode(reason) raise Fail("offline") @@ -305,7 +306,7 @@ class Hoster(Plugin): Fail and indicates file ist temporary offline, the core may take consequences """ if reason: - self.pyfile.error = fs_encode(reason) + self.pyfile.error = encode(reason) raise Fail("temp. offline") @@ -489,7 +490,12 @@ class Hoster(Plugin): return name finally: if delete and do_delete: - os.remove(lastDownload) + try: + os.remove(lastDownload) + except OSError, e: + self.log_warning(_("Error removing: %s") % lastDownload, e) + if self.pyload.debug: + traceback.print_exc() def direct_link(self, url, follow_location=None): -- cgit v1.2.3 From 94d017cd2a5c1f194960827a8c7e46afc3682008 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 24 Jul 2015 06:55:49 +0200 Subject: Hotfixes (2) --- module/plugins/internal/Hoster.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index d26592718..1cc127522 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -13,12 +13,12 @@ if os.name != "nt": import grp import pwd -from module.plugins.internal import Captcha -from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip +from module.plugins.internal.Captcha import Captcha +from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, chunks, encode, fixurl as _fixurl, replace_patterns, seconds_to_midnight, set_cookies, parse_html_form, parse_html_tag_attr_value, timestamp) -from module.utils import fs_decode, fs_encode, save_join as fs_join +from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename #@TODO: Remove in 0.4.10 @@ -48,7 +48,7 @@ class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" __version__ = "0.05" - __status__ = "stable" + __status__ = "testing" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] @@ -66,11 +66,11 @@ class Hoster(Plugin): self.info = {} #: Provide information in dict here #: Engage wan reconnection - self.want_reconnect = False + self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10 #: Enable simultaneous processing of multiple downloads - self.multi_dl = True - self.limit_dl = 0 + self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10 + self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 #: Chunk limit self.chunk_limit = 1 @@ -206,8 +206,8 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): reconnect = bool(reconnect) - self.log_debug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.want_reconnect)) - self.want_reconnect = reconnect + self.log_debug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + self.wantReconnect = reconnect def set_wait(self, seconds, reconnect=None): @@ -247,7 +247,7 @@ class Hoster(Plugin): pyfile.setStatus("waiting") self.log_info(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), - _("Reconnect: %s") % self.want_reconnect) + _("Reconnect: %s") % self.wantReconnect) if self.account: self.log_debug("Ignore reconnection due account logged") @@ -266,7 +266,7 @@ class Hoster(Plugin): if self.thread.m.reconnecting.isSet(): self.waiting = False - self.want_reconnect = False + self.wantReconnect = False raise Reconnect time.sleep(1) @@ -396,7 +396,7 @@ class Hoster(Plugin): filename = os.path.join(location, name) - self.pyload.addonManager.dispatchEvent("download-start", self.pyfile, url, filename) + self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename) try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, @@ -532,7 +532,7 @@ class Hoster(Plugin): value = value.strip() if key in header: - if type(header[key]) == list: + if type(header[key]) is list: header[key].append(value) else: header[key] = [header[key], value] @@ -550,7 +550,7 @@ class Hoster(Plugin): baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) location = urlparse.urljoin(baseurl, location) - if 'code' in header and header['code'] == 302: + if 'code' in header and header['code'] is 302: link = location if follow_location: @@ -597,7 +597,7 @@ class Hoster(Plugin): if traffic is None: return False - elif traffic == -1: + elif traffic is -1: return True else: size = self.pyfile.size / 1024 @@ -612,6 +612,11 @@ class Hoster(Plugin): return self.pyfile.package().password or "" + #: Deprecated method, use `check_for_same_files` instead + def checkForSameFiles(self, *args, **kwargs): + return self.check_for_same_files(*args, **kwargs) + + def check_for_same_files(self, starting=False): """ Checks if same file was/is downloaded within same package @@ -622,7 +627,7 @@ class Hoster(Plugin): pack = self.pyfile.package() for pyfile in self.pyload.files.cache.values(): - if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile != self.pyfile and pyfile.name is self.pyfile.name and pyfile.package().folder is pack.folder: if pyfile.status in (0, 12): #: Finished or downloading self.skip(pyfile.pluginname) elif pyfile.status in (5, 7) and starting: #: A download is waiting/starting and was appenrently started before -- cgit v1.2.3 From 761ca5c66e07559925ebbdbc6531f9ca658b12ce Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 24 Jul 2015 16:11:58 +0200 Subject: Code cosmetics --- module/plugins/internal/Hoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 1cc127522..dbc5e88a8 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -408,7 +408,7 @@ class Hoster(Plugin): if newname: newname = urlparse.urlparse(newname).path.split('/')[-1] - if disposition and newname != name: + if disposition and newname not is name: self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) self.pyfile.name = newname filename = os.path.join(location, newname) @@ -550,7 +550,7 @@ class Hoster(Plugin): baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) location = urlparse.urljoin(baseurl, location) - if 'code' in header and header['code'] is 302: + if 'code' in header and header['code'] == 302: link = location if follow_location: @@ -597,7 +597,7 @@ class Hoster(Plugin): if traffic is None: return False - elif traffic is -1: + elif traffic == -1: return True else: size = self.pyfile.size / 1024 -- cgit v1.2.3 From dd13825fbd3df9e441200638cd2a92e3924dfff6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 24 Jul 2015 23:57:04 +0200 Subject: Fix typo --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index dbc5e88a8..0d2e4dcc4 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -408,7 +408,7 @@ class Hoster(Plugin): if newname: newname = urlparse.urlparse(newname).path.split('/')[-1] - if disposition and newname not is name: + if disposition and newname is not name: self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) self.pyfile.name = newname filename = os.path.join(location, newname) -- cgit v1.2.3 From 8f17f875f6e28f73ddb10da59c6464bd04922222 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Jul 2015 04:59:27 +0200 Subject: Account rewritten --- module/plugins/internal/Hoster.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 0d2e4dcc4..1d2728bd3 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -96,10 +96,10 @@ class Hoster(Plugin): self.account = None if self.account: - self.user, data = self.account.select_account() + self.user, data = self.account.select() #: Browser instance, see `network.Browser` - self.req = self.account.get_account_request(self.user) + self.req = self.account.get_request(self.user) self.chunk_limit = -1 #: Chunk limit, -1 for unlimited #: Enables resume (will be ignored if server dont accept chunks) @@ -593,7 +593,7 @@ class Hoster(Plugin): if not self.account: return True - traffic = self.account.get_account_info(self.user, True)['trafficleft'] + traffic = self.account.get_data(self.user, True)['trafficleft'] if traffic is None: return False @@ -612,7 +612,7 @@ class Hoster(Plugin): return self.pyfile.package().password or "" - #: Deprecated method, use `check_for_same_files` instead + #: Deprecated method, use `check_for_same_files` instead (Remove in 0.4.10) def checkForSameFiles(self, *args, **kwargs): return self.check_for_same_files(*args, **kwargs) -- cgit v1.2.3 From a95c217627a1cb651b24e69f20640df40797aff9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Jul 2015 09:34:18 +0200 Subject: Account rewritten (2) --- module/plugins/internal/Hoster.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 1d2728bd3..840d73847 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -171,7 +171,7 @@ class Hoster(Plugin): self.req.renewHTTPRequest() if self.account: - self.account.check_login(self.user) + self.account.is_logged(self.user) else: self.req.clearCookies() @@ -206,7 +206,10 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): reconnect = bool(reconnect) - self.log_debug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + + self.log_info(_("Reconnect: %s") % reconnect) + self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) + self.wantReconnect = reconnect @@ -220,8 +223,8 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_debug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), - "Wait: %d (+1) seconds" % wait_time) + self.log_info(_("Wait: %d seconds") % wait_time) + self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until @@ -229,13 +232,13 @@ class Hoster(Plugin): self.set_reconnect(reconnect) - def wait(self, seconds=0, reconnect=None): + def wait(self, seconds=None, reconnect=None): """ Waits the time previously set """ pyfile = self.pyfile - if seconds > 0: + if seconds is not None: self.set_wait(seconds) if reconnect is not None: @@ -243,12 +246,9 @@ class Hoster(Plugin): self.waiting = True - status = pyfile.status + status = pyfile.status #@NOTE: Remove in 0.4.10 pyfile.setStatus("waiting") - self.log_info(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), - _("Reconnect: %s") % self.wantReconnect) - if self.account: self.log_debug("Ignore reconnection due account logged") @@ -256,11 +256,9 @@ class Hoster(Plugin): if pyfile.abort: self.abort() - time.sleep(1) + time.sleep(3) else: while pyfile.waitUntil > time.time(): - self.thread.m.reconnecting.wait(1) - if pyfile.abort: self.abort() @@ -268,12 +266,12 @@ class Hoster(Plugin): self.waiting = False self.wantReconnect = False raise Reconnect - - time.sleep(1) + + self.thread.m.reconnecting.wait(3) + time.sleep(3) self.waiting = False - - pyfile.status = status + pyfile.status = status #@NOTE: Remove in 0.4.10 def skip(self, reason=""): @@ -388,7 +386,7 @@ class Hoster(Plugin): os.chown(location, uid, gid) except Exception, e: - self.fail(e) + self.fail(str(e)) #@TODO: Remove `str` in 0.4.10 #: Convert back to unicode location = fs_decode(location) @@ -508,7 +506,7 @@ class Hoster(Plugin): redirect = max(follow_location, 1) else: - redirect = self.get_config("maxredirs", plugin="UserAgentSwitcher") + redirect = self.get_config("maxredirs", 10, "UserAgentSwitcher") for i in xrange(redirect): try: -- cgit v1.2.3 From 952001324e1faf584b1adcb01c4a0406a3722932 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Jul 2015 09:42:49 +0200 Subject: =?UTF-8?q?Don't=20user=20dictionary=E2=80=99s=20iterator=20method?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 840d73847..10cab5616 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -362,7 +362,7 @@ class Hoster(Plugin): self.fail(_("No url given")) if self.pyload.debug: - self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) self.captcha.correct() self.check_for_same_files() @@ -474,7 +474,7 @@ class Hoster(Plugin): #: Produces encoding errors, better log to other file in the future? # self.log_debug("Content: %s" % content) - for name, rule in rules.iteritems(): + for name, rule in rules.items(): if isinstance(rule, basestring): if rule in content: do_delete = True -- cgit v1.2.3 From f83389333ec10376452aa5f6d5ccd3963c6bafa1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Jul 2015 10:28:30 +0200 Subject: Update internal plugins --- module/plugins/internal/Hoster.py | 67 +++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 27 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 10cab5616..c35178547 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.05" + __version__ = "0.06" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -179,6 +179,7 @@ class Hoster(Plugin): self.pyfile.setStatus("starting") + self.log_debug("PROCESS URL " + self.pyfile.url) return self.process(self.pyfile) @@ -195,19 +196,10 @@ class Hoster(Plugin): return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) - def reset_account(self): - """ - Don't use account and retry download - """ - self.account = None - self.req = self.pyload.requestFactory.getRequest(self.__name__) - self.retry() - - def set_reconnect(self, reconnect): reconnect = bool(reconnect) - self.log_info(_("Reconnect: %s") % reconnect) + self.log_info(_("RECONNECT ") + ("enabled" if reconnect else "disabled")) self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) self.wantReconnect = reconnect @@ -223,7 +215,7 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_info(_("Wait: %d seconds") % wait_time) + self.log_info(_("WAIT %d seconds") % wait_time) self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until @@ -249,14 +241,16 @@ class Hoster(Plugin): status = pyfile.status #@NOTE: Remove in 0.4.10 pyfile.setStatus("waiting") - if self.account: - self.log_debug("Ignore reconnection due account logged") + if not self.wantReconnect or self.account: + if self.account: + self.log_warning("Ignore reconnection due logged account") while pyfile.waitUntil > time.time(): if pyfile.abort: self.abort() time.sleep(3) + else: while pyfile.waitUntil > time.time(): if pyfile.abort: @@ -266,7 +260,7 @@ class Hoster(Plugin): self.waiting = False self.wantReconnect = False raise Reconnect - + self.thread.m.reconnecting.wait(3) time.sleep(3) @@ -321,7 +315,7 @@ class Hoster(Plugin): self.retries[id] = 0 if 0 < max_tries <= self.retries[id]: - self.fail(reason or _("Max retries reached"), _("retry")) + self.fail(reason or _("Max retries reached")) self.wait(wait_time, False) @@ -329,6 +323,21 @@ class Hoster(Plugin): raise Retry(reason) + def restart(self, reason=None, reset=False): + if not reason: + reason = _("Fallback to free download") if reset else _("Restart") + + if reset: + if not self.premium: + return + + self.premium = False + self.account = None + self.req = self.pyload.requestFactory.getRequest(self.__name__) + + raise Retry(reason) + + def fixurl(self, url): url = _fixurl(url) @@ -362,7 +371,8 @@ class Hoster(Plugin): self.fail(_("No url given")) if self.pyload.debug: - self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + self.log_debug("DOWNLOAD URL " + url, + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) self.captcha.correct() self.check_for_same_files() @@ -432,7 +442,7 @@ class Hoster(Plugin): return self.last_download - def check_download(self, rules, delete=True, file_size=0, size_tolerance=1000, read_size=100000): + def check_download(self, rules, delete=False, file_size=0, size_tolerance=2048, read_size=100000): """ Checks the content of the last downloaded file, re match is saved to `lastCheck` @@ -444,14 +454,14 @@ class Hoster(Plugin): :return: dictionary key of the first rule that matched """ do_delete = False - lastDownload = fs_encode(self.last_download) + last_download = fs_encode(self.last_download) - if not self.last_download or not os.path.exists(lastDownload): + if not self.last_download or not os.path.exists(last_download): self.last_download = "" self.fail(self.pyfile.error or _("No file downloaded")) try: - download_size = os.stat(lastDownload).st_size + download_size = os.stat(last_download).st_size if download_size < 1: do_delete = True @@ -462,14 +472,13 @@ class Hoster(Plugin): if diff > size_tolerance: do_delete = True - self.fail(_("File size mismatch")) + self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") + % (file_size, download_size)) elif diff != 0: self.log_warning(_("File size is not equal to expected size")) - self.log_debug("Download Check triggered") - - with open(lastDownload, "rb") as f: + with open(last_download, "rb") as f: content = f.read(read_size) #: Produces encoding errors, better log to other file in the future? @@ -489,12 +498,16 @@ class Hoster(Plugin): finally: if delete and do_delete: try: - os.remove(lastDownload) + os.remove(last_download) + except OSError, e: - self.log_warning(_("Error removing: %s") % lastDownload, e) + self.log_warning(_("Error removing: %s") % last_download, e) if self.pyload.debug: traceback.print_exc() + else: + self.log_info(_("File deleted")) + def direct_link(self, url, follow_location=None): link = "" -- cgit v1.2.3 From 34f48259060656077b5cb45edd8f9d92bb0282de Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Jul 2015 20:19:03 +0200 Subject: Bunch of fixups --- module/plugins/internal/Hoster.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index c35178547..8a11499fa 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -15,9 +15,9 @@ if os.name != "nt": from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, encode, fixurl as _fixurl, replace_patterns, seconds_to_midnight, - set_cookies, parse_html_form, parse_html_tag_attr_value, - timestamp) + chunks, encode, exists, fixurl as _fixurl, replace_patterns, + seconds_to_midnight, set_cookies, parse_html_form, + parse_html_tag_attr_value, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename @@ -249,7 +249,7 @@ class Hoster(Plugin): if pyfile.abort: self.abort() - time.sleep(3) + time.sleep(2) else: while pyfile.waitUntil > time.time(): @@ -261,8 +261,8 @@ class Hoster(Plugin): self.wantReconnect = False raise Reconnect - self.thread.m.reconnecting.wait(3) - time.sleep(3) + self.thread.m.reconnecting.wait(2) + time.sleep(2) self.waiting = False pyfile.status = status #@NOTE: Remove in 0.4.10 @@ -386,7 +386,7 @@ class Hoster(Plugin): location = fs_join(download_folder, self.pyfile.package().folder) - if not os.path.exists(location): + if not exists(location): try: os.makedirs(location, int(self.pyload.config.get("permission", "folder"), 8)) @@ -456,7 +456,7 @@ class Hoster(Plugin): do_delete = False last_download = fs_encode(self.last_download) - if not self.last_download or not os.path.exists(last_download): + if not self.last_download or not exists(last_download): self.last_download = "" self.fail(self.pyfile.error or _("No file downloaded")) @@ -506,6 +506,7 @@ class Hoster(Plugin): traceback.print_exc() else: + self.last_download = "" self.log_info(_("File deleted")) @@ -647,14 +648,14 @@ class Hoster(Plugin): download_folder = self.pyload.config.get("general", "download_folder") location = fs_join(download_folder, pack.folder, self.pyfile.name) - if starting and self.pyload.config.get("download", "skip_existing") and os.path.exists(location): + if starting and self.pyload.config.get("download", "skip_existing") and exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: self.skip("File exists") pyfile = self.pyload.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: - if os.path.exists(location): + if exists(location): self.skip(pyfile[0]) self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name) -- cgit v1.2.3 From 95ed76d34290e08876dccce6840c3e09138a2047 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Jul 2015 23:47:47 +0200 Subject: Spare code fixes --- module/plugins/internal/Hoster.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 8a11499fa..3d4c11af9 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -272,15 +272,17 @@ class Hoster(Plugin): """ Skip and give reason """ - raise Skip(encode(reason)) + raise Skip(encode(reason)) #@TODO: Remove `encode` in 0.4.10 def abort(self, reason=""): """ Abort and give reason """ + #@TODO: Remove in 0.4.10 if reason: self.pyfile.error = encode(reason) + raise Abort @@ -288,8 +290,10 @@ class Hoster(Plugin): """ Fail and indicate file is offline """ + #@TODO: Remove in 0.4.10 if reason: self.pyfile.error = encode(reason) + raise Fail("offline") @@ -297,8 +301,10 @@ class Hoster(Plugin): """ Fail and indicates file ist temporary offline, the core may take consequences """ + #@TODO: Remove in 0.4.10 if reason: self.pyfile.error = encode(reason) + raise Fail("temp. offline") @@ -320,7 +326,7 @@ class Hoster(Plugin): self.wait(wait_time, False) self.retries[id] += 1 - raise Retry(reason) + raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 def restart(self, reason=None, reset=False): @@ -335,7 +341,7 @@ class Hoster(Plugin): self.account = None self.req = self.pyload.requestFactory.getRequest(self.__name__) - raise Retry(reason) + raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 def fixurl(self, url): -- cgit v1.2.3 From 6f387ff66b93e24cc52db7a5ea277265d35f051d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 00:29:19 +0200 Subject: Missed to bump up some plugin __version__ --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 3d4c11af9..f3b78a348 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.07" __status__ = "testing" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From ca5eb403dd6797a3517f6eb5aae7922f586c5c45 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 18:55:09 +0200 Subject: Fix http://forum.pyload.org/viewtopic.php?f=12&t=4410 --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index f3b78a348..15344b86b 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.07" + __version__ = "0.08" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -448,7 +448,7 @@ class Hoster(Plugin): return self.last_download - def check_download(self, rules, delete=False, file_size=0, size_tolerance=2048, read_size=100000): + def check_download(self, rules, delete=False, file_size=0, size_tolerance=1024, read_size=100000): """ Checks the content of the last downloaded file, re match is saved to `lastCheck` -- cgit v1.2.3 From 9e44724d6dda3e1f47cf4d6de4b3e521a244ba4b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 19:15:28 +0200 Subject: Fix content-disposition --- module/plugins/internal/Hoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 15344b86b..fdfe92593 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.08" + __version__ = "0.09" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -355,7 +355,7 @@ class Hoster(Plugin): return url - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ Downloads the content at url to download folder @@ -420,7 +420,7 @@ class Hoster(Plugin): self.pyfile.size = self.req.size if newname: - newname = urlparse.urlparse(newname).path.split('/')[-1] + newname = urlparse.urlparse(newname).path.split('/')[-1].split('*=')[-1] #@TODO: Remove in 0.4.10 if disposition and newname is not name: self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) -- cgit v1.2.3 From 5745baca2dd9c8831631489781ef950af5184081 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Jul 2015 22:06:31 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1520 --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index fdfe92593..6fe68bb6f 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.09" + __version__ = "0.10" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -420,7 +420,7 @@ class Hoster(Plugin): self.pyfile.size = self.req.size if newname: - newname = urlparse.urlparse(newname).path.split('/')[-1].split('*=')[-1] #@TODO: Remove in 0.4.10 + newname = urlparse.urlparse(newname).path.split('*=')[-1].split('/')[-1] #@TODO: Remove in 0.4.10 if disposition and newname is not name: self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) -- cgit v1.2.3 From 064c4bca39b545cdab7e0b45816ed1a260e192a2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 29 Jul 2015 14:05:49 +0200 Subject: Fix http://forum.pyload.org/viewtopic.php?f=12&t=4421 --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 6fe68bb6f..d30c56035 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.10" + __version__ = "0.11" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -420,7 +420,7 @@ class Hoster(Plugin): self.pyfile.size = self.req.size if newname: - newname = urlparse.urlparse(newname).path.split('*=')[-1].split('/')[-1] #@TODO: Remove in 0.4.10 + newname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] #@TODO: Remove in 0.4.10 if disposition and newname is not name: self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) -- cgit v1.2.3 From 503906b69e7ffcad398a2529c64b806cf7c5b558 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 30 Jul 2015 21:04:38 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1624 --- module/plugins/internal/Hoster.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index d30c56035..1b96f1201 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.11" + __version__ = "0.12" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -385,11 +385,7 @@ class Hoster(Plugin): self.pyfile.setStatus("downloading") - if disposition: - self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name - download_folder = self.pyload.config.get("general", "download_folder") - location = fs_join(download_folder, self.pyfile.package().folder) if not exists(location): @@ -402,7 +398,7 @@ class Hoster(Plugin): os.chown(location, uid, gid) except Exception, e: - self.fail(str(e)) #@TODO: Remove `str` in 0.4.10 + self.fail(e) #: Convert back to unicode location = fs_decode(location) -- cgit v1.2.3 From 092112b44af84c7a59b1fa2cfff5c5875e778a8f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 02:21:35 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1625 --- module/plugins/internal/Hoster.py | 66 +++++++++++++++------------------------ 1 file changed, 26 insertions(+), 40 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 1b96f1201..c54adf4b3 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.12" + __version__ = "0.13" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -179,7 +179,7 @@ class Hoster(Plugin): self.pyfile.setStatus("starting") - self.log_debug("PROCESS URL " + self.pyfile.url) + self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) return self.process(self.pyfile) @@ -190,12 +190,6 @@ class Hoster(Plugin): raise NotImplementedError - def get_chunk_count(self): - if self.chunk_limit <= 0: - return self.pyload.config.get("download", "chunks") - return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) - - def set_reconnect(self, reconnect): reconnect = bool(reconnect) @@ -385,29 +379,27 @@ class Hoster(Plugin): self.pyfile.setStatus("downloading") - download_folder = self.pyload.config.get("general", "download_folder") - location = fs_join(download_folder, self.pyfile.package().folder) + download_folder = self.pyload.config.get("general", "download_folder") + download_location = fs_join(download_folder, self.pyfile.package().folder) - if not exists(location): + if not exists(download_location): try: - os.makedirs(location, int(self.pyload.config.get("permission", "folder"), 8)) - - if self.pyload.config.get("permission", "change_dl") and os.name != "nt": - uid = pwd.getpwnam(self.pyload.config.get("permission", "user"))[2] - gid = grp.getgrnam(self.pyload.config.get("permission", "group"))[2] - os.chown(location, uid, gid) + os.makedirs(download_location) except Exception, e: self.fail(e) - #: Convert back to unicode - location = fs_decode(location) - name = safe_filename(self.pyfile.name) + self.set_permissions(download_location) + location = fs_decode(download_location) + name = safe_filename(urlparse.urlparse(self.pyfile.name).path.split('/')[-1] or self.pyfile.name) filename = os.path.join(location, name) self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename) + if self.pyfile.abort: + self.abort() + try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.get_chunk_count(), resume=self.resume_download, @@ -415,32 +407,26 @@ class Hoster(Plugin): finally: self.pyfile.size = self.req.size - if newname: - newname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] #@TODO: Remove in 0.4.10 - - if disposition and newname is not name: - self.log_info(_("%(name)s saved as %(newname)s") % {'name': name, 'newname': newname}) - self.pyfile.name = newname - filename = os.path.join(location, newname) + #@TODO: Recheck in 0.4.10 + if disposition: + finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] - fs_filename = fs_encode(filename) + if finalname != newname != name: + try: + os.rename(fs_join(location, newname), fs_join(location, finalname)) - if self.pyload.config.get("permission", "change_file"): - try: - os.chmod(fs_filename, int(self.pyload.config.get("permission", "file"), 8)) - except Exception, e: - self.log_warning(_("Setting file mode failed"), e) + except OSError, e: + self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e) + finalname = newname - if self.pyload.config.get("permission", "change_dl") and os.name != "nt": - try: - uid = pwd.getpwnam(self.pyload.config.get("permission", "user"))[2] - gid = grp.getgrnam(self.pyload.config.get("permission", "group"))[2] - os.chown(fs_filename, uid, gid) + self.log_info(_("`%s` saved as `%s`") % (name, finalname)) + self.pyfile.name = finalname + filename = os.path.join(location, finalname) - except Exception, e: - self.log_warning(_("Setting User and Group failed"), e) + self.set_permissions(fs_encode(filename)) self.last_download = filename + return self.last_download -- cgit v1.2.3 From 8a8323f3fd86985a5ca876e8d54f1699e75b24a8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 07:37:10 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1628 --- module/plugins/internal/Hoster.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index c54adf4b3..4d69720ac 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -9,10 +9,6 @@ import time import traceback import urlparse -if os.name != "nt": - import grp - import pwd - from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, chunks, encode, exists, fixurl as _fixurl, replace_patterns, -- cgit v1.2.3 From cc3c575a68fd4b585f0e4af586e027fddb0fe605 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 31 Jul 2015 10:38:51 +0200 Subject: Fix and improve account logic --- module/plugins/internal/Hoster.py | 92 ++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 44 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 4d69720ac..1aea3804d 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.13" + __version__ = "0.14" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -68,43 +68,14 @@ class Hoster(Plugin): self.multiDL = True #@TODO: Change to `multi_dl` in 0.4.10 self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 - #: Chunk limit - self.chunk_limit = 1 - self.resume_download = False - #: time.time() + wait in seconds self.wait_until = 0 - self.waiting = False - - #: Captcha reader instance - self.ocr = None + self.waiting = False #: Account handler instance, see :py:class:`Account` - self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - - #: Premium status - self.premium = False - - #: username/login - self.user = None - - if self.account and not self.account.can_use(): - self.account = None - - if self.account: - self.user, data = self.account.select() - - #: Browser instance, see `network.Browser` - self.req = self.account.get_request(self.user) - self.chunk_limit = -1 #: Chunk limit, -1 for unlimited - - #: Enables resume (will be ignored if server dont accept chunks) - self.resume_download = True - - #: Premium status - self.premium = self.account.is_premium(self.user) - else: - self.req = self.pyload.requestFactory.getRequest(self.__name__) + self.account = None + self.user = None + self.req = None #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile @@ -129,6 +100,7 @@ class Hoster(Plugin): #: Dict of the amount of retries already made self.retries = {} + self._setup() self.init() @@ -136,9 +108,9 @@ class Hoster(Plugin): def get_info(cls, url="", html=""): url = _fixurl(url) url_p = urlparse.urlparse(url) - return {'name' : (url_p.path.split('/')[-1] - or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] - or url_p.netloc.split('.', 1)[0]), + return {'name' : (url_p.path.split('/')[-1] or + url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or + url_p.netloc.split('.', 1)[0]), 'size' : 0, 'status': 3 if url else 8, 'url' : url} @@ -158,6 +130,41 @@ class Hoster(Plugin): pass + def _setup(self): + if self.account: + self.chunk_limit = -1 #: -1 for unlimited + self.resume_download = True + self.premium = self.account.is_premium(self.user) + else: + self.chunk_limit = 1 + self.resume_download = False + self.premium = False + + + def load_account(self): + oldaccount = self.account + + if not self.account: + self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) + + if self.account: + if self.user: + self.account = self.account.is_logged(self.user, relogin=True) + else: + self.user, data = self.account.select() + if not self.user: + self.account = False + + if self.account == oldaccount: + self.req.clearCookies() + + elif self.account: + self.req = self.account.get_request(self.user) #: Browser instance, see `network.Browser` + + else: + self.req = self.pyload.requestFactory.getRequest(self.__name__) + + def preprocessing(self, thread): """ Handles important things to do before starting @@ -166,16 +173,13 @@ class Hoster(Plugin): self.req.renewHTTPRequest() - if self.account: - self.account.is_logged(self.user) - else: - self.req.clearCookies() - + self.load_account() + self._setup() self.setup() self.pyfile.setStatus("starting") - self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) + return self.process(self.pyfile) @@ -426,7 +430,7 @@ class Hoster(Plugin): return self.last_download - def check_download(self, rules, delete=False, file_size=0, size_tolerance=1024, read_size=100000): + def check_download(self, rules, delete=False, file_size=0, size_tolerance=1024, read_size=1048576): """ Checks the content of the last downloaded file, re match is saved to `lastCheck` -- cgit v1.2.3 From a9e1aacd52e53b734c1e18299141900c1c8c0c32 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 1 Aug 2015 02:36:15 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1636 --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 1aea3804d..92afd3672 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -332,7 +332,7 @@ class Hoster(Plugin): return self.premium = False - self.account = None + self.account = False self.req = self.pyload.requestFactory.getRequest(self.__name__) raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 -- cgit v1.2.3 From b612fbda4fa6ed8cb0b4e2e3015e8127a20aaea2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 1 Aug 2015 22:34:51 +0200 Subject: Fix load_account routine --- module/plugins/internal/Hoster.py | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 92afd3672..ea54d9589 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.14" + __version__ = "0.15" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -142,27 +142,22 @@ class Hoster(Plugin): def load_account(self): - oldaccount = self.account + if self.req: + self.req.close() if not self.account: self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) if self.account: - if self.user: - self.account = self.account.is_logged(self.user, relogin=True) - else: + if not self.user: self.user, data = self.account.select() - if not self.user: - self.account = False - if self.account == oldaccount: - self.req.clearCookies() + if not self.user or not self.account.is_logged(self.user, relogin=True): + self.account = False - elif self.account: - self.req = self.account.get_request(self.user) #: Browser instance, see `network.Browser` - - else: - self.req = self.pyload.requestFactory.getRequest(self.__name__) + #: Browser instance, see `network.Browser` + self.req = self.pyload.requestFactory.getRequest(self.__name__, + self.user if self.account else None) def preprocessing(self, thread): @@ -171,8 +166,6 @@ class Hoster(Plugin): """ self.thread = thread - self.req.renewHTTPRequest() - self.load_account() self._setup() self.setup() @@ -516,10 +509,9 @@ class Hoster(Plugin): header = self.load(url, just_header=True) except Exception: #: Bad bad bad... rewrite this part in 0.4.10 - req = pyreq.getHTTPRequest() - res = self.load(url, just_header=True) - - req.close() + res = self.load(url, + just_header=True, + req=self.pyload.requestFactory.getRequest()) header = {'code': req.code} for line in res.splitlines(): @@ -657,7 +649,8 @@ class Hoster(Plugin): del self.pyfile if hasattr(self, "req"): - self.req.close() + if self.req: + self.req.close() del self.req if hasattr(self, "thread"): -- cgit v1.2.3 From b0d9532f675854a6a32d22f7b6b7b4cd32683443 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 07:10:51 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1640 --- module/plugins/internal/Hoster.py | 64 ++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 38 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index ea54d9589..d707154bb 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.15" + __version__ = "0.16" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -75,7 +75,7 @@ class Hoster(Plugin): #: Account handler instance, see :py:class:`Account` self.account = None self.user = None - self.req = None + self.req = None #: Browser instance, see `network.Browser` #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile @@ -98,7 +98,8 @@ class Hoster(Plugin): self.html = None #: Dict of the amount of retries already made - self.retries = {} + self.retries = {} + self.retry_free = False #@TODO: Recheck in 0.4.10 self._setup() self.init() @@ -132,10 +133,12 @@ class Hoster(Plugin): def _setup(self): if self.account: + self.req = self.pyload.requestFactory.getRequest(self.__name__, self.user) self.chunk_limit = -1 #: -1 for unlimited self.resume_download = True self.premium = self.account.is_premium(self.user) else: + self.req = self.pyload.requestFactory.getRequest(self.__name__) self.chunk_limit = 1 self.resume_download = False self.premium = False @@ -150,15 +153,11 @@ class Hoster(Plugin): if self.account: if not self.user: - self.user, data = self.account.select() + self.user = self.account.select()[0] - if not self.user or not self.account.is_logged(self.user, relogin=True): + if not self.user or not self.account.is_logged(self.user, True): self.account = False - #: Browser instance, see `network.Browser` - self.req = self.pyload.requestFactory.getRequest(self.__name__, - self.user if self.account else None) - def preprocessing(self, thread): """ @@ -166,10 +165,20 @@ class Hoster(Plugin): """ self.thread = thread - self.load_account() + if self.retry_free: + self.account = False + else: + self.load_account() #@TODO: Move to PluginThread in 0.4.10 + self.retry_free = False + self._setup() self.setup() + self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 + + if self.pyfile.abort: + self.abort() + self.pyfile.setStatus("starting") self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) @@ -316,17 +325,15 @@ class Hoster(Plugin): raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 - def restart(self, reason=None, reset=False): + def restart(self, reason=None, nopremium=False): if not reason: - reason = _("Fallback to free download") if reset else _("Restart") - - if reset: - if not self.premium: - return + reason = _("Fallback to free download") if nopremium else _("Restart") - self.premium = False - self.account = False - self.req = self.pyload.requestFactory.getRequest(self.__name__) + if nopremium: + if self.premium: + self.retry_free = True + else: + self.fail(reason, _("Download was already free")) raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 @@ -639,22 +646,3 @@ class Hoster(Plugin): self.skip(pyfile[0]) self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name) - - - def clean(self): - """ - Clean everything and remove references - """ - if hasattr(self, "pyfile"): - del self.pyfile - - if hasattr(self, "req"): - if self.req: - self.req.close() - del self.req - - if hasattr(self, "thread"): - del self.thread - - if hasattr(self, "html"): - del self.html -- cgit v1.2.3 From 84f2193d76f7c6f47c834dc8902d8ead8e45a11a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 07:44:07 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1640 (2) --- module/plugins/internal/Hoster.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index d707154bb..edc6f5d30 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.16" + __version__ = "0.17" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -58,8 +58,7 @@ class Hoster(Plugin): def __init__(self, pyfile): - self.pyload = pyfile.m.core - self.info = {} #: Provide information in dict here + self._init(pyfile.m.core) #: Engage wan reconnection self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10 -- cgit v1.2.3 From 5e15580202c44628f2fbfabad0c3f693975fb3c9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Aug 2015 17:13:17 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1663 --- module/plugins/internal/Hoster.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index edc6f5d30..67feec38c 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.17" + __version__ = "0.18" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -373,6 +373,9 @@ class Hoster(Plugin): self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + name = _fixurl(self.pyfile.name) + self.pyfile.name = urlparse.urlparse(name).path.split('/')[-1] or name + self.captcha.correct() self.check_for_same_files() @@ -384,15 +387,13 @@ class Hoster(Plugin): if not exists(download_location): try: os.makedirs(download_location) - except Exception, e: self.fail(e) self.set_permissions(download_location) location = fs_decode(download_location) - name = safe_filename(urlparse.urlparse(self.pyfile.name).path.split('/')[-1] or self.pyfile.name) - filename = os.path.join(location, name) + filename = os.path.join(location, safe_filename(self.pyfile.name)) #@TODO: Move `safe_filename` check to HTTPDownload in 0.4.10 self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename) @@ -407,10 +408,10 @@ class Hoster(Plugin): self.pyfile.size = self.req.size #@TODO: Recheck in 0.4.10 - if disposition: + if disposition and newname: finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] - if finalname != newname != name: + if finalname != newname != self.pyfile.name: try: os.rename(fs_join(location, newname), fs_join(location, finalname)) @@ -418,7 +419,7 @@ class Hoster(Plugin): self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e) finalname = newname - self.log_info(_("`%s` saved as `%s`") % (name, finalname)) + self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname)) self.pyfile.name = finalname filename = os.path.join(location, finalname) -- cgit v1.2.3 From e5ce0acf056dc96c40d5616ab6d2b82f998eefbe Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Aug 2015 00:37:57 +0200 Subject: Use set_cookie instead cj.setCookie --- module/plugins/internal/Hoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 67feec38c..87ee2188c 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -12,7 +12,7 @@ import urlparse from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, chunks, encode, exists, fixurl as _fixurl, replace_patterns, - seconds_to_midnight, set_cookies, parse_html_form, + seconds_to_midnight, set_cookie, set_cookies, parse_html_form, parse_html_tag_attr_value, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename -- cgit v1.2.3 From 4323507752c54c03cf109ee4c5b74d11bae753dd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Aug 2015 23:57:23 +0200 Subject: Fix https://github.com/pyload/pyload/issues/1707 --- module/plugins/internal/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal/Hoster.py') diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 87ee2188c..a0cdb1e2e 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -43,7 +43,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.18" + __version__ = "0.19" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -332,7 +332,7 @@ class Hoster(Plugin): if self.premium: self.retry_free = True else: - self.fail(reason, _("Download was already free")) + self.fail("%s | %s" % (reason, _("Download was already free"))) raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 -- cgit v1.2.3