From 5499be89203a18ca61a21cfc7266cf0f4ebe6547 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 15 Dec 2011 23:18:21 +0100 Subject: refractoring --- module/plugins/Hoster.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 814a70949..aa50099fb 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -19,15 +19,15 @@ from module.plugins.Plugin import Plugin -def getInfo(self): - #result = [ .. (name, size, status, url) .. ] - return - class Hoster(Plugin): - __name__ = "Hoster" - __version__ = "0.1" - __pattern__ = None - __type__ = "hoster" - __description__ = """Base hoster plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + + @staticmethod + def getInfo(urls): + """This method is used to retrieve the online status of files for hoster plugins. + It has to *yield* list of tuples with the result in this format (name, size, status, url), + where status is one of API pyfile statusses. + + :param urls: List of urls + :return: + """ + pass \ No newline at end of file -- cgit v1.2.3 From d35c003cc53d4723d1dfe0d81eeb9bea78cee594 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 31 Dec 2011 16:01:24 +0100 Subject: new crypter plugin API, now decrypting possible for now. --- module/plugins/Hoster.py | 444 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 438 insertions(+), 6 deletions(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index aa50099fb..54c2efdfd 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -13,13 +13,39 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - - @author: mkaay + + @author: RaNaN, spoob, mkaay """ -from module.plugins.Plugin import Plugin +from time import time, sleep +from random import randint + +import os + +if os.name != "nt": + from module.utils.fs import chown + from pwd import getpwnam + from grp import getgrnam + +from Base import Base, Fail, Retry +from module.utils import chunks #legacy import +from module.utils.fs import save_join, save_path, fs_encode, fs_decode,\ + remove, makedirs, chmod, stat, exists, join + + +class Abort(Exception): + """ raised when aborted """ + +class Reconnect(Exception): + """ raised when reconnected """ -class Hoster(Plugin): +class SkipDownload(Exception): + """ raised when download should be skipped """ + +class Hoster(Base): + """ + Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading. + """ @staticmethod def getInfo(urls): @@ -28,6 +54,412 @@ class Hoster(Plugin): where status is one of API pyfile statusses. :param urls: List of urls - :return: + :return: yield list of tuple with results (name, size, status, url) + """ + pass + + def __init__(self, pyfile): + Base.__init__(self, pyfile.m.core) + + self.wantReconnect = False + #: enables simultaneous processing of multiple downloads + self.multiDL = True + self.limitDL = 0 + #: chunk limit + self.chunkLimit = 1 + #: enables resume (will be ignored if server dont accept chunks) + self.resumeDownload = False + + #: time() + wait in seconds + self.waitUntil = 0 + self.waiting = False + + self.ocr = None #captcha reader instance + #: account handler instance, see :py:class:`Account` + self.account = self.core.accountManager.getAccountForPlugin(self.__name__) + + #: premium status + self.premium = False + #: username/login + self.user = None + + if self.account and not self.account.isUsable(): self.account = None + if self.account: + self.user = self.account.loginname + #: Browser instance, see `network.Browser` + self.req = self.account.getAccountRequest() + # Default: -1, True, True + self.chunkLimit, self.resumeDownload, self.multiDL = self.account.getDownloadSettings() + self.premium = self.account.isPremium() + else: + self.req = self.core.requestFactory.getRequest(self.__name__) + + #: associated pyfile instance, see `PyFile` + self.pyfile = pyfile + self.thread = None # holds thread in future + + #: location where the last call to download was saved + self.lastDownload = "" + #: re match of the last call to `checkDownload` + self.lastCheck = None + #: js engine, see `JsEngine` + self.js = self.core.js + self.cTask = None #captcha task + + self.retries = 0 # amount of retries already made + self.html = None # some plugins store html code here + + self.init() + + def getChunkCount(self): + if self.chunkLimit <= 0: + return self.config["download"]["chunks"] + return min(self.config["download"]["chunks"], self.chunkLimit) + + def __call__(self): + return self.__name__ + + def init(self): + """initialize the plugin (in addition to `__init__`)""" + pass + + def setup(self): + """ setup for enviroment and other things, called before downloading (possibly more than one time)""" + pass + + def preprocessing(self, thread): + """ handles important things to do before starting """ + self.thread = thread + + if self.account: + # will force a relogin or reload of account info if necessary + self.account.getAccountInfo() + else: + self.req.clearCookies() + + self.setup() + + self.pyfile.setStatus("starting") + + return self.process(self.pyfile) + + + def process(self, pyfile): + """the 'main' method of every plugin, you **have to** overwrite it""" + raise NotImplementedError + + def resetAccount(self): + """ dont use account and retry download """ + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retry() + + def checksum(self, local_file=None): + """ + return codes: + 0 - checksum ok + 1 - checksum wrong + 5 - can't get checksum + 10 - not implemented + 20 - unknown error + """ + #@TODO checksum check hook + + return True, 10 + + + def setWait(self, seconds, reconnect=False): + """Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + if reconnect: + self.wantReconnect = True + self.pyfile.waitUntil = time() + int(seconds) + + def wait(self): + """ waits the time previously set """ + self.waiting = True + self.pyfile.setStatus("waiting") + + while self.pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) + + if self.pyfile.abort: raise Abort + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + self.waiting = False + self.pyfile.setStatus("starting") + + def offline(self): + """ fail and indicate file is offline """ + raise Fail("offline") + + def tempOffline(self): + """ fail and indicates file ist temporary offline, the core may take consequences """ + raise Fail("temp. offline") + + def retry(self, max_tries=3, wait_time=1, reason=""): + """Retries and begin again from the beginning + + :param max_tries: number of maximum retries + :param wait_time: time to wait in seconds + :param reason: reason for retrying, will be passed to fail if max_tries reached + """ + if 0 < max_tries <= self.retries: + if not reason: reason = "Max retries reached" + raise Fail(reason) + + self.wantReconnect = False + self.setWait(wait_time) + self.wait() + + self.retries += 1 + raise Retry(reason) + + def invalidCaptcha(self): + if self.cTask: + self.cTask.invalid() + + def correctCaptcha(self): + if self.cTask: + self.cTask.correct() + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', + result_type='textual'): + """ Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time())[-6:].replace(".", "") + temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp_file.write(img) + temp_file.close() + + has_plugin = self.__name__ in self.core.pluginManager.getPlugins("captcha") + + if self.core.captcha: + Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) + else: + Ocr = None + + if Ocr and not forceUser: + sleep(randint(3000, 5000) / 1000.0) + if self.pyfile.abort: raise Abort + + ocr = Ocr() + result = ocr.get_captcha(temp_file.name) + else: + captchaManager = self.core.captchaManager + task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) + self.cTask = task + captchaManager.handleCaptcha(task) + + while task.isWaiting(): + if self.pyfile.abort: + captchaManager.removeTask(task) + raise Abort + sleep(1) + + captchaManager.removeTask(task) + + if task.error and has_plugin: #ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) + + result = task.result + self.log.debug("Received captcha result: %s" % str(result)) + + if not self.core.debug: + try: + remove(temp_file.name) + except: + pass + + return result + + + def load(self, *args, **kwargs): + """ See 'Base' load method for more info """ + if self.pyfile.abort: raise Abort + return Base.load(self, *args, **kwargs) + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + """Downloads the content at url to download folder + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param disposition: if True and server provides content-disposition header\ + the filename will be changed if needed + :return: The location where the file was saved + """ + + self.checkForSameFiles() + + self.pyfile.setStatus("downloading") + + download_folder = self.config['general']['download_folder'] + + location = save_join(download_folder, self.pyfile.package().folder) + + if not exists(location): + makedirs(location, int(self.core.config["permission"]["folder"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(location, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + # convert back to unicode + location = fs_decode(location) + name = save_path(self.pyfile.name) + + filename = join(location, name) + + self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) + + try: + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.setProgress, disposition=disposition) + finally: + self.pyfile.size = self.req.size + + if disposition and newname and newname != name: #triple check, just to be sure + self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) + + fs_filename = fs_encode(filename) + + if self.core.config["permission"]["change_file"]: + chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(fs_filename, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + self.lastDownload = filename + return self.lastDownload + + def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): + """ checks the content of the last downloaded file, re match is saved to `lastCheck` + + :param rules: dict with names and rules to match (compiled regexp or strings) + :param api_size: expected file size + :param max_size: if the file is larger then it wont be checked + :param delete: delete if matched + :param read_size: amount of bytes to read from files larger then max_size + :return: dictionary key of the first rule that matched + """ + lastDownload = fs_encode(self.lastDownload) + if not exists(lastDownload): return None + + size = stat(lastDownload) + size = size.st_size + + if api_size and api_size <= size: return None + elif size > max_size and not read_size: return None + self.log.debug("Download Check triggered") + f = open(lastDownload, "rb") + content = f.read(read_size if read_size else -1) + f.close() + #produces encoding errors, better log to other file in the future? + #self.log.debug("Content: %s" % content) + for name, rule in rules.iteritems(): + if type(rule) in (str, unicode): + if rule in content: + if delete: + remove(lastDownload) + return name + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + if delete: + remove(lastDownload) + self.lastCheck = m + return name + + + def getPassword(self): + """ get the password the user provided in the package""" + password = self.pyfile.package().password + if not password: return "" + return password + + + def checkForSameFiles(self, starting=False): + """ checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises SkipDownload: """ - pass \ No newline at end of file + + pack = self.pyfile.package() + + for pyfile in self.core.files.cache.values(): + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #finished or downloading + raise SkipDownload(pyfile.pluginname) + elif pyfile.status in ( + 5, 7) and starting: #a download is waiting/starting and was appenrently started before + raise SkipDownload(pyfile.pluginname) + + download_folder = self.config['general']['download_folder'] + location = save_join(download_folder, pack.folder, self.pyfile.name) + + if starting and self.core.config['download']['skip_existing'] and exists(location): + size = os.stat(location).st_size + if size >= self.pyfile.size: + raise SkipDownload("File exists.") + + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + if pyfile: + if exists(location): + raise SkipDownload(pyfile[0]) + + self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) + + def clean(self): + """ clean everything and remove references """ + if hasattr(self, "pyfile"): + del self.pyfile + if hasattr(self, "req"): + self.req.close() + del self.req + if hasattr(self, "thread"): + del self.thread + if hasattr(self, "html"): + del self.html -- cgit v1.2.3 From 1ecdd9f6b53fec45e1d48592e3ff56aa7a576bec Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 8 Jan 2012 16:47:52 +0100 Subject: some cleanups, closed #490 --- module/plugins/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 54c2efdfd..bef4b1949 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -29,7 +29,7 @@ if os.name != "nt": from Base import Base, Fail, Retry from module.utils import chunks #legacy import -from module.utils.fs import save_join, save_path, fs_encode, fs_decode,\ +from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\ remove, makedirs, chmod, stat, exists, join @@ -339,7 +339,7 @@ class Hoster(Base): # convert back to unicode location = fs_decode(location) - name = save_path(self.pyfile.name) + name = save_filename(self.pyfile.name) filename = join(location, name) -- cgit v1.2.3 From 692d015627ecf03fbc23cfdb4afcf398b9a09a51 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 12 Jan 2012 17:26:28 +0100 Subject: scripts for testing and syntax unit test --- module/plugins/Hoster.py | 1 - 1 file changed, 1 deletion(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index bef4b1949..7c43c6444 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -28,7 +28,6 @@ if os.name != "nt": from grp import getgrnam from Base import Base, Fail, Retry -from module.utils import chunks #legacy import from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\ remove, makedirs, chmod, stat, exists, join -- cgit v1.2.3 From c7ad1cc5b4a5d190a060e3ddd9274c3065da6708 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Fri, 13 Jan 2012 23:24:21 +0100 Subject: plugin unit test, closed #499, #500 --- module/plugins/Hoster.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 7c43c6444..4a5d15759 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -28,9 +28,12 @@ if os.name != "nt": from grp import getgrnam from Base import Base, Fail, Retry +from module.utils import chunks as _chunks from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\ remove, makedirs, chmod, stat, exists, join +# Import for Hoster Plugins +chunks = _chunks class Abort(Exception): """ raised when aborted """ -- cgit v1.2.3 From 21791b33f033c7a182e64ca6f878bf0ec1d8f88d Mon Sep 17 00:00:00 2001 From: RaNaN Date: Fri, 20 Jan 2012 00:01:15 +0100 Subject: parallel dl limit for hoster --- module/plugins/Hoster.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 4a5d15759..c30fed412 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -65,7 +65,6 @@ class Hoster(Base): self.wantReconnect = False #: enables simultaneous processing of multiple downloads - self.multiDL = True self.limitDL = 0 #: chunk limit self.chunkLimit = 1 @@ -113,11 +112,30 @@ class Hoster(Base): self.init() + def getMultiDL(self): + self.logDebug("Deprectated attribute multiDL, use limitDL instead") + return self.limitDL <= 0 + + def setMultiDL(self, val): + self.logDebug("Deprectated attribute multiDL, use limitDL instead") + self.limitDL = 0 if val else 1 + + multiDL = property(getMultiDL, setMultiDL) + def getChunkCount(self): if self.chunkLimit <= 0: return self.config["download"]["chunks"] return min(self.config["download"]["chunks"], self.chunkLimit) + def getDownloadLimit(self): + if self.account: + limit = self.account.options.get("limitDL", 0) + if limit == "": limit = 0 + return int(limit) + else: + return self.limitDL + + def __call__(self): return self.__name__ -- cgit v1.2.3 From 718be218273d0acd96c0b0c4739758302044daad Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Feb 2012 12:32:14 +0000 Subject: Don't fail miserably on hoster's internal server error. --- module/plugins/Hoster.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index c30fed412..9f3548350 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -161,7 +161,14 @@ class Hoster(Base): self.pyfile.setStatus("starting") - return self.process(self.pyfile) + try: + return self.process(self.pyfile) + except Exception, e: + # Can't seem to import BadHeader + if e.__class__.__name__ == 'BadHeader' and e.code == 500: + self.logInfo("Internal Server Error") + self.tempOffline() + raise e def process(self, pyfile): -- cgit v1.2.3 From 526c6cdbdbe1a352333df162e2c1761bb49dabf2 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Feb 2012 12:38:42 +0000 Subject: Add the internal server error messages to the pyfile. --- module/plugins/Hoster.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 9f3548350..1f21a27c8 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -167,6 +167,7 @@ class Hoster(Base): # Can't seem to import BadHeader if e.__class__.__name__ == 'BadHeader' and e.code == 500: self.logInfo("Internal Server Error") + self.pyfile.error = _("Internal Server Error") self.tempOffline() raise e -- cgit v1.2.3 From ebe0e6039d822e9c16a6095dba8691066bc3b466 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Feb 2012 12:56:40 +0000 Subject: Catch internal server errors on the right place. --- module/plugins/Hoster.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 1f21a27c8..05f55ebc8 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -161,16 +161,7 @@ class Hoster(Base): self.pyfile.setStatus("starting") - try: - return self.process(self.pyfile) - except Exception, e: - # Can't seem to import BadHeader - if e.__class__.__name__ == 'BadHeader' and e.code == 500: - self.logInfo("Internal Server Error") - self.pyfile.error = _("Internal Server Error") - self.tempOffline() - raise e - + return self.process(self.pyfile) def process(self, pyfile): """the 'main' method of every plugin, you **have to** overwrite it""" -- cgit v1.2.3 From 0a453a4ae0294910eb8a1076d9f291b78c7e7eb3 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 13 Feb 2012 14:36:21 +0100 Subject: little account, hoster fix --- module/plugins/Hoster.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 05f55ebc8..fc9e23132 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -90,7 +90,7 @@ class Hoster(Base): #: Browser instance, see `network.Browser` self.req = self.account.getAccountRequest() # Default: -1, True, True - self.chunkLimit, self.resumeDownload, self.multiDL = self.account.getDownloadSettings() + self.chunkLimit, self.limitDL, self.resumeDownload = self.account.getDownloadSettings() self.premium = self.account.isPremium() else: self.req = self.core.requestFactory.getRequest(self.__name__) @@ -131,7 +131,10 @@ class Hoster(Base): if self.account: limit = self.account.options.get("limitDL", 0) if limit == "": limit = 0 - return int(limit) + if self.limitDL > 0: # a limit is already set, we use the minimum + return min(int(limit), self.limitDL) + else: + return int(limit) else: return self.limitDL -- cgit v1.2.3 From 4df2b77fdf42046fe19bd371be7c7255986b5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 6 Mar 2012 13:36:39 +0100 Subject: renamed hooks to addons, new filemanager and database, many new api methods you will loose ALL your LINKS, webinterface will NOT work --- module/plugins/Hoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index fc9e23132..32c587aa5 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -185,7 +185,7 @@ class Hoster(Base): 10 - not implemented 20 - unknown error """ - #@TODO checksum check hook + #@TODO checksum check addon return True, 10 @@ -365,7 +365,7 @@ class Hoster(Base): filename = join(location, name) - self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) + self.core.addonManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, -- cgit v1.2.3 From 50d4df8b4d48b855bd18e9922355b7f3f2b4da4e Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 20 Mar 2012 14:57:45 +0100 Subject: captcha decrypting for all plugin types, new interaction manager --- module/plugins/Hoster.py | 96 +++--------------------------------------------- 1 file changed, 6 insertions(+), 90 deletions(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 32c587aa5..b330743e6 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -17,10 +17,8 @@ @author: RaNaN, spoob, mkaay """ -from time import time, sleep -from random import randint - import os +from time import time if os.name != "nt": from module.utils.fs import chown @@ -35,9 +33,6 @@ from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\ # Import for Hoster Plugins chunks = _chunks -class Abort(Exception): - """ raised when aborted """ - class Reconnect(Exception): """ raised when reconnected """ @@ -170,6 +165,9 @@ class Hoster(Base): """the 'main' method of every plugin, you **have to** overwrite it""" raise NotImplementedError + def abort(self): + return self.pyfile.abort + def resetAccount(self): """ dont use account and retry download """ self.account = None @@ -208,7 +206,7 @@ class Hoster(Base): while self.pyfile.waitUntil > time(): self.thread.m.reconnecting.wait(2) - if self.pyfile.abort: raise Abort + self.checkAbort() if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False @@ -243,88 +241,6 @@ class Hoster(Base): self.retries += 1 raise Retry(reason) - def invalidCaptcha(self): - if self.cTask: - self.cTask.invalid() - - def correctCaptcha(self): - if self.cTask: - self.cTask.correct() - - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual'): - """ Loads a captcha and decrypts it with ocr, plugin, user input - - :param url: url of captcha image - :param get: get part for request - :param post: post part for request - :param cookies: True if cookies should be enabled - :param forceUser: if True, ocr is not used - :param imgtype: Type of the Image - :param result_type: 'textual' if text is written on the captcha\ - or 'positional' for captcha where the user have to click\ - on a specific region on the captcha - - :return: result of decrypting - """ - - img = self.load(url, get=get, post=post, cookies=cookies) - - id = ("%.2f" % time())[-6:].replace(".", "") - temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - temp_file.write(img) - temp_file.close() - - has_plugin = self.__name__ in self.core.pluginManager.getPlugins("captcha") - - if self.core.captcha: - Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) - else: - Ocr = None - - if Ocr and not forceUser: - sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: raise Abort - - ocr = Ocr() - result = ocr.get_captcha(temp_file.name) - else: - captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) - self.cTask = task - captchaManager.handleCaptcha(task) - - while task.isWaiting(): - if self.pyfile.abort: - captchaManager.removeTask(task) - raise Abort - sleep(1) - - captchaManager.removeTask(task) - - if task.error and has_plugin: #ignore default error message since the user could use OCR - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) - - result = task.result - self.log.debug("Received captcha result: %s" % str(result)) - - if not self.core.debug: - try: - remove(temp_file.name) - except: - pass - - return result - - - def load(self, *args, **kwargs): - """ See 'Base' load method for more info """ - if self.pyfile.abort: raise Abort - return Base.load(self, *args, **kwargs) def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder @@ -338,8 +254,8 @@ class Hoster(Base): the filename will be changed if needed :return: The location where the file was saved """ - self.checkForSameFiles() + self.checkAbort() self.pyfile.setStatus("downloading") -- cgit v1.2.3 From b40b32ee05f611323a7827fad2a25fa0a28dcb24 Mon Sep 17 00:00:00 2001 From: X3n0m0rph59 Date: Sun, 22 Apr 2012 19:56:17 +0200 Subject: a huge pile of spelling fixes --- module/plugins/Hoster.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index b330743e6..737bdcdb4 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -48,7 +48,7 @@ class Hoster(Base): def getInfo(urls): """This method is used to retrieve the online status of files for hoster plugins. It has to *yield* list of tuples with the result in this format (name, size, status, url), - where status is one of API pyfile statusses. + where status is one of API pyfile statuses. :param urls: List of urls :return: yield list of tuple with results (name, size, status, url) @@ -108,11 +108,11 @@ class Hoster(Base): self.init() def getMultiDL(self): - self.logDebug("Deprectated attribute multiDL, use limitDL instead") + self.logDebug("Deprecated attribute multiDL, use limitDL instead") return self.limitDL <= 0 def setMultiDL(self, val): - self.logDebug("Deprectated attribute multiDL, use limitDL instead") + self.logDebug("Deprecated attribute multiDL, use limitDL instead") self.limitDL = 0 if val else 1 multiDL = property(getMultiDL, setMultiDL) @@ -142,7 +142,7 @@ class Hoster(Base): pass def setup(self): - """ setup for enviroment and other things, called before downloading (possibly more than one time)""" + """ setup for environment and other things, called before downloading (possibly more than one time)""" pass def preprocessing(self, thread): @@ -150,7 +150,7 @@ class Hoster(Base): self.thread = thread if self.account: - # will force a relogin or reload of account info if necessary + # will force a re-login or reload of account info if necessary self.account.getAccountInfo() else: self.req.clearCookies() @@ -169,7 +169,7 @@ class Hoster(Base): return self.pyfile.abort def resetAccount(self): - """ dont use account and retry download """ + """ don't use account and retry download """ self.account = None self.req = self.core.requestFactory.getRequest(self.__name__) self.retry() @@ -372,7 +372,7 @@ class Hoster(Base): if pyfile.status in (0, 12): #finished or downloading raise SkipDownload(pyfile.pluginname) elif pyfile.status in ( - 5, 7) and starting: #a download is waiting/starting and was appenrently started before + 5, 7) and starting: #a download is waiting/starting and was apparently started before raise SkipDownload(pyfile.pluginname) download_folder = self.config['general']['download_folder'] -- cgit v1.2.3 From e8eaa91da9e1236d8009df0d79bebe023de8933f Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 6 May 2012 13:04:15 +0200 Subject: little documentation update --- module/plugins/Hoster.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'module/plugins/Hoster.py') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 737bdcdb4..ad4f8f16b 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -100,7 +100,6 @@ class Hoster(Base): self.lastCheck = None #: js engine, see `JsEngine` self.js = self.core.js - self.cTask = None #captcha task self.retries = 0 # amount of retries already made self.html = None # some plugins store html code here @@ -108,13 +107,12 @@ class Hoster(Base): self.init() def getMultiDL(self): - self.logDebug("Deprecated attribute multiDL, use limitDL instead") return self.limitDL <= 0 def setMultiDL(self, val): - self.logDebug("Deprecated attribute multiDL, use limitDL instead") self.limitDL = 0 if val else 1 + #: virtual attribute using self.limitDL on behind multiDL = property(getMultiDL, setMultiDL) def getChunkCount(self): -- cgit v1.2.3