From 958bf611f5d9d117f19f824990ec6fd6b537e967 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 22 Dec 2011 23:45:38 +0100 Subject: accountmanager v2, delete your accounts.conf and re-enter them in pyload, new nice debug functions, try core.shell() and core.breakpoint() --- module/plugins/internal/MultiHoster.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index d50df3943..872e0b770 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -5,6 +5,7 @@ import re from module.utils import remove_chars from module.plugins.Hook import Hook +from module.plugins.PluginManager import PluginTuple class MultiHoster(Hook): """ @@ -43,7 +44,7 @@ class MultiHoster(Hook): def coreReady(self): pluginMap = {} - for name in self.core.pluginManager.hosterPlugins.keys(): + for name in self.core.pluginManager.getPlugins("hoster").keys(): pluginMap[name.lower()] = name new_supported = [] @@ -66,25 +67,19 @@ class MultiHoster(Hook): # inject plugin plugin self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - dict["new_module"] = module - dict["new_name"] = self.__name__ + self.core.pluginManager.injectPlugin("hoster", hoster, module, self.__name__) self.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) # create new regexp regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict["pattern"] = regexp - dict["re"] = re.compile(regexp) + hoster = self.core.pluginManager.getPlugins("hoster") + p = hoster[self.__name__] + new = PluginTuple(p.version, re.compile(regexp), p.deps, p.user, p.path) + hoster[self.__name__] = new def unload(self): for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - if "module" in dict: - del dict["module"] - - del dict["new_module"] - del dict["new_name"] \ No newline at end of file + self.core.pluginManager.restoreState("hoster", hoster) \ No newline at end of file -- cgit v1.2.3 From b877847094b0ba03a098dff0fd769eb456b48dd1 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Fri, 6 Jan 2012 17:54:53 +0100 Subject: several improvements, also closes #486, #487 --- module/plugins/internal/MultiHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 872e0b770..1629fdc5f 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -80,6 +80,6 @@ class MultiHoster(Hook): hoster[self.__name__] = new - def unload(self): + def deactivate(self): for hoster in self.supported: self.core.pluginManager.restoreState("hoster", hoster) \ No newline at end of file -- cgit v1.2.3 From 1bb6ebf544b43cacf7c0755c5a8608b79b95e2d6 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 20:11:16 +0100 Subject: MultiHoster plugin type, some fixes, new documentation structure --- module/plugins/internal/MultiHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 1629fdc5f..2252c4460 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -23,7 +23,7 @@ class MultiHoster(Hook): try: self.hosters = self.getHoster() except Exception, e: - self.logError("%s" % str(e)) + self.logError(e) return [] for rep in self.replacements: -- cgit v1.2.3 From bac28b7740aae772636d8b90e291d9c17dfd59a7 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 8 Jan 2012 14:44:59 +0100 Subject: new MultiHoster hook --- module/plugins/internal/MultiHoster.py | 85 ---------------------------------- 1 file changed, 85 deletions(-) delete mode 100644 module/plugins/internal/MultiHoster.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py deleted file mode 100644 index 2252c4460..000000000 --- a/module/plugins/internal/MultiHoster.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.utils import remove_chars -from module.plugins.Hook import Hook -from module.plugins.PluginManager import PluginTuple - -class MultiHoster(Hook): - """ - Generic MultiHoster plugin - """ - - interval = 0 - hosters = [] - replacements = [] - supported = [] - - def getHosterCached(self): - if not self.hosters: - - try: - self.hosters = self.getHoster() - except Exception, e: - self.logError(e) - return [] - - for rep in self.replacements: - if rep[0] in self.hosters: - self.hosters.remove(rep[0]) - if rep[1] not in self.hosters: - self.hosters.append(rep[1]) - - return self.hosters - - - def getHoster(self): - """Load list of supported hoster - - :return: List of domain names - """ - raise NotImplementedError - - def coreReady(self): - pluginMap = {} - for name in self.core.pluginManager.getPlugins("hoster").keys(): - pluginMap[name.lower()] = name - - new_supported = [] - - for hoster in self.getHosterCached(): - name = remove_chars(hoster.lower(), "-.") - - if name in pluginMap: - self.supported.append(pluginMap[name]) - else: - new_supported.append(hoster) - - if not self.supported and not new_supported: - self.logError(_("No Hoster loaded")) - return - - module = self.core.pluginManager.getPlugin(self.__name__) - klass = getattr(module, self.__name__) - - # inject plugin plugin - self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) - for hoster in self.supported: - self.core.pluginManager.injectPlugin("hoster", hoster, module, self.__name__) - - self.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) - - # create new regexp - regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) - - hoster = self.core.pluginManager.getPlugins("hoster") - p = hoster[self.__name__] - new = PluginTuple(p.version, re.compile(regexp), p.deps, p.user, p.path) - hoster[self.__name__] = new - - - def deactivate(self): - for hoster in self.supported: - self.core.pluginManager.restoreState("hoster", hoster) \ No newline at end of file -- cgit v1.2.3 From 1ecdd9f6b53fec45e1d48592e3ff56aa7a576bec Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 8 Jan 2012 16:47:52 +0100 Subject: some cleanups, closed #490 --- module/plugins/internal/UnRar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index feac4c176..9f57a9ad6 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -23,7 +23,7 @@ from os.path import join from glob import glob from subprocess import Popen, PIPE -from module.utils import save_join, decode +from module.utils.fs import save_join, decode from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError class UnRar(AbtractExtractor): -- cgit v1.2.3 From a72688b208ed4ba3e98234e995f7bc1eb4afec42 Mon Sep 17 00:00:00 2001 From: zoidberg10 Date: Mon, 16 Jan 2012 20:44:19 +0100 Subject: merge in plugin updates --- module/plugins/internal/SimpleHoster.py | 43 +++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 10 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index aae76e781..12b629a81 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -17,30 +17,35 @@ @author: zoidberg """ from urlparse import urlparse -from re import search, sub +import re from module.plugins.Hoster import Hoster -from module.utils import html_unescape, parseFileSize +from module.utils import html_unescape, fixup, parseFileSize from module.network.RequestFactory import getURL def reSub(string, ruleslist): for r in ruleslist: rf, rt = r - string = sub(rf, rt, string) + string = re.sub(rf, rt, string) + #self.logDebug(rf, rt, string) return string + +def parseHtmlTagAttrValue(attr_name, tag): + m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^\s\"'][^>\s]+)\1" % attr_name, tag) + return m.group(2) if m else '' def parseFileInfo(self, url = '', html = '', infomode = False): if not html and hasattr(self, "html"): html = self.html info = {"name" : url, "size" : 0, "status" : 3} online = False - if hasattr(self, "FILE_OFFLINE_PATTERN") and search(self.FILE_OFFLINE_PATTERN, html): + if hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html): # File offline info['status'] = 1 else: for pattern in ("FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): try: - info = dict(info, **search(getattr(self, pattern), html).groupdict()) + info = dict(info, **re.search(getattr(self, pattern), html).groupdict()) online = True except AttributeError: continue @@ -77,7 +82,7 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" - __version__ = "0.14" + __version__ = "0.16" __pattern__ = None __type__ = "hoster" __description__ = """Base hoster plugin""" @@ -93,7 +98,7 @@ class SimpleHoster(Hoster): """ FILE_SIZE_REPLACEMENTS = [] - FILE_NAME_REPLACEMENTS = [] + FILE_NAME_REPLACEMENTS = [("&#?\w+;", fixup)] FILE_URL_REPLACEMENTS = [] def setup(self): @@ -103,14 +108,14 @@ class SimpleHoster(Hoster): pyfile.url = reSub(pyfile.url, self.FILE_URL_REPLACEMENTS) self.html = self.load(pyfile.url, decode = True) self.file_info = self.getFileInfo() - if self.account: + if self.premium: self.handlePremium() else: self.handleFree() def getFileInfo(self): self.logDebug("URL: %s" % self.pyfile.url) - if hasattr(self, "TEMP_OFFLINE_PATTERN") and search(self.TEMP_OFFLINE_PATTERN, self.html): + if hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, self.html): self.tempOffline() file_info = parseFileInfo(self, infomode = True) @@ -140,4 +145,22 @@ class SimpleHoster(Hoster): self.fail("Premium download not implemented") def parseError(self, msg): - raise PluginParseError(msg) \ No newline at end of file + raise PluginParseError(msg) + + def parseHtmlForm(self, attr_str): + inputs = {} + action = None + form = re.search(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, self.html, re.S) + if form: + action = parseHtmlTagAttrValue("action", form.group('tag')) + for input in re.finditer(r'(<(?:input|textarea)[^>]*>)', form.group('content')): + name = parseHtmlTagAttrValue("name", input.group(1)) + if name: + inputs[name] = parseHtmlTagAttrValue("value", input.group(1)) + return action, inputs + + def checkTrafficLeft(self): + traffic = self.account.getAccountInfo(self.user, True)["trafficleft"] + size = self.pyfile.size / 1024 + self.logInfo("Filesize: %i KiB, Traffic left for user %s: %i KiB" % (size, self.user, traffic)) + return size <= traffic \ No newline at end of file -- cgit v1.2.3 From a2ab9ced6ffb038a051e465f06d8b5ca6512a79b Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 22 Jan 2012 13:59:46 +0100 Subject: try to fix unrar encoding issues --- module/plugins/internal/UnRar.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 9f57a9ad6..ce271bec4 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -23,7 +23,7 @@ from os.path import join from glob import glob from subprocess import Popen, PIPE -from module.utils.fs import save_join, decode +from module.utils.fs import save_join, decode, fs_encode from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError class UnRar(AbtractExtractor): @@ -39,7 +39,7 @@ class UnRar(AbtractExtractor): @staticmethod def checkDeps(): if os.name == "nt": - UnRar.CMD = join(pypath, "UnRAR.exe") + UnRar.CMD = save_join(pypath, "UnRAR.exe") p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) p.communicate() else: @@ -80,7 +80,7 @@ class UnRar(AbtractExtractor): self.password = "" #save the correct password def checkArchive(self): - p = self.call_unrar("l", "-v", self.file) + p = self.call_unrar("l", "-v", fs_encode(self.file)) out, err = p.communicate() if self.re_wrongpwd.search(err): self.passwordProtected = True @@ -102,7 +102,7 @@ class UnRar(AbtractExtractor): def checkPassword(self, password): #at this point we can only verify header protected files if self.headerProtected: - p = self.call_unrar("l", "-v", self.file, password=password) + p = self.call_unrar("l", "-v", fs_encode(self.file), password=password) out, err = p.communicate() if self.re_wrongpwd.search(err): return False @@ -115,7 +115,7 @@ class UnRar(AbtractExtractor): # popen thinks process is still alive (just like pexpect) - very strange behavior # so for now progress can not be determined correctly - p = self.call_unrar(command, self.file, self.out, password=password) + p = self.call_unrar(command, fs_encode(self.file), self.out, password=password) renice(p.pid, self.renice) progress(0) @@ -141,7 +141,7 @@ class UnRar(AbtractExtractor): def listContent(self): command = "vb" if self.fullpath else "lb" - p = self.call_unrar(command, "-v", self.file, password=self.password) + p = self.call_unrar(command, "-v", fs_encode(self.file), password=self.password) out, err = p.communicate() if "Cannot open" in err: -- cgit v1.2.3 From 5aa52ea8d5c3d28f350dc47d23d14f230be499eb Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 22 Jan 2012 16:31:12 +0000 Subject: Fix encoding issues in RAR plugin log messages --- module/plugins/internal/UnRar.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index ce271bec4..ebb5cdbd6 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -23,7 +23,7 @@ from os.path import join from glob import glob from subprocess import Popen, PIPE -from module.utils.fs import save_join, decode, fs_encode +from module.utils.fs import save_join, decode, fs_encode, fs_decode from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError class UnRar(AbtractExtractor): @@ -66,9 +66,9 @@ class UnRar(AbtractExtractor): if match: #only add first parts if int(match[0][1]) == 1: - result.append((file, id)) + result.append((fs_decode(file), id)) else: - result.append((file, id)) + result.append((fs_decode(file), id)) return result @@ -136,8 +136,8 @@ class UnRar(AbtractExtractor): def getDeleteFiles(self): if ".part" in self.file: - return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.IGNORECASE)) - return [self.file] + return glob(re.sub("(?<=\.part)([01]+)", "*", fd_decode(self.file), re.IGNORECASE)) + return [fs_decode(self.file)] def listContent(self): command = "vb" if self.fullpath else "lb" @@ -176,7 +176,7 @@ class UnRar(AbtractExtractor): #NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.m.logDebug(" ".join(call)) + self.m.logDebug(" ".join([decode(arg) for arg in call])) p = Popen(call, stdout=PIPE, stderr=PIPE) @@ -188,4 +188,4 @@ def renice(pid, value): try: Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) except: - print "Renice failed" \ No newline at end of file + print "Renice failed" -- cgit v1.2.3 From ddb01583858279a51c1579c6d0cafc5d5f32f8d5 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 22 Jan 2012 17:56:15 +0100 Subject: removed some fs_decode functions --- module/plugins/internal/UnRar.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index ebb5cdbd6..a315fbea3 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -19,11 +19,10 @@ import os import re -from os.path import join from glob import glob from subprocess import Popen, PIPE -from module.utils.fs import save_join, decode, fs_encode, fs_decode +from module.utils.fs import save_join, decode, fs_encode from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError class UnRar(AbtractExtractor): @@ -66,9 +65,9 @@ class UnRar(AbtractExtractor): if match: #only add first parts if int(match[0][1]) == 1: - result.append((fs_decode(file), id)) + result.append((file, id)) else: - result.append((fs_decode(file), id)) + result.append((file, id)) return result @@ -136,8 +135,8 @@ class UnRar(AbtractExtractor): def getDeleteFiles(self): if ".part" in self.file: - return glob(re.sub("(?<=\.part)([01]+)", "*", fd_decode(self.file), re.IGNORECASE)) - return [fs_decode(self.file)] + return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.IGNORECASE)) + return [self.file] def listContent(self): command = "vb" if self.fullpath else "lb" -- cgit v1.2.3 From 4df2b77fdf42046fe19bd371be7c7255986b5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 6 Mar 2012 13:36:39 +0100 Subject: renamed hooks to addons, new filemanager and database, many new api methods you will loose ALL your LINKS, webinterface will NOT work --- module/plugins/internal/AbstractExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index 2130f910e..ceb188193 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -30,7 +30,7 @@ class AbtractExtractor: def __init__(self, m, file, out, fullpath, overwrite, renice): """Initialize extractor for specific file - :param m: ExtractArchive Hook plugin + :param m: ExtractArchive Addon plugin :param file: Absolute filepath :param out: Absolute path to destination directory :param fullpath: extract to fullpath -- cgit v1.2.3 From 50d4df8b4d48b855bd18e9922355b7f3f2b4da4e Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 20 Mar 2012 14:57:45 +0100 Subject: captcha decrypting for all plugin types, new interaction manager --- module/plugins/internal/NetloadInOCR.py | 27 +++ module/plugins/internal/OCR.py | 314 +++++++++++++++++++++++++++ module/plugins/internal/ShareonlineBizOCR.py | 53 +++++ 3 files changed, 394 insertions(+) create mode 100644 module/plugins/internal/NetloadInOCR.py create mode 100644 module/plugins/internal/OCR.py create mode 100644 module/plugins/internal/ShareonlineBizOCR.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/NetloadInOCR.py b/module/plugins/internal/NetloadInOCR.py new file mode 100644 index 000000000..e50978701 --- /dev/null +++ b/module/plugins/internal/NetloadInOCR.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from OCR import OCR + +class NetloadInOCR(OCR): + __version__ = 0.1 + + def __init__(self): + OCR.__init__(self) + + def get_captcha(self, image): + self.load_image(image) + self.to_greyscale() + self.clean(3) + self.clean(3) + self.run_tesser(True, True, False, False) + + self.result_captcha = self.result_captcha.replace(" ", "")[:4] # cut to 4 numbers + + return self.result_captcha + +if __name__ == '__main__': + import urllib + ocr = NetloadInOCR() + urllib.urlretrieve("http://netload.in/share/includes/captcha.php", "captcha.png") + + print ocr.get_captcha('captcha.png') diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py new file mode 100644 index 000000000..9f8b7ef8c --- /dev/null +++ b/module/plugins/internal/OCR.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2009 kingzero, RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### +from __future__ import with_statement +import os +from os.path import join +from os.path import abspath +import logging +import subprocess +#import tempfile + +import Image +import TiffImagePlugin +import PngImagePlugin +import GifImagePlugin +import JpegImagePlugin + + +class OCR(object): + __version__ = 0.1 + + def __init__(self): + self.logger = logging.getLogger("log") + + def load_image(self, image): + self.image = Image.open(image) + self.pixels = self.image.load() + self.result_captcha = '' + + def unload(self): + """delete all tmp images""" + pass + + def threshold(self, value): + self.image = self.image.point(lambda a: a * value + 10) + + def run(self, command): + """Run a command""" + + popen = subprocess.Popen(command, bufsize = -1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + popen.wait() + output = popen.stdout.read() +" | "+ popen.stderr.read() + popen.stdout.close() + popen.stderr.close() + self.logger.debug("Tesseract ReturnCode %s Output: %s" % (popen.returncode, output)) + + def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True): + #self.logger.debug("create tmp tif") + + + #tmp = tempfile.NamedTemporaryFile(suffix=".tif") + tmp = open(join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") + tmp.close() + #self.logger.debug("create tmp txt") + #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") + tmpTxt = open(join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") + tmpTxt.close() + + self.logger.debug("save tiff") + self.image.save(tmp.name, 'TIFF') + + if os.name == "nt": + tessparams = [join(pypath,"tesseract","tesseract.exe")] + else: + tessparams = ['tesseract'] + + tessparams.extend( [abspath(tmp.name), abspath(tmpTxt.name).replace(".txt", "")] ) + + if subset and (digits or lowercase or uppercase): + #self.logger.debug("create temp subset config") + #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") + tmpSub = open(join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") + tmpSub.write("tessedit_char_whitelist ") + if digits: + tmpSub.write("0123456789") + if lowercase: + tmpSub.write("abcdefghijklmnopqrstuvwxyz") + if uppercase: + tmpSub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + tmpSub.write("\n") + tessparams.append("nobatch") + tessparams.append(abspath(tmpSub.name)) + tmpSub.close() + + self.logger.debug("run tesseract") + self.run(tessparams) + self.logger.debug("read txt") + + try: + with open(tmpTxt.name, 'r') as f: + self.result_captcha = f.read().replace("\n", "") + except: + self.result_captcha = "" + + self.logger.debug(self.result_captcha) + try: + os.remove(tmp.name) + os.remove(tmpTxt.name) + if subset and (digits or lowercase or uppercase): + os.remove(tmpSub.name) + except: + pass + + def get_captcha(self, name): + raise NotImplementedError + + def to_greyscale(self): + if self.image.mode != 'L': + self.image = self.image.convert('L') + + self.pixels = self.image.load() + + def eval_black_white(self, limit): + self.pixels = self.image.load() + w, h = self.image.size + for x in xrange(w): + for y in xrange(h): + if self.pixels[x, y] > limit: + self.pixels[x, y] = 255 + else: + self.pixels[x, y] = 0 + + def clean(self, allowed): + pixels = self.pixels + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 255: continue + # no point in processing white pixels since we only want to remove black pixel + count = 0 + + try: + if pixels[x-1, y-1] != 255: count += 1 + if pixels[x-1, y] != 255: count += 1 + if pixels[x-1, y + 1] != 255: count += 1 + if pixels[x, y + 1] != 255: count += 1 + if pixels[x + 1, y + 1] != 255: count += 1 + if pixels[x + 1, y] != 255: count += 1 + if pixels[x + 1, y-1] != 255: count += 1 + if pixels[x, y-1] != 255: count += 1 + except: + pass + + # not enough neighbors are dark pixels so mark this pixel + # to be changed to white + if count < allowed: + pixels[x, y] = 1 + + # second pass: this time set all 1's to 255 (white) + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 1: pixels[x, y] = 255 + + self.pixels = pixels + + def derotate_by_average(self): + """rotate by checking each angle and guess most suitable""" + + w, h = self.image.size + pixels = self.pixels + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 155 + + highest = {} + counts = {} + + for angle in range(-45, 45): + + tmpimage = self.image.rotate(angle) + + pixels = tmpimage.load() + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + + count = {} + + for x in xrange(w): + count[x] = 0 + for y in xrange(h): + if pixels[x, y] == 155: + count[x] += 1 + + sum = 0 + cnt = 0 + + for x in count.values(): + if x != 0: + sum += x + cnt += 1 + + avg = sum / cnt + counts[angle] = cnt + highest[angle] = 0 + for x in count.values(): + if x > highest[angle]: + highest[angle] = x + + highest[angle] = highest[angle] - avg + + hkey = 0 + hvalue = 0 + + for key, value in highest.iteritems(): + if value > hvalue: + hkey = key + hvalue = value + + self.image = self.image.rotate(hkey) + pixels = self.image.load() + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + if pixels[x, y] == 155: + pixels[x, y] = 0 + + self.pixels = pixels + + def split_captcha_letters(self): + captcha = self.image + started = False + letters = [] + width, height = captcha.size + bottomY, topY = 0, height + pixels = captcha.load() + + for x in xrange(width): + black_pixel_in_col = False + for y in xrange(height): + if pixels[x, y] != 255: + if not started: + started = True + firstX = x + lastX = x + + if y > bottomY: bottomY = y + if y < topY: topY = y + if x > lastX: lastX = x + + black_pixel_in_col = True + + if black_pixel_in_col == False and started == True: + rect = (firstX, topY, lastX, bottomY) + new_captcha = captcha.crop(rect) + + w, h = new_captcha.size + if w > 5 and h > 5: + letters.append(new_captcha) + + started = False + bottomY, topY = 0, height + + return letters + + def correct(self, values, var=None): + + if var: + result = var + else: + result = self.result_captcha + + for key, item in values.iteritems(): + + if key.__class__ == str: + result = result.replace(key, item) + else: + for expr in key: + result = result.replace(expr, item) + + if var: + return result + else: + self.result_captcha = result + + +if __name__ == '__main__': + ocr = OCR() + ocr.load_image("B.jpg") + ocr.to_greyscale() + ocr.eval_black_white(140) + ocr.derotate_by_average() + ocr.run_tesser() + print "Tesseract", ocr.result_captcha + ocr.image.save("derotated.jpg") + diff --git a/module/plugins/internal/ShareonlineBizOCR.py b/module/plugins/internal/ShareonlineBizOCR.py new file mode 100644 index 000000000..c5c2e92e8 --- /dev/null +++ b/module/plugins/internal/ShareonlineBizOCR.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2009 kingzero, RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### +from OCR import OCR + +class ShareonlineBizOCR(OCR): + __version__ = 0.1 + + def __init__(self): + OCR.__init__(self) + + def get_captcha(self, image): + self.load_image(image) + self.to_greyscale() + self.image = self.image.resize((160, 50)) + self.pixels = self.image.load() + self.threshold(1.85) + #self.eval_black_white(240) + #self.derotate_by_average() + + letters = self.split_captcha_letters() + + final = "" + for letter in letters: + self.image = letter + self.run_tesser(True, True, False, False) + final += self.result_captcha + + return final + + #tesseract at 60% + +if __name__ == '__main__': + import urllib + ocr = ShareonlineBizOCR() + urllib.urlretrieve("http://www.share-online.biz/captcha.php", "captcha.jpeg") + print ocr.get_captcha('captcha.jpeg') -- cgit v1.2.3 From b40b32ee05f611323a7827fad2a25fa0a28dcb24 Mon Sep 17 00:00:00 2001 From: X3n0m0rph59 Date: Sun, 22 Apr 2012 19:56:17 +0200 Subject: a huge pile of spelling fixes --- module/plugins/internal/AbstractExtractor.py | 12 ++++++------ module/plugins/internal/SimpleHoster.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index ceb188193..3cd635eff 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -13,7 +13,7 @@ class WrongPassword(Exception): class AbtractExtractor: @staticmethod def checkDeps(): - """ Check if system statisfy dependencies + """ Check if system satisfies dependencies :return: boolean """ return True @@ -21,7 +21,7 @@ class AbtractExtractor: @staticmethod def getTargets(files_ids): """ Filter suited targets from list of filename id tuple list - :param files_ids: List of filepathes + :param files_ids: List of file paths :return: List of targets, id tuple list """ raise NotImplementedError @@ -30,10 +30,10 @@ class AbtractExtractor: def __init__(self, m, file, out, fullpath, overwrite, renice): """Initialize extractor for specific file - :param m: ExtractArchive Addon plugin - :param file: Absolute filepath + :param m: ExtractArchive addon plugin + :param file: Absolute file path :param out: Absolute path to destination directory - :param fullpath: extract to fullpath + :param fullpath: Extract to fullpath :param overwrite: Overwrite existing archives :param renice: Renice value """ @@ -52,7 +52,7 @@ class AbtractExtractor: def checkArchive(self): - """Check if password if needed. Raise ArchiveError if integrity is + """Check if password is needed. Raise ArchiveError if integrity is questionable. :return: boolean diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 69909a8a1..20263064a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -103,7 +103,7 @@ class SimpleHoster(Hoster): or FILE_NAME_INFO = r'(?Pfile_name)' and FILE_SIZE_INFO = r'(?Pfile_size) (?Punits)' FILE_OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN = r'Server maintainance' + TEMP_OFFLINE_PATTERN = r'Server maintenance' """ FILE_SIZE_REPLACEMENTS = [] -- cgit v1.2.3