diff options
| -rw-r--r-- | module/network/HTTPRequest.py | 6 | ||||
| -rw-r--r-- | module/plugins/Base.py | 2 | ||||
| -rw-r--r-- | module/plugins/Plugin.py | 526 | ||||
| -rw-r--r-- | module/web/ServerThread.py | 7 | ||||
| -rw-r--r-- | module/web/api_app.py | 10 | ||||
| -rw-r--r-- | tests/crypterlinks.txt | 1 | ||||
| -rw-r--r-- | tests/helper/PluginTester.py | 6 | ||||
| -rwxr-xr-x | tests/hosterlinks.txt | 5 |
8 files changed, 22 insertions, 541 deletions
diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index a0b419763..2f084efb5 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -28,12 +28,12 @@ from cStringIO import StringIO from module.plugins.Hoster import Abort def myquote(url): - return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") + return quote(url.encode('utf8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") def myurlencode(data): data = dict(data) - return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ - y.encode('utf_8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) + return urlencode(dict((x.encode('utf8') if isinstance(x, unicode) else x, \ + y.encode('utf8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) bad_headers = range(400, 404) + range(405, 418) + range(500, 506) diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 53840ee18..48a3707b1 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -167,8 +167,6 @@ class Base(object): """ if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") - if type(url) == unicode: url = str(url) - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.core.debug: diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py deleted file mode 100644 index 3cfb89c21..000000000 --- a/module/plugins/Plugin.py +++ /dev/null @@ -1,526 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN, spoob, mkaay -""" - -from time import time, sleep -from random import randint - -import os -from os import remove, makedirs, chmod, stat -from os.path import exists, join - -if os.name != "nt": - from os import chown - from pwd import getpwnam - from grp import getgrnam - - -from Base import Base -from module.utils import save_join, save_path, fs_encode, fs_decode, chunks - -class Abort(Exception): - """ raised when aborted """ - - -class Fail(Exception): - """ raised when failed """ - - -class Reconnect(Exception): - """ raised when reconnected """ - - -class Retry(Exception): - """ raised when start again from beginning """ - - -class SkipDownload(Exception): - """ raised when download should be skipped """ - - -class Plugin(Base): - """ - Base plugin for hoster/crypter. - Overwrite `process` / `decrypt` in your subclassed plugin. - """ - def __init__(self, pyfile): - Base.__init__(self, pyfile.m.core) - - self.wantReconnect = False - #: enables simultaneous processing of multiple downloads - self.multiDL = True - self.limitDL = 0 - #: chunk limit - self.chunkLimit = 1 - self.resumeDownload = False - - #: time() + wait in seconds - self.waitUntil = 0 - self.waiting = False - - self.ocr = None #captcha reader instance - #: account handler instance, see :py:class:`Account` - self.account = self.core.accountManager.getAccountForPlugin(self.__name__) - - #: premium status - self.premium = False - #: username/login - self.user = None - - if self.account and not self.account.isUsable(): self.account = None - if self.account: - self.user, data = self.account.loginname, {} #TODO change plugins to not use data anymore - #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest() - self.chunkLimit = -1 # chunk limit, -1 for unlimited - #: enables resume (will be ignored if server dont accept chunks) - self.resumeDownload = True - self.multiDL = True #every hoster with account should provide multiple downloads - #: premium status - self.premium = self.account.isPremium() - else: - self.req = self.core.requestFactory.getRequest(self.__name__) - - #: associated pyfile instance, see `PyFile` - self.pyfile = pyfile - self.thread = None # holds thread in future - - #: location where the last call to download was saved - self.lastDownload = "" - #: re match of the last call to `checkDownload` - self.lastCheck = None - #: js engine, see `JsEngine` - self.js = self.core.js - self.cTask = None #captcha task - - self.retries = 0 # amount of retries already made - self.html = None # some plugins store html code here - - self.init() - - def getChunkCount(self): - if self.chunkLimit <= 0: - return self.config["download"]["chunks"] - return min(self.config["download"]["chunks"], self.chunkLimit) - - def __call__(self): - return self.__name__ - - def init(self): - """initialize the plugin (in addition to `__init__`)""" - pass - - def setup(self): - """ setup for enviroment and other things, called before downloading (possibly more than one time)""" - pass - - def preprocessing(self, thread): - """ handles important things to do before starting """ - self.thread = thread - - if self.account: - # will force a relogin or reload of account info if necessary - self.account.getAccountInfo() - else: - self.req.clearCookies() - - self.setup() - - self.pyfile.setStatus("starting") - - return self.process(self.pyfile) - - - def process(self, pyfile): - """the 'main' method of every plugin, you **have to** overwrite it""" - raise NotImplementedError - - def resetAccount(self): - """ dont use account and retry download """ - self.account = None - self.req = self.core.requestFactory.getRequest(self.__name__) - self.retry() - - def checksum(self, local_file=None): - """ - return codes: - 0 - checksum ok - 1 - checksum wrong - 5 - can't get checksum - 10 - not implemented - 20 - unknown error - """ - #@TODO checksum check hook - - return True, 10 - - - def setWait(self, seconds, reconnect=False): - """Set a specific wait time later used with `wait` - - :param seconds: wait time in seconds - :param reconnect: True if a reconnect would avoid wait time - """ - if reconnect: - self.wantReconnect = True - self.pyfile.waitUntil = time() + int(seconds) - - def wait(self): - """ waits the time previously set """ - self.waiting = True - self.pyfile.setStatus("waiting") - - while self.pyfile.waitUntil > time(): - self.thread.m.reconnecting.wait(2) - - if self.pyfile.abort: raise Abort - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - self.waiting = False - self.pyfile.setStatus("starting") - - def fail(self, reason): - """ fail and give reason """ - raise Fail(reason) - - def offline(self): - """ fail and indicate file is offline """ - raise Fail("offline") - - def tempOffline(self): - """ fail and indicates file ist temporary offline, the core may take consequences """ - raise Fail("temp. offline") - - def retry(self, max_tries=3, wait_time=1, reason=""): - """Retries and begin again from the beginning - - :param max_tries: number of maximum retries - :param wait_time: time to wait in seconds - :param reason: reason for retrying, will be passed to fail if max_tries reached - """ - if 0 < max_tries <= self.retries: - if not reason: reason = "Max retries reached" - raise Fail(reason) - - self.wantReconnect = False - self.setWait(wait_time) - self.wait() - - self.retries += 1 - raise Retry(reason) - - def invalidCaptcha(self): - if self.cTask: - self.cTask.invalid() - - def correctCaptcha(self): - if self.cTask: - self.cTask.correct() - - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual'): - """ Loads a captcha and decrypts it with ocr, plugin, user input - - :param url: url of captcha image - :param get: get part for request - :param post: post part for request - :param cookies: True if cookies should be enabled - :param forceUser: if True, ocr is not used - :param imgtype: Type of the Image - :param result_type: 'textual' if text is written on the captcha\ - or 'positional' for captcha where the user have to click\ - on a specific region on the captcha - - :return: result of decrypting - """ - - img = self.load(url, get=get, post=post, cookies=cookies) - - id = ("%.2f" % time())[-6:].replace(".", "") - temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - temp_file.write(img) - temp_file.close() - - has_plugin = self.__name__ in self.core.pluginManager.getPlugins("captcha") - - if self.core.captcha: - Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) - else: - Ocr = None - - if Ocr and not forceUser: - sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: raise Abort - - ocr = Ocr() - result = ocr.get_captcha(temp_file.name) - else: - captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) - self.cTask = task - captchaManager.handleCaptcha(task) - - while task.isWaiting(): - if self.pyfile.abort: - captchaManager.removeTask(task) - raise Abort - sleep(1) - - captchaManager.removeTask(task) - - if task.error and has_plugin: #ignore default error message since the user could use OCR - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) - - result = task.result - self.log.debug("Received captcha result: %s" % str(result)) - - if not self.core.debug: - try: - remove(temp_file.name) - except: - pass - - return result - - - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): - """Load content at url and returns it - - :param url: - :param get: - :param post: - :param ref: - :param cookies: - :param just_header: if True only the header will be retrieved and returned as dict - :param decode: Wether to decode the output according to http header, should be True in most cases - :return: Loaded content - """ - if self.pyfile.abort: raise Abort - #utf8 vs decode -> please use decode attribute in all future plugins - if type(url) == unicode: url = str(url) - - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) - - if self.core.debug: - from inspect import currentframe - - frame = currentframe() - if not exists(join("tmp", self.__name__)): - makedirs(join("tmp", self.__name__)) - - f = open( - join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) - , "wb") - del frame # delete the frame or it wont be cleaned - - try: - tmp = res.encode("utf8") - except: - tmp = res - - f.write(tmp) - f.close() - - if just_header: - #parse header - header = {"code": self.req.code} - for line in res.splitlines(): - line = line.strip() - if not line or ":" not in line: continue - - key, none, value = line.partition(":") - key = key.lower().strip() - value = value.strip() - - if key in header: - if type(header[key]) == list: - header[key].append(value) - else: - header[key] = [header[key], value] - else: - header[key] = value - res = header - - return res - - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): - """Downloads the content at url to download folder - - :param url: - :param get: - :param post: - :param ref: - :param cookies: - :param disposition: if True and server provides content-disposition header\ - the filename will be changed if needed - :return: The location where the file was saved - """ - - self.checkForSameFiles() - - self.pyfile.setStatus("downloading") - - download_folder = self.config['general']['download_folder'] - - location = save_join(download_folder, self.pyfile.package().folder) - - if not exists(location): - makedirs(location, int(self.core.config["permission"]["folder"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - - chown(location, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) - - # convert back to unicode - location = fs_decode(location) - name = save_path(self.pyfile.name) - - filename = join(location, name) - - self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) - - try: - newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, - chunks=self.getChunkCount(), resume=self.resumeDownload, - progressNotify=self.pyfile.setProgress, disposition=disposition) - finally: - self.pyfile.size = self.req.size - - if disposition and newname and newname != name: #triple check, just to be sure - self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) - self.pyfile.name = newname - filename = join(location, newname) - - fs_filename = fs_encode(filename) - - if self.core.config["permission"]["change_file"]: - chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - - chown(fs_filename, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) - - self.lastDownload = filename - return self.lastDownload - - def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): - """ checks the content of the last downloaded file, re match is saved to `lastCheck` - - :param rules: dict with names and rules to match (compiled regexp or strings) - :param api_size: expected file size - :param max_size: if the file is larger then it wont be checked - :param delete: delete if matched - :param read_size: amount of bytes to read from files larger then max_size - :return: dictionary key of the first rule that matched - """ - lastDownload = fs_encode(self.lastDownload) - if not exists(lastDownload): return None - - size = stat(lastDownload) - size = size.st_size - - if api_size and api_size <= size: return None - elif size > max_size and not read_size: return None - self.log.debug("Download Check triggered") - f = open(lastDownload, "rb") - content = f.read(read_size if read_size else -1) - f.close() - #produces encoding errors, better log to other file in the future? - #self.log.debug("Content: %s" % content) - for name, rule in rules.iteritems(): - if type(rule) in (str, unicode): - if rule in content: - if delete: - remove(lastDownload) - return name - elif hasattr(rule, "search"): - m = rule.search(content) - if m: - if delete: - remove(lastDownload) - self.lastCheck = m - return name - - - def getPassword(self): - """ get the password the user provided in the package""" - password = self.pyfile.package().password - if not password: return "" - return password - - - def checkForSameFiles(self, starting=False): - """ checks if same file was/is downloaded within same package - - :param starting: indicates that the current download is going to start - :raises SkipDownload: - """ - - pack = self.pyfile.package() - - for pyfile in self.core.files.cache.values(): - if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: - if pyfile.status in (0, 12): #finished or downloading - raise SkipDownload(pyfile.pluginname) - elif pyfile.status in ( - 5, 7) and starting: #a download is waiting/starting and was appenrently started before - raise SkipDownload(pyfile.pluginname) - - download_folder = self.config['general']['download_folder'] - location = save_join(download_folder, pack.folder, self.pyfile.name) - - if starting and self.core.config['download']['skip_existing'] and exists(location): - size = os.stat(location).st_size - if size >= self.pyfile.size: - raise SkipDownload("File exists.") - - pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) - if pyfile: - if exists(location): - raise SkipDownload(pyfile[0]) - - self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) - - def clean(self): - """ clean everything and remove references """ - if hasattr(self, "pyfile"): - del self.pyfile - if hasattr(self, "req"): - self.req.close() - del self.req - if hasattr(self, "thread"): - del self.thread - if hasattr(self, "html"): - del self.html diff --git a/module/web/ServerThread.py b/module/web/ServerThread.py index 84667e5f6..8b59ca01b 100644 --- a/module/web/ServerThread.py +++ b/module/web/ServerThread.py @@ -93,6 +93,13 @@ class WebServer(threading.Thread): webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) def start_fcgi(self): + + from flup.server.threadedserver import ThreadedServer + + def noop(*args, **kwargs): + pass + + ThreadedServer._installSignalHandlers = noop self.core.log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) webinterface.run_fcgi(host=self.host, port=self.port) diff --git a/module/web/api_app.py b/module/web/api_app.py index 160a984df..affcdb39a 100644 --- a/module/web/api_app.py +++ b/module/web/api_app.py @@ -25,16 +25,16 @@ class TBaseEncoder(json.JSONEncoder): # accepting positional arguments, as well as kwargs via post and get - -@route("/api/:func:args#[a-zA-Z0-9\-_/\"'\[\]%{}]*#") -@route("/api/:func:args#[a-zA-Z0-9\-_/\"'\[\]%{}]*#", method="POST") +# only forbidden path symbol are "?", which is used to seperate GET data and # +@route("/api/<func><args:re:[^#?]*>") +@route("/api/<func><args:re:[^#?]*>", method="POST") def call_api(func, args=""): response.headers.replace("Content-type", "application/json") response.headers.append("Cache-Control", "no-cache, must-revalidate") s = request.environ.get('beaker.session') if 'session' in request.POST: - # removes "' so it works on json strings + # removes "' so it works on json strings s = s.get_by_id(remove_chars(request.POST['session'], "'\"")) if not s or not s.get("authenticated", False): @@ -65,7 +65,7 @@ def callApi(func, *args, **kwargs): result = getattr(PYLOAD, func)(*[literal_eval(x) for x in args], **dict([(x, literal_eval(y)) for x, y in kwargs.iteritems()])) - # null is invalid json response + # null is invalid json response if result is None: result = True return json.dumps(result, cls=TBaseEncoder) diff --git a/tests/crypterlinks.txt b/tests/crypterlinks.txt index 38692c756..4ff651888 100644 --- a/tests/crypterlinks.txt +++ b/tests/crypterlinks.txt @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Crypter links, append ||fail or ||#N to mark error or number of expected results (single links+packages) diff --git a/tests/helper/PluginTester.py b/tests/helper/PluginTester.py index b70c0d061..ef61385be 100644 --- a/tests/helper/PluginTester.py +++ b/tests/helper/PluginTester.py @@ -111,14 +111,14 @@ def respond(ticket, value): def invalidCaptcha(self): log(DEBUG, "Captcha invalid") if self.cTask: - respond(self.ticket, 0) + respond(self.cTask, 0) Hoster.invalidCaptcha = invalidCaptcha def correctCaptcha(self): log(DEBUG, "Captcha correct") if self.cTask: - respond(self.ticket, 1) + respond(self.cTask, 1) Hoster.correctCaptcha = correctCaptcha @@ -148,4 +148,4 @@ class PluginTester(TestCase): exc = exc_info() if exc != (None, None, None): debug = self.thread.writeDebugReport() - log(DEBUG, debug)
\ No newline at end of file + log(DEBUG, debug) diff --git a/tests/hosterlinks.txt b/tests/hosterlinks.txt index f255661ab..c55634f0a 100755 --- a/tests/hosterlinks.txt +++ b/tests/hosterlinks.txt @@ -1,6 +1,7 @@ +# -*- coding: utf-8 -*- # Valid files, with md5 hash -# Please only use files around 5-15 MB +# Please only use files around 5-15 MB and with explicit permission for redistribution http://download.pyload.org/random.bin random.bin d76505d0869f9f928a17d42d66326307 @@ -45,4 +46,4 @@ http://letitbit.net/download/67793.60a7d3745791db7271a6e6c92cfe/Mořská_želva_ http://www.mediafire.com/?n09th58z1x5r585 http://www.quickshare.cz/stahnout-soubor/676150:morska-zelva----_-oi-moi-dua-tar_6MB http://www.uloz.to/12553820/morska-zelva-oi-moi-dua-tar -http://www.wupload.com/file/2642593407/
\ No newline at end of file +http://www.wupload.com/file/2642593407/ |