diff options
| -rw-r--r-- | module/HookManager.py | 2 | ||||
| -rw-r--r-- | module/network/HTTPRequest.py | 2 | ||||
| -rw-r--r-- | module/plugins/Hoster.py | 3 | ||||
| -rw-r--r-- | module/plugins/crypter/FilesonicComFolder.py | 12 | ||||
| -rw-r--r-- | module/plugins/hoster/FilesMailRu.py | 3 | ||||
| -rw-r--r-- | module/plugins/hoster/HotfileCom.py | 3 | ||||
| -rw-r--r-- | module/plugins/hoster/NetloadIn.py | 4 | ||||
| -rw-r--r-- | module/plugins/hoster/ShareonlineBiz.py | 4 | ||||
| -rw-r--r-- | module/plugins/hoster/UploadedTo.py | 3 | ||||
| -rw-r--r-- | module/threads/BaseThread.py | 1 | ||||
| -rw-r--r-- | module/utils/__init__.py | 7 | ||||
| -rw-r--r-- | module/utils/fs.py | 2 | ||||
| -rw-r--r-- | tests/CrypterPluginTester.py | 71 | ||||
| -rw-r--r-- | tests/HosterPluginTester.py | 115 | ||||
| -rw-r--r-- | tests/crypterlinks.txt | 4 | ||||
| -rw-r--r-- | tests/helper/PluginTester.py | 52 | ||||
| -rw-r--r-- | tests/helper/Stubs.py | 119 | ||||
| -rw-r--r-- | tests/helper/__init__.py | 0 | ||||
| -rwxr-xr-x | tests/hosterlinks.txt (renamed from tests/testlinks.txt) | 24 | ||||
| -rwxr-xr-x | tests/plugin_tests.sh | 5 | ||||
| -rw-r--r-- | tests/stubs/__init__.py | 1 | ||||
| -rw-r--r-- | tests/test_syntax.py | 5 |
22 files changed, 400 insertions, 42 deletions
diff --git a/module/HookManager.py b/module/HookManager.py index 8afd6fe26..0ad37b321 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -200,7 +200,7 @@ class HookManager: def activePlugins(self): """ returns all active plugins """ - return [x for x in self.plugins if x.isActivated()] + return [x for x in self.plugins.itervalues() if x.isActivated()] def getAllInfo(self): """returns info stored by hook plugins""" diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 4684397d9..7887081e7 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -206,7 +206,7 @@ class HTTPRequest(): finally: self.c.setopt(pycurl.FOLLOWLOCATION, 1) self.c.setopt(pycurl.NOBODY, 0) - self.c.setopt(pycurl.CUSTOMREQUEST, 0) + self.c.unsetopt(pycurl.CUSTOMREQUEST) else: self.c.perform() diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 7c43c6444..4a5d15759 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -28,9 +28,12 @@ if os.name != "nt": from grp import getgrnam from Base import Base, Fail, Retry +from module.utils import chunks as _chunks from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\ remove, makedirs, chmod, stat, exists, join +# Import for Hoster Plugins +chunks = _chunks class Abort(Exception): """ raised when aborted """ diff --git a/module/plugins/crypter/FilesonicComFolder.py b/module/plugins/crypter/FilesonicComFolder.py index b967a74a1..02ae66295 100644 --- a/module/plugins/crypter/FilesonicComFolder.py +++ b/module/plugins/crypter/FilesonicComFolder.py @@ -4,8 +4,6 @@ import re from module.plugins.Crypter import Crypter class FilesonicComFolder(Crypter): - __name__ = "FilesonicComFolder" - __type__ = "crypter" __pattern__ = r"http://(\w*\.)?(sharingmatrix|filesonic|wupload)\.[^/]*/folder/\w+/?" __version__ = "0.11" __description__ = """Filesonic.com/Wupload.com Folder Plugin""" @@ -15,9 +13,8 @@ class FilesonicComFolder(Crypter): FOLDER_PATTERN = r'<table>\s*<caption>Files Folder</caption>(.*?)</table>' LINK_PATTERN = r'<a href="([^"]+)">' - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - + def decryptURL(self, url): + html = self.load(url) new_links = [] folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) @@ -26,6 +23,7 @@ class FilesonicComFolder(Crypter): new_links.extend(re.findall(self.LINK_PATTERN, folder.group(1))) if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) + return new_links else: - self.fail('Could not extract any links')
\ No newline at end of file + self.fail('Could not extract any links') + diff --git a/module/plugins/hoster/FilesMailRu.py b/module/plugins/hoster/FilesMailRu.py index 6002ab3dc..1284329b5 100644 --- a/module/plugins/hoster/FilesMailRu.py +++ b/module/plugins/hoster/FilesMailRu.py @@ -2,9 +2,8 @@ # -*- coding: utf-8 -*- import re -from module.plugins.Hoster import Hoster +from module.plugins.Hoster import Hoster, chunks from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks def getInfo(urls): result = [] diff --git a/module/plugins/hoster/HotfileCom.py b/module/plugins/hoster/HotfileCom.py index 9c056d899..d36a4df2e 100644 --- a/module/plugins/hoster/HotfileCom.py +++ b/module/plugins/hoster/HotfileCom.py @@ -2,11 +2,10 @@ # -*- coding: utf-8 -*- import re -from module.plugins.Hoster import Hoster +from module.plugins.Hoster import Hoster, chunks from module.plugins.ReCaptcha import ReCaptcha from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks def getInfo(urls): api_url_base = "http://api.hotfile.com/" diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py index b2bec873d..382328496 100644 --- a/module/plugins/hoster/NetloadIn.py +++ b/module/plugins/hoster/NetloadIn.py @@ -5,11 +5,9 @@ import re from time import sleep, time +from module.utils import chunks from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks - - def getInfo(urls): ## returns list of tupels (name, size (in bytes), status (see FileDatabase), url) diff --git a/module/plugins/hoster/ShareonlineBiz.py b/module/plugins/hoster/ShareonlineBiz.py index 641a9b025..2d1fc8d85 100644 --- a/module/plugins/hoster/ShareonlineBiz.py +++ b/module/plugins/hoster/ShareonlineBiz.py @@ -7,10 +7,8 @@ import hashlib import random from time import sleep -from module.plugins.Hoster import Hoster +from module.plugins.Hoster import Hoster, chunks from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks - def getInfo(urls): api_url_base = "http://api.share-online.biz/linkcheck.php" diff --git a/module/plugins/hoster/UploadedTo.py b/module/plugins/hoster/UploadedTo.py index 39483cf86..751dcda25 100644 --- a/module/plugins/hoster/UploadedTo.py +++ b/module/plugins/hoster/UploadedTo.py @@ -2,11 +2,10 @@ import re -from module.utils import html_unescape, parseFileSize +from module.utils import html_unescape, parseFileSize, chunks from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks from module.plugins.ReCaptcha import ReCaptcha key = "bGhGMkllZXByd2VEZnU5Y2NXbHhYVlZ5cEE1bkEzRUw=".decode('base64') diff --git a/module/threads/BaseThread.py b/module/threads/BaseThread.py index 526913e9b..f6fac46a0 100644 --- a/module/threads/BaseThread.py +++ b/module/threads/BaseThread.py @@ -68,6 +68,7 @@ class BaseThread(Thread): f.close() self.log.info("Debug Report written to %s" % dump_name) + return dump_name def getFileDump(self, pyfile): dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( diff --git a/module/utils/__init__.py b/module/utils/__init__.py index b68928f04..bf11fbc69 100644 --- a/module/utils/__init__.py +++ b/module/utils/__init__.py @@ -160,6 +160,13 @@ def accumulate(it, inv_map=None): def to_string(value): return str(value) if not isinstance(value, basestring) else value +def to_int(string): + """ return int from string or 0 """ + try: + return int(string) + except ValueError: + return 0 + def from_string(value, typ=None): """ cast value to given type, unicode for strings """ diff --git a/module/utils/fs.py b/module/utils/fs.py index 03832e368..c1927423a 100644 --- a/module/utils/fs.py +++ b/module/utils/fs.py @@ -35,7 +35,7 @@ def remove(path): def exists(path): return os.path.exists(fs_encode(path)) -def makedirs(path, mode=0660): +def makedirs(path, mode=0777): return os.makedirs(fs_encode(path), mode) def listdir(path): diff --git a/tests/CrypterPluginTester.py b/tests/CrypterPluginTester.py index 124cb4d0a..27013ede7 100644 --- a/tests/CrypterPluginTester.py +++ b/tests/CrypterPluginTester.py @@ -1,6 +1,71 @@ # -*- coding: utf-8 -*- -from unittest import TestCase +from os.path import dirname, join +from nose.tools import nottest -class DecryptPluginTester(TestCase): - pass
\ No newline at end of file +from logging import log, DEBUG + +from helper.Stubs import Core +from helper.PluginTester import PluginTester + +from module.plugins.Base import Fail +from module.utils import accumulate, to_int + +class CrypterPluginTester(PluginTester): + + @nottest + def test_plugin(self, name, url, flag): + + log(DEBUG, "%s: %s", name, url) + + plugin = self.core.pluginManager.getPluginClass(name) + p = plugin(self.core, None, "") + self.thread.plugin = p + + try: + result = p._decrypt([url]) + + if to_int(flag): + assert to_int(flag) == len(result) + + except Exception, e: + if isinstance(e, Fail) and flag == "fail": + pass + else: + raise + + +# setup methods + +c = Core() + +f = open(join(dirname(__file__), "crypterlinks.txt")) +links = [x.strip() for x in f.readlines()] +urls = [] +flags = {} + +for l in links: + if not l or l.startswith("#"): continue + if l.startswith("http"): + if "||" in l: + l, flag = l.split("||") + flags[l] = flag + + urls.append(l) + +h, crypter = c.pluginManager.parseUrls(urls) +plugins = accumulate(crypter) +for plugin, urls in plugins.iteritems(): + + for i, url in enumerate(urls): + + + def meta(plugin, url, flag, sig): + def _test(self): + self.test_plugin(plugin, url, flag) + + _test.func_name = sig + return _test + + sig = "test_%s_LINK%d" % (plugin, i) + setattr(CrypterPluginTester, sig, meta(plugin, url, flags.get(url, None), sig))
\ No newline at end of file diff --git a/tests/HosterPluginTester.py b/tests/HosterPluginTester.py index faaaf799c..e4738ad5e 100644 --- a/tests/HosterPluginTester.py +++ b/tests/HosterPluginTester.py @@ -1,3 +1,118 @@ # -*- coding: utf-8 -*- +from os import remove +from os.path import dirname +from logging import log, DEBUG +from hashlib import md5 +from time import time +from nose.tools import nottest + +from helper.Stubs import Core +from helper.PluginTester import PluginTester + +from module.PyFile import PyFile +from module.plugins.Base import Fail +from module.utils import accumulate +from module.utils.fs import save_join, join, exists + +DL_DIR = join("Downloads", "tmp") + +class HosterPluginTester(PluginTester): + + files = {} + + def setUp(self): + PluginTester.setUp(self) + for f in self.files: + pass + if exists(join(DL_DIR, f)): remove(join(DL_DIR, f)) + + @nottest + def test_plugin(self, name, url, flag): + + # Print to stdout to see whats going on + print "%s: %s" % (name, url) + log(DEBUG, "%s: %s", name, url) + + # url and plugin should be only important thing + pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0) + pyfile.initPlugin() + + self.thread.pyfile = pyfile + self.thread.plugin = pyfile.plugin + + try: + a = time() + pyfile.plugin.preprocessing(self.thread) + + + log(DEBUG, "downloading took %ds" % (time()-a)) + log(DEBUG, "size %d kb" % (pyfile.size / 1024)) + + if pyfile.name not in self.files: + raise Exception("Filename %s wrong." % pyfile.name) + + if not exists(save_join(DL_DIR, pyfile.name)): + raise Exception("File %s does not exists." % pyfile.name) + + hash = md5() + f = open(save_join(DL_DIR, pyfile.name)) + while True: + buf = f.read(4096) + if not buf: break + hash.update(buf) + + if hash.hexdigest() != self.files[pyfile.name]: + raise Exception("Hash does not match.") + + + + except Exception, e: + if isinstance(e, Fail) and flag == "fail": + pass + elif isinstance(e, Fail) and flag == "offline" and e.message == "offline": + pass + else: + raise + + +# setup methods + +c = Core() + +f = open(join(dirname(__file__), "hosterlinks.txt")) +links = [x.strip() for x in f.readlines()] +urls = [] +flags = {} + +for l in links: + if not l or l.startswith("#"): continue + if l.startswith("http"): + if "||" in l: + l, flag = l.split("||") + flags[l] = flag + urls.append(l) + + elif len(l.split(" ")) == 2: + name, hash = l.split(" ") + HosterPluginTester.files[name] = hash + + +hoster, c = c.pluginManager.parseUrls(urls) + +plugins = accumulate(hoster) +for plugin, urls in plugins.iteritems(): + + for i, url in enumerate(urls): + + + def meta(plugin, url, flag, sig): + def _test(self): + self.test_plugin(plugin, url, flag) + + _test.func_name = sig + return _test + + sig = "test_%s_LINK%d" % (plugin, i) + setattr(HosterPluginTester, sig, meta(plugin, url, flags.get(url, None), sig))
\ No newline at end of file diff --git a/tests/crypterlinks.txt b/tests/crypterlinks.txt new file mode 100644 index 000000000..38692c756 --- /dev/null +++ b/tests/crypterlinks.txt @@ -0,0 +1,4 @@ + +# Crypter links, append ||fail or ||#N to mark error or number of expected results (single links+packages) + +http://www.filesonic.com/folder/19906605||2 diff --git a/tests/helper/PluginTester.py b/tests/helper/PluginTester.py new file mode 100644 index 000000000..997a0923f --- /dev/null +++ b/tests/helper/PluginTester.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +from unittest import TestCase +from os.path import abspath +from sys import exc_clear, exc_info +from logging import log, DEBUG +from time import sleep, time + +from Stubs import Thread, Core, noop + +from module.plugins.Hoster import Hoster, Abort, Fail + +def _wait(self): + """ waits the time previously set """ + self.waiting = True + + waittime = self.pyfile.waitUntil - time() + log(DEBUG, "waiting %ss" % waittime) + + if self.wantReconnect: + raise Fail("Would wait for reconnect %ss" % waittime ) + if self.wantReconnect or waittime > 300: + raise Fail("Would wait %ss" % waittime ) + + while self.pyfile.waitUntil > time(): + sleep(1) + if self.pyfile.abort: raise Abort + + self.waiting = False + self.pyfile.setStatus("starting") + +Hoster.wait = _wait + +Hoster.checkForSameFiles = noop + +class PluginTester(TestCase): + + @classmethod + def setUpClass(cls): + cls.core = Core() + + def setUp(self): + self.thread = Thread(self.core) + exc_clear() + + def tearDown(self): + exc = exc_info() + if exc != (None, None, None): + debug = self.thread.writeDebugReport() + log(DEBUG, debug) + # generate attachment + print "\n[[ATTACHMENT|%s]]\n" % abspath(debug)
\ No newline at end of file diff --git a/tests/helper/Stubs.py b/tests/helper/Stubs.py new file mode 100644 index 000000000..eb3cc98c1 --- /dev/null +++ b/tests/helper/Stubs.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- + +import sys +from os.path import abspath, dirname, join +from time import strftime + +sys.path.append(abspath(join(dirname(__file__), "..", "..", "module", "lib"))) +sys.path.append(abspath(join(dirname(__file__), "..", ".."))) + +import __builtin__ + +from module.PyPackage import PyPackage +from module.threads.BaseThread import BaseThread +from module.config.ConfigParser import ConfigParser +from module.network.RequestFactory import RequestFactory +from module.plugins.PluginManager import PluginManager +from module.common.JsEngine import JsEngine + +from logging import log, DEBUG, INFO, WARN, ERROR + + +# Do nothing +def noop(*args, **kwargs): + pass + +ConfigParser.save = noop + +class LogStub: + def debug(self, *args): + log(DEBUG, *args) + + def info(self, *args): + log(INFO, *args) + + def error(self, *args): + log(ERROR, *args) + + def warning(self, *args): + log(WARN, *args) + + +class NoLog: + def debug(self, *args): + pass + + def info(self, *args): + pass + + def error(self, *args): + log(ERROR, *args) + + def warning(self, *args): + log(WARN, *args) + + +class Core: + def __init__(self): + self.log = NoLog() + + self.api = self + self.core = self + self.debug = True + self.captcha = True + self.config = ConfigParser() + self.pluginManager = PluginManager(self) + self.requestFactory = RequestFactory(self) + __builtin__.pyreq = self.requestFactory + self.accountManager = AccountManager() + self.hookManager = self.eventManager = self.interActionManager = NopClass() + self.js = JsEngine() + self.cache = {} + self.packageCache = {} + + self.log = LogStub() + + def getServerVersion(self): + return "TEST_RUNNER on %s" % strftime("%d %h %Y") + + def path(self, path): + return path + + def updateLink(self, *args): + pass + + def updatePackage(self, *args): + pass + + def getPackage(self, id): + return PyPackage(self, 0, "tmp", "tmp", "", "", 0, 0) + + + +class NopClass: + def __getattr__(self, item): + return noop + +class AccountManager: + + def getAccountForPlugin(self, name): + return None + +class Thread(BaseThread): + def __init__(self, core): + BaseThread.__init__(self, core) + self.plugin = None + + + def writeDebugReport(self): + if hasattr(self, "pyfile"): + dump = BaseThread.writeDebugReport(self, self.plugin.__name__, pyfile=self.pyfile) + else: + dump = BaseThread.writeDebugReport(self, self.plugin.__name__, plugin=self.plugin) + + return dump + +__builtin__._ = lambda x: x +__builtin__.pypath = "" +__builtin__.hookManager = NopClass() +__builtin__.pyreq = None
\ No newline at end of file diff --git a/tests/helper/__init__.py b/tests/helper/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/helper/__init__.py diff --git a/tests/testlinks.txt b/tests/hosterlinks.txt index 428cf63ea..0da4074dc 100755 --- a/tests/testlinks.txt +++ b/tests/hosterlinks.txt @@ -1,24 +1,22 @@ -sha1: - 961486646bf3c1d5d7a45ec32bb62e1bc4f2d894 random.bin -md5: - d76505d0869f9f928a17d42d66326307 random.bin -please save bandwith on our server, -use this link only for remote uploads to new hoster -http://get.pyload.org/static/random.bin ---------------------------------------- +# Valid files, with md5 hash +# Please only use files around 5-15 MB +random.bin d76505d0869f9f928a17d42d66326307 + +# Hoster links, append ||offline or ||fail to mark your expectation + http://netload.in/datei9XirAJZs79/random.bin.htm -http://ul.to/file/o41isx +http://ul.to/file/o41isx||offline http://rapidshare.com/files/445996776/random.bin -http://dl.free.fr/d4aL5dyXY +http://dl.free.fr/d4aL5dyXY||offline http://www.duckload.com/dl/QggW2 -http://files.mail.ru/32EW66 -http://www.fileserve.com/file/MxjZXjX +http://files.mail.ru/32EW66||offline +http://www.fileserve.com/file/MxjZXjX||offline http://www.4shared.com/file/-O5CBhQV/random.html http://hotfile.com/dl/101569859/2e01f04/random.bin.html http://www.megaupload.com/?d=1JZLOP3B http://www.share.cx/files/235687689252/random.bin.html -http://www.share-online.biz/download.php?id=PTCOX1GL6XAH +http://www.share-online.biz/download.php?id=PTCOX1GL6XAH||offline http://www.shragle.com/files/f899389b/random.bin http://www10.zippyshare.com/v/76557688/file.html http://yourfiles.to/?d=312EC6E911 diff --git a/tests/plugin_tests.sh b/tests/plugin_tests.sh new file mode 100755 index 000000000..a0260b5bb --- /dev/null +++ b/tests/plugin_tests.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +NS=nosetests +which nosetests2 > /dev/null && NS=nosetests2 +$NS tests/HosterPluginTester.py tests/CrypterPluginTester.py -s --with-xunit --with-coverage --cover-erase --cover-package=module.plugins +coverage xml diff --git a/tests/stubs/__init__.py b/tests/stubs/__init__.py deleted file mode 100644 index 4b31e848b..000000000 --- a/tests/stubs/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'christian' diff --git a/tests/test_syntax.py b/tests/test_syntax.py index 82c4194da..4a131ef6f 100644 --- a/tests/test_syntax.py +++ b/tests/test_syntax.py @@ -8,9 +8,8 @@ from unittest import TestCase PATH = abspath(join(dirname(abspath(__file__)), "..", "")) -# gettext -__builtin__._ = lambda x: x -__builtin__.hookManager = _ +# needed to register globals +from helper import Stubs class TestSyntax(TestCase): pass |