From 7e7adc64713f74976d5994af36b5f758620fb37b Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 13 Oct 2012 21:39:58 +0200 Subject: added JSON and WS client, re organized tests, new classes for easier api tests --- module/common/APIExerciser.py | 155 -------------------- module/common/__init__.py | 2 - module/database/DatabaseBackend.py | 9 +- module/remote/JSONClient.py | 52 +++++++ module/remote/WSClient.py | 41 ++++++ module/remote/WebSocketBackend.py | 6 +- module/remote/json_converter.py | 26 ++-- module/remote/wsbackend/AbstractHandler.py | 29 ++-- module/remote/wsbackend/ApiHandler.py | 22 +-- module/remote/wsbackend/AsyncHandler.py | 112 +++++++++++++++ module/remote/wsbackend/EventHandler.py | 41 ------ module/web/api_app.py | 2 + module/web/utils.py | 8 +- pyLoadCore.py | 2 +- tests/__init__.py | 0 tests/api/ApiProxy.py | 68 +++++++++ tests/api/ApiTester.py | 35 +++++ tests/api/__init__.py | 0 tests/api/test_JSONBackend.py | 27 ++++ tests/api/test_api.py | 39 +++++ tests/api/test_noargs.py | 29 ++++ tests/config/db.version | 1 - tests/helper/Stubs.py | 3 + tests/manager/__init__.py | 0 tests/manager/test_filemanager.py | 223 +++++++++++++++++++++++++++++ tests/manager/test_interactionManager.py | 58 ++++++++ tests/other/__init__.py | 0 tests/other/test_configparser.py | 53 +++++++ tests/other/test_database.py | 192 +++++++++++++++++++++++++ tests/other/test_syntax.py | 43 ++++++ tests/test_api.py | 18 --- tests/test_backends.py | 59 -------- tests/test_configparser.py | 48 ------- tests/test_database.py | 195 ------------------------- tests/test_filemanager.py | 223 ----------------------------- tests/test_interactionManager.py | 58 -------- tests/test_syntax.py | 43 ------ 37 files changed, 1033 insertions(+), 889 deletions(-) delete mode 100644 module/common/APIExerciser.py create mode 100644 module/remote/JSONClient.py create mode 100644 module/remote/WSClient.py create mode 100644 module/remote/wsbackend/AsyncHandler.py delete mode 100644 module/remote/wsbackend/EventHandler.py create mode 100644 tests/__init__.py create mode 100644 tests/api/ApiProxy.py create mode 100644 tests/api/ApiTester.py create mode 100644 tests/api/__init__.py create mode 100644 tests/api/test_JSONBackend.py create mode 100644 tests/api/test_api.py create mode 100644 tests/api/test_noargs.py delete mode 100644 tests/config/db.version create mode 100644 tests/manager/__init__.py create mode 100644 tests/manager/test_filemanager.py create mode 100644 tests/manager/test_interactionManager.py create mode 100644 tests/other/__init__.py create mode 100644 tests/other/test_configparser.py create mode 100644 tests/other/test_database.py create mode 100644 tests/other/test_syntax.py delete mode 100644 tests/test_api.py delete mode 100644 tests/test_backends.py delete mode 100644 tests/test_configparser.py delete mode 100644 tests/test_database.py delete mode 100644 tests/test_filemanager.py delete mode 100644 tests/test_interactionManager.py delete mode 100644 tests/test_syntax.py diff --git a/module/common/APIExerciser.py b/module/common/APIExerciser.py deleted file mode 100644 index 37ba82b47..000000000 --- a/module/common/APIExerciser.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- - -import string -from threading import Thread -from random import choice, random, sample, randint -from time import time, sleep -from math import floor -import gc - -from traceback import print_exc, format_exc - -def createURLs(): - """ create some urls, some may fail """ - urls = [] - for x in range(0, randint(20, 100)): - name = "DEBUG_API" - if randint(0, 5) == 5: - name = "" #this link will fail - - urls.append(name + "".join(sample(string.ascii_letters, randint(10, 20)))) - - return urls - -AVOID = (0,3,8) - -idPool = 0 -sumCalled = 0 - - -def startApiExerciser(core, n): - for i in range(n): - APIExerciser(core).start() - -class APIExerciser(Thread): - - - def __init__(self, core, thrift=False, user=None, pw=None): - global idPool - - Thread.__init__(self) - self.setDaemon(True) - self.core = core - self.count = 0 #number of methods - self.time = time() - - if thrift: - self.api = ThriftClient(user=user, password=pw) - else: - self.api = core.api - - - self.id = idPool - - idPool += 1 - - #self.start() - - def run(self): - - self.core.log.info("API Exerciser started %d" % self.id) - - out = open("error.log", "ab") - #core errors are not logged of course - out.write("\n" + "Starting\n") - out.flush() - - while True: - try: - self.testAPI() - except Exception: - self.core.log.error("Exerciser %d throw an exception" % self.id) - print_exc() - out.write(format_exc() + 2 * "\n") - out.flush() - - if not self.count % 100: - self.core.log.info("Exerciser %d tested %d api calls" % (self.id, self.count)) - if not self.count % 1000: - out.flush() - - if not sumCalled % 1000: #not thread safe - self.core.log.info("Exercisers tested %d api calls" % sumCalled) - persec = sumCalled / (time() - self.time) - self.core.log.info("Approx. %.2f calls per second." % persec) - self.core.log.info("Approx. %.2f ms per call." % (1000 / persec)) - self.core.log.info("Collected garbage: %d" % gc.collect()) - - - #sleep(random() / 500) - - def testAPI(self): - global sumCalled - - m = ["statusDownloads", "statusServer", "addPackage", "getPackageData", "getFileData", "deleteFiles", - "deletePackages", "getQueue", "getCollector", "getQueueData", "getCollectorData", "isCaptchaWaiting", - "getCaptchaTask", "stopAllDownloads", "getAllInfo", "getServices" , "getAccounts", "getAllUserData"] - - method = choice(m) - #print "Testing:", method - - if hasattr(self, method): - res = getattr(self, method)() - else: - res = getattr(self.api, method)() - - self.count += 1 - sumCalled += 1 - - #print res - - def addPackage(self): - name = "".join(sample(string.ascii_letters, 10)) - urls = createURLs() - - self.api.addPackage(name, urls, choice([Destination.Queue, Destination.Collector]), "") - - - def deleteFiles(self): - info = self.api.getQueueData() - if not info: return - - pack = choice(info) - fids = pack.links - - if len(fids): - fids = [f.fid for f in sample(fids, randint(1, max(len(fids) / 2, 1)))] - self.api.deleteFiles(fids) - - - def deletePackages(self): - info = choice([self.api.getQueue(), self.api.getCollector()]) - if not info: return - - pids = [p.pid for p in info] - if len(pids): - pids = sample(pids, randint(1, max(floor(len(pids) / 2.5), 1))) - self.api.deletePackages(pids) - - def getFileData(self): - info = self.api.getQueueData() - if info: - p = choice(info) - if p.links: - self.api.getFileData(choice(p.links).fid) - - def getPackageData(self): - info = self.api.getQueue() - if info: - self.api.getPackageData(choice(info).pid) - - def getAccounts(self): - self.api.getAccounts(False) - - def getCaptchaTask(self): - self.api.getCaptchaTask(False) diff --git a/module/common/__init__.py b/module/common/__init__.py index de6d13128..e69de29bb 100644 --- a/module/common/__init__.py +++ b/module/common/__init__.py @@ -1,2 +0,0 @@ -__author__ = 'christian' - \ No newline at end of file diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index b22f8ffc5..6e67c799a 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -190,19 +190,18 @@ class DatabaseBackend(Thread): j = self.jobs.get() if j == "quit": self.c.close() + self.conn.commit() self.conn.close() + self.closing.set() break j.processJob() def shutdown(self): self.running.clear() - self._shutdown() - - @queue - def _shutdown(self): - self.conn.commit() + self.closing = Event() self.jobs.put("quit") + self.closing.wait(1) def _checkVersion(self): """ get db version""" diff --git a/module/remote/JSONClient.py b/module/remote/JSONClient.py new file mode 100644 index 000000000..52b712c81 --- /dev/null +++ b/module/remote/JSONClient.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from json_converter import loads, dumps +from urllib import urlopen, urlencode +from httplib import UNAUTHORIZED, FORBIDDEN + +class JSONClient: + URL = "http://localhost:8001/api" + + def __init__(self, url=None): + self.url = url or self.URL + self.session = None + + def request(self, path, data): + ret = urlopen(self.url + path, urlencode(data)) + if ret.code == 404: + raise AttributeError("Unknown Method") + if ret.code == 500: + raise Exception("Remote Exception") + if ret.code == UNAUTHORIZED: + raise Exception("Unauthorized") + if ret.code == FORBIDDEN: + raise Exception("Forbidden") + return ret.read() + + def login(self, username, password): + self.session = loads(self.request("/login", {'username': username, 'password': password})) + return self.session + + def logout(self): + self.call("logout") + self.session = None + + def call(self, func, *args, **kwargs): + # Add the current session + kwargs["session"] = self.session + path = "/" + func + "/" + "/".join(dumps(x) for x in args) + data = dict((k, dumps(v)) for k, v in kwargs.iteritems()) + rep = self.request(path, data) + return loads(rep) + + def __getattr__(self, item): + def call(*args, **kwargs): + return self.call(item, *args, **kwargs) + + return call + +if __name__ == "__main__": + api = JSONClient() + api.login("User", "test") + print api.getServerVersion() \ No newline at end of file diff --git a/module/remote/WSClient.py b/module/remote/WSClient.py new file mode 100644 index 000000000..c06bab661 --- /dev/null +++ b/module/remote/WSClient.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from json_converter import loads, dumps +from websocket import create_connection + +class WSClient: + URL = "ws://localhost:7227/api" + + def __init__(self, url=None): + self.url = url or self.URL + self.ws = None + + def login(self, username, password): + self.ws = create_connection(self.URL) + return self.call("login", username, password) + + def logout(self): + self.call("logout") + self.ws.close() + + def call(self, func, *args, **kwargs): + self.ws.send(dumps([func, args, kwargs])) + code, result = loads(self.ws.recv()) + if code == 404: + raise AttributeError("Unknown Method") + elif code == 505: + raise Exception("Remote Exception") + + return result + + def __getattr__(self, item): + def call(*args, **kwargs): + return self.call(item, *args, **kwargs) + + return call + +if __name__ == "__main__": + api = WSClient() + api.login("User", "test") + print api.getServerVersion() \ No newline at end of file diff --git a/module/remote/WebSocketBackend.py b/module/remote/WebSocketBackend.py index 6e01dad87..2d22664c6 100644 --- a/module/remote/WebSocketBackend.py +++ b/module/remote/WebSocketBackend.py @@ -33,14 +33,14 @@ class WebSocketBackend(BackendBase): from wsbackend.Server import WebSocketServer, DefaultOptions from wsbackend.Dispatcher import Dispatcher from wsbackend.ApiHandler import ApiHandler - from wsbackend.EventHandler import EventHandler + from wsbackend.AsyncHandler import AsyncHandler options = DefaultOptions() options.server_host = host options.port = port options.dispatcher = Dispatcher() - options.dispatcher.addHandler('/api', ApiHandler(self.core.api)) - options.dispatcher.addHandler('/events', EventHandler(self.core.api)) + options.dispatcher.addHandler(ApiHandler.PATH, ApiHandler(self.core.api)) + options.dispatcher.addHandler(AsyncHandler.PATH, AsyncHandler(self.core.api)) self.server = WebSocketServer(options) diff --git a/module/remote/json_converter.py b/module/remote/json_converter.py index 57e85fd16..ea76842a6 100644 --- a/module/remote/json_converter.py +++ b/module/remote/json_converter.py @@ -1,7 +1,13 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from module.common.json_layer import json +try: + from module.common.json_layer import json +except ImportError: + import json + + +import ttypes from ttypes import BaseObject # json encoder that accepts TBase objects @@ -16,18 +22,12 @@ class BaseEncoder(json.JSONEncoder): return json.JSONEncoder.default(self, o) -class BaseDecoder(json.JSONDecoder): - - def __init__(self, *args, **kwargs): - json.JSONDecoder.__init__(self, *args, **kwargs) - self.object_hook = self.convertObject - - def convertObject(self, dct): - if '@class' in dct: - # TODO: convert - pass - return dct +def convert_obj(dct): + if '@class' in dct: + cls = getattr(ttypes, dct['@class']) + del dct['@class'] + return cls(**dct) def dumps(*args, **kwargs): kwargs['cls'] = BaseEncoder @@ -35,5 +35,5 @@ def dumps(*args, **kwargs): def loads(*args, **kwargs): - kwargs['cls'] = BaseDecoder + kwargs['object_hook'] = convert_obj return json.loads(*args, **kwargs) \ No newline at end of file diff --git a/module/remote/wsbackend/AbstractHandler.py b/module/remote/wsbackend/AbstractHandler.py index 291dbf100..276f6fa38 100644 --- a/module/remote/wsbackend/AbstractHandler.py +++ b/module/remote/wsbackend/AbstractHandler.py @@ -25,10 +25,18 @@ class AbstractHandler: """ Abstract Handler providing common methods shared across WebSocket handlers """ + PATH = "/" + + OK = 200 + UNAUTHORIZED = 401 + FORBIDDEN = 403 + NOT_FOUND = 404 + ERROR = 500 def __init__(self, api): self.log = get_class_logger() self.api = api + self.core = api.core def do_extra_handshake(self, req): self.log.debug("WS Connected: %s" % req) @@ -58,22 +66,23 @@ class AbstractHandler: o = loads(msg) except ValueError, e: #invalid json object self.log.debug("Invalid Request: %s" % e) + self.send_result(req, self.ERROR, "No JSON request") return None, None, None - if type(o) != list and len(o) > 2: + if type(o) != list and len(o) not in range(1,4): self.log.debug("Invalid Api call: %s" % o) - self.send_result(req, 500, "Invalid Api call") + self.send_result(req, self.ERROR, "Invalid Api call") return None, None, None if len(o) == 1: # arguments omitted - o.append([]) - - func, args = o - if type(args) == list: - kwargs = {} + return o[0], [], {} + elif len(o) == 2: + func, args = o + if type(args) == list: + return func, args, {} + else: + return func, [], args else: - args, kwargs = [], args - - return func, args, kwargs + return tuple(o) def send_result(self, req, code, result): return send_message(req, dumps([code, result])) \ No newline at end of file diff --git a/module/remote/wsbackend/ApiHandler.py b/module/remote/wsbackend/ApiHandler.py index 57d9ecd5b..e8ba80982 100644 --- a/module/remote/wsbackend/ApiHandler.py +++ b/module/remote/wsbackend/ApiHandler.py @@ -34,14 +34,15 @@ class ApiHandler(AbstractHandler): Non json request will be ignored. """ + PATH = "/api" + def transfer_data(self, req): while True: try: line = receive_message(req) except TypeError, e: # connection closed self.log.debug("WS Error: %s" % e) - self.passive_closing_handshake(req) - return + return self.passive_closing_handshake(req) self.handle_message(line, req) @@ -55,30 +56,31 @@ class ApiHandler(AbstractHandler): user = self.api.checkAuth(*args, **kwargs) if user: req.api = self.api.withUserContext(user.uid) - return self.send_result(req, 200, True) + return self.send_result(req, self.OK, True) else: - return self.send_result(req, 403, "Forbidden") + return self.send_result(req, self.FORBIDDEN, "Forbidden") elif func == 'logout': req.api = None - return self.send_result(req, 200, True) + return self.send_result(req, self.OK, True) else: if not req.api: - return self.send_result(req, 403, "Forbidden") + return self.send_result(req, self.FORBIDDEN, "Forbidden") if not self.api.isAuthorized(func, req.api.user): - return self.send_result(req, 401, "Unauthorized") + return self.send_result(req, self.UNAUTHORIZED, "Unauthorized") try: result = getattr(req.api, func)(*args, **kwargs) except AttributeError: - return self.send_result(req, 404, "Not Found") + return self.send_result(req, self.NOT_FOUND, "Not Found") except Exception, e: - return self.send_result(req, 500, str(e)) + self.core.print_exc() + return self.send_result(req, self.ERROR, str(e)) # None is invalid json type if result is None: result = True - return self.send_result(req, 200, result) \ No newline at end of file + return self.send_result(req, self.OK, result) \ No newline at end of file diff --git a/module/remote/wsbackend/AsyncHandler.py b/module/remote/wsbackend/AsyncHandler.py new file mode 100644 index 000000000..a8382a211 --- /dev/null +++ b/module/remote/wsbackend/AsyncHandler.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This file is part of pyLoad. +# pyLoad is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + +from Queue import Queue +from threading import Lock + +from mod_pywebsocket.msgutil import receive_message + +from module.utils import lock +from AbstractHandler import AbstractHandler + +class Mode: + STANDBY = 1 + RUNNING = 2 + +class AsyncHandler(AbstractHandler): + """ + Handler that provides asynchronous information about server status, running downloads, occurred events. + + Progress information are continuous and will be pushed in a fixed interval when available. + After connect you have to login and can set the interval by sending the json command ["setInterval", xy]. + To start receiving updates call "start", afterwards no more incoming messages will be accept! + """ + + PATH = "/async" + COMMAND = "start" + + PROGRESS_INTERVAL = 1 + STATUS_INTERVAL = 60 + + def __init__(self, api): + AbstractHandler.__init__(self, api) + self.clients = [] + self.lock = Lock() + + @lock + def on_open(self, req): + req.queue = Queue() + req.interval = self.PROGRESS_INTERVAL + req.mode = Mode.STANDBY + self.clients.append(req) + + @lock + def on_close(self, req): + self.clients.remove(req) + + @lock + def add_event(self, event): + for req in self.clients: + req.queue.put(event) + + def transfer_data(self, req): + while True: + + if req.mode == Mode.STANDBY: + try: + line = receive_message(req) + except TypeError, e: # connection closed + self.log.debug("WS Error: %s" % e) + return self.passive_closing_handshake(req) + + self.mode_standby(line, req) + else: + if self.mode_running(req): + return self.passive_closing_handshake(req) + + def mode_standby(self, msg, req): + """ accepts calls before pushing updates """ + func, args, kwargs = self.handle_call(msg, req) + if not func: + return # Result was already sent + + if func == 'login': + user = self.api.checkAuth(*args, **kwargs) + if user: + req.api = self.api.withUserContext(user.uid) + return self.send_result(req, self.OK, True) + + else: + return self.send_result(req, self.FORBIDDEN, "Forbidden") + + elif func == 'logout': + req.api = None + return self.send_result(req, self.OK, True) + + else: + if not req.api: + return self.send_result(req, self.FORBIDDEN, "Forbidden") + if func == "setInterval": + req.interval = args[0] + elif func == self.COMMAND: + req.mode = Mode.RUNNING + + + def mode_running(self, req): + """ Listen for events, closes socket when returning True """ + self.send_result(req, "update", "test") \ No newline at end of file diff --git a/module/remote/wsbackend/EventHandler.py b/module/remote/wsbackend/EventHandler.py deleted file mode 100644 index 2550ff2eb..000000000 --- a/module/remote/wsbackend/EventHandler.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -############################################################################### -# Copyright(c) 2008-2012 pyLoad Team -# http://www.pyload.org -# -# This file is part of pyLoad. -# pyLoad is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# Subjected to the terms and conditions in LICENSE -# -# @author: RaNaN -############################################################################### - -from threading import Lock - - -from module.utils import lock -from AbstractHandler import AbstractHandler - -class EventHandler(AbstractHandler): - - def __init__(self, api): - AbstractHandler.__init__(self, api) - self.clients = [] - self.lock = Lock() - - @lock - def on_open(self, req): - self.clients.append(req) - - @lock - def on_close(self, req): - self.clients.remove(req) - - def handle_message(self, line, req): - pass \ No newline at end of file diff --git a/module/web/api_app.py b/module/web/api_app.py index c3237c743..4a84c85ca 100644 --- a/module/web/api_app.py +++ b/module/web/api_app.py @@ -98,3 +98,5 @@ def logout(): s = request.environ.get('beaker.session') s.delete() + + return dumps(True) diff --git a/module/web/utils.py b/module/web/utils.py index 1c0b81338..ac85d9f91 100644 --- a/module/web/utils.py +++ b/module/web/utils.py @@ -39,10 +39,10 @@ def set_session(request, user): return s def get_user_api(s): - uid = s.get("uid", None) - if uid is not None: - api = PYLOAD.withUserContext(uid) - return api + if s: + uid = s.get("uid", None) + if uid is not None: + return PYLOAD.withUserContext(uid) return None def is_mobile(): diff --git a/pyLoadCore.py b/pyLoadCore.py index a0f682bcb..a44e9a2d5 100755 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -48,7 +48,6 @@ from module.network.RequestFactory import RequestFactory from module.web.ServerThread import WebServer from module.Scheduler import Scheduler from module.common.JsEngine import JsEngine -from module import remote from module.remote.RemoteManager import RemoteManager import module.common.pylgettext as gettext @@ -582,6 +581,7 @@ class Core(object): finally: self.files.syncSave() + self.db.shutdown() self.shuttedDown = True self.deletePidFile() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/api/ApiProxy.py b/tests/api/ApiProxy.py new file mode 100644 index 000000000..74c938870 --- /dev/null +++ b/tests/api/ApiProxy.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + + +from module.remote.ttypes_debug import classes, methods + +class ApiProxy: + """ Proxy that does type checking on the api """ + + def __init__(self, api, user="User", pw="test"): + self.api = api + self.user = user + self.pw = pw + + if user and pw is not None: + self.api.login(user, pw) + + def assert_type(self, result, type): + if not type: return # void + try: + # Complex attribute + if isinstance(type, tuple): + # Optional result + if type[0] is None: + # Only check if not None + if result is not None: self.assert_type(result, type[1]) + + # List + elif type[0] == list: + assert isinstance(result, list) + for item in result: + self.assert_type(item, type[1]) + # Dict + elif type[0] == dict: + assert isinstance(result, dict) + for k, v in result.iteritems(): + self.assert_type(k, type[1]) + self.assert_type(v, type[2]) + + # Struct - Api class + elif hasattr(result, "__name__") and result.__name__ in classes: + for attr, atype in zip(result.__slots__, classes[result.__name__]): + self.assert_type(getattr(result, attr), atype) + else: # simple object + assert isinstance(result, type) + except AssertionError: + print "Assertion for %s as %s failed" % (result, type) + raise + + + def call(self, func, *args, **kwargs): + result = getattr(self.api, func)(*args, **kwargs) + self.assert_type(result, methods[func]) + + return result + + + def __getattr__(self, item): + def call(*args, **kwargs): + return self.call(item, *args, **kwargs) + + return call + +if __name__ == "__main__": + + from module.remote.JSONClient import JSONClient + + api = ApiProxy(JSONClient(), "User", "test") + api.getServerVersion() \ No newline at end of file diff --git a/tests/api/ApiTester.py b/tests/api/ApiTester.py new file mode 100644 index 000000000..b17a7655e --- /dev/null +++ b/tests/api/ApiTester.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +from module.remote.JSONClient import JSONClient +from module.remote.WSClient import WSClient + +from ApiProxy import ApiProxy + +class ApiTester: + + tester= [] + + @classmethod + def register(cls, tester): + cls.tester.append(tester) + + @classmethod + def get_methods(cls): + """ All available methods for testing """ + methods = [] + for t in cls.tester: + methods.extend(getattr(t, attr) for attr in dir(t) if attr.startswith("test_")) + return methods + + def __init__(self): + ApiTester.register(self) + self.api = None + + def setApi(self, api): + self.api = api + + def enableJSON(self): + self.api = ApiProxy(JSONClient()) + + def enableWS(self): + self.api = ApiProxy(WSClient()) diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/api/test_JSONBackend.py b/tests/api/test_JSONBackend.py new file mode 100644 index 000000000..a3805497b --- /dev/null +++ b/tests/api/test_JSONBackend.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from nose.tools import raises + +from module.remote.JSONClient import JSONClient + +class TestJSONBackend: + + def setUp(self): + self.client = JSONClient() + + def test_login(self): + self.client.login("User", "test") + self.client.getServerVersion() + self.client.logout() + + def test_wronglogin(self): + ret = self.client.login("WrongUser", "wrongpw") + assert ret == False + + @raises(Exception) + def test_access(self): + self.client.getServerVersion() + + @raises(Exception) + def test_unknown_method(self): + self.client.sdfdsg() diff --git a/tests/api/test_api.py b/tests/api/test_api.py new file mode 100644 index 000000000..68448b891 --- /dev/null +++ b/tests/api/test_api.py @@ -0,0 +1,39 @@ + +from unittest import TestCase +from random import choice + +from pyLoadCore import Core + +from ApiTester import ApiTester + +class TestAPI(TestCase): + """ + Test all available testers randomly and on all backends + """ + core = None + + @classmethod + def setUpClass(cls): + from test_noargs import TestNoArgs + + cls.core = Core() + cls.core.start(False, False, True) + for Test in (TestNoArgs,): + t = Test() + t.enableJSON() + t = Test() + t.enableWS() + t = Test() + t.setApi(cls.core.api) + + cls.methods = ApiTester.get_methods() + + @classmethod + def tearDownClass(cls): + cls.core.shutdown() + + def test_random(self, n=10000): + + for i in range(n): + func = choice(self.methods) + func() diff --git a/tests/api/test_noargs.py b/tests/api/test_noargs.py new file mode 100644 index 000000000..02e49cf13 --- /dev/null +++ b/tests/api/test_noargs.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +import inspect + +from ApiTester import ApiTester + +from module.remote.ttypes import Iface + +IGNORE = ('kill', 'restart') + +class TestNoArgs(ApiTester): + def setUp(self): + self.enableJSON() + +# Setup test_methods dynamically, only these which require no arguments +for name in dir(Iface): + if name.startswith("_") or name in IGNORE: continue + + spec = inspect.getargspec(getattr(Iface, name)) + if len(spec.args) == 1 and (not spec.varargs or len(spec.varargs) == 0): + def meta_test(name): #retain local scope + def test(self): + getattr(self.api, name)() + test.func_name = "test_%s" % name + return test + + setattr(TestNoArgs, "test_%s" % name, meta_test(name)) + + del meta_test \ No newline at end of file diff --git a/tests/config/db.version b/tests/config/db.version deleted file mode 100644 index bf0d87ab1..000000000 --- a/tests/config/db.version +++ /dev/null @@ -1 +0,0 @@ -4 \ No newline at end of file diff --git a/tests/helper/Stubs.py b/tests/helper/Stubs.py index 5c44cfb58..4ebd12592 100644 --- a/tests/helper/Stubs.py +++ b/tests/helper/Stubs.py @@ -4,6 +4,7 @@ import sys from os.path import abspath, dirname, join from time import strftime from traceback import format_exc +from collections import defaultdict sys.path.append(abspath(join(dirname(__file__), "..", "..", "module", "lib"))) sys.path.append(abspath(join(dirname(__file__), "..", ".."))) @@ -73,6 +74,8 @@ class Core: self.cache = {} self.packageCache = {} + self.statusMsg = defaultdict(lambda: "statusmsg") + self.log = LogStub() def getServerVersion(self): diff --git a/tests/manager/__init__.py b/tests/manager/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/manager/test_filemanager.py b/tests/manager/test_filemanager.py new file mode 100644 index 000000000..81acea4d0 --- /dev/null +++ b/tests/manager/test_filemanager.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- + +from random import choice + +from tests.helper.Stubs import Core +from tests.helper.BenchmarkTest import BenchmarkTest + +from module.database import DatabaseBackend +# disable asyncronous queries +DatabaseBackend.async = DatabaseBackend.queue + +from module.Api import DownloadState +from module.FileManager import FileManager + + +class TestFileManager(BenchmarkTest): + bench = ["add_packages", "add_files", "get_files_root", "get", + "get_package_content", "get_package_tree", + "order_package", "order_files", "move"] + + pids = [-1] + count = 100 + + @classmethod + def setUpClass(cls): + c = Core() + # db needs seperate initialisation + cls.db = c.db = DatabaseBackend(c) + cls.db.setup() + cls.db.purgeAll() + + cls.m = cls.db.manager = FileManager(c) + + @classmethod + def tearDownClass(cls): + cls.db.purgeAll() + cls.db.shutdown() + + + # benchmarker ignore setup + def setUp(self): + self.db.purgeAll() + self.pids = [-1] + + self.count = 20 + self.test_add_packages() + self.test_add_files() + + def test_add_packages(self): + for i in range(100): + pid = self.m.addPackage("name", "folder", choice(self.pids), "", "", "", False) + self.pids.append(pid) + + if -1 in self.pids: + self.pids.remove(-1) + + def test_add_files(self): + for pid in self.pids: + self.m.addLinks([("plugin %d" % i, "url %s" % i) for i in range(self.count)], pid) + + count = self.m.getQueueCount() + files = self.count * len(self.pids) + # in test runner files get added twice + assert count == files or count == files * 2 + + def test_get(self): + info = self.m.getPackageInfo(choice(self.pids)) + assert info.stats.linkstotal == self.count + + fid = choice(info.fids) + f = self.m.getFile(fid) + assert f.fid in self.m.files + + f.name = "new name" + f.sync() + finfo = self.m.getFileInfo(fid) + assert finfo is not None + assert finfo.name == "new name" + + p = self.m.getPackage(choice(self.pids)) + assert p is not None + assert p.pid in self.m.packages + p.sync() + + p.delete() + + self.m.getTree(-1, True, None) + + def test_get_filtered(self): + all = self.m.getTree(-1, True, None) + finished = self.m.getTree(-1, True, DownloadState.Finished) + unfinished = self.m.getTree(-1, True, DownloadState.Unfinished) + + assert len(finished.files) + len(unfinished.files) == len(all.files) == self.m.getFileCount() + + + def test_get_files_root(self): + view = self.m.getTree(-1, True, None) + + for pid in self.pids: + assert pid in view.packages + + assert len(view.packages) == len(self.pids) + + p = choice(view.packages.values()) + assert len(p.fids) == self.count + assert p.stats.linkstotal == self.count + + + def test_get_package_content(self): + view = self.m.getTree(choice(self.pids), False, None) + p = view.root + + assert len(view.packages) == len(p.pids) + for pid in p.pids: assert pid in view.packages + + def test_get_package_tree(self): + view = self.m.getTree(choice(self.pids), True, None) + for pid in view.root.pids: assert pid in view.packages + for fid in view.root.fids: assert fid in view.files + + def test_delete(self): + self.m.deleteFile(self.count * 5) + self.m.deletePackage(choice(self.pids)) + + def test_order_package(self): + parent = self.m.addPackage("order", "", -1, "", "", "", False) + self.m.addLinks([("url", "plugin") for i in range(100)], parent) + + pids = [self.m.addPackage("c", "", parent, "", "", "", False) for i in range(5)] + v = self.m.getTree(parent, False, None) + self.assert_ordered(pids, 0, 5, v.root.pids, v.packages, True) + + pid = v.packages.keys()[0] + self.assert_pack_ordered(parent, pid, 3) + self.assert_pack_ordered(parent, pid, 0) + self.assert_pack_ordered(parent, pid, 0) + self.assert_pack_ordered(parent, pid, 4) + pid = v.packages.keys()[2] + self.assert_pack_ordered(parent, pid, 4) + self.assert_pack_ordered(parent, pid, 3) + self.assert_pack_ordered(parent, pid, 2) + + + def test_order_files(self): + parent = self.m.addPackage("order", "", -1, "", "", "", False) + self.m.addLinks([("url", "plugin") for i in range(100)], parent) + v = self.m.getTree(parent, False, None) + + fids = v.root.fids[10:20] + v = self.assert_files_ordered(parent, fids, 0) + + fids = v.root.fids[20:30] + + self.m.orderFiles(fids, parent, 99) + v = self.m.getTree(parent, False, None) + assert fids[-1] == v.root.fids[-1] + assert fids[0] == v.root.fids[90] + self.assert_ordered(fids, 90, 100, v.root.fids, v.files) + + fids = v.root.fids[80:] + v = self.assert_files_ordered(parent, fids, 20) + + self.m.orderFiles(fids, parent, 80) + v = self.m.getTree(parent, False, None) + self.assert_ordered(fids, 61, 81, v.root.fids, v.files) + + fids = v.root.fids[50:51] + self.m.orderFiles(fids, parent, 99) + v = self.m.getTree(parent, False, None) + self.assert_ordered(fids, 99, 100, v.root.fids, v.files) + + fids = v.root.fids[50:51] + v = self.assert_files_ordered(parent, fids, 0) + + + def assert_files_ordered(self, parent, fids, pos): + fs = [self.m.getFile(choice(fids)) for i in range(5)] + self.m.orderFiles(fids, parent, pos) + v = self.m.getTree(parent, False, False) + self.assert_ordered(fids, pos, pos+len(fids), v.root.fids, v.files) + + return v + + def assert_pack_ordered(self, parent, pid, pos): + self.m.orderPackage(pid, pos) + v = self.m.getTree(parent, False, False) + self.assert_ordered([pid], pos, pos+1, v.root.pids, v.packages, True) + + # assert that ordering is total, complete with no gaps + def assert_ordered(self, part, start, end, data, dict, pack=False): + assert data[start:end] == part + if pack: + assert sorted([p.packageorder for p in dict.values()]) == range(len(dict)) + assert [dict[pid].packageorder for pid in part] == range(start, end) + else: + assert sorted([f.fileorder for f in dict.values()]) == range(len(dict)) + assert [dict[fid].fileorder for fid in part] == range(start, end) + + + def test_move(self): + + pid = self.pids[-1] + pid2 = self.pids[1] + + self.m.movePackage(pid, -1) + v = self.m.getTree(-1, False, False) + + assert v.root.pids[-1] == pid + assert sorted([p.packageorder for p in v.packages.values()]) == range(len(v.packages)) + + v = self.m.getTree(pid, False, False) + fids = v.root.fids[10:20] + self.m.moveFiles(fids, pid2) + v = self.m.getTree(pid2, False, False) + + assert sorted([f.fileorder for f in v.files.values()]) == range(len(v.files)) + assert len(v.files) == self.count + len(fids) + + + +if __name__ == "__main__": + TestFileManager.benchmark() \ No newline at end of file diff --git a/tests/manager/test_interactionManager.py b/tests/manager/test_interactionManager.py new file mode 100644 index 000000000..db233bb25 --- /dev/null +++ b/tests/manager/test_interactionManager.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- + +from unittest import TestCase +from tests.helper.Stubs import Core + +from module.Api import Input, Output +from module.interaction.InteractionManager import InteractionManager + +class TestInteractionManager(TestCase): + + @classmethod + def setUpClass(cls): + cls.core = Core() + + def setUp(self): + self.im = InteractionManager(self.core) + + + def test_notifications(self): + + n = self.im.createNotification("test", "notify") + assert self.im.getNotifications() == [n] + + for i in range(10): + self.im.createNotification("title", "test") + + assert len(self.im.getNotifications()) == 11 + + + def test_captcha(self): + assert self.im.getTask() is None + + t = self.im.newCaptchaTask("1", "", "") + assert t.output == Output.Captcha + self.im.handleTask(t) + assert t is self.im.getTask() + + t2 = self.im.newCaptchaTask("2", "", "") + self.im.handleTask(t2) + + assert self.im.getTask(Output.Query) is None + assert self.im.getTask() is t + + self.im.removeTask(t) + assert self.im.getTask() is t2 + + self.im.getTaskByID(t2.iid) + assert self.im.getTask() is None + + + def test_query(self): + assert self.im.getTask() is None + t = self.im.newQueryTask(Input.Text, None, "text") + assert t.description == "text" + self.im.handleTask(t) + + assert self.im.getTask(Output.Query) is t + assert self.im.getTask(Output.Captcha) is None \ No newline at end of file diff --git a/tests/other/__init__.py b/tests/other/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/other/test_configparser.py b/tests/other/test_configparser.py new file mode 100644 index 000000000..acb05c63e --- /dev/null +++ b/tests/other/test_configparser.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +from collections import defaultdict +from tests.helper.Stubs import Core + +from module.database import DatabaseBackend +from module.config.ConfigParser import ConfigParser + +# TODO +class TestConfigParser(): + + db = None + + @classmethod + def setUpClass(cls): + cls.db = DatabaseBackend(Core()) + cls.db.manager = cls.db.core + cls.db.manager.statusMsg = defaultdict(lambda: "statusmsg") + cls.config = ConfigParser() + cls.db.setup() + cls.db.clearAllConfigs() + + @classmethod + def tearDownClass(cls): + cls.db.shutdown() + + def test_db(self): + + self.db.saveConfig("plugin", "some value", 0) + self.db.saveConfig("plugin", "some other value", 1) + + assert self.db.loadConfig("plugin", 0) == "some value" + assert self.db.loadConfig("plugin", 1) == "some other value" + + d = self.db.loadAllConfigs() + assert d[0]["plugin"] == "some value" + + self.db.deleteConfig("plugin") + + assert not self.db.loadAllConfigs() + + + def test_dict(self): + + assert self.config["general"]["language"] + self.config["general"]["language"] = "de" + assert self.config["general"]["language"] == "de" + + def test_config(self): + pass + + def test_userconfig(self): + pass \ No newline at end of file diff --git a/tests/other/test_database.py b/tests/other/test_database.py new file mode 100644 index 000000000..dd733b4ff --- /dev/null +++ b/tests/other/test_database.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- + +from tests.helper.Stubs import Core +from tests.helper.BenchmarkTest import BenchmarkTest + +from module.Api import DownloadState +from module.database import DatabaseBackend + +# disable asyncronous queries +DatabaseBackend.async = DatabaseBackend.queue + +from random import choice + +class TestDatabase(BenchmarkTest): + bench = ["insert", "insert_links", "insert_many", "get_packages", + "get_files", "get_files_queued", "get_package_childs", "get_package_files", + "get_package_data", "get_file_data", "find_files", "collector", "purge"] + pids = None + fids = None + owner = 123 + pstatus = 0 + + @classmethod + def setUpClass(cls): + cls.pids = [-1] + cls.fids = [] + + cls.db = DatabaseBackend(Core()) + cls.db.manager = cls.db.core + + cls.db.setup() + cls.db.purgeAll() + + @classmethod + def tearDownClass(cls): + cls.db.purgeAll() + cls.db.shutdown() + + # benchmarker ignore setup + def setUp(self): + self.db.purgeAll() + self.pids = [-1] + self.fids = [] + + self.test_insert(20) + self.test_insert_many() + self.fids = self.db.getAllFiles().keys() + + + def test_insert(self, n=200): + for i in range(n): + pid = self.db.addPackage("name", "folder", choice(self.pids), "password", "site", "comment", self.pstatus, + self.owner) + self.pids.append(pid) + + def test_insert_links(self): + for i in range(10000): + fid = self.db.addLink("url %s" % i, "name", "plugin", choice(self.pids), self.owner) + self.fids.append(fid) + + def test_insert_many(self): + for pid in self.pids: + self.db.addLinks([("url %s" % i, "plugin") for i in range(50)], pid, self.owner) + + def test_get_packages(self): + packs = self.db.getAllPackages() + n = len(packs) + assert n == len(self.pids) - 1 + + print "Fetched %d packages" % n + self.assert_pack(choice(packs.values())) + + def test_get_files(self): + files = self.db.getAllFiles() + n = len(files) + assert n >= len(self.pids) + + print "Fetched %d files" % n + self.assert_file(choice(files.values())) + + def test_get_files_queued(self): + files = self.db.getAllFiles(state=DownloadState.Unfinished) + print "Fetched %d files queued" % len(files) + + def test_delete(self): + pid = choice(self.pids) + self.db.deletePackage(pid) + self.pids.remove(pid) + + def test_get_package_childs(self): + pid = choice(self.pids) + packs = self.db.getAllPackages(root=pid) + + print "Package %d has %d packages" % (pid, len(packs)) + self.assert_pack(choice(packs.values())) + + def test_get_package_files(self): + pid = choice(self.pids) + files = self.db.getAllFiles(package=pid) + + print "Package %d has %d files" % (pid, len(files)) + self.assert_file(choice(files.values())) + + def test_get_package_data(self, stats=False): + pid = choice(self.pids) + p = self.db.getPackageInfo(pid, stats) + self.assert_pack(p) + # test again with stat + if not stats: + self.test_get_package_data(True) + + def test_get_file_data(self): + fid = choice(self.fids) + f = self.db.getFileInfo(fid) + self.assert_file(f) + + def test_find_files(self): + files = self.db.getAllFiles(search="1") + print "Found %s files" % len(files) + f = choice(files.values()) + + assert "1" in f.name + + def test_collector(self): + self.db.saveCollector(0, "data") + assert self.db.retrieveCollector(0) == "data" + self.db.deleteCollector(0) + + def test_purge(self): + self.db.purgeLinks() + + + def test_user_context(self): + self.db.purgeAll() + + p1 = self.db.addPackage("name", "folder", 0, "password", "site", "comment", self.pstatus, 0) + self.db.addLink("url", "name", "plugin", p1, 0) + p2 = self.db.addPackage("name", "folder", 0, "password", "site", "comment", self.pstatus, 1) + self.db.addLink("url", "name", "plugin", p2, 1) + + assert len(self.db.getAllPackages(owner=0)) == 1 == len(self.db.getAllFiles(owner=0)) + assert len(self.db.getAllPackages(root=0, owner=0)) == 1 == len(self.db.getAllFiles(package=p1, owner=0)) + assert len(self.db.getAllPackages(owner=1)) == 1 == len(self.db.getAllFiles(owner=1)) + assert len(self.db.getAllPackages(root=0, owner=1)) == 1 == len(self.db.getAllFiles(package=p2, owner=1)) + assert len(self.db.getAllPackages()) == 2 == len(self.db.getAllFiles()) + + self.db.deletePackage(p1, 1) + assert len(self.db.getAllPackages(owner=0)) == 1 == len(self.db.getAllFiles(owner=0)) + self.db.deletePackage(p1, 0) + assert len(self.db.getAllPackages(owner=1)) == 1 == len(self.db.getAllFiles(owner=1)) + self.db.deletePackage(p2) + + assert len(self.db.getAllPackages()) == 0 + + def test_count(self): + self.db.purgeAll() + + assert self.db.filecount() == 0 + assert self.db.queuecount() == 0 + assert self.db.processcount() == 0 + + def assert_file(self, f): + try: + assert f is not None + self.assert_int(f, ("fid", "status", "size", "media", "fileorder", "added", "package", "owner")) + assert f.status in range(5) + assert f.owner == self.owner + assert f.media in range(1024) + assert f.package in self.pids + assert f.added > 10 ** 6 # date is usually big integer + except: + print f + raise + + def assert_pack(self, p): + try: + assert p is not None + self.assert_int(p, ("pid", "root", "added", "status", "packageorder", "owner")) + assert p.pid in self.pids + assert p.owner == self.owner + assert p.status in range(5) + assert p.root in self.pids + assert p.added > 10 ** 6 + except: + print p + raise + + def assert_int(self, obj, list): + for attr in list: assert type(getattr(obj, attr)) == int + +if __name__ == "__main__": + TestDatabase.benchmark() \ No newline at end of file diff --git a/tests/other/test_syntax.py b/tests/other/test_syntax.py new file mode 100644 index 000000000..fbf7edf8f --- /dev/null +++ b/tests/other/test_syntax.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +from os import walk +from os.path import abspath, dirname, join + +from unittest import TestCase + +PATH = abspath(join(dirname(abspath(__file__)), "..", "..", "")) + +# needed to register globals +from tests.helper import Stubs + +class TestSyntax(TestCase): + pass + + +for path, dirs, files in walk(join(PATH, "module")): + + for f in files: + if not f.endswith(".py") or f.startswith("__"): continue + fpath = join(path, f) + pack = fpath.replace(PATH, "")[1:-3] #replace / and .py + imp = pack.replace("/", ".") + packages = imp.split(".") + #__import__(imp) + + # to much sideeffect when importing + if "web" in packages or "lib" in packages: continue + if "ThriftTest" in packages: continue + + # currying + def meta(imp, sig): + def _test(self=None): + __import__(imp) + + _test.func_name = sig + return _test + + # generate test methods + sig = "test_%s_%s" % (packages[-2], packages[-1]) + + + setattr(TestSyntax, sig, meta(imp, sig)) \ No newline at end of file diff --git a/tests/test_api.py b/tests/test_api.py deleted file mode 100644 index 0171b46bb..000000000 --- a/tests/test_api.py +++ /dev/null @@ -1,18 +0,0 @@ - -from unittest import TestCase - -from pyLoadCore import Core -from module.common.APIExerciser import APIExerciser - -class TestApi(TestCase): - - @classmethod - def setUpClass(cls): - cls.core = Core() - cls.core.start(False, False, True) - - def test_random(self): - api = APIExerciser(self.core) - - for i in range(2000): - api.testAPI() diff --git a/tests/test_backends.py b/tests/test_backends.py deleted file mode 100644 index 71ccedd2f..000000000 --- a/tests/test_backends.py +++ /dev/null @@ -1,59 +0,0 @@ -# -*- coding: utf-8 -*- - - -from urllib import urlencode -from urllib2 import urlopen, HTTPError -from json import loads - -from logging import log - -from module.common import APIExerciser - -url = "http://localhost:8001/api/%s" - -class TestBackends(): - - def setUp(self): - u = urlopen(url % "login", data=urlencode({"username": "TestUser", "password": "sometestpw"})) - self.key = loads(u.read()) - assert self.key is not False - - def test_random(self): - api = APIExerciser.APIExerciser(None, True, "TestUser", "sometestpw") - - assert api.api.login("crapp", "wrong pw") is False - - for i in range(0, 1000): - api.testAPI() - - def call(self, name, post=None): - if not post: post = {} - post["session"] = self.key - u = urlopen(url % name, data=urlencode(post)) - return loads(u.read()) - - def test_wronglogin(self): - u = urlopen(url % "login", data=urlencode({"username": "crap", "password": "wrongpw"})) - assert loads(u.read()) is False - - def test_access(self): - try: - urlopen(url % "getServerVersion") - except HTTPError, e: - assert e.code == 403 - else: - assert False - - def test_status(self): - ret = self.call("statusServer") - log(1, str(ret)) - assert "pause" in ret - assert "queue" in ret - - def test_unknown_method(self): - try: - self.call("notExisting") - except HTTPError, e: - assert e.code == 404 - else: - assert False diff --git a/tests/test_configparser.py b/tests/test_configparser.py deleted file mode 100644 index d797c7912..000000000 --- a/tests/test_configparser.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- - -from collections import defaultdict -from helper.Stubs import Core - -from module.database import DatabaseBackend -from module.config.ConfigParser import ConfigParser - -# TODO -class TestConfigParser(): - - @classmethod - def setUpClass(cls): - cls.db = DatabaseBackend(Core()) - cls.db.manager = cls.db.core - cls.db.manager.statusMsg = defaultdict(lambda: "statusmsg") - cls.config = ConfigParser() - cls.db.setup() - cls.db.clearAllConfigs() - - - def test_db(self): - - self.db.saveConfig("plugin", "some value", 0) - self.db.saveConfig("plugin", "some other value", 1) - - assert self.db.loadConfig("plugin", 0) == "some value" - assert self.db.loadConfig("plugin", 1) == "some other value" - - d = self.db.loadAllConfigs() - assert d[0]["plugin"] == "some value" - - self.db.deleteConfig("plugin") - - assert not self.db.loadAllConfigs() - - - def test_dict(self): - - assert self.config["general"]["language"] - self.config["general"]["language"] = "de" - assert self.config["general"]["language"] == "de" - - def test_config(self): - pass - - def test_userconfig(self): - pass \ No newline at end of file diff --git a/tests/test_database.py b/tests/test_database.py deleted file mode 100644 index fb134ff41..000000000 --- a/tests/test_database.py +++ /dev/null @@ -1,195 +0,0 @@ -# -*- coding: utf-8 -*- - -from collections import defaultdict - -from helper.Stubs import Core -from helper.BenchmarkTest import BenchmarkTest - -from module.Api import DownloadState -from module.database import DatabaseBackend - -# disable asyncronous queries -DatabaseBackend.async = DatabaseBackend.queue - -from random import choice - -class TestDatabase(BenchmarkTest): - bench = ["insert", "insert_links", "insert_many", "get_packages", - "get_files", "get_files_queued", "get_package_childs", "get_package_files", - "get_package_data", "get_file_data", "find_files", "collector", "purge"] - pids = None - fids = None - owner = 123 - pstatus = 0 - - @classmethod - def setUpClass(cls): - cls.pids = [-1] - cls.fids = [] - - cls.db = DatabaseBackend(Core()) - cls.db.manager = cls.db.core - cls.db.manager.statusMsg = defaultdict(lambda: "statusmsg") - - cls.db.setup() - cls.db.purgeAll() - - @classmethod - def tearDownClass(cls): - cls.db.purgeAll() - cls.db.shutdown() - - # benchmarker ignore setup - def setUp(self): - self.db.purgeAll() - self.pids = [-1] - self.fids = [] - - self.test_insert(20) - self.test_insert_many() - self.fids = self.db.getAllFiles().keys() - - - def test_insert(self, n=200): - for i in range(n): - pid = self.db.addPackage("name", "folder", choice(self.pids), "password", "site", "comment", self.pstatus, - self.owner) - self.pids.append(pid) - - def test_insert_links(self): - for i in range(10000): - fid = self.db.addLink("url %s" % i, "name", "plugin", choice(self.pids), self.owner) - self.fids.append(fid) - - def test_insert_many(self): - for pid in self.pids: - self.db.addLinks([("url %s" % i, "plugin") for i in range(50)], pid, self.owner) - - def test_get_packages(self): - packs = self.db.getAllPackages() - n = len(packs) - assert n == len(self.pids) - 1 - - print "Fetched %d packages" % n - self.assert_pack(choice(packs.values())) - - def test_get_files(self): - files = self.db.getAllFiles() - n = len(files) - assert n >= len(self.pids) - - print "Fetched %d files" % n - self.assert_file(choice(files.values())) - - def test_get_files_queued(self): - files = self.db.getAllFiles(state=DownloadState.Unfinished) - print "Fetched %d files queued" % len(files) - - def test_delete(self): - pid = choice(self.pids) - self.db.deletePackage(pid) - self.pids.remove(pid) - - def test_get_package_childs(self): - pid = choice(self.pids) - packs = self.db.getAllPackages(root=pid) - - print "Package %d has %d packages" % (pid, len(packs)) - self.assert_pack(choice(packs.values())) - - def test_get_package_files(self): - pid = choice(self.pids) - files = self.db.getAllFiles(package=pid) - - print "Package %d has %d files" % (pid, len(files)) - self.assert_file(choice(files.values())) - - def test_get_package_data(self, stats=False): - pid = choice(self.pids) - p = self.db.getPackageInfo(pid, stats) - self.assert_pack(p) - # test again with stat - if not stats: - self.test_get_package_data(True) - - def test_get_file_data(self): - fid = choice(self.fids) - f = self.db.getFileInfo(fid) - self.assert_file(f) - - def test_find_files(self): - files = self.db.getAllFiles(search="1") - print "Found %s files" % len(files) - f = choice(files.values()) - - assert "1" in f.name - - def test_collector(self): - self.db.saveCollector(0, "data") - assert self.db.retrieveCollector(0) == "data" - self.db.deleteCollector(0) - - def test_purge(self): - self.db.purgeLinks() - - - def test_user_context(self): - self.db.purgeAll() - - p1 = self.db.addPackage("name", "folder", 0, "password", "site", "comment", self.pstatus, 0) - self.db.addLink("url", "name", "plugin", p1, 0) - p2 = self.db.addPackage("name", "folder", 0, "password", "site", "comment", self.pstatus, 1) - self.db.addLink("url", "name", "plugin", p2, 1) - - assert len(self.db.getAllPackages(owner=0)) == 1 == len(self.db.getAllFiles(owner=0)) - assert len(self.db.getAllPackages(root=0, owner=0)) == 1 == len(self.db.getAllFiles(package=p1, owner=0)) - assert len(self.db.getAllPackages(owner=1)) == 1 == len(self.db.getAllFiles(owner=1)) - assert len(self.db.getAllPackages(root=0, owner=1)) == 1 == len(self.db.getAllFiles(package=p2, owner=1)) - assert len(self.db.getAllPackages()) == 2 == len(self.db.getAllFiles()) - - self.db.deletePackage(p1, 1) - assert len(self.db.getAllPackages(owner=0)) == 1 == len(self.db.getAllFiles(owner=0)) - self.db.deletePackage(p1, 0) - assert len(self.db.getAllPackages(owner=1)) == 1 == len(self.db.getAllFiles(owner=1)) - self.db.deletePackage(p2) - - assert len(self.db.getAllPackages()) == 0 - - def test_count(self): - self.db.purgeAll() - - assert self.db.filecount() == 0 - assert self.db.queuecount() == 0 - assert self.db.processcount() == 0 - - def assert_file(self, f): - try: - assert f is not None - self.assert_int(f, ("fid", "status", "size", "media", "fileorder", "added", "package", "owner")) - assert f.status in range(5) - assert f.owner == self.owner - assert f.media in range(1024) - assert f.package in self.pids - assert f.added > 10 ** 6 # date is usually big integer - except: - print f - raise - - def assert_pack(self, p): - try: - assert p is not None - self.assert_int(p, ("pid", "root", "added", "status", "packageorder", "owner")) - assert p.pid in self.pids - assert p.owner == self.owner - assert p.status in range(5) - assert p.root in self.pids - assert p.added > 10 ** 6 - except: - print p - raise - - def assert_int(self, obj, list): - for attr in list: assert type(getattr(obj, attr)) == int - -if __name__ == "__main__": - TestDatabase.benchmark() \ No newline at end of file diff --git a/tests/test_filemanager.py b/tests/test_filemanager.py deleted file mode 100644 index 7b82840b1..000000000 --- a/tests/test_filemanager.py +++ /dev/null @@ -1,223 +0,0 @@ -# -*- coding: utf-8 -*- - -from random import choice - -from helper.Stubs import Core -from helper.BenchmarkTest import BenchmarkTest - -from module.database import DatabaseBackend -# disable asyncronous queries -DatabaseBackend.async = DatabaseBackend.queue - -from module.Api import DownloadState -from module.FileManager import FileManager - - -class TestFileManager(BenchmarkTest): - bench = ["add_packages", "add_files", "get_files_root", "get", - "get_package_content", "get_package_tree", - "order_package", "order_files", "move"] - - pids = [-1] - count = 100 - - @classmethod - def setUpClass(cls): - c = Core() - # db needs seperate initialisation - cls.db = c.db = DatabaseBackend(c) - cls.db.setup() - cls.db.purgeAll() - - cls.m = cls.db.manager = FileManager(c) - - @classmethod - def tearDownClass(cls): - cls.db.purgeAll() - cls.db.shutdown() - - - # benchmarker ignore setup - def setUp(self): - self.db.purgeAll() - self.pids = [-1] - - self.count = 20 - self.test_add_packages() - self.test_add_files() - - def test_add_packages(self): - for i in range(100): - pid = self.m.addPackage("name", "folder", choice(self.pids), "", "", "", False) - self.pids.append(pid) - - if -1 in self.pids: - self.pids.remove(-1) - - def test_add_files(self): - for pid in self.pids: - self.m.addLinks([("plugin %d" % i, "url %s" % i) for i in range(self.count)], pid) - - count = self.m.getQueueCount() - files = self.count * len(self.pids) - # in test runner files get added twice - assert count == files or count == files * 2 - - def test_get(self): - info = self.m.getPackageInfo(choice(self.pids)) - assert info.stats.linkstotal == self.count - - fid = choice(info.fids) - f = self.m.getFile(fid) - assert f.fid in self.m.files - - f.name = "new name" - f.sync() - finfo = self.m.getFileInfo(fid) - assert finfo is not None - assert finfo.name == "new name" - - p = self.m.getPackage(choice(self.pids)) - assert p is not None - assert p.pid in self.m.packages - p.sync() - - p.delete() - - self.m.getTree(-1, True, None) - - def test_get_filtered(self): - all = self.m.getTree(-1, True, None) - finished = self.m.getTree(-1, True, DownloadState.Finished) - unfinished = self.m.getTree(-1, True, DownloadState.Unfinished) - - assert len(finished.files) + len(unfinished.files) == len(all.files) == self.m.getFileCount() - - - def test_get_files_root(self): - view = self.m.getTree(-1, True, None) - - for pid in self.pids: - assert pid in view.packages - - assert len(view.packages) == len(self.pids) - - p = choice(view.packages.values()) - assert len(p.fids) == self.count - assert p.stats.linkstotal == self.count - - - def test_get_package_content(self): - view = self.m.getTree(choice(self.pids), False, None) - p = view.root - - assert len(view.packages) == len(p.pids) - for pid in p.pids: assert pid in view.packages - - def test_get_package_tree(self): - view = self.m.getTree(choice(self.pids), True, None) - for pid in view.root.pids: assert pid in view.packages - for fid in view.root.fids: assert fid in view.files - - def test_delete(self): - self.m.deleteFile(self.count * 5) - self.m.deletePackage(choice(self.pids)) - - def test_order_package(self): - parent = self.m.addPackage("order", "", -1, "", "", "", False) - self.m.addLinks([("url", "plugin") for i in range(100)], parent) - - pids = [self.m.addPackage("c", "", parent, "", "", "", False) for i in range(5)] - v = self.m.getTree(parent, False, None) - self.assert_ordered(pids, 0, 5, v.root.pids, v.packages, True) - - pid = v.packages.keys()[0] - self.assert_pack_ordered(parent, pid, 3) - self.assert_pack_ordered(parent, pid, 0) - self.assert_pack_ordered(parent, pid, 0) - self.assert_pack_ordered(parent, pid, 4) - pid = v.packages.keys()[2] - self.assert_pack_ordered(parent, pid, 4) - self.assert_pack_ordered(parent, pid, 3) - self.assert_pack_ordered(parent, pid, 2) - - - def test_order_files(self): - parent = self.m.addPackage("order", "", -1, "", "", "", False) - self.m.addLinks([("url", "plugin") for i in range(100)], parent) - v = self.m.getTree(parent, False, None) - - fids = v.root.fids[10:20] - v = self.assert_files_ordered(parent, fids, 0) - - fids = v.root.fids[20:30] - - self.m.orderFiles(fids, parent, 99) - v = self.m.getTree(parent, False, None) - assert fids[-1] == v.root.fids[-1] - assert fids[0] == v.root.fids[90] - self.assert_ordered(fids, 90, 100, v.root.fids, v.files) - - fids = v.root.fids[80:] - v = self.assert_files_ordered(parent, fids, 20) - - self.m.orderFiles(fids, parent, 80) - v = self.m.getTree(parent, False, None) - self.assert_ordered(fids, 61, 81, v.root.fids, v.files) - - fids = v.root.fids[50:51] - self.m.orderFiles(fids, parent, 99) - v = self.m.getTree(parent, False, None) - self.assert_ordered(fids, 99, 100, v.root.fids, v.files) - - fids = v.root.fids[50:51] - v = self.assert_files_ordered(parent, fids, 0) - - - def assert_files_ordered(self, parent, fids, pos): - fs = [self.m.getFile(choice(fids)) for i in range(5)] - self.m.orderFiles(fids, parent, pos) - v = self.m.getTree(parent, False, False) - self.assert_ordered(fids, pos, pos+len(fids), v.root.fids, v.files) - - return v - - def assert_pack_ordered(self, parent, pid, pos): - self.m.orderPackage(pid, pos) - v = self.m.getTree(parent, False, False) - self.assert_ordered([pid], pos, pos+1, v.root.pids, v.packages, True) - - # assert that ordering is total, complete with no gaps - def assert_ordered(self, part, start, end, data, dict, pack=False): - assert data[start:end] == part - if pack: - assert sorted([p.packageorder for p in dict.values()]) == range(len(dict)) - assert [dict[pid].packageorder for pid in part] == range(start, end) - else: - assert sorted([f.fileorder for f in dict.values()]) == range(len(dict)) - assert [dict[fid].fileorder for fid in part] == range(start, end) - - - def test_move(self): - - pid = self.pids[-1] - pid2 = self.pids[1] - - self.m.movePackage(pid, -1) - v = self.m.getTree(-1, False, False) - - assert v.root.pids[-1] == pid - assert sorted([p.packageorder for p in v.packages.values()]) == range(len(v.packages)) - - v = self.m.getTree(pid, False, False) - fids = v.root.fids[10:20] - self.m.moveFiles(fids, pid2) - v = self.m.getTree(pid2, False, False) - - assert sorted([f.fileorder for f in v.files.values()]) == range(len(v.files)) - assert len(v.files) == self.count + len(fids) - - - -if __name__ == "__main__": - TestFileManager.benchmark() \ No newline at end of file diff --git a/tests/test_interactionManager.py b/tests/test_interactionManager.py deleted file mode 100644 index 920d84b9d..000000000 --- a/tests/test_interactionManager.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -from unittest import TestCase -from helper.Stubs import Core - -from module.Api import Input, Output -from module.interaction.InteractionManager import InteractionManager - -class TestInteractionManager(TestCase): - - @classmethod - def setUpClass(cls): - cls.core = Core() - - def setUp(self): - self.im = InteractionManager(self.core) - - - def test_notifications(self): - - n = self.im.createNotification("test", "notify") - assert self.im.getNotifications() == [n] - - for i in range(10): - self.im.createNotification("title", "test") - - assert len(self.im.getNotifications()) == 11 - - - def test_captcha(self): - assert self.im.getTask() is None - - t = self.im.newCaptchaTask("1", "", "") - assert t.output == Output.Captcha - self.im.handleTask(t) - assert t is self.im.getTask() - - t2 = self.im.newCaptchaTask("2", "", "") - self.im.handleTask(t2) - - assert self.im.getTask(Output.Query) is None - assert self.im.getTask() is t - - self.im.removeTask(t) - assert self.im.getTask() is t2 - - self.im.getTaskByID(t2.iid) - assert self.im.getTask() is None - - - def test_query(self): - assert self.im.getTask() is None - t = self.im.newQueryTask(Input.Text, None, "text") - assert t.description == "text" - self.im.handleTask(t) - - assert self.im.getTask(Output.Query) is t - assert self.im.getTask(Output.Captcha) is None \ No newline at end of file diff --git a/tests/test_syntax.py b/tests/test_syntax.py deleted file mode 100644 index a4cc53ee5..000000000 --- a/tests/test_syntax.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- - -from os import walk -from os.path import abspath, dirname, join - -from unittest import TestCase - -PATH = abspath(join(dirname(abspath(__file__)), "..", "")) - -# needed to register globals -from helper import Stubs - -class TestSyntax(TestCase): - pass - - -for path, dirs, files in walk(join(PATH, "module")): - - for f in files: - if not f.endswith(".py") or f.startswith("__"): continue - fpath = join(path, f) - pack = fpath.replace(PATH, "")[1:-3] #replace / and .py - imp = pack.replace("/", ".") - packages = imp.split(".") - #__import__(imp) - - # to much sideeffect when importing - if "web" in packages or "lib" in packages: continue - if "ThriftTest" in packages: continue - - # currying - def meta(imp, sig): - def _test(self=None): - __import__(imp) - - _test.func_name = sig - return _test - - # generate test methods - sig = "test_%s_%s" % (packages[-2], packages[-1]) - - - setattr(TestSyntax, sig, meta(imp, sig)) \ No newline at end of file -- cgit v1.2.3