diff options
Diffstat (limited to 'module')
33 files changed, 454 insertions, 260 deletions
diff --git a/module/Api.py b/module/Api.py index 577c420c3..96b10be9c 100644 --- a/module/Api.py +++ b/module/Api.py @@ -21,15 +21,12 @@ from types import MethodType from remote.apitypes import * -from utils import bits_set +from utils import bits_set, primary_uid # contains function names mapped to their permissions # unlisted functions are for admins only perm_map = {} -# store which methods needs user context -user_context = {} - # decorator only called on init, never initialized, so has no effect on runtime def RequirePerm(bits): class _Dec(object): @@ -39,12 +36,6 @@ def RequirePerm(bits): return _Dec -# TODO: not needed anymore -# decorator to annotate user methods, these methods must have user=None kwarg. -class UserContext(object): - def __new__(cls, f, *args, **kwargs): - user_context[f.__name__] = True - return f urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&]*)", re.IGNORECASE) @@ -93,8 +84,8 @@ class Api(Iface): return None #TODO return default user? @property - def userHandle(self): - return self.user.primary if self.user is not None else None + def primaryUID(self): + return primary_uid(self.user) @classmethod def initComponents(cls): diff --git a/module/FileManager.py b/module/FileManager.py index 74ff2ebeb..082bdb4d4 100644 --- a/module/FileManager.py +++ b/module/FileManager.py @@ -28,9 +28,8 @@ from datatypes.PyPackage import PyPackage, RootPackage # invalidates the cache def invalidate(func): def new(*args): - args[0].filecount = -1 - args[0].downloadcount = -1 - args[0].queuecount = -1 + args[0].downloadstats = {} + args[0].queuestats = {} args[0].jobCache = {} return func(*args) @@ -65,9 +64,8 @@ class FileManager: self.lock = RLock() #self.lock._Verbose__verbose = True - self.filecount = -1 # if an invalid value is set get current value from db - self.downloadcount = -1 # number of downloads - self.queuecount = -1 # number of package to be loaded + self.downloadstats = {} # cached dl stats + self.queuestats = {} # cached queue stats self.db = self.core.db @@ -99,7 +97,7 @@ class FileManager: def addLinks(self, data, package): """Add links, data = (plugin, url) tuple. Internal method should use API.""" self.db.addLinks(data, package, OWNER) - self.evm.dispatchEvent("packageUpdated", package) + self.evm.dispatchEvent("package:updated", package) @invalidate @@ -109,7 +107,7 @@ class FileManager: PackageStatus.Paused if paused else PackageStatus.Ok, OWNER) p = self.db.getPackageInfo(pid) - self.evm.dispatchEvent("packageInserted", pid, p.root, p.packageorder) + self.evm.dispatchEvent("package:inserted", pid, p.root, p.packageorder) return pid @@ -294,28 +292,20 @@ class FileManager: return pyfile - - def getFileCount(self): - """returns number of files""" - - if self.filecount == -1: - self.filecount = self.db.filecount() - - return self.filecount - - def getDownloadCount(self): + #TODO + def getDownloadStats(self, user=None): """ return number of downloads """ - if self.downloadcount == -1: - self.downloadcount = self.db.downloadcount() + if user not in self.downloadstats: + self.downloadstats[user] = self.db.downloadstats() - return self.downloadcount + return self.downloadstats[user] - def getQueueCount(self, force=False): + def getQueueStats(self, user=None, force=False): """number of files that have to be processed""" - if self.queuecount == -1 or force: - self.queuecount = self.db.queuecount() + if user not in self.queuestats or force: + self.queuestats[user] = self.db.queuestats() - return self.queuecount + return self.queuestats[user] def scanDownloadFolder(self): pass @@ -345,7 +335,7 @@ class FileManager: if pack.root == root and pack.packageorder > oldorder: pack.packageorder -= 1 - self.evm.dispatchEvent("packageDeleted", pid) + self.evm.dispatchEvent("package:deleted", pid) @lock @invalidate @@ -370,7 +360,7 @@ class FileManager: if pyfile.packageid == pid and pyfile.fileorder > order: pyfile.fileorder -= 1 - self.evm.dispatchEvent("fileDeleted", fid, pid) + self.evm.dispatchEvent("file:deleted", fid, pid) @lock def releaseFile(self, fid): @@ -387,24 +377,25 @@ class FileManager: def updateFile(self, pyfile): """updates file""" self.db.updateFile(pyfile) - self.evm.dispatchEvent("fileUpdated", pyfile.fid, pyfile.packageid) + self.evm.dispatchEvent("file:updated", pyfile.fid, pyfile.packageid) def updatePackage(self, pypack): """updates a package""" self.db.updatePackage(pypack) - self.evm.dispatchEvent("packageUpdated", pypack.pid) + self.evm.dispatchEvent("package:updated", pypack.pid) @invalidate def updateFileInfo(self, data, pid): """ updates file info (name, size, status,[ hash,] url)""" self.db.updateLinkInfo(data) - self.evm.dispatchEvent("packageUpdated", pid) + self.evm.dispatchEvent("package:updated", pid) def checkAllLinksFinished(self): """checks if all files are finished and dispatch event""" - if not self.getQueueCount(True): - self.core.addonManager.dispatchEvent("allDownloadsFinished") + # TODO: user context? + if not self.getQueueStats(None, True)[0]: + self.core.addonManager.dispatchEvent("downloads:finished") self.core.log.debug("All downloads finished") return True @@ -416,8 +407,9 @@ class FileManager: # reset count so statistic will update (this is called when dl was processed) self.resetCount() + # TODO: user context? if not self.db.processcount(fid): - self.core.addonManager.dispatchEvent("allDownloadsProcessed") + self.core.addonManager.dispatchEvent("downloads:processed") self.core.log.debug("All downloads processed") return True @@ -449,7 +441,7 @@ class FileManager: if pid in self.packages: self.packages[pid].setFinished = False - self.evm.dispatchEvent("packageUpdated", pid) + self.evm.dispatchEvent("package:updated", pid) @lock @invalidate @@ -463,7 +455,7 @@ class FileManager: f.abortDownload() self.db.restartFile(fid) - self.evm.dispatchEvent("fileUpdated", fid) + self.evm.dispatchEvent("file:updated", fid) @lock @@ -486,7 +478,7 @@ class FileManager: self.db.commit() - self.evm.dispatchEvent("packageReordered", pid, position, p.root) + self.evm.dispatchEvent("package:reordered", pid, position, p.root) @lock @invalidate @@ -526,7 +518,7 @@ class FileManager: self.db.commit() - self.evm.dispatchEvent("filesReordered", pid) + self.evm.dispatchEvent("file:reordered", pid) @lock @invalidate @@ -569,6 +561,7 @@ class FileManager: return True + @invalidate def reCheckPackage(self, pid): """ recheck links in package """ data = self.db.getPackageData(pid) diff --git a/module/api/ApiComponent.py b/module/api/ApiComponent.py index c3b8c974b..3948086c2 100644 --- a/module/api/ApiComponent.py +++ b/module/api/ApiComponent.py @@ -18,6 +18,6 @@ class ApiComponent(Iface): self.core = core assert isinstance(user, User) self.user = user - self.userHandle = 0 + self.primaryUID = 0 # No instantiating! raise Exception()
\ No newline at end of file diff --git a/module/api/ConfigApi.py b/module/api/ConfigApi.py index 55e0aa49b..9df9455a2 100644 --- a/module/api/ConfigApi.py +++ b/module/api/ConfigApi.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from module.Api import Api, UserContext, RequirePerm, Permission, ConfigHolder, ConfigItem, ConfigInfo +from module.Api import Api, RequirePerm, Permission, ConfigHolder, ConfigItem, ConfigInfo from module.utils import to_string from ApiComponent import ApiComponent @@ -9,7 +9,6 @@ from ApiComponent import ApiComponent class ConfigApi(ApiComponent): """ Everything related to configuration """ - @UserContext def getConfigValue(self, section, option): """Retrieve config value. @@ -21,7 +20,6 @@ class ConfigApi(ApiComponent): value = self.core.config.get(section, option, self.user) return to_string(value) - @UserContext def setConfigValue(self, section, option, value): """Set new config value. @@ -56,7 +54,6 @@ class ConfigApi(ApiComponent): return [ConfigInfo(section, config.name, config.description, False, False) for section, config, values in self.core.config.iterCoreSections()] - @UserContext @RequirePerm(Permission.Plugins) def getPluginConfig(self): """All plugins and addons the current user has configured @@ -75,7 +72,6 @@ class ConfigApi(ApiComponent): return data - @UserContext @RequirePerm(Permission.Plugins) def getAvailablePlugins(self): """List of all available plugins, that are configurable @@ -88,7 +84,6 @@ class ConfigApi(ApiComponent): self.core.pluginManager.isUserPlugin(name)) for name, config, values in self.core.config.iterSections(self.user)] - @UserContext @RequirePerm(Permission.Plugins) def configurePlugin(self, plugin): """Get complete config options for desired section @@ -99,7 +94,6 @@ class ConfigApi(ApiComponent): pass - @UserContext @RequirePerm(Permission.Plugins) def saveConfig(self, config): """Used to save a configuration, core config can only be saved by admins @@ -108,7 +102,6 @@ class ConfigApi(ApiComponent): """ pass - @UserContext @RequirePerm(Permission.Plugins) def deleteConfig(self, plugin): """Deletes modified config diff --git a/module/api/CoreApi.py b/module/api/CoreApi.py index 4de8c1f96..9338954d0 100644 --- a/module/api/CoreApi.py +++ b/module/api/CoreApi.py @@ -1,12 +1,13 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from module.Api import Api, RequirePerm, Permission, ServerStatus +from module.Api import Api, RequirePerm, Permission, ServerStatus, PackageStats from module.utils.fs import join, free_space from module.utils import compare_time from ApiComponent import ApiComponent + class CoreApi(ApiComponent): """ This module provides methods for general interaction with the core, like status or progress retrieval """ @@ -18,7 +19,8 @@ class CoreApi(ApiComponent): @RequirePerm(Permission.All) def getWSAddress(self): """Gets and address for the websocket based on configuration""" - # TODO + # TODO SSL (wss) + return "ws://%%s:%d" % self.core.config['remote']['port'] @RequirePerm(Permission.All) def getServerStatus(self): @@ -26,10 +28,17 @@ class CoreApi(ApiComponent): :return: `ServerStatus` """ - serverStatus = ServerStatus(self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, - not self.core.threadManager.pause and self.isTimeDownload(), self.core.threadManager.pause, - self.core.config['reconnect']['activated'] and self.isTimeReconnect()) + queue = self.core.files.getQueueStats(self.primaryUID) + total = self.core.files.getDownloadStats(self.primaryUID) + + serverStatus = ServerStatus(0, + PackageStats(total[0], total[0] - queue[0], total[1], total[1] - queue[1]), + 0, + not self.core.threadManager.pause and self.isTimeDownload(), + self.core.threadManager.pause, + self.core.config['reconnect']['activated'] and self.isTimeReconnect()) + # TODO multi user for pyfile in self.core.threadManager.getActiveDownloads(): serverStatus.speed += pyfile.getSpeed() #bytes/s @@ -117,5 +126,6 @@ class CoreApi(ApiComponent): end = self.core.config['reconnect']['endTime'].split(":") return compare_time(start, end) and self.core.config["reconnect"]["activated"] + if Api.extend(CoreApi): del CoreApi
\ No newline at end of file diff --git a/module/api/FileApi.py b/module/api/FileApi.py index 8f09f3cb7..a5d5a8535 100644 --- a/module/api/FileApi.py +++ b/module/api/FileApi.py @@ -80,7 +80,7 @@ class FileApi(ApiComponent): @RequirePerm(Permission.All) def searchSuggestions(self, pattern): - names = self.core.db.getMatchingFilenames(pattern, self.userHandle) + names = self.core.db.getMatchingFilenames(pattern, self.primaryUID) # TODO: stemming and reducing the names to provide better suggestions return uniqify(names) diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 557d9c034..632961c2a 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -24,30 +24,46 @@ zero_stats = PackageStats(0, 0, 0, 0) class FileMethods(DatabaseMethods): + @queue - def filecount(self, user=None): - """returns number of files""" + def filecount(self): + """returns number of files, currently only used for debugging""" self.c.execute("SELECT COUNT(*) FROM files") return self.c.fetchone()[0] @queue - def downloadcount(self, user=None): - """ number of downloads """ - self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus != 0") - return self.c.fetchone()[0] + def downloadstats(self, user=None): + """ number of downloads and size """ + if user is None: + self.c.execute("SELECT COUNT(*), SUM(f.size) FROM files f WHERE dlstatus != 0") + else: + self.c.execute( + "SELECT COUNT(*), SUM(f.size) FROM files f, packages p WHERE f.package = p.pid AND dlstatus != 0", + user) + + r = self.c.fetchone() + # sum is None when no elements are added + return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) @queue - def queuecount(self, user=None): - """ number of files in queue not finished yet""" + def queuestats(self, user=None): + """ number and size of files in queue not finished yet""" # status not in NA, finished, skipped - self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus NOT IN (0,5,6)") - return self.c.fetchone()[0] + if user is None: + self.c.execute("SELECT COUNT(*), SUM(f.size) FROM files f WHERE dlstatus NOT IN (0,5,6)") + else: + self.c.execute( + "SELECT COUNT(*), SUM(f.size) FROM files f, package p WHERE f.package = p.pid AND p.owner=? AND dlstatus NOT IN (0,5,6)", + user) + + r = self.c.fetchone() + return (r[0], r[1] if r[1] is not None else 0) if r else (0, 0) @queue def processcount(self, fid=-1, user=None): """ number of files which have to be processed """ # status in online, queued, starting, waiting, downloading - self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (fid, )) + self.c.execute("SELECT COUNT(*), SUM(size) FROM files WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (fid, )) return self.c.fetchone()[0] # TODO: think about multiuser side effects on *count methods @@ -184,8 +200,8 @@ class FileMethods(DatabaseMethods): :param tags: optional tag list """ qry = ( - 'SELECT pid, name, folder, root, owner, site, comment, password, added, tags, status, shared, packageorder ' - 'FROM packages%s ORDER BY root, packageorder') + 'SELECT pid, name, folder, root, owner, site, comment, password, added, tags, status, shared, packageorder ' + 'FROM packages%s ORDER BY root, packageorder') if root is None: stats = self.getPackageStats(owner=owner) diff --git a/module/remote/apitypes.py b/module/remote/apitypes.py index bc53f5f7c..aaec2b3ce 100644 --- a/module/remote/apitypes.py +++ b/module/remote/apitypes.py @@ -297,13 +297,13 @@ class ProgressInfo(BaseObject): self.download = download class ServerStatus(BaseObject): - __slots__ = ['queuedDownloads', 'totalDownloads', 'speed', 'pause', 'download', 'reconnect'] + __slots__ = ['speed', 'files', 'notifications', 'paused', 'download', 'reconnect'] - def __init__(self, queuedDownloads=None, totalDownloads=None, speed=None, pause=None, download=None, reconnect=None): - self.queuedDownloads = queuedDownloads - self.totalDownloads = totalDownloads + def __init__(self, speed=None, files=None, notifications=None, paused=None, download=None, reconnect=None): self.speed = speed - self.pause = pause + self.files = files + self.notifications = notifications + self.paused = paused self.download = download self.reconnect = reconnect diff --git a/module/remote/apitypes_debug.py b/module/remote/apitypes_debug.py index 974a68c29..6d30f1da6 100644 --- a/module/remote/apitypes_debug.py +++ b/module/remote/apitypes_debug.py @@ -37,7 +37,7 @@ classes = { 'PackageInfo' : [int, basestring, basestring, int, int, basestring, basestring, basestring, int, (list, basestring), int, bool, int, PackageStats, (list, int), (list, int)], 'PackageStats' : [int, int, int, int], 'ProgressInfo' : [basestring, basestring, basestring, int, int, int, (None, DownloadProgress)], - 'ServerStatus' : [int, int, int, bool, bool, bool], + 'ServerStatus' : [int, PackageStats, int, bool, bool, bool], 'ServiceDoesNotExists' : [basestring, basestring], 'ServiceException' : [basestring], 'TreeCollection' : [PackageInfo, (dict, int, FileInfo), (dict, int, PackageInfo)], diff --git a/module/remote/json_converter.py b/module/remote/json_converter.py index 256674c34..50f0309bd 100644 --- a/module/remote/json_converter.py +++ b/module/remote/json_converter.py @@ -14,7 +14,7 @@ from apitypes import ExceptionObject # compact json separator separators = (',', ':') -# json encoder that accepts TBase objects +# json encoder that accepts api objects class BaseEncoder(json.JSONEncoder): def default(self, o): @@ -26,17 +26,35 @@ class BaseEncoder(json.JSONEncoder): return json.JSONEncoder.default(self, o) +# more compact representation, only clients with information of the classes can handle it +class BaseEncoderCompact(json.JSONEncoder): + + def default(self, o): + if isinstance(o, BaseObject) or isinstance(o, ExceptionObject): + ret = {"@compact" : [o.__class__.__name__]} + ret["@compact"].extend(getattr(o, attr) for attr in o.__slots__) + return ret + + return json.JSONEncoder.default(self, o) def convert_obj(dct): if '@class' in dct: cls = getattr(apitypes, dct['@class']) del dct['@class'] return cls(**dct) + elif '@compact' in dct: + cls = getattr(apitypes, dct['@compact'][0]) + return cls(*dct['@compact'][1:]) return dct def dumps(*args, **kwargs): - kwargs['cls'] = BaseEncoder + if 'compact' in kwargs: + kwargs['cls'] = BaseEncoderCompact + del kwargs['compact'] + else: + kwargs['cls'] = BaseEncoder + kwargs['separators'] = separators return json.dumps(*args, **kwargs) diff --git a/module/remote/pyload.thrift b/module/remote/pyload.thrift index c66ec20d6..dc6b1c406 100644 --- a/module/remote/pyload.thrift +++ b/module/remote/pyload.thrift @@ -128,15 +128,6 @@ struct ProgressInfo { 7: optional DownloadProgress download } -struct ServerStatus { - 1: i16 queuedDownloads, - 2: i16 totalDownloads, - 3: ByteCount speed, - 4: bool pause, - 5: bool download, - 6: bool reconnect -} - // download info for specific file struct DownloadInfo { 1: string url, @@ -203,6 +194,15 @@ struct LinkStatus { 6: string packagename, } +struct ServerStatus { + 1: ByteCount speed, + 2: PackageStats files, + 3: i16 notifications, + 4: bool paused, + 5: bool download, + 6: bool reconnect, +} + struct InteractionTask { 1: InteractionID iid, 2: Input input, diff --git a/module/remote/wsbackend/AbstractHandler.py b/module/remote/wsbackend/AbstractHandler.py index f843fc278..45fbb134c 100644 --- a/module/remote/wsbackend/AbstractHandler.py +++ b/module/remote/wsbackend/AbstractHandler.py @@ -41,11 +41,38 @@ class AbstractHandler: def do_extra_handshake(self, req): self.log.debug("WS Connected: %s" % req) + req.api = None #when api is set client is logged in + + # allow login via session when webinterface is active + if self.core.config['webinterface']['activated']: + cookie = req.headers_in.getheader('Cookie') + s = self.load_session(cookie) + if s: + uid = s.get('uid', None) + req.api = self.api.withUserContext(uid) + self.log.debug("WS authenticated with cookie: %d" % uid) + self.on_open(req) def on_open(self, req): pass + def load_session(self, cookies): + from Cookie import SimpleCookie + from beaker.session import Session + from module.web.webinterface import session + + cookies = SimpleCookie(cookies) + sid = cookies.get(session.options['key']) + if not sid: + return None + + s = Session({}, use_cookies=False, id=sid.value, **session.options) + if s.is_new: + return None + + return s + def passive_closing_handshake(self, req): self.log.debug("WS Closed: %s" % req) self.on_close(req) @@ -59,8 +86,6 @@ class AbstractHandler: def handle_call(self, msg, req): """ Parses the msg for an argument call. If func is null an response was already sent. - :param msg: - :param req: :return: func, args, kwargs """ try: @@ -70,11 +95,15 @@ class AbstractHandler: self.send_result(req, self.ERROR, "No JSON request") return None, None, None - if type(o) != list and len(o) not in range(1,4): + if not isinstance(o, basestring) and type(o) != list and len(o) not in range(1, 4): self.log.debug("Invalid Api call: %s" % o) self.send_result(req, self.ERROR, "Invalid Api call") return None, None, None - if len(o) == 1: # arguments omitted + + # called only with name, no args + if isinstance(o, basestring): + return o, [], {} + elif len(o) == 1: # arguments omitted return o[0], [], {} elif len(o) == 2: func, args = o @@ -85,5 +114,20 @@ class AbstractHandler: else: return tuple(o) + def do_login(self, req, args, kwargs): + user = self.api.checkAuth(*args, **kwargs) + if user: + req.api = self.api.withUserContext(user.uid) + return self.send_result(req, self.OK, True) + else: + return self.send_result(req, self.FORBIDDEN, "Forbidden") + + def do_logout(self, req): + req.api = None + return self.send_result(req, self.OK, True) + def send_result(self, req, code, result): - return send_message(req, dumps([code, result]))
\ No newline at end of file + return send_message(req, dumps([code, result])) + + def send(self, req, obj): + return send_message(req, dumps(obj))
\ No newline at end of file diff --git a/module/remote/wsbackend/ApiHandler.py b/module/remote/wsbackend/ApiHandler.py index eec546d47..e985e10be 100644 --- a/module/remote/wsbackend/ApiHandler.py +++ b/module/remote/wsbackend/ApiHandler.py @@ -55,18 +55,9 @@ class ApiHandler(AbstractHandler): return # handle_call already sent the result if func == 'login': - user = self.api.checkAuth(*args, **kwargs) - if user: - req.api = self.api.withUserContext(user.uid) - return self.send_result(req, self.OK, True) - - else: - return self.send_result(req, self.OK, False) - + return self.do_login(req, args, kwargs) elif func == 'logout': - req.api = None - return self.send_result(req, self.OK, True) - + return self.do_logout(req) else: if not req.api: return self.send_result(req, self.FORBIDDEN, "Forbidden") diff --git a/module/remote/wsbackend/AsyncHandler.py b/module/remote/wsbackend/AsyncHandler.py index a8382a211..2f9b43ad2 100644 --- a/module/remote/wsbackend/AsyncHandler.py +++ b/module/remote/wsbackend/AsyncHandler.py @@ -16,7 +16,7 @@ # @author: RaNaN ############################################################################### -from Queue import Queue +from Queue import Queue, Empty from threading import Lock from mod_pywebsocket.msgutil import receive_message @@ -34,13 +34,13 @@ class AsyncHandler(AbstractHandler): Progress information are continuous and will be pushed in a fixed interval when available. After connect you have to login and can set the interval by sending the json command ["setInterval", xy]. - To start receiving updates call "start", afterwards no more incoming messages will be accept! + To start receiving updates call "start", afterwards no more incoming messages will be accepted! """ PATH = "/async" COMMAND = "start" - PROGRESS_INTERVAL = 1 + PROGRESS_INTERVAL = 2 STATUS_INTERVAL = 60 def __init__(self, api): @@ -57,7 +57,10 @@ class AsyncHandler(AbstractHandler): @lock def on_close(self, req): - self.clients.remove(req) + try: + self.clients.remove(req) + except ValueError: # ignore when not in list + pass @lock def add_event(self, event): @@ -86,21 +89,15 @@ class AsyncHandler(AbstractHandler): return # Result was already sent if func == 'login': - user = self.api.checkAuth(*args, **kwargs) - if user: - req.api = self.api.withUserContext(user.uid) - return self.send_result(req, self.OK, True) - - else: - return self.send_result(req, self.FORBIDDEN, "Forbidden") + return self.do_login(req, args, kwargs) elif func == 'logout': - req.api = None - return self.send_result(req, self.OK, True) + return self.do_logout(req) else: if not req.api: return self.send_result(req, self.FORBIDDEN, "Forbidden") + if func == "setInterval": req.interval = args[0] elif func == self.COMMAND: @@ -109,4 +106,14 @@ class AsyncHandler(AbstractHandler): def mode_running(self, req): """ Listen for events, closes socket when returning True """ - self.send_result(req, "update", "test")
\ No newline at end of file + try: + ev = req.queue.get(True, req.interval) + self.send(req, ev) + + except Empty: + # TODO: server status is not enough + # modify core api to include progress? think of other needed information to show + # notifications + + self.send(req, self.api.getServerStatus()) + self.send(req, self.api.getProgressInfo())
\ No newline at end of file diff --git a/module/threads/BaseThread.py b/module/threads/BaseThread.py index 3e27eec96..c64678a72 100644 --- a/module/threads/BaseThread.py +++ b/module/threads/BaseThread.py @@ -1,10 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import os -import sys -import locale - from threading import Thread from time import strftime, gmtime from sys import exc_info @@ -12,6 +8,7 @@ from types import MethodType from pprint import pformat from traceback import format_exc +from module.utils import primary_uid from module.utils.fs import listdir, join, save_join, stat, exists class BaseThread(Thread): @@ -24,6 +21,13 @@ class BaseThread(Thread): self.core = manager.core self.log = manager.core.log + #: Owner of the thread, every type should set it + self.owner = None + + @property + def user(self): + return primary_uid(self.owner) + def getProgress(self): """ retrieves progress information about the current running task diff --git a/module/web/api_app.py b/module/web/api_app.py index 75a817c46..52903e92b 100644 --- a/module/web/api_app.py +++ b/module/web/api_app.py @@ -65,6 +65,8 @@ def callApi(api, func, *args, **kwargs): print "Invalid API call", func return HTTPError(404, dumps("Not Found")) + # TODO: accept same payload as WS backends, combine into json_converter + # TODO: arguments as json dictionaries # TODO: encoding result = getattr(api, func)(*[loads(x) for x in args], **dict([(x, loads(y)) for x, y in kwargs.iteritems()])) diff --git a/module/web/pyload_app.py b/module/web/pyload_app.py index f8578fcf0..0c3af103f 100644 --- a/module/web/pyload_app.py +++ b/module/web/pyload_app.py @@ -44,7 +44,8 @@ def pre_processor(): return {"user": user, 'server': status, - 'url': request.url } + 'url': request.url , + 'ws': PYLOAD.getWSAddress()} def base(messages): diff --git a/module/web/static/css/default/style.less b/module/web/static/css/default/style.less index d3f23478f..260f9fa52 100644 --- a/module/web/static/css/default/style.less +++ b/module/web/static/css/default/style.less @@ -422,7 +422,7 @@ footer { // background-color: @greyDark; background: url("../../img/default/bgpatterndark.png") repeat;
color: @grey;
height: @footer-height;
- margin-top: -@footer-height + 10px;
+ margin-top: -@footer-height;
position: relative;
width: 100%;
line-height: 16px;
diff --git a/module/web/static/js/app.js b/module/web/static/js/app.js index b081022af..59ad04fc9 100644 --- a/module/web/static/js/app.js +++ b/module/web/static/js/app.js @@ -28,10 +28,14 @@ define([ // Add Global Helper functions _.extend(Application.prototype, Backbone.Events, { - restartFailed: function(pids, options) { + apiCall: function(method, args, options) { options || (options = {}); - options.url = 'api/restartFailed'; - $.ajax(options); + + + }, + + openWebSocket: function(path) { + return new WebSocket(window.wsAddress.replace('%s', window.location.hostname) + path); } }); diff --git a/module/web/static/js/helpers/formatSize.js b/module/web/static/js/helpers/formatSize.js index a792392b7..a50588bc6 100644 --- a/module/web/static/js/helpers/formatSize.js +++ b/module/web/static/js/helpers/formatSize.js @@ -2,7 +2,7 @@ define('helpers/formatSize', ['handlebars'], function(Handlebars) { var sizes = ["B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB"]; function formatSize(bytes, options) { - if (bytes === 0) return '0 B'; + if (!bytes || bytes === 0) return '0 B'; var i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024))); // round to two digits return (bytes / Math.pow(1024, i)).toFixed(2) + ' ' + sizes[i]; diff --git a/module/web/static/js/helpers/formatTime.js b/module/web/static/js/helpers/formatTime.js new file mode 100644 index 000000000..cb635ede9 --- /dev/null +++ b/module/web/static/js/helpers/formatTime.js @@ -0,0 +1,40 @@ +// Format bytes in human readable format +define('helpers/formatTime', ['handlebars'], function(Handlebars) { + + // TODO: seconds are language dependant + // time could be better formatted + function seconds2time (seconds) { + var hours = Math.floor(seconds / 3600); + var minutes = Math.floor((seconds - (hours * 3600)) / 60); + seconds = seconds - (hours * 3600) - (minutes * 60); + var time = ""; + + if (hours != 0) { + time = hours+":"; + } + if (minutes != 0 || time !== "") { + minutes = (minutes < 10 && time !== "") ? "0"+minutes : String(minutes); + time += minutes+":"; + } + if (time === "") { + time = seconds+"s"; + } + else { + time += (seconds < 10) ? "0"+seconds : String(seconds); + } + return time; + } + + + function formatTime(seconds, options) { + if (seconds === Infinity) + return '∞'; + else if (!seconds || seconds <= 0) + return "-"; + + return seconds2time(seconds); + } + + Handlebars.registerHelper('formatTime', formatTime); + return formatTime; +});
\ No newline at end of file diff --git a/module/web/static/js/models/File.js b/module/web/static/js/models/File.js index 42275a452..fa0945713 100644 --- a/module/web/static/js/models/File.js +++ b/module/web/static/js/models/File.js @@ -31,8 +31,13 @@ define(['jquery', 'backbone', 'underscore', 'utils/apitypes'], function($, Backb }, - destroy: function() { + destroy: function(options) { + options || (options = {}); + // TODO: as post data + options.url = 'api/deleteFiles/[' + this.get('fid') + ']'; + options.type = "post"; + return Backbone.Model.prototype.destroy.call(this, options); }, restart: function(options) { diff --git a/module/web/static/js/models/ServerStatus.js b/module/web/static/js/models/ServerStatus.js index 35257fcb1..2430a9ffd 100644 --- a/module/web/static/js/models/ServerStatus.js +++ b/module/web/static/js/models/ServerStatus.js @@ -1,15 +1,15 @@ -define(['jquery', 'backbone', 'underscore', 'collections/ProgressList'], - function($, Backbone, _, ProgressList) { +define(['jquery', 'backbone', 'underscore'], + function($, Backbone, _) { return Backbone.Model.extend({ defaults: { - queuedDownloads: -1, - totalDownloads: -1, - speed: -1, - pause: false, + speed: 0, + files: null, + notifications: -1, + paused: false, download: false, - reconnect: false, + reconnect: false }, // Model Constructor @@ -24,16 +24,23 @@ define(['jquery', 'backbone', 'underscore', 'collections/ProgressList'], return Backbone.Model.prototype.fetch.call(this, options); }, - parse: function(resp, xhr) { - // Package is loaded from tree collection - if (_.has(resp, 'root')) { - resp.root.files = new FileList(_.values(resp.files)); - // circular dependencies needs to be avoided - var PackageList = require('collections/PackageList'); - resp.root.packs = new PackageList(_.values(resp.packages)); - return resp.root; - } - return Backbone.model.prototype.fetch.call(this, resp, xhr); + toJSON: function(options) { + var obj = Backbone.Model.prototype.toJSON.call(this, options); + + // stats are not available + if (obj.files === null) + return obj; + + obj.files.linksleft = obj.files.linkstotal - obj.files.linksdone; + obj.files.sizeleft = obj.files.sizetotal - obj.files.sizedone; + if (obj.speed && obj.speed > 0) + obj.files.eta = Math.round(obj.files.sizeleft / obj.speed); + else if (obj.files.sizeleft > 0) + obj.files.eta = Infinity; + else + obj.files.eta = 0; + + return obj; } }); diff --git a/module/web/static/js/utils/initHB.js b/module/web/static/js/utils/initHB.js index f3a0955b3..c977f063d 100644 --- a/module/web/static/js/utils/initHB.js +++ b/module/web/static/js/utils/initHB.js @@ -1,6 +1,6 @@ // Loads all helper and set own handlebars rules define(['underscore', 'handlebars', - 'helpers/formatSize', 'helpers/fileHelper'], + 'helpers/formatSize', 'helpers/fileHelper', 'helpers/formatTime'], function(_, Handlebars) { // Replace with own lexer rules compiled from handlebars.l Handlebars.Parser.lexer.rules = [/^(?:[^\x00]*?(?=(<%)))/, /^(?:[^\x00]+)/, /^(?:[^\x00]{2,}?(?=(\{\{|$)))/, /^(?:\{\{>)/, /^(?:<%=)/, /^(?:<%\/)/, /^(?:\{\{\^)/, /^(?:<%\s*else\b)/, /^(?:\{<%%)/, /^(?:\{\{&)/, /^(?:<%![\s\S]*?%>)/, /^(?:<%)/, /^(?:=)/, /^(?:\.(?=[%} ]))/, /^(?:\.\.)/, /^(?:[\/.])/, /^(?:\s+)/, /^(?:%%>)/, /^(?:%>)/, /^(?:"(\\["]|[^"])*")/, /^(?:'(\\[']|[^'])*')/, /^(?:@[a-zA-Z]+)/, /^(?:true(?=[%}\s]))/, /^(?:false(?=[%}\s]))/, /^(?:[0-9]+(?=[%}\s]))/, /^(?:[a-zA-Z0-9_$-]+(?=[=%}\s\/.]))/, /^(?:\[[^\]]*\])/, /^(?:.)/, /^(?:$)/]; diff --git a/module/web/static/js/views/abstract/itemView.js b/module/web/static/js/views/abstract/itemView.js index 75b058874..394044ec4 100644 --- a/module/web/static/js/views/abstract/itemView.js +++ b/module/web/static/js/views/abstract/itemView.js @@ -23,6 +23,13 @@ define(['jquery', 'backbone', 'underscore'], function($, Backbone, _) { this.$el.slideDown(); }, + unrender: function() { + var self = this; + this.$el.slideUp(function() { + self.destroy(); + }); + }, + deleteItem: function(e) { if (e) e.stopPropagation(); diff --git a/module/web/static/js/views/dashboardView.js b/module/web/static/js/views/dashboardView.js index d9ea8d444..d9ff1c5fc 100644 --- a/module/web/static/js/views/dashboardView.js +++ b/module/web/static/js/views/dashboardView.js @@ -1,5 +1,5 @@ define(['jquery', 'backbone', 'underscore', 'app', 'models/TreeCollection', - 'views/packageView', 'views/fileView', 'views/selectionView', 'views/filterView'], + 'views/packageView', 'views/fileView', 'views/selectionView', 'views/filterView', 'select2'], function($, Backbone, _, App, TreeCollection, packageView, fileView, selectionView, filterView) { // Renders whole dashboard @@ -51,6 +51,8 @@ define(['jquery', 'backbone', 'underscore', 'app', 'models/TreeCollection', }); }}); + + this.$('.input').select2({tags: ["a", "b", "sdf"]}); }, render: function() { diff --git a/module/web/static/js/views/fileView.js b/module/web/static/js/views/fileView.js index 17da74de3..2459b6cd6 100644 --- a/module/web/static/js/views/fileView.js +++ b/module/web/static/js/views/fileView.js @@ -9,14 +9,15 @@ define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', 'views/abst // template: _.template($("#template-file").html()), template: _.compile($("#template-file").html()), events: { - 'click .checkbox': 'select' + 'click .checkbox': 'select', + 'click .iconf-trash': 'deleteItem' }, initialize: function() { this.listenTo(this.model, 'change', this.render); // This will be triggered manually and changed before with silent=true this.listenTo(this.model, 'change:visible', this.visibility_changed); - this.listenTo(this.model, 'remove', this.destroy); + this.listenTo(this.model, 'remove', this.unrender); this.listenTo(App.vent, 'dashboard:destroyContent', this.destroy); }, diff --git a/module/web/static/js/views/headerView.js b/module/web/static/js/views/headerView.js index cfceca6cd..c22f173c4 100644 --- a/module/web/static/js/views/headerView.js +++ b/module/web/static/js/views/headerView.js @@ -1,102 +1,153 @@ -define(['jquery', 'underscore', 'backbone', 'flot'], function($, _, Backbone) { - // Renders the header with all information - return Backbone.View.extend({ - - el: 'header', - - events: { - 'click i.iconf-list': 'toggle_taskList', - 'click .popover .close': 'hide_taskList', - 'click .btn-grabber': 'open_grabber' - }, - - // Will hold the link grabber - grabber: null, - notifications: null, - selections: null, - - initialize: function() { - - this.notifications = this.$('#notification-area').calculateHeight().height(0); - this.selections = this.$('#selection-area').calculateHeight().height(0); - - var totalPoints = 100; - var data = []; - - function getRandomData() { - if (data.length > 0) - data = data.slice(1); - - // do a random walk - while (data.length < totalPoints) { - var prev = data.length > 0 ? data[data.length - 1] : 50; - var y = prev + Math.random() * 10 - 5; - if (y < 0) - y = 0; - if (y > 100) - y = 100; - data.push(y); +define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus', 'flot'], + function($, _, Backbone, App, ServerStatus) { + // Renders the header with all information + return Backbone.View.extend({ + + el: 'header', + + events: { + 'click i.iconf-list': 'toggle_taskList', + 'click .popover .close': 'hide_taskList', + 'click .btn-grabber': 'open_grabber' + }, + + templateStatus: _.compile($('#template-header-status').html()), + + // Will hold the link grabber + grabber: null, + notifications: null, + ws: null, + + // Status model + status: null, + + initialize: function() { + this.notifications = this.$('#notification-area').calculateHeight().height(0); + + this.status = new ServerStatus(); + this.listenTo(this.status, 'change', this.render); + + // TODO: button to start stop refresh + var ws = App.openWebSocket('/async'); + ws.onopen = function() { + ws.send(JSON.stringify('start')); + }; + // TODO compare with polling + ws.onmessage = _.bind(this.onData, this); + + this.ws = ws; + + this.initGraph(); + }, + + initGraph: function() { + var totalPoints = 100; + var data = []; + + function getRandomData() { + if (data.length > 0) + data = data.slice(1); + + // do a random walk + while (data.length < totalPoints) { + var prev = data.length > 0 ? data[data.length - 1] : 50; + var y = prev + Math.random() * 10 - 5; + if (y < 0) + y = 0; + if (y > 100) + y = 100; + data.push(y); + } + + // zip the generated y values with the x values + var res = []; + for (var i = 0; i < data.length; ++i) + res.push([i, data[i]]) + return res; } - // zip the generated y values with the x values - var res = []; - for (var i = 0; i < data.length; ++i) - res.push([i, data[i]]) - return res; - } - - var updateInterval = 1500; - - var speedgraph = $.plot(this.$el.find("#speedgraph"), [getRandomData()], { - series: { - lines: { show: true, lineWidth: 2 }, - shadowSize: 0, - color: "#fee247" - }, - xaxis: { ticks: [], mode: "time" }, - yaxis: { ticks: [], min: 0, autoscaleMargin: 0.1 }, - grid: { - show: true, + var updateInterval = 1500; + + var speedgraph = $.plot(this.$el.find("#speedgraph"), [getRandomData()], { + series: { + lines: { show: true, lineWidth: 2 }, + shadowSize: 0, + color: "#fee247" + }, + xaxis: { ticks: [], mode: "time" }, + yaxis: { ticks: [], min: 0, autoscaleMargin: 0.1 }, + grid: { + show: true, // borderColor: "#757575", - borderColor: "white", - borderWidth: 1, - labelMargin: 0, - axisMargin: 0, - minBorderMargin: 0 + borderColor: "white", + borderWidth: 1, + labelMargin: 0, + axisMargin: 0, + minBorderMargin: 0 + } + }); + + function update() { + speedgraph.setData([ getRandomData() ]); + // since the axes don't change, we don't need to call plot.setupGrid() + speedgraph.draw(); + + setTimeout(update, updateInterval); } - }); - function update() { - speedgraph.setData([ getRandomData() ]); - // since the axes don't change, we don't need to call plot.setupGrid() - speedgraph.draw(); +// update(); - setTimeout(update, updateInterval); - } + }, + + render: function() { +// console.log('Render header'); + + this.$('.status-block').html( + this.templateStatus(this.status.toJSON()) + ); + }, + + toggle_taskList: function() { + this.$('.popover').animate({opacity: 'toggle'}); + }, - update(); + hide_taskList: function() { + this.$('.popover').fadeOut(); + }, - }, + open_grabber: function() { + var self = this; + _.requireOnce(['views/linkGrabberModal'], function(modalView) { + if (self.grabber === null) + self.grabber = new modalView(); - render: function() { - }, + self.grabber.show(); + }); + }, - toggle_taskList: function() { - this.$('.popover').animate({opacity: 'toggle'}); - }, + onData: function(evt) { + var data = JSON.parse(evt.data); + if (data === null) return; - hide_taskList: function() { - this.$('.popover').fadeOut(); - }, + if (data['@class'] === "ServerStatus") { + this.status.set(data); + } + else if (data['@class'] === 'progress') + this.onProgressUpdate(data); + else if (data['@class'] === 'event') + this.onEvent(data); + else + console.log('Unknown Async input'); + + }, + + onProgressUpdate: function(progress) { - open_grabber: function() { - var self = this; - _.requireOnce(['views/linkGrabberModal'], function(modalView) { - if (self.grabber === null) - self.grabber = new modalView(); + }, + + onEvent: function(event) { + + } - self.grabber.show(); - }); - } - }); -});
\ No newline at end of file + }); + });
\ No newline at end of file diff --git a/module/web/static/js/views/packageView.js b/module/web/static/js/views/packageView.js index cfd671611..534fe2ad4 100644 --- a/module/web/static/js/views/packageView.js +++ b/module/web/static/js/views/packageView.js @@ -43,10 +43,7 @@ define(['jquery', 'app', 'views/abstract/itemView', 'underscore'], }, unrender: function() { - var self = this; - this.$el.slideUp(function() { - self.destroy(); - }); + itemView.prototype.unrender.apply(this); // TODO: display other package App.vent.trigger('dashboard:loading', null); diff --git a/module/web/static/js/views/selectionView.js b/module/web/static/js/views/selectionView.js index 2237c5f92..480b7127b 100644 --- a/module/web/static/js/views/selectionView.js +++ b/module/web/static/js/views/selectionView.js @@ -19,6 +19,8 @@ define(['jquery', 'backbone', 'underscore', 'app'], current: 0, initialize: function() { + this.$el.calculateHeight().height(0); + var render = _.bind(this.render, this); App.vent.on('dashboard:updated', render); @@ -69,8 +71,8 @@ define(['jquery', 'backbone', 'underscore', 'app'], this.current = files + packs; }, - // Deselects all items, optional only files - deselect: function(filesOnly) { + // Deselects all items + deselect: function() { this.get_files().map(function(file) { file.set('selected', false); }); @@ -90,6 +92,7 @@ define(['jquery', 'backbone', 'underscore', 'app'], }, trash: function() { + // TODO: delete many at once, check if package is parent this.get_files().map(function(file) { file.destroy(); }); diff --git a/module/web/templates/default/base.html b/module/web/templates/default/base.html index 621059c8c..e8661cbbc 100644 --- a/module/web/templates/default/base.html +++ b/module/web/templates/default/base.html @@ -21,6 +21,9 @@ <script src="/static/js/libs/less-1.3.0.min.js" type="text/javascript"></script>
<script type="text/javascript" data-main="static/js/config" src="/static/js/libs/require-2.1.5.js"></script>
<script>
+ window.wsAddress = "{{ ws }}";
+ window.pathPrefix = ""; // TODO
+
require(['default'], function(App) {
App.init();
{% block require %}
@@ -28,6 +31,13 @@ });
</script>
+ <script type="text/template" id="template-header-status">
+ <span class="pull-right eta"><% formatTime files.eta %></span><br>
+ <span class="pull-right remeaning"><% formatSize files.sizeleft %></span><br>
+ <span class="pull-right"><span
+ style="font-weight:bold;color: #fff !important;"><% files.linksleft %></span> of <% files.linkstotal %></span>
+ </script>
+
{% block head %}
{% endblock %}
</head>
@@ -67,16 +77,11 @@ <div id="speedgraph" class="visible-desktop"></div>
- <div class="header_block right-border">
- <span class="pull-right">8:15:01</span><br>
- <span class="pull-right">Started</span><br>
- <span class="pull-right"><span
- style="font-weight:bold;color: #fff !important;">5</span> of 12</span>
-
+ <div class="header_block right-border status-block">
</div>
<div class="header_block left-border">
- <i class="icon-time icon-white"></i> Remaining:<br>
- <i class="icon-retweet icon-white"></i> Status:<br>
+ <i class="icon-time icon-white"></i> approx. ETA :<br>
+ <i class=" icon-hdd icon-white"></i> Remeaning:<br>
<i class="icon-download-alt icon-white"></i> Downloads: <br>
</div>
diff --git a/module/web/templates/default/dashboard.html b/module/web/templates/default/dashboard.html index 8c20973e4..7fe9c9635 100644 --- a/module/web/templates/default/dashboard.html +++ b/module/web/templates/default/dashboard.html @@ -177,6 +177,7 @@ <div class="sidebar-header">
<i class="iconf-hdd"></i> Local
<div class="pull-right" style="font-size: medium; line-height: normal">
+{# <input type="text" class="input">#}
<i class="iconf-chevron-down" style="font-size: 20px"></i>
</div>
<div class="clearfix"></div>
diff --git a/module/web/webinterface.py b/module/web/webinterface.py index cec0f24a4..f18157cd7 100644 --- a/module/web/webinterface.py +++ b/module/web/webinterface.py @@ -113,7 +113,8 @@ session_opts = { 'session.auto': False
}
-web = StripPathMiddleware(SessionMiddleware(app(), session_opts))
+session = SessionMiddleware(app(), session_opts)
+web = StripPathMiddleware(session)
web = GZipMiddleWare(web)
if PREFIX:
|