From 6f48bf541b7b0eeb0c6968ca63125bf73e016643 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 12 May 2015 03:15:13 +0200 Subject: Fix some directory names --- docs/access_api.rst | 8 +- docs/module_overview.rst | 2 +- docs/write_addons.rst | 8 +- pyload/Api/__init__.py | 1051 +++++++++++++++++++++++++++++++++++ pyload/Api/types.py | 562 +++++++++++++++++++ pyload/Core.py | 4 +- pyload/Database/Backend.py | 329 +++++++++++ pyload/Database/File.py | 985 ++++++++++++++++++++++++++++++++ pyload/Database/Storage.py | 36 ++ pyload/Database/User.py | 93 ++++ pyload/Database/__init__.py | 6 + pyload/Datatype/File.py | 299 ++++++++++ pyload/Datatype/Package.py | 73 +++ pyload/Datatype/__init__.py | 4 + pyload/Thread/Info.py | 2 +- pyload/Thread/Plugin.py | 2 +- pyload/Thread/Server.py | 2 +- pyload/api/__init__.py | 1051 ----------------------------------- pyload/api/types.py | 562 ------------------- pyload/database/Backend.py | 329 ----------- pyload/database/File.py | 985 -------------------------------- pyload/database/Storage.py | 36 -- pyload/database/User.py | 93 ---- pyload/database/__init__.py | 6 - pyload/datatype/File.py | 299 ---------- pyload/datatype/Package.py | 73 --- pyload/datatype/__init__.py | 4 - pyload/plugin/addon/IRCInterface.py | 2 +- pyload/webui/app/api.py | 2 +- pyload/webui/app/utils.py | 2 +- 30 files changed, 3455 insertions(+), 3455 deletions(-) create mode 100644 pyload/Api/__init__.py create mode 100644 pyload/Api/types.py create mode 100644 pyload/Database/Backend.py create mode 100644 pyload/Database/File.py create mode 100644 pyload/Database/Storage.py create mode 100644 pyload/Database/User.py create mode 100644 pyload/Database/__init__.py create mode 100644 pyload/Datatype/File.py create mode 100644 pyload/Datatype/Package.py create mode 100644 pyload/Datatype/__init__.py delete mode 100644 pyload/api/__init__.py delete mode 100644 pyload/api/types.py delete mode 100644 pyload/database/Backend.py delete mode 100644 pyload/database/File.py delete mode 100644 pyload/database/Storage.py delete mode 100644 pyload/database/User.py delete mode 100644 pyload/database/__init__.py delete mode 100644 pyload/datatype/File.py delete mode 100644 pyload/datatype/Package.py delete mode 100644 pyload/datatype/__init__.py diff --git a/docs/access_api.rst b/docs/access_api.rst index 938b73d04..1e19ea71b 100644 --- a/docs/access_api.rst +++ b/docs/access_api.rst @@ -13,12 +13,12 @@ First of all, you need to know what you can do with our API. It lets you do all retrieving download status, manage queue, manage accounts, modify config and so on. This document is not intended to explain every function in detail, for a complete listing -see :class:`Api `. +see :class:`Api `. Of course its possible to access the ``core.api`` attribute in plugins and addons, but much more interesting is the possibillity to call function from different programs written in many different languages. -pyLoad uses thrift as backend and provides its :class:`Api ` as service. +pyLoad uses thrift as backend and provides its :class:`Api ` as service. More information about thrift can be found here http://wiki.apache.org/thrift/. @@ -92,7 +92,7 @@ so pyLoad can authenticate you. Calling Methods =============== -In general you can use any method listed at the :class:`Api ` documentation, which is also available to +In general you can use any method listed at the :class:`Api ` documentation, which is also available to the thriftbackend. Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address @@ -107,7 +107,7 @@ Passing parameters To pass arguments you have two choices. Either use positional arguments, eg ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword arguments -supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names in the :class:`Api ` +supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names in the :class:`Api ` documentation. It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because diff --git a/docs/module_overview.rst b/docs/module_overview.rst index a69aa9d9d..c6f19d7a0 100644 --- a/docs/module_overview.rst +++ b/docs/module_overview.rst @@ -8,7 +8,7 @@ You can find an overview of some important classes here: .. autosummary:: :toctree: pyload - pyload.api.Api + pyload.Api.Api pyload.plugin.Plugin.Base pyload.plugin.Plugin.Plugin pyload.plugin.Crypter.Crypter diff --git a/docs/write_addons.rst b/docs/write_addons.rst index 9b94a0233..6fc77abee 100644 --- a/docs/write_addons.rst +++ b/docs/write_addons.rst @@ -6,7 +6,7 @@ Addons A Addon is a python file which is located at :file:`pyload/plugin/addon`. The :class:`AddonManager ` will load it automatically on startup. Only one instance exists over the complete lifetime of pyload. Your addon can interact on various events called by the :class:`AddonManager `, -do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. +do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. The UpdateManager, CaptchaTrader, UnRar and many more are realised as addon. Addon header @@ -113,7 +113,7 @@ available as event and not accessible through overwriting addon methods. However Providing RPC services ---------------------- -You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. +You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. So probably clients want to be able to interact with your addon to request it's state or invoke some action. Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: @@ -130,7 +130,7 @@ Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: def invoke(self, arg): print "Invoked with", arg -Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. +Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. Here is a basic example: :: # Assuming client is a ThriftClient or Api object @@ -140,7 +140,7 @@ Here is a basic example: :: Providing status information ---------------------------- -Your addon can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. +Your addon can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. Just store everything in ``self.info``. :: diff --git a/pyload/Api/__init__.py b/pyload/Api/__init__.py new file mode 100644 index 000000000..44bae48bc --- /dev/null +++ b/pyload/Api/__init__.py @@ -0,0 +1,1051 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from __future__ import with_statement + +import base64 +import os +import re +import time +import urlparse + +from pyload.Datatype import PyFile +from pyload.utils.packagetools import parseNames +from pyload.network.RequestFactory import getURL +from pyload.remote import activated +from pyload.utils import compare_time, freeSpace, safe_filename + +if activated: + try: + from thrift.protocol import TBase + from pyload.remote.thriftbackend.thriftgen.pyload.ttypes import * + from pyload.remote.thriftbackend.thriftgen.pyload.Pyload import Iface + + BaseObject = TBase + + except ImportError: + from pyload.Api.types import * + + print "Thrift not imported" + +else: + from pyload.Api.types import * + +# contains function names mapped to their permissions +# unlisted functions are for admins only +permMap = {} + + +# decorator only called on init, never initialized, so has no effect on runtime +def permission(bits): + class _Dec(object): + + def __new__(cls, func, *args, **kwargs): + permMap[func.__name__] = bits + return func + return _Dec + + +urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&\[\]\|]*)", re.IGNORECASE) + + +class PERMS(object): + ALL = 0 #: requires no permission, but login + ADD = 1 #: can add packages + DELETE = 2 #: can delete packages + STATUS = 4 #: see and change server status + LIST = 16 #: see queue and collector + MODIFY = 32 #: moddify some attribute of downloads + DOWNLOAD = 64 #: can download from webinterface + SETTINGS = 128 #: can access settings + ACCOUNTS = 256 #: can access accounts + LOGS = 512 #: can see server logs + + +class ROLE(object): + ADMIN = 0 #: admin has all permissions implicit + USER = 1 + + +def has_permission(userperms, perms): + # bytewise or perms before if needed + return perms == (userperms & perms) + + +class Api(Iface): + """ + **pyLoads API** + + This is accessible either internal via core.api or via thrift backend. + + see Thrift specification file remote/thriftbackend/pyload.thrift\ + for information about data structures and what methods are usuable with rpc. + + Most methods requires specific permissions, please look at the source code if you need to know.\ + These can be configured via webinterface. + Admin user have all permissions, and are the only ones who can access the methods with no specific permission. + """ + EXTERNAL = Iface #: let the json api know which methods are external + + + def __init__(self, core): + self.core = core + + + def _convertPyFile(self, p): + fdata = FileData(p['id'], p['url'], p['name'], p['plugin'], p['size'], + p['format_size'], p['status'], p['statusmsg'], + p['package'], p['error'], p['order']) + return fdata + + + def _convertConfigFormat(self, c): + sections = {} + for sectionName, sub in c.iteritems(): + section = ConfigSection(sectionName, sub['desc']) + items = [] + for key, data in sub.iteritems(): + if key in ("desc", "outline"): + continue + item = ConfigItem() + item.name = key + item.description = data['desc'] + item.value = str(data['value']) if not isinstance(data['value'], basestring) else data['value'] + item.type = data['type'] + items.append(item) + section.items = items + sections[sectionName] = section + if "outline" in sub: + section.outline = sub['outline'] + return sections + + + @permission(PERMS.SETTINGS) + def getConfigValue(self, category, option, section="core"): + """Retrieve config value. + + :param category: name of category, or plugin + :param option: config option + :param section: 'plugin' or 'core' + :return: config value as string + """ + if section == "core": + value = self.core.config[category][option] + else: + value = self.core.config.getPlugin(category, option) + return str(value) + + + @permission(PERMS.SETTINGS) + def setConfigValue(self, category, option, value, section="core"): + """Set new config value. + + :param section: + :param option: + :param value: new config value + :param section: 'plugin' or 'core + """ + self.core.addonManager.dispatchEvent("config-changed", category, option, value, section) + if section == "core": + self.core.config[category][option] = value + if option in ("limit_speed", "max_speed"): #: not so nice to update the limit + self.core.requestFactory.updateBucket() + elif section == "plugin": + self.core.config.setPlugin(category, option, value) + + + @permission(PERMS.SETTINGS) + def getConfig(self): + """Retrieves complete config of core. + + :return: list of `ConfigSection` + """ + return self._convertConfigFormat(self.core.config.config) + + + def getConfigDict(self): + """Retrieves complete config in dict format, not for RPC. + + :return: dict + """ + return self.core.config.config + + + @permission(PERMS.SETTINGS) + def getPluginConfig(self): + """Retrieves complete config for all plugins. + + :return: list of `ConfigSection` + """ + return self._convertConfigFormat(self.core.config.plugin) + + + def getPluginConfigDict(self): + """Plugin config as dict, not for RPC. + + :return: dict + """ + return self.core.config.plugin + + + @permission(PERMS.STATUS) + def pauseServer(self): + """Pause server: Tt wont start any new downloads, but nothing gets aborted.""" + self.core.threadManager.pause = True + + + @permission(PERMS.STATUS) + def unpauseServer(self): + """Unpause server: New Downloads will be started.""" + self.core.threadManager.pause = False + + + @permission(PERMS.STATUS) + def togglePause(self): + """Toggle pause state. + + :return: new pause state + """ + self.core.threadManager.pause ^= True + return self.core.threadManager.pause + + + @permission(PERMS.STATUS) + def toggleReconnect(self): + """Toggle reconnect activation. + + :return: new reconnect state + """ + self.core.config['reconnect']['activated'] ^= True + return self.core.config.get("reconnect", "activated") + + + @permission(PERMS.LIST) + def statusServer(self): + """Some general information about the current status of pyLoad. + + :return: `ServerStatus` + """ + serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), + self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, + not self.core.threadManager.pause and self.isTimeDownload(), + self.core.config.get("reconnect", "activated") and self.isTimeReconnect()) + for pyfile in [x.active for x in self.core.threadManager.threads if x.active and isinstance(x.active, PyFile)]: + serverStatus.speed += pyfile.getSpeed() #: bytes/s + return serverStatus + + + @permission(PERMS.STATUS) + def freeSpace(self): + """Available free space at download directory in bytes""" + return freeSpace(self.core.config.get("general", "download_folder")) + + + @permission(PERMS.ALL) + def getServerVersion(self): + """pyLoad Core version """ + return self.core.version + + + def kill(self): + """Clean way to quit pyLoad""" + self.core.do_kill = True + + + def restart(self): + """Restart pyload core""" + self.core.do_restart = True + + + @permission(PERMS.LOGS) + def getLog(self, offset=0): + """Returns most recent log entries. + + :param offset: line offset + :return: List of log entries + """ + filename = os.path.join(self.core.config.get("log", "log_folder"), 'log.txt') + try: + with open(filename, "r") as fh: + lines = fh.readlines() + if offset >= len(lines): + return [] + return lines[offset:] + except Exception: + return ['No log available'] + + + @permission(PERMS.STATUS) + def isTimeDownload(self): + """Checks if pyload will start new downloads according to time in config. + + :return: bool + """ + start = self.core.config.get("downloadTime", "start").split(":") + end = self.core.config.get("downloadTime", "end").split(":") + return compare_time(start, end) + + + @permission(PERMS.STATUS) + def isTimeReconnect(self): + """Checks if pyload will try to make a reconnect + + :return: bool + """ + start = self.core.config.get("reconnect", "startTime").split(":") + end = self.core.config.get("reconnect", "endTime").split(":") + return compare_time(start, end) and self.core.config.get("reconnect", "activated") + + + @permission(PERMS.LIST) + def statusDownloads(self): + """ Status off all currently running downloads. + + :return: list of `DownloadStatus` + """ + data = [] + for pyfile in self.core.threadManager.getActiveFiles(): + if not isinstance(pyfile, PyFile): + continue + data.append(DownloadInfo( + pyfile.id, pyfile.name, pyfile.getSpeed(), pyfile.getETA(), pyfile.formatETA(), + pyfile.getBytesLeft(), pyfile.getSize(), pyfile.formatSize(), pyfile.getPercent(), + pyfile.status, pyfile.getStatusName(), pyfile.formatWait(), + pyfile.waitUntil, pyfile.packageid, pyfile.package().name, pyfile.pluginname)) + return data + + + @permission(PERMS.ADD) + def addPackage(self, name, links, dest=Destination.Queue): + """Adds a package, with links to desired destination. + + :param name: name of the new package + :param links: list of urls + :param dest: `Destination` + :return: package id of the new package + """ + if self.core.config.get("general", "folder_per_package"): + folder = urlparse.urlparse(name).path.split("/")[-1] + else: + folder = "" + + folder = safe_filename(folder) + + pid = self.core.files.addPackage(name, folder, dest) + + self.core.files.addLinks(links, pid) + + self.core.log.info(_("Added package %(name)s containing %(count)d links") % {"name": name.decode('utf-8'), "count": len(links)}) + + self.core.files.save() + + return pid + + + @permission(PERMS.ADD) + def parseURLs(self, html=None, url=None): + """Parses html content or any arbitaty text for links and returns result of `checkURLs` + + :param html: html source + :return: + """ + urls = [] + if html: + urls += [x[0] for x in urlmatcher.findall(html)] + if url: + page = getURL(url) + urls += [x[0] for x in urlmatcher.findall(page)] + # remove duplicates + return self.checkURLs(set(urls)) + + + @permission(PERMS.ADD) + def checkURLs(self, urls): + """ Gets urls and returns pluginname mapped to list of matches urls. + + :param urls: + :return: {plugin: urls} + """ + data = self.core.pluginManager.parseUrls(urls) + plugins = {} + + for url, plugintype, pluginname in data: + try: + plugins[plugintype][pluginname].append(url) + except Exception: + plugins[plugintype][pluginname] = [url] + + return plugins + + + @permission(PERMS.ADD) + def checkOnlineStatus(self, urls): + """ initiates online status check + + :param urls: + :return: initial set of data as `OnlineCheck` instance containing the result id + """ + data = self.core.pluginManager.parseUrls(urls) + + rid = self.core.threadManager.createResultThread(data, False) + + tmp = [(url, (url, OnlineStatus(url, (plugintype, pluginname), "unknown", 3, 0))) for url, plugintype, pluginname in data] + data = parseNames(tmp) + result = {} + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + return OnlineCheck(rid, result) + + + @permission(PERMS.ADD) + def checkOnlineStatusContainer(self, urls, container, data): + """ checks online status of urls and a submited container file + + :param urls: list of urls + :param container: container file name + :param data: file content + :return: online check + """ + with open(os.path.join(self.core.config.get("general", "download_folder"), "tmp_" + container), "wb") as th: + th.write(str(data)) + return self.checkOnlineStatus(urls + [th.name]) + + + @permission(PERMS.ADD) + def pollResults(self, rid): + """ Polls the result available for ResultID + + :param rid: `ResultID` + :return: `OnlineCheck`, if rid is -1 then no more data available + """ + result = self.core.threadManager.getInfoResult(rid) + if "ALL_INFO_FETCHED" in result: + del result['ALL_INFO_FETCHED'] + return OnlineCheck(-1, result) + else: + return OnlineCheck(rid, result) + + + @permission(PERMS.ADD) + def generatePackages(self, links): + """ Parses links, generates packages names from urls + + :param links: list of urls + :return: package names mapped to urls + """ + return parseNames((x, x) for x in links) + + + @permission(PERMS.ADD) + def generateAndAddPackages(self, links, dest=Destination.Queue): + """Generates and add packages + + :param links: list of urls + :param dest: `Destination` + :return: list of package ids + """ + return [self.addPackage(name, urls, dest) for name, urls + in self.generatePackages(links).iteritems()] + + + @permission(PERMS.ADD) + def checkAndAddPackages(self, links, dest=Destination.Queue): + """Checks online status, retrieves names, and will add packages.\ + Because of this packages are not added immediatly, only for internal use. + + :param links: list of urls + :param dest: `Destination` + :return: None + """ + data = self.core.pluginManager.parseUrls(links) + self.core.threadManager.createResultThread(data, True) + + + @permission(PERMS.LIST) + def getPackageData(self, pid): + """Returns complete information about package, and included files. + + :param pid: package id + :return: `PackageData` with .links attribute + """ + data = self.core.files.getPackageData(int(pid)) + if not data: + raise PackageDoesNotExists(pid) + return PackageData(data['id'], data['name'], data['folder'], data['site'], data['password'], + data['queue'], data['order'], + links=[self._convertPyFile(x) for x in data['links'].itervalues()]) + + + @permission(PERMS.LIST) + def getPackageInfo(self, pid): + """Returns information about package, without detailed information about containing files + + :param pid: package id + :return: `PackageData` with .fid attribute + """ + data = self.core.files.getPackageData(int(pid)) + + if not data: + raise PackageDoesNotExists(pid) + return PackageData(data['id'], data['name'], data['folder'], data['site'], data['password'], + data['queue'], data['order'], + fids=[int(x) for x in data['links']]) + + + @permission(PERMS.LIST) + def getFileData(self, fid): + """Get complete information about a specific file. + + :param fid: file id + :return: `FileData` + """ + info = self.core.files.getFileData(int(fid)) + if not info: + raise FileDoesNotExists(fid) + return self._convertPyFile(info.values()[0]) + + + @permission(PERMS.DELETE) + def deleteFiles(self, fids): + """Deletes several file entries from pyload. + + :param fids: list of file ids + """ + for fid in fids: + self.core.files.deleteLink(int(fid)) + self.core.files.save() + + + @permission(PERMS.DELETE) + def deletePackages(self, pids): + """Deletes packages and containing links. + + :param pids: list of package ids + """ + for pid in pids: + self.core.files.deletePackage(int(pid)) + self.core.files.save() + + + @permission(PERMS.LIST) + def getQueue(self): + """Returns info about queue and packages, **not** about files, see `getQueueData` \ + or `getPackageData` instead. + + :return: list of `PackageInfo` + """ + return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], + pack['password'], pack['queue'], pack['order'], + pack['linksdone'], pack['sizedone'], pack['sizetotal'], + pack['linkstotal']) + for pack in self.core.files.getInfoData(Destination.Queue).itervalues()] + + + @permission(PERMS.LIST) + def getQueueData(self): + """Return complete data about everything in queue, this is very expensive use it sparely.\ + See `getQueue` for alternative. + + :return: list of `PackageData` + """ + return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], + pack['password'], pack['queue'], pack['order'], + pack['linksdone'], pack['sizedone'], pack['sizetotal'], + links=[self._convertPyFile(x) for x in pack['links'].itervalues()]) + for pack in self.core.files.getCompleteData(Destination.Queue).itervalues()] + + + @permission(PERMS.LIST) + def getCollector(self): + """same as `getQueue` for collector. + + :return: list of `PackageInfo` + """ + return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], + pack['password'], pack['queue'], pack['order'], + pack['linksdone'], pack['sizedone'], pack['sizetotal'], + pack['linkstotal']) + for pack in self.core.files.getInfoData(Destination.Collector).itervalues()] + + + @permission(PERMS.LIST) + def getCollectorData(self): + """same as `getQueueData` for collector. + + :return: list of `PackageInfo` + """ + return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], + pack['password'], pack['queue'], pack['order'], + pack['linksdone'], pack['sizedone'], pack['sizetotal'], + links=[self._convertPyFile(x) for x in pack['links'].itervalues()]) + for pack in self.core.files.getCompleteData(Destination.Collector).itervalues()] + + + @permission(PERMS.ADD) + def addFiles(self, pid, links): + """Adds files to specific package. + + :param pid: package id + :param links: list of urls + """ + self.core.files.addLinks(links, int(pid)) + self.core.log.info(_("Added %(count)d links to package #%(package)d ") % {"count": len(links), "package": pid}) + self.core.files.save() + + + @permission(PERMS.MODIFY) + def pushToQueue(self, pid): + """Moves package from Collector to Queue. + + :param pid: package id + """ + self.core.files.setPackageLocation(pid, Destination.Queue) + + + @permission(PERMS.MODIFY) + def pullFromQueue(self, pid): + """Moves package from Queue to Collector. + + :param pid: package id + """ + self.core.files.setPackageLocation(pid, Destination.Collector) + + + @permission(PERMS.MODIFY) + def restartPackage(self, pid): + """Restarts a package, resets every containing files. + + :param pid: package id + """ + self.core.files.restartPackage(int(pid)) + + + @permission(PERMS.MODIFY) + def restartFile(self, fid): + """Resets file status, so it will be downloaded again. + + :param fid: file id + """ + self.core.files.restartFile(int(fid)) + + + @permission(PERMS.MODIFY) + def recheckPackage(self, pid): + """Proofes online status of all files in a package, also a default action when package is added. + + :param pid: + :return: + """ + self.core.files.reCheckPackage(int(pid)) + + + @permission(PERMS.MODIFY) + def stopAllDownloads(self): + """Aborts all running downloads.""" + + pyfiles = self.core.files.cache.values() + for pyfile in pyfiles: + pyfile.abortDownload() + + + @permission(PERMS.MODIFY) + def stopDownloads(self, fids): + """Aborts specific downloads. + + :param fids: list of file ids + :return: + """ + pyfiles = self.core.files.cache.values() + for pyfile in pyfiles: + if pyfile.id in fids: + pyfile.abortDownload() + + + @permission(PERMS.MODIFY) + def setPackageName(self, pid, name): + """Renames a package. + + :param pid: package id + :param name: new package name + """ + pack = self.core.files.getPackage(pid) + pack.name = name + pack.sync() + + + @permission(PERMS.MODIFY) + def movePackage(self, destination, pid): + """Set a new package location. + + :param destination: `Destination` + :param pid: package id + """ + if destination in (0, 1): + self.core.files.setPackageLocation(pid, destination) + + + @permission(PERMS.MODIFY) + def moveFiles(self, fids, pid): + """Move multiple files to another package + + :param fids: list of file ids + :param pid: destination package + :return: + """ + # TODO: implement + pass + + + @permission(PERMS.ADD) + def uploadContainer(self, filename, data): + """Uploads and adds a container file to pyLoad. + + :param filename: filename, extension is important so it can correctly decrypted + :param data: file content + """ + with open(os.path.join(self.core.config.get("general", "download_folder"), "tmp_" + filename), "wb") as th: + th.write(str(data)) + self.addPackage(th.name, [th.name], Destination.Queue) + + + @permission(PERMS.MODIFY) + def orderPackage(self, pid, position): + """Gives a package a new position. + + :param pid: package id + :param position: + """ + self.core.files.reorderPackage(pid, position) + + + @permission(PERMS.MODIFY) + def orderFile(self, fid, position): + """Gives a new position to a file within its package. + + :param fid: file id + :param position: + """ + self.core.files.reorderFile(fid, position) + + + @permission(PERMS.MODIFY) + def setPackageData(self, pid, data): + """Allows to modify several package attributes. + + :param pid: package id + :param data: dict that maps attribute to desired value + """ + package = self.core.files.getPackage(pid) + if not package: + raise PackageDoesNotExists(pid) + for key, value in data.iteritems(): + if key == "id": + continue + setattr(package, key, value) + package.sync() + self.core.files.save() + + + @permission(PERMS.DELETE) + def deleteFinished(self): + """Deletes all finished files and completly finished packages. + + :return: list of deleted package ids + """ + return self.core.files.deleteFinishedLinks() + + + @permission(PERMS.MODIFY) + def restartFailed(self): + """Restarts all failed failes.""" + self.core.files.restartFailed() + + + @permission(PERMS.LIST) + def getPackageOrder(self, destination): + """Returns information about package order. + + :param destination: `Destination` + :return: dict mapping order to package id + """ + packs = self.core.files.getInfoData(destination) + order = {} + for pid in packs: + pack = self.core.files.getPackageData(int(pid)) + while pack['order'] in order.keys(): #: just in case + pack['order'] += 1 + order[pack['order']] = pack['id'] + return order + + + @permission(PERMS.LIST) + def getFileOrder(self, pid): + """Information about file order within package. + + :param pid: + :return: dict mapping order to file id + """ + rawdata = self.core.files.getPackageData(int(pid)) + order = {} + for id, pyfile in rawdata['links'].iteritems(): + while pyfile['order'] in order.keys(): #: just in case + pyfile['order'] += 1 + order[pyfile['order']] = pyfile['id'] + return order + + + @permission(PERMS.STATUS) + def isCaptchaWaiting(self): + """Indicates wether a captcha task is available + + :return: bool + """ + self.core.lastClientConnected = time.time() + task = self.core.captchaManager.getTask() + return not task is None + + + @permission(PERMS.STATUS) + def getCaptchaTask(self, exclusive=False): + """Returns a captcha task + + :param exclusive: unused + :return: `CaptchaTask` + """ + self.core.lastClientConnected = time.time() + task = self.core.captchaManager.getTask() + if task: + task.setWatingForUser(exclusive=exclusive) + data, type, result = task.getCaptcha() + ctask = CaptchaTask(int(task.id), base64.standard_b64encode(data), type, result) + return ctask + return CaptchaTask(-1) + + + @permission(PERMS.STATUS) + def getCaptchaTaskStatus(self, tid): + """Get information about captcha task + + :param tid: task id + :return: string + """ + self.core.lastClientConnected = time.time() + task = self.core.captchaManager.getTaskByID(tid) + return task.getStatus() if task else "" + + + @permission(PERMS.STATUS) + def setCaptchaResult(self, tid, result): + """Set result for a captcha task + + :param tid: task id + :param result: captcha result + """ + self.core.lastClientConnected = time.time() + task = self.core.captchaManager.getTaskByID(tid) + if task: + task.setResult(result) + self.core.captchaManager.removeTask(task) + + + @permission(PERMS.STATUS) + def getEvents(self, uuid): + """Lists occured events, may be affected to changes in future. + + :param uuid: + :return: list of `Events` + """ + events = self.core.pullManager.getEvents(uuid) + new_events = [] + + + def convDest(d): + return Destination.Queue if d == "queue" else Destination.Collector + + for e in events: + event = EventInfo() + event.eventname = e[0] + if e[0] in ("update", "remove", "insert"): + event.id = e[3] + event.type = ElementType.Package if e[2] == "pack" else ElementType.File + event.destination = convDest(e[1]) + elif e[0] == "order": + if e[1]: + event.id = e[1] + event.type = ElementType.Package if e[2] == "pack" else ElementType.File + event.destination = convDest(e[3]) + elif e[0] == "reload": + event.destination = convDest(e[1]) + new_events.append(event) + return new_events + + + @permission(PERMS.ACCOUNTS) + def getAccounts(self, refresh): + """Get information about all entered accounts. + + :param refresh: reload account info + :return: list of `AccountInfo` + """ + accs = self.core.accountManager.getAccountInfos(False, refresh) + for group in accs.values(): + accounts = [AccountInfo(acc['validuntil'], acc['login'], acc['options'], acc['valid'], + acc['trafficleft'], acc['maxtraffic'], acc['premium'], acc['type']) + for acc in group] + return accounts or list() + + + @permission(PERMS.ALL) + def getAccountTypes(self): + """All available account types. + + :return: list + """ + return self.core.accountManager.accounts.keys() + + + @permission(PERMS.ACCOUNTS) + def updateAccount(self, plugin, account, password=None, options={}): + """Changes pw/options for specific account.""" + self.core.accountManager.updateAccount(plugin, account, password, options) + + + @permission(PERMS.ACCOUNTS) + def removeAccount(self, plugin, account): + """Remove account from pyload. + + :param plugin: pluginname + :param account: accountname + """ + self.core.accountManager.removeAccount(plugin, account) + + + @permission(PERMS.ALL) + def login(self, username, password, remoteip=None): + """Login into pyLoad, this **must** be called when using rpc before any methods can be used. + + :param username: + :param password: + :param remoteip: Omit this argument, its only used internal + :return: bool indicating login was successful + """ + return bool(self.checkAuth(username, password, remoteip)) + + + def checkAuth(self, username, password, remoteip=None): + """Check authentication and returns details + + :param username: + :param password: + :param remoteip: + :return: dict with info, empty when login is incorrect + """ + if self.core.config.get("remote", "nolocalauth") and remoteip == "127.0.0.1": + return "local" + else: + return self.core.db.checkAuth(username, password) + + + def isAuthorized(self, func, userdata): + """checks if the user is authorized for specific method + + :param func: function name + :param userdata: dictionary of user data + :return: boolean + """ + if userdata == "local" or userdata['role'] == ROLE.ADMIN: + return True + elif func in permMap and has_permission(userdata['permission'], permMap[func]): + return True + else: + return False + + + @permission(PERMS.ALL) + def getUserData(self, username, password): + """similar to `checkAuth` but returns UserData thrift type """ + user = self.checkAuth(username, password) + if user: + return UserData(user['name'], user['email'], user['role'], user['permission'], user['template']) + else: + return UserData() + + + def getAllUserData(self): + """returns all known user and info""" + return dict((user, UserData(user, data['email'], data['role'], data['permission'], data['template'])) for user, data + in self.core.db.getAllUserData().iteritems()) + + + @permission(PERMS.STATUS) + def getServices(self): + """ A dict of available services, these can be defined by addon plugins. + + :return: dict with this style: {"plugin": {"method": "description"}} + """ + return dict((plugin, funcs) for plugin, funcs in self.core.addonManager.methods.iteritems()) + + + @permission(PERMS.STATUS) + def hasService(self, plugin, func): + """Checks wether a service is available. + + :param plugin: + :param func: + :return: bool + """ + cont = self.core.addonManager.methods + return plugin in cont and func in cont[plugin] + + + @permission(PERMS.STATUS) + def call(self, info): + """Calls a service (a method in addon plugin). + + :param info: `ServiceCall` + :return: result + :raises: ServiceDoesNotExists, when its not available + :raises: ServiceException, when a exception was raised + """ + plugin = info.plugin + func = info.func + args = info.arguments + parse = info.parseArguments + if not self.hasService(plugin, func): + raise ServiceDoesNotExists(plugin, func) + try: + ret = self.core.addonManager.callRPC(plugin, func, args, parse) + except Exception, e: + raise ServiceException(e.message) + + + @permission(PERMS.STATUS) + def getAllInfo(self): + """Returns all information stored by addon plugins. Values are always strings + + :return: {"plugin": {"name": value}} + """ + return self.core.addonManager.getAllInfo() + + + @permission(PERMS.STATUS) + def getInfoByPlugin(self, plugin): + """Returns information stored by a specific plugin. + + :param plugin: pluginname + :return: dict of attr names mapped to value {"name": value} + """ + return self.core.addonManager.getInfo(plugin) + + + def changePassword(self, user, oldpw, newpw): + """ changes password for specific user """ + return self.core.db.changePassword(user, oldpw, newpw) + + + def setUserPermission(self, user, perm, role): + self.core.db.setPermission(user, perm) + self.core.db.setRole(user, role) diff --git a/pyload/Api/types.py b/pyload/Api/types.py new file mode 100644 index 000000000..9381df3c7 --- /dev/null +++ b/pyload/Api/types.py @@ -0,0 +1,562 @@ +# -*- coding: utf-8 -*- +# Autogenerated by pyload +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + +class BaseObject(object): + __slots__ = [] + + +class Destination(object): + Collector = 0 + Queue = 1 + + +class DownloadStatus(object): + Aborted = 9 + Custom = 11 + Decrypting = 10 + Downloading = 12 + Failed = 8 + Finished = 0 + Offline = 1 + Online = 2 + Processing = 13 + Queued = 3 + Skipped = 4 + Starting = 7 + TempOffline = 6 + Unknown = 14 + Waiting = 5 + + +class ElementType(object): + File = 1 + Package = 0 + + +class Input(object): + BOOL = 4 + CHOICE = 6 + CLICK = 5 + LIST = 8 + MULTIPLE = 7 + NONE = 0 + PASSWORD = 3 + TABLE = 9 + TEXT = 1 + TEXTBOX = 2 + + +class Output(object): + CAPTCHA = 1 + NOTIFICATION = 4 + QUESTION = 2 + + +class AccountInfo(BaseObject): + __slots__ = ['validuntil', 'login', 'options', 'valid', 'trafficleft', 'maxtraffic', 'premium', 'type'] + + + def __init__(self, validuntil=None, login=None, options=None, valid=None, trafficleft=None, maxtraffic=None, premium=None, type=None): + self.validuntil = validuntil + self.login = login + self.options = options + self.valid = valid + self.trafficleft = trafficleft + self.maxtraffic = maxtraffic + self.premium = premium + self.type = type + + +class CaptchaTask(BaseObject): + __slots__ = ['tid', 'data', 'type', 'resultType'] + + + def __init__(self, tid=None, data=None, type=None, resultType=None): + self.tid = tid + self.data = data + self.type = type + self.resultType = resultType + + +class ConfigItem(BaseObject): + __slots__ = ['name', 'description', 'value', 'type'] + + + def __init__(self, name=None, description=None, value=None, type=None): + self.name = name + self.description = description + self.value = value + self.type = type + + +class ConfigSection(BaseObject): + __slots__ = ['name', 'description', 'items', 'outline'] + + + def __init__(self, name=None, description=None, items=None, outline=None): + self.name = name + self.description = description + self.items = items + self.outline = outline + + +class DownloadInfo(BaseObject): + __slots__ = ['fid', 'name', 'speed', 'eta', 'format_eta', 'bleft', 'size', 'format_size', 'percent', 'status', 'statusmsg', 'format_wait', 'wait_until', 'packageID', 'packageName', 'plugin'] + + + def __init__(self, fid=None, name=None, speed=None, eta=None, format_eta=None, bleft=None, size=None, format_size=None, percent=None, status=None, statusmsg=None, format_wait=None, wait_until=None, packageID=None, packageName=None, plugin=None): + self.fid = fid + self.name = name + self.speed = speed + self.eta = eta + self.format_eta = format_eta + self.bleft = bleft + self.size = size + self.format_size = format_size + self.percent = percent + self.status = status + self.statusmsg = statusmsg + self.format_wait = format_wait + self.wait_until = wait_until + self.packageID = packageID + self.packageName = packageName + self.plugin = plugin + + +class EventInfo(BaseObject): + __slots__ = ['eventname', 'id', 'type', 'destination'] + + + def __init__(self, eventname=None, id=None, type=None, destination=None): + self.eventname = eventname + self.id = id + self.type = type + self.destination = destination + + +class FileData(BaseObject): + __slots__ = ['fid', 'url', 'name', 'plugin', 'size', 'format_size', 'status', 'statusmsg', 'packageID', 'error', 'order'] + + + def __init__(self, fid=None, url=None, name=None, plugin=None, size=None, format_size=None, status=None, statusmsg=None, packageID=None, error=None, order=None): + self.fid = fid + self.url = url + self.name = name + self.plugin = plugin + self.size = size + self.format_size = format_size + self.status = status + self.statusmsg = statusmsg + self.packageID = packageID + self.error = error + self.order = order + + +class FileDoesNotExists(Exception): + __slots__ = ['fid'] + + + def __init__(self, fid=None): + self.fid = fid + + +class InteractionTask(BaseObject): + __slots__ = ['iid', 'input', 'structure', 'preset', 'output', 'data', 'title', 'description', 'plugin'] + + + def __init__(self, iid=None, input=None, structure=None, preset=None, output=None, data=None, title=None, description=None, plugin=None): + self.iid = iid + self.input = input + self.structure = structure + self.preset = preset + self.output = output + self.data = data + self.title = title + self.description = description + self.plugin = plugin + + +class OnlineCheck(BaseObject): + __slots__ = ['rid', 'data'] + + + def __init__(self, rid=None, data=None): + self.rid = rid + self.data = data + + +class OnlineStatus(BaseObject): + __slots__ = ['name', 'plugin', 'packagename', 'status', 'size'] + + + def __init__(self, name=None, plugin=(None, None), packagename=None, status=None, size=None): + self.name = name + self.plugin = plugin + self.packagename = packagename + self.status = status + self.size = size + + +class PackageData(BaseObject): + __slots__ = ['pid', 'name', 'folder', 'site', 'password', 'dest', 'order', 'linksdone', 'sizedone', 'sizetotal', 'linkstotal', 'links', 'fids'] + + + def __init__(self, pid=None, name=None, folder=None, site=None, password=None, dest=None, order=None, linksdone=None, sizedone=None, sizetotal=None, linkstotal=None, links=None, fids=None): + self.pid = pid + self.name = name + self.folder = folder + self.site = site + self.password = password + self.dest = dest + self.order = order + self.linksdone = linksdone + self.sizedone = sizedone + self.sizetotal = sizetotal + self.linkstotal = linkstotal + self.links = links + self.fids = fids + + +class PackageDoesNotExists(Exception): + __slots__ = ['pid'] + + + def __init__(self, pid=None): + self.pid = pid + + +class ServerStatus(BaseObject): + __slots__ = ['pause', 'active', 'queue', 'total', 'speed', 'download', 'reconnect'] + + + def __init__(self, pause=None, active=None, queue=None, total=None, speed=None, download=None, reconnect=None): + self.pause = pause + self.active = active + self.queue = queue + self.total = total + self.speed = speed + self.download = download + self.reconnect = reconnect + + +class ServiceCall(BaseObject): + __slots__ = ['plugin', 'func', 'arguments', 'parseArguments'] + + + def __init__(self, plugin=None, func=None, arguments=None, parseArguments=None): + self.plugin = plugin + self.func = func + self.arguments = arguments + self.parseArguments = parseArguments + + +class ServiceDoesNotExists(Exception): + __slots__ = ['plugin', 'func'] + + + def __init__(self, plugin=None, func=None): + self.plugin = plugin + self.func = func + + +class ServiceException(Exception): + __slots__ = ['msg'] + + + def __init__(self, msg=None): + self.msg = msg + + +class UserData(BaseObject): + __slots__ = ['name', 'email', 'role', 'permission', 'templateName'] + + + def __init__(self, name=None, email=None, role=None, permission=None, templateName=None): + self.name = name + self.email = email + self.role = role + self.permission = permission + self.templateName = templateName + + +class Iface(object): + + def addFiles(self, pid, links): + pass + + + def addPackage(self, name, links, dest): + pass + + + def call(self, info): + pass + + + def checkOnlineStatus(self, urls): + pass + + + def checkOnlineStatusContainer(self, urls, filename, data): + pass + + + def checkURLs(self, urls): + pass + + + def deleteFiles(self, fids): + pass + + + def deleteFinished(self): + pass + + + def deletePackages(self, pids): + pass + + + def freeSpace(self): + pass + + + def generateAndAddPackages(self, links, dest): + pass + + + def generatePackages(self, links): + pass + + + def getAccountTypes(self): + pass + + + def getAccounts(self, refresh): + pass + + + def getAllInfo(self): + pass + + + def getAllUserData(self): + pass + + + def getCaptchaTask(self, exclusive): + pass + + + def getCaptchaTaskStatus(self, tid): + pass + + + def getCollector(self): + pass + + + def getCollectorData(self): + pass + + + def getConfig(self): + pass + + + def getConfigValue(self, category, option, section): + pass + + + def getEvents(self, uuid): + pass + + + def getFileData(self, fid): + pass + + + def getFileOrder(self, pid): + pass + + + def getInfoByPlugin(self, plugin): + pass + + + def getLog(self, offset): + pass + + + def getPackageData(self, pid): + pass + + + def getPackageInfo(self, pid): + pass + + + def getPackageOrder(self, destination): + pass + + + def getPluginConfig(self): + pass + + + def getQueue(self): + pass + + + def getQueueData(self): + pass + + + def getServerVersion(self): + pass + + + def getServices(self): + pass + + + def getUserData(self, username, password): + pass + + + def hasService(self, plugin, func): + pass + + + def isCaptchaWaiting(self): + pass + + + def isTimeDownload(self): + pass + + + def isTimeReconnect(self): + pass + + + def kill(self): + pass + + + def login(self, username, password): + pass + + + def moveFiles(self, fids, pid): + pass + + + def movePackage(self, destination, pid): + pass + + + def orderFile(self, fid, position): + pass + + + def orderPackage(self, pid, position): + pass + + + def parseURLs(self, html, url): + pass + + + def pauseServer(self): + pass + + + def pollResults(self, rid): + pass + + + def pullFromQueue(self, pid): + pass + + + def pushToQueue(self, pid): + pass + + + def recheckPackage(self, pid): + pass + + + def removeAccount(self, plugin, account): + pass + + + def restart(self): + pass + + + def restartFailed(self): + pass + + + def restartFile(self, fid): + pass + + + def restartPackage(self, pid): + pass + + + def setCaptchaResult(self, tid, result): + pass + + + def setConfigValue(self, category, option, value, section): + pass + + + def setPackageData(self, pid, data): + pass + + + def setPackageName(self, pid, name): + pass + + + def statusDownloads(self): + pass + + + def statusServer(self): + pass + + + def stopAllDownloads(self): + pass + + + def stopDownloads(self, fids): + pass + + + def togglePause(self): + pass + + + def toggleReconnect(self): + pass + + + def unpauseServer(self): + pass + + + def updateAccount(self, plugin, account, password, options): + pass + + + def uploadContainer(self, filename, data): + pass diff --git a/pyload/Core.py b/pyload/Core.py index eb6a968b4..545c1d63d 100755 --- a/pyload/Core.py +++ b/pyload/Core.py @@ -360,11 +360,11 @@ class Core(object): self.lastClientConnected = 0 # later imported because they would trigger api import, and remote value not set correctly - from pyload.api import Api + from pyload.Api import Api from pyload.manager.Addon import AddonManager from pyload.manager.Thread import ThreadManager - if pyload.api.activated != self.remote: + if pyload.Api.activated != self.remote: self.log.warning("Import error: API remote status not correct.") self.api = Api(self) diff --git a/pyload/Database/Backend.py b/pyload/Database/Backend.py new file mode 100644 index 000000000..0fc961973 --- /dev/null +++ b/pyload/Database/Backend.py @@ -0,0 +1,329 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +from __future__ import with_statement + +try: + from pysqlite2 import dbapi2 as sqlite3 +except Exception: + import sqlite3 + +import shutil +import threading +import traceback + +from Queue import Queue + +from pyload.utils import chmod + + +DB_VERSION = 4 + + +class style(object): + db = None + + + @classmethod + def setDB(cls, db): + cls.db = db + + + @classmethod + def inner(cls, f): + + + @staticmethod + def x(*args, **kwargs): + if cls.db: + return f(cls.db, *args, **kwargs) + + return x + + + @classmethod + def queue(cls, f): + + + @staticmethod + def x(*args, **kwargs): + if cls.db: + return cls.db.queue(f, *args, **kwargs) + + return x + + + @classmethod + def async(cls, f): + + + @staticmethod + def x(*args, **kwargs): + if cls.db: + return cls.db.async(f, *args, **kwargs) + return x + + +class DatabaseJob(object): + + def __init__(self, f, *args, **kwargs): + self.done = threading.Event() + + self.f = f + self.args = args + self.kwargs = kwargs + + self.result = None + self.exception = False + + # import inspect + # self.frame = inspect.currentframe() + + + def __repr__(self): + import os + + frame = self.frame.f_back + output = "" + for _i in xrange(5): + output += "\t%s:%s, %s\n" % (os.path.basename(frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name) + frame = frame.f_back + del frame + del self.frame + + return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result) + + + def processJob(self): + try: + self.result = self.f(*self.args, **self.kwargs) + except Exception, e: + traceback.print_exc() + try: + print "Database Error @", self.f.__name__, self.args[1:], self.kwargs, e + except Exception: + pass + + self.exception = e + finally: + self.done.set() + + + def wait(self): + self.done.wait() + + +class DatabaseBackend(threading.Thread): + subs = [] + + + def __init__(self, core): + threading.Thread.__init__(self) + self.setDaemon(True) + self.core = core + + self.jobs = Queue() + + self.setuplock = threading.Event() + + style.setDB(self) + + + def setup(self): + self.start() + self.setuplock.wait() + + + def run(self): + """main loop, which executes commands""" + convert = self._checkVersion() #: returns None or current version + + self.conn = sqlite3.connect("files.db") + os.chmod("files.db", 0600) + + self.c = self.conn.cursor() #: compatibility + + if convert is not None: + self._convertDB(convert) + + self._createTables() + self._migrateUser() + + self.conn.commit() + + self.setuplock.set() + + while True: + j = self.jobs.get() + if j == "quit": + self.c.close() + self.conn.close() + break + j.processJob() + + + @style.queue + def shutdown(self): + self.conn.commit() + self.jobs.put("quit") + + + def _checkVersion(self): + """ check db version and delete it if needed""" + if not os.path.exists("files.version"): + with open("files.version", "wb") as f: + f.write(str(DB_VERSION)) + return + + with open("files.version", "rb") as f: + v = int(f.read().strip() or 0) + + if v < DB_VERSION: + if v < 2: + try: + self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) + except Exception: + print "Filedatabase was deleted due to incompatible version." + reshutil.move("files.version") + shutil.move("files.db", "files.backup.db") + + with open("files.version", "wb") as f: + f.write(str(DB_VERSION)) + + return v + + + def _convertDB(self, v): + try: + getattr(self, "_convertV%i" % v)() + except Exception: + try: + self.core.log.error(_("Filedatabase could NOT be converted.")) + except Exception: + print "Filedatabase could NOT be converted." + + # convert scripts start --------------------------------------------------- + + + def _convertV2(self): + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + try: + self.manager.core.log.info(_("Database was converted from v2 to v3.")) + except Exception: + print "Database was converted from v2 to v3." + self._convertV3() + + + def _convertV3(self): + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + try: + self.manager.core.log.info(_("Database was converted from v3 to v4.")) + except Exception: + print "Database was converted from v3 to v4." + + # convert scripts end ----------------------------------------------------- + + + def _createTables(self): + """create tables for database""" + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') + self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + + self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \ + SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\ + FROM packages p JOIN links l ON p.id = l.package LEFT OUTER JOIN\ + (SELECT p.id AS id, COUNT(*) AS linksdone, SUM(l.size) AS sizedone \ + FROM packages p JOIN links l ON p.id = l.package AND l.status IN (0, 4, 13) GROUP BY p.id) s ON s.id = p.id \ + GROUP BY p.id') + + # try to lower ids + self.c.execute('SELECT max(id) FROM LINKS') + fid = self.c.fetchone()[0] + fid = int(fid) if fid else 0 + self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links")) + + self.c.execute('SELECT max(id) FROM packages') + pid = self.c.fetchone()[0] + pid = int(pid) if pid else 0 + self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (pid, "packages")) + + self.c.execute('VACUUM') + + + def _migrateUser(self): + if os.path.exists("pyload.db"): + try: + self.core.log.info(_("Converting old Django DB")) + except Exception: + print "Converting old Django DB" + conn = sqlite3.connect('pyload.db') + c = conn.cursor() + c.execute("SELECT username, password, email FROM auth_user WHERE is_superuser") + users = [] + for r in c: + pw = r[1].split("$") + users.append((r[0], pw[1] + pw[2], r[2])) + c.close() + conn.close() + + self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users) + shutil.move("pyload.db", "pyload.old.db") + + + def createCursor(self): + return self.conn.cursor() + + + @style.async + def commit(self): + self.conn.commit() + + + @style.queue + def syncSave(self): + self.conn.commit() + + + @style.async + def rollback(self): + self.conn.rollback() + + + def async(self, f, *args, **kwargs): + args = (self,) + args + job = DatabaseJob(f, *args, **kwargs) + self.jobs.put(job) + + + def queue(self, f, *args, **kwargs): + args = (self,) + args + job = DatabaseJob(f, *args, **kwargs) + self.jobs.put(job) + job.wait() + return job.result + + + @classmethod + def registerSub(cls, klass): + cls.subs.append(klass) + + + @classmethod + def unregisterSub(cls, klass): + cls.subs.remove(klass) + + + def __getattr__(self, attr): + for sub in DatabaseBackend.subs: + if hasattr(sub, attr): + return getattr(sub, attr) diff --git a/pyload/Database/File.py b/pyload/Database/File.py new file mode 100644 index 000000000..f28535ffa --- /dev/null +++ b/pyload/Database/File.py @@ -0,0 +1,985 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +import threading + +from pyload.Datatype import PyFile, PyPackage +from pyload.Database import DatabaseBackend, style +from pyload.manager.Event import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent +from pyload.utils import formatSize, lock + +try: + from pysqlite2 import dbapi2 as sqlite3 +except Exception: + import sqlite3 + + +class FileHandler(object): + """Handles all request made to obtain information, + modify status or other request for links or packages""" + + def __init__(self, core): + """Constructor""" + self.core = core + + # translations + self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), + _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), + _("downloading"), _("processing"), _("unknown")] + + self.cache = {} #: holds instances for files + self.packageCache = {} #: same for packages + #@TODO: purge the cache + + self.jobCache = {} + + self.lock = threading.RLock() #@TODO: should be a Lock w/o R + # self.lock._Verbose__verbose = True + + self.filecount = -1 #: if an invalid value is set get current value from db + self.queuecount = -1 #: number of package to be loaded + self.unchanged = False #: determines if any changes was made since last call + + self.db = self.core.db + + + def change(func): + + + def new(*args): + args[0].unchanged = False + args[0].filecount = -1 + args[0].queuecount = -1 + args[0].jobCache = {} + return func(*args) + + return new + + + #-------------------------------------------------------------------------- + + def save(self): + """saves all data to backend""" + self.db.commit() + + + #-------------------------------------------------------------------------- + + def syncSave(self): + """saves all data to backend and waits until all data are written""" + pyfiles = self.cache.values() + for pyfile in pyfiles: + pyfile.sync() + + pypacks = self.packageCache.values() + for pypack in pypacks: + pypack.sync() + + self.db.syncSave() + + + @lock + def getCompleteData(self, queue=1): + """gets a complete data representation""" + + data = self.db.getAllLinks(queue) + packs = self.db.getAllPackages(queue) + + data.update([(x.id, x.toDbDict()[x.id]) for x in self.cache.values()]) + + for x in self.packageCache.itervalues(): + if x.queue != queue or x.id not in packs: + continue + packs[x.id].update(x.toDict()[x.id]) + + for key, value in data.iteritems(): + if value['package'] in packs: + packs[value['package']]['links'][key] = value + + return packs + + + @lock + def getInfoData(self, queue=1): + """gets a data representation without links""" + + packs = self.db.getAllPackages(queue) + for x in self.packageCache.itervalues(): + if x.queue != queue or x.id not in packs: + continue + packs[x.id].update(x.toDict()[x.id]) + + return packs + + + @lock + @change + def addLinks(self, urls, package): + """adds links""" + + self.core.addonManager.dispatchEvent("links-added", urls, package) + + data = self.core.pluginManager.parseUrls(urls) + + self.db.addLinks(data, package) + self.core.threadManager.createInfoThread(data, package) + + #@TODO: change from reloadAll event to package update event + self.core.pullManager.addEvent(ReloadAllEvent("collector")) + + + #-------------------------------------------------------------------------- + + @lock + @change + def addPackage(self, name, folder, queue=0): + """adds a package, default to link collector""" + lastID = self.db.addPackage(name, folder, queue) + p = self.db.getPackage(lastID) + e = InsertEvent("pack", lastID, p.order, "collector" if not queue else "queue") + self.core.pullManager.addEvent(e) + return lastID + + + #-------------------------------------------------------------------------- + + @lock + @change + def deletePackage(self, id): + """delete package and all contained links""" + + p = self.getPackage(id) + if not p: + if id in self.packageCache: + del self.packageCache[id] + return + + oldorder = p.order + queue = p.queue + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + + pyfiles = self.cache.values() + + for pyfile in pyfiles: + if pyfile.packageid == id: + pyfile.abortDownload() + pyfile.release() + + self.db.deletePackage(p) + self.core.pullManager.addEvent(e) + self.core.addonManager.dispatchEvent("package-deleted", id) + + if id in self.packageCache: + del self.packageCache[id] + + packs = self.packageCache.values() + for pack in packs: + if pack.queue == queue and pack.order > oldorder: + pack.order -= 1 + pack.notifyChange() + + + #-------------------------------------------------------------------------- + + @lock + @change + def deleteLink(self, id): + """deletes links""" + + f = self.getFile(id) + if not f: + return None + + pid = f.packageid + e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") + + oldorder = f.order + + if id in self.core.threadManager.processingIds(): + self.cache[id].abortDownload() + + if id in self.cache: + del self.cache[id] + + self.db.deleteLink(f) + + self.core.pullManager.addEvent(e) + + p = self.getPackage(pid) + if not len(p.getChildren()): + p.delete() + + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid == pid and pyfile.order > oldorder: + pyfile.order -= 1 + pyfile.notifyChange() + + + #-------------------------------------------------------------------------- + + def releaseLink(self, id): + """removes pyfile from cache""" + if id in self.cache: + del self.cache[id] + + + #-------------------------------------------------------------------------- + + def releasePackage(self, id): + """removes package from cache""" + if id in self.packageCache: + del self.packageCache[id] + + + #-------------------------------------------------------------------------- + + def updateLink(self, pyfile): + """updates link""" + self.db.updateLink(pyfile) + + e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") + self.core.pullManager.addEvent(e) + + + #-------------------------------------------------------------------------- + + def updatePackage(self, pypack): + """updates a package""" + self.db.updatePackage(pypack) + + e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") + self.core.pullManager.addEvent(e) + + + #-------------------------------------------------------------------------- + + def getPackage(self, id): + """return package instance""" + + if id in self.packageCache: + return self.packageCache[id] + else: + return self.db.getPackage(id) + + + #-------------------------------------------------------------------------- + + def getPackageData(self, id): + """returns dict with package information""" + pack = self.getPackage(id) + + if not pack: + return None + + pack = pack.toDict()[id] + + data = self.db.getPackageData(id) + + tmplist = [] + + cache = self.cache.values() + for x in cache: + if int(x.toDbDict()[x.id]['package']) == int(id): + tmplist.append((x.id, x.toDbDict()[x.id])) + data.update(tmplist) + + pack['links'] = data + + return pack + + + #-------------------------------------------------------------------------- + + def getFileData(self, id): + """returns dict with file information""" + if id in self.cache: + return self.cache[id].toDbDict() + + return self.db.getLinkData(id) + + + #-------------------------------------------------------------------------- + + def getFile(self, id): + """returns pyfile instance""" + if id in self.cache: + return self.cache[id] + else: + return self.db.getFile(id) + + + #-------------------------------------------------------------------------- + + @lock + def getJob(self, occ): + """get suitable job""" + + #@TODO: clean mess + #@TODO: improve selection of valid jobs + + if occ in self.jobCache: + if self.jobCache[occ]: + id = self.jobCache[occ].pop() + if id == "empty": + pyfile = None + self.jobCache[occ].append("empty") + else: + pyfile = self.getFile(id) + else: + jobs = self.db.getJob(occ) + jobs.reverse() + if not jobs: + self.jobCache[occ].append("empty") + pyfile = None + else: + self.jobCache[occ].extend(jobs) + pyfile = self.getFile(self.jobCache[occ].pop()) + + else: + self.jobCache = {} #: better not caching to much + jobs = self.db.getJob(occ) + jobs.reverse() + self.jobCache[occ] = jobs + + if not jobs: + self.jobCache[occ].append("empty") + pyfile = None + else: + pyfile = self.getFile(self.jobCache[occ].pop()) + + #@TODO: maybe the new job has to be approved... + + # pyfile = self.getFile(self.jobCache[occ].pop()) + return pyfile + + + @lock + def getDecryptJob(self): + """return job for decrypting""" + if "decrypt" in self.jobCache: + return None + + plugins = self.core.pluginManager.crypterPlugins.keys() + self.core.pluginManager.containerPlugins.keys() + plugins = str(tuple(plugins)) + + jobs = self.db.getPluginJob(plugins) + if jobs: + return self.getFile(jobs[0]) + else: + self.jobCache['decrypt'] = "empty" + return None + + + def getFileCount(self): + """returns number of files""" + + if self.filecount == -1: + self.filecount = self.db.filecount(1) + + return self.filecount + + + def getQueueCount(self, force=False): + """number of files that have to be processed""" + if self.queuecount == -1 or force: + self.queuecount = self.db.queuecount(1) + + return self.queuecount + + + def checkAllLinksFinished(self): + """checks if all files are finished and dispatch event""" + + if not self.getQueueCount(True): + self.core.addonManager.dispatchEvent("all_downloads-finished") + self.core.log.debug("All downloads finished") + return True + + return False + + + def checkAllLinksProcessed(self, fid): + """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" + + # reset count so statistic will update (this is called when dl was processed) + self.resetCount() + + if not self.db.processcount(1, fid): + self.core.addonManager.dispatchEvent("all_downloads-processed") + self.core.log.debug("All downloads processed") + return True + + return False + + + def resetCount(self): + self.queuecount = -1 + + + @lock + @change + def restartPackage(self, id): + """restart package""" + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid == id: + self.restartFile(pyfile.id) + + self.db.restartPackage(id) + + if id in self.packageCache: + self.packageCache[id].setFinished = False + + e = UpdateEvent("pack", id, "collector" if not self.getPackage(id).queue else "queue") + self.core.pullManager.addEvent(e) + + + @lock + @change + def restartFile(self, id): + """ restart file""" + if id in self.cache: + self.cache[id].status = 3 + self.cache[id].name = self.cache[id].url + self.cache[id].error = "" + self.cache[id].abortDownload() + + self.db.restartFile(id) + + e = UpdateEvent("file", id, "collector" if not self.getFile(id).package().queue else "queue") + self.core.pullManager.addEvent(e) + + + @lock + @change + def setPackageLocation(self, id, queue): + """push package to queue""" + + p = self.db.getPackage(id) + oldorder = p.order + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + self.db.clearPackageOrder(p) + + p = self.db.getPackage(id) + + p.queue = queue + self.db.updatePackage(p) + + self.db.reorderPackage(p, -1, True) + + packs = self.packageCache.values() + for pack in packs: + if pack.queue != queue and pack.order > oldorder: + pack.order -= 1 + pack.notifyChange() + + self.db.commit() + self.releasePackage(id) + p = self.getPackage(id) + + e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + + @lock + @change + def reorderPackage(self, id, position): + p = self.getPackage(id) + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + self.db.reorderPackage(p, position) + + packs = self.packageCache.values() + for pack in packs: + if pack.queue != p.queue or pack.order < 0 or pack == p: + continue + if p.order > position: + if position <= pack.order < p.order: + pack.order += 1 + pack.notifyChange() + elif p.order < position: + if position >= pack.order > p.order: + pack.order -= 1 + pack.notifyChange() + + p.order = position + self.db.commit() + + e = InsertEvent("pack", id, position, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + + @lock + @change + def reorderFile(self, id, position): + f = self.getFileData(id) + f = f[id] + + e = RemoveEvent("file", id, "collector" if not self.getPackage(f['package']).queue else "queue") + self.core.pullManager.addEvent(e) + + self.db.reorderLink(f, position) + + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid != f['package'] or pyfile.order < 0: + continue + if f['order'] > position: + if position <= pyfile.order < f['order']: + pyfile.order += 1 + pyfile.notifyChange() + elif f['order'] < position: + if position >= pyfile.order > f['order']: + pyfile.order -= 1 + pyfile.notifyChange() + + if id in self.cache: + self.cache[id].order = position + + self.db.commit() + + e = InsertEvent("file", id, position, "collector" if not self.getPackage(f['package']).queue else "queue") + self.core.pullManager.addEvent(e) + + + @change + def updateFileInfo(self, data, pid): + """ updates file info (name, size, status, url)""" + ids = self.db.updateLinkInfo(data) + e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue") + self.core.pullManager.addEvent(e) + + + def checkPackageFinished(self, pyfile): + """ checks if package is finished and calls AddonManager """ + + ids = self.db.getUnfinished(pyfile.packageid) + if not ids or (pyfile.id in ids and len(ids) == 1): + if not pyfile.package().setFinished: + self.core.log.info(_("Package finished: %s") % pyfile.package().name) + self.core.addonManager.packageFinished(pyfile.package()) + pyfile.package().setFinished = True + + + def reCheckPackage(self, pid): + """ recheck links in package """ + data = self.db.getPackageData(pid) + + urls = [] + + for pyfile in data.itervalues(): + if pyfile['status'] not in (0, 12, 13): + urls.append((pyfile['url'], pyfile['plugin'])) + + self.core.threadManager.createInfoThread(urls, pid) + + + @lock + @change + def deleteFinishedLinks(self): + """ deletes finished links and packages, return deleted packages """ + + old_packs = self.getInfoData(0) + old_packs.update(self.getInfoData(1)) + + self.db.deleteFinished() + + new_packs = self.db.getAllPackages(0) + new_packs.update(self.db.getAllPackages(1)) + # get new packages only from db + + deleted = [id for id in old_packs.iterkeys() if id not in new_packs] + for id_deleted in deleted: + self.deletePackage(int(id_deleted)) + + return deleted + + + @lock + @change + def restartFailed(self): + """ restart all failed links """ + self.db.restartFailed() + + +class FileMethods(object): + + + @style.queue + def filecount(self, queue): + """returns number of files in queue""" + self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", + (queue,)) + return self.c.fetchone()[0] + + + @style.queue + def queuecount(self, queue): + """ number of files in queue not finished yet""" + self.c.execute( + "SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0, 4)", + (queue,)) + return self.c.fetchone()[0] + + + @style.queue + def processcount(self, queue, fid): + """ number of files which have to be proccessed """ + self.c.execute( + "SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2, 3, 5, 7, 12) AND l.id != ?", + (queue, str(fid))) + return self.c.fetchone()[0] + + + @style.inner + def _nextPackageOrder(self, queue=0): + self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) + max = self.c.fetchone()[0] + if max is not None: + return max + 1 + else: + return 0 + + + @style.inner + def _nextFileOrder(self, package): + self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) + max = self.c.fetchone()[0] + if max is not None: + return max + 1 + else: + return 0 + + + @style.queue + def addLink(self, url, name, plugin, package): + order = self._nextFileOrder(package) + self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', + (url, name, ".".join(plugintype, pluginname), package, order)) + return self.c.lastrowid + + + @style.queue + def addLinks(self, links, package): + """ links is a list of tupels (url, plugin)""" + order = self._nextFileOrder(package) + orders = [order + x for x in xrange(len(links))] + links = [(x[0], x[0], ".".join((x[1], x[2])), package, o) for x, o in zip(links, orders)] + self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) + + + @style.queue + def addPackage(self, name, folder, queue): + order = self._nextPackageOrder(queue) + self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', + (name, folder, queue, order)) + return self.c.lastrowid + + + @style.queue + def deletePackage(self, p): + self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) + self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', + (p.order, p.queue)) + + + @style.queue + def deleteLink(self, f): + self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) + self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', + (f.order, str(f.packageid))) + + + @style.queue + def getAllLinks(self, q): + """return information about all links in queue q + + q0 queue + q1 collector + + format: + + { + id: {'name': name, ... 'package': id }, ... + } + + """ + self.c.execute( + 'SELECT l.id, l.url, l.name, l.size, l.status, l.error, l.plugin, l.package, l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', + (q,)) + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': tuple(r[6].split('.')), + 'package': r[7], + 'order': r[8], + } + + return data + + + @style.queue + def getAllPackages(self, q): + """return information about packages in queue q + (only useful in get all data) + + q0 queue + q1 collector + + format: + + { + id: {'name': name ... 'links': {}}, ... + } + """ + self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ + FROM packages p JOIN pstats s ON p.id = s.id \ + WHERE p.queue=? ORDER BY p.packageorder', str(q)) + + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'name': r[1], + 'folder': r[2], + 'site': r[3], + 'password': r[4], + 'queue': r[5], + 'order': r[6], + 'sizetotal': int(r[7]), + 'sizedone': r[8] if r[8] else 0, #: these can be None + 'linksdone': r[9] if r[9] else 0, + 'linkstotal': r[10], + 'links': {} + } + + return data + + + @style.queue + def getLinkData(self, id): + """get link information as dict""" + self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?', + (str(id),)) + data = {} + r = self.c.fetchone() + if not r: + return None + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': tuple(r[6].split('.')), + 'package': r[7], + 'order': r[8], + } + + return data + + + @style.queue + def getPackageData(self, id): + """get data about links for a package""" + self.c.execute( + 'SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE package=? ORDER BY linkorder', + (str(id),)) + + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': tuple(r[6].split('.')), + 'package': r[7], + 'order': r[8], + } + + return data + + + @style.async + def updateLink(self, f): + self.c.execute('UPDATE links SET url=?, name=?, size=?, status=?, error=?, package=? WHERE id=?', + (f.url, f.name, f.size, f.status, str(f.error), str(f.packageid), str(f.id))) + + + @style.queue + def updatePackage(self, p): + self.c.execute('UPDATE packages SET name=?, folder=?, site=?, password=?, queue=? WHERE id=?', + (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) + + @style.queue + def updateLinkInfo(self, data): + """ data is list of tupels (name, size, status, url) """ + self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1, 2, 3, 14)', data) + ids = [] + self.c.execute('SELECT id FROM links WHERE url IN (\'%s\')' % "','".join([x[3] for x in data])) + for r in self.c: + ids.append(int(r[0])) + return ids + + + @style.queue + def reorderPackage(self, p, position, noMove=False): + if position == -1: + position = self._nextPackageOrder(p.queue) + if not noMove: + if p.order > position: + self.c.execute( + 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND queue=? AND packageorder >= 0', + (position, p.order, p.queue)) + elif p.order < position: + self.c.execute( + 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND queue=? AND packageorder >= 0', + (position, p.order, p.queue)) + + self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) + + + @style.queue + def reorderLink(self, f, position): + """ reorder link with f as dict for pyfile """ + if f['order'] > position: + self.c.execute('UPDATE links SET linkorder=linkorder+1 WHERE linkorder >= ? AND linkorder < ? AND package=?', + (position, f['order'], f['package'])) + elif f['order'] < position: + self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder <= ? AND linkorder > ? AND package=?', + (position, f['order'], f['package'])) + + self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f['id'])) + + + @style.queue + def clearPackageOrder(self, p): + self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', + (p.order, p.queue, str(p.id))) + + + @style.async + def restartFile(self, id): + self.c.execute('UPDATE links SET status=3, error="" WHERE id=?', (str(id),)) + + + @style.async + def restartPackage(self, id): + self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) + + + @style.queue + def getPackage(self, id): + """return package instance from id""" + self.c.execute("SELECT name, folder, site, password, queue, packageorder FROM packages WHERE id=?", (str(id),)) + r = self.c.fetchone() + if not r: + return None + return PyPackage(self.manager, id, *r) + + + #-------------------------------------------------------------------------- + + @style.queue + def getFile(self, id): + """return link instance from id""" + self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", + (str(id),)) + r = self.c.fetchone() + if not r: + return None + r = list(r) + r[5] = tuple(r[5].split('.')) + return PyFile(self.manager, id, *r) + + + @style.queue + def getJob(self, occ): + """return pyfile ids, which are suitable for download and dont use a occupied plugin""" + + #@TODO: improve this hardcoded method + pre = "('CCF', 'DLC', 'LinkList', 'RSDF', 'TXT')" #: plugins which are processed in collector + + cmd = "(" + for i, item in enumerate(occ): + if i: cmd += ", " + cmd += "'%s'" % item + + cmd += ")" + + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) + + self.c.execute(cmd) #: very bad! + + return [x[0] for x in self.c] + + + @style.queue + def getPluginJob(self, plugins): + """returns pyfile ids with suited plugins""" + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins + + self.c.execute(cmd) #: very bad! + + return [x[0] for x in self.c] + + + @style.queue + def getUnfinished(self, pid): + """return list of max length 3 ids with pyfiles in package not finished or processed""" + + self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) + return [r[0] for r in self.c] + + + @style.queue + def deleteFinished(self): + self.c.execute("DELETE FROM links WHERE status IN (0, 4)") + self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") + + + @style.queue + def restartFailed(self): + self.c.execute("UPDATE links SET status=3, error='' WHERE status IN (6, 8, 9)") + + + @style.queue + def findDuplicates(self, id, folder, filename): + """ checks if filename exists with different id and same package """ + self.c.execute( + "SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", + (folder, id, filename)) + return self.c.fetchone() + + + @style.queue + def purgeLinks(self): + self.c.execute("DELETE FROM links;") + self.c.execute("DELETE FROM packages;") + + +DatabaseBackend.registerSub(FileMethods) diff --git a/pyload/Database/Storage.py b/pyload/Database/Storage.py new file mode 100644 index 000000000..a19f67606 --- /dev/null +++ b/pyload/Database/Storage.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# @author: mkaay + +from pyload.Database import style +from pyload.Database import DatabaseBackend + + +class StorageMethods(object): + + @style.queue + def setStorage(db, identifier, key, value): + db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) + if db.c.fetchone() is not None: + db.c.execute("UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key)) + else: + db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) + + + @style.queue + def getStorage(db, identifier, key=None): + if key is not None: + db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) + row = db.c.fetchone() + if row is not None: + return row[0] + else: + db.c.execute("SELECT key, value FROM storage WHERE identifier=?", (identifier,)) + return {row[0]: row[1] for row in db.c} + + + @style.queue + def delStorage(db, identifier, key): + db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) + + +DatabaseBackend.registerSub(StorageMethods) diff --git a/pyload/Database/User.py b/pyload/Database/User.py new file mode 100644 index 000000000..dc60ce23a --- /dev/null +++ b/pyload/Database/User.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# @author: mkaay + +from hashlib import sha1 +import random + +from pyload.Database import DatabaseBackend, style + + +class UserMethods(object): + + + @style.queue + def checkAuth(db, user, password): + c = db.c + c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user,)) + r = c.fetchone() + if not r: + return {} + + salt = r[2][:5] + pw = r[2][5:] + h = sha1(salt + password) + if h.hexdigest() == pw: + return {"id": r[0], "name": r[1], "role": r[3], + "permission": r[4], "template": r[5], "email": r[6]} + else: + return {} + + + @style.queue + def addUser(db, user, password): + salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for _i in xrange(0, 5)]) + h = sha1(salt + password) + password = salt + h.hexdigest() + + c = db.c + c.execute('SELECT name FROM users WHERE name=?', (user,)) + if c.fetchone() is not None: + c.execute('UPDATE users SET password=? WHERE name=?', (password, user)) + else: + c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) + + + @style.queue + def changePassword(db, user, oldpw, newpw): + db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user,)) + r = db.c.fetchone() + if not r: + return False + + salt = r[2][:5] + pw = r[2][5:] + h = sha1(salt + oldpw) + if h.hexdigest() == pw: + salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for _i in xrange(0, 5)]) + h = sha1(salt + newpw) + password = salt + h.hexdigest() + + db.c.execute("UPDATE users SET password=? WHERE name=?", (password, user)) + return True + + return False + + + @style.async + def setPermission(db, user, perms): + db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) + + + @style.async + def setRole(db, user, role): + db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) + + + @style.queue + def listUsers(db): + db.c.execute('SELECT name FROM users') + return [row[0] for row in db.c] + + + @style.queue + def getAllUserData(db): + db.c.execute("SELECT name, permission, role, template, email FROM users") + return {r[0]: {"permission": r[1], "role": r[2], "template": r[3], "email": r[4]} for r in db.c} + + + @style.queue + def removeUser(db, user): + db.c.execute('DELETE FROM users WHERE name=?', (user,)) + + +DatabaseBackend.registerSub(UserMethods) diff --git a/pyload/Database/__init__.py b/pyload/Database/__init__.py new file mode 100644 index 000000000..b4200a698 --- /dev/null +++ b/pyload/Database/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- + +from pyload.Database.Backend import DatabaseBackend, style +from pyload.Database.File import FileHandler +from pyload.Database.Storage import StorageMethods +from pyload.Database.User import UserMethods diff --git a/pyload/Datatype/File.py b/pyload/Datatype/File.py new file mode 100644 index 000000000..cb6989d98 --- /dev/null +++ b/pyload/Datatype/File.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +import time +import threading + +from pyload.manager.Event import UpdateEvent +from pyload.utils import formatSize, lock + + +statusMap = { + "finished" : 0, + "offline" : 1, + "online" : 2, + "queued" : 3, + "skipped" : 4, + "waiting" : 5, + "temp. offline": 6, + "starting" : 7, + "failed" : 8, + "aborted" : 9, + "decrypting" : 10, + "custom" : 11, + "downloading" : 12, + "processing" : 13, + "unknown" : 14, +} + + +def setSize(self, value): + self._size = int(value) + + +class PyFile(object): + """ + Represents a file object at runtime + """ + __slots__ = ("m", "id", "url", "name", "size", "_size", "status", "plugintype", "pluginname", + "packageid", "error", "order", "lock", "plugin", "waitUntil", + "active", "abort", "statusname", "reconnected", "progress", + "maxprogress", "pluginmodule", "pluginclass") + + + def __init__(self, manager, id, url, name, size, status, error, plugin, package, order): + self.m = manager + + self.id = int(id) + self.url = url + self.name = name + self.size = size + self.status = status + self.plugintype, self.pluginname = plugin + self.packageid = package #: should not be used, use package() instead + self.error = error + self.order = order + # database information ends here + + self.lock = threading.RLock() + + self.plugin = None + # self.download = None + + self.waitUntil = 0 #: time.time() + time to wait + + # status attributes + self.active = False #: obsolete? + self.abort = False + self.reconnected = False + + self.statusname = None + + self.progress = 0 + self.maxprogress = 100 + + self.m.cache[int(id)] = self + + # will convert all sizes to ints + size = property(lambda self: self._size, setSize) + + + def __repr__(self): + return "PyFile %s: %s@%s" % (self.id, self.name, self.pluginname) + + + @lock + def initPlugin(self): + """ inits plugin instance """ + if not self.plugin: + self.pluginmodule = self.m.core.pluginManager.getPlugin(self.plugintype, self.pluginname) + self.pluginclass = getattr(self.pluginmodule, self.m.core.pluginManager.getPluginName(self.plugintype, self.pluginname)) + self.plugin = self.pluginclass(self) + + + @lock + def hasPlugin(self): + """Thread safe way to determine this file has initialized plugin attribute + + :return: + """ + return hasattr(self, "plugin") and self.plugin + + + def package(self): + """ return package instance""" + return self.m.getPackage(self.packageid) + + + def setStatus(self, status): + self.status = statusMap[status] + self.sync() #@TODO: needed aslong no better job approving exists + + + def setCustomStatus(self, msg, status="processing"): + self.statusname = msg + self.setStatus(status) + + + def getStatusName(self): + if self.status not in (13, 14) or not self.statusname: + return self.m.statusMsg[self.status] + else: + return self.statusname + + + def hasStatus(self, status): + return statusMap[status] == self.status + + + def sync(self): + """sync PyFile instance with database""" + self.m.updateLink(self) + + + @lock + def release(self): + """sync and remove from cache""" + # file has valid package + if self.packageid > 0: + self.sync() + + if hasattr(self, "plugin") and self.plugin: + self.plugin.clean() + del self.plugin + + self.m.releaseLink(self.id) + + + def delete(self): + """delete pyfile from database""" + self.m.deleteLink(self.id) + + + def toDict(self): + """return dict with all information for interface""" + return self.toDbDict() + + + def toDbDict(self): + """return data as dict for databse + + format: + + { + id: {'url': url, 'name': name ... } + } + + """ + return { + self.id: { + 'id': self.id, + 'url': self.url, + 'name': self.name, + 'plugin': self.pluginname, + 'size': self.getSize(), + 'format_size': self.formatSize(), + 'status': self.status, + 'statusmsg': self.getStatusName(), + 'package': self.packageid, + 'error': self.error, + 'order': self.order + } + } + + + def abortDownload(self): + """abort pyfile if possible""" + while self.id in self.m.core.threadManager.processingIds(): + self.abort = True + if self.plugin and self.plugin.req: + self.plugin.req.abortDownloads() + time.sleep(0.1) + + self.abort = False + if self.hasPlugin() and self.plugin.req: + self.plugin.req.abortDownloads() + + self.release() + + + def finishIfDone(self): + """set status to finish and release file if every thread is finished with it""" + + if self.id in self.m.core.threadManager.processingIds(): + return False + + self.setStatus("finished") + self.release() + self.m.checkAllLinksFinished() + return True + + + def checkIfProcessed(self): + self.m.checkAllLinksProcessed(self.id) + + + def formatWait(self): + """ formats and return wait time in humanreadable format """ + seconds = self.waitUntil - time.time() + + if seconds < 0: + return "00:00:00" + + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + + + def formatSize(self): + """ formats size to readable format """ + return formatSize(self.getSize()) + + + def formatETA(self): + """ formats eta to readable format """ + seconds = self.getETA() + + if seconds < 0: + return "00:00:00" + + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + + + def getSpeed(self): + """ calculates speed """ + try: + return self.plugin.req.speed + except Exception: + return 0 + + + def getETA(self): + """ gets established time of arrival""" + try: + return self.getBytesLeft() / self.getSpeed() + except Exception: + return 0 + + + def getBytesLeft(self): + """ gets bytes left """ + try: + return self.getSize() - self.plugin.req.arrived + except Exception: + return 0 + + + def getPercent(self): + """ get % of download """ + if self.status == 12: + try: + return self.plugin.req.percent + except Exception: + return 0 + else: + return self.progress + + + def getSize(self): + """ get size of download """ + try: + if self.plugin.req.size: + return self.plugin.req.size + else: + return self.size + except Exception: + return self.size + + + def notifyChange(self): + e = UpdateEvent("file", self.id, "collector" if not self.package().queue else "queue") + self.m.core.pullManager.addEvent(e) + + + def setProgress(self, value): + if not value == self.progress: + self.progress = value + self.notifyChange() diff --git a/pyload/Datatype/Package.py b/pyload/Datatype/Package.py new file mode 100644 index 000000000..5ba42f596 --- /dev/null +++ b/pyload/Datatype/Package.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +from pyload.manager.Event import UpdateEvent +from pyload.utils import safe_filename + + +class PyPackage(object): + """ + Represents a package object at runtime + """ + + def __init__(self, manager, id, name, folder, site, password, queue, order): + self.m = manager + self.m.packageCache[int(id)] = self + + self.id = int(id) + self.name = name + self._folder = folder + self.site = site + self.password = password + self.queue = queue + self.order = order + self.setFinished = False + + + @property + def folder(self): + return safe_filename(self._folder) + + + def toDict(self): + """ Returns a dictionary representation of the data. + + :return: dict: {id: { attr: value }} + """ + return { + self.id: { + 'id': self.id, + 'name': self.name, + 'folder': self.folder, + 'site': self.site, + 'password': self.password, + 'queue': self.queue, + 'order': self.order, + 'links': {} + } + } + + + def getChildren(self): + """get information about contained links""" + return self.m.getPackageData(self.id)["links"] + + + def sync(self): + """sync with db""" + self.m.updatePackage(self) + + + def release(self): + """sync and delete from cache""" + self.sync() + self.m.releasePackage(self.id) + + + def delete(self): + self.m.deletePackage(self.id) + + + def notifyChange(self): + e = UpdateEvent("pack", self.id, "collector" if not self.queue else "queue") + self.m.core.pullManager.addEvent(e) diff --git a/pyload/Datatype/__init__.py b/pyload/Datatype/__init__.py new file mode 100644 index 000000000..29f0d40aa --- /dev/null +++ b/pyload/Datatype/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +from pyload.Datatype.File import PyFile +from pyload.Datatype.Package import PyPackage diff --git a/pyload/Thread/Info.py b/pyload/Thread/Info.py index 36779a962..c1960fa9a 100644 --- a/pyload/Thread/Info.py +++ b/pyload/Thread/Info.py @@ -11,7 +11,7 @@ from copy import copy from pprint import pformat from types import MethodType -from pyload.api import OnlineStatus +from pyload.Api import OnlineStatus from pyload.Datatype import PyFile from pyload.Thread.Plugin import PluginThread diff --git a/pyload/Thread/Plugin.py b/pyload/Thread/Plugin.py index c7350a735..0f1b14d26 100644 --- a/pyload/Thread/Plugin.py +++ b/pyload/Thread/Plugin.py @@ -14,7 +14,7 @@ from copy import copy from pprint import pformat from types import MethodType -from pyload.api import OnlineStatus +from pyload.Api import OnlineStatus from pyload.Datatype import PyFile from pyload.plugin.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload from pyload.utils.packagetools import parseNames diff --git a/pyload/Thread/Server.py b/pyload/Thread/Server.py index 6b6ef6989..a26ffe6a1 100644 --- a/pyload/Thread/Server.py +++ b/pyload/Thread/Server.py @@ -31,7 +31,7 @@ class WebServer(threading.Thread): def run(self): - import pyload.Webui as webinterface + import pyload.webui as webinterface global webinterface reset = False diff --git a/pyload/api/__init__.py b/pyload/api/__init__.py deleted file mode 100644 index 55c21572f..000000000 --- a/pyload/api/__init__.py +++ /dev/null @@ -1,1051 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -from __future__ import with_statement - -import base64 -import os -import re -import time -import urlparse - -from pyload.Datatype import PyFile -from pyload.utils.packagetools import parseNames -from pyload.network.RequestFactory import getURL -from pyload.remote import activated -from pyload.utils import compare_time, freeSpace, safe_filename - -if activated: - try: - from thrift.protocol import TBase - from pyload.remote.thriftbackend.thriftgen.pyload.ttypes import * - from pyload.remote.thriftbackend.thriftgen.pyload.Pyload import Iface - - BaseObject = TBase - - except ImportError: - from pyload.api.types import * - - print "Thrift not imported" - -else: - from pyload.api.types import * - -# contains function names mapped to their permissions -# unlisted functions are for admins only -permMap = {} - - -# decorator only called on init, never initialized, so has no effect on runtime -def permission(bits): - class _Dec(object): - - def __new__(cls, func, *args, **kwargs): - permMap[func.__name__] = bits - return func - return _Dec - - -urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&\[\]\|]*)", re.IGNORECASE) - - -class PERMS(object): - ALL = 0 #: requires no permission, but login - ADD = 1 #: can add packages - DELETE = 2 #: can delete packages - STATUS = 4 #: see and change server status - LIST = 16 #: see queue and collector - MODIFY = 32 #: moddify some attribute of downloads - DOWNLOAD = 64 #: can download from webinterface - SETTINGS = 128 #: can access settings - ACCOUNTS = 256 #: can access accounts - LOGS = 512 #: can see server logs - - -class ROLE(object): - ADMIN = 0 #: admin has all permissions implicit - USER = 1 - - -def has_permission(userperms, perms): - # bytewise or perms before if needed - return perms == (userperms & perms) - - -class Api(Iface): - """ - **pyLoads API** - - This is accessible either internal via core.api or via thrift backend. - - see Thrift specification file remote/thriftbackend/pyload.thrift\ - for information about data structures and what methods are usuable with rpc. - - Most methods requires specific permissions, please look at the source code if you need to know.\ - These can be configured via webinterface. - Admin user have all permissions, and are the only ones who can access the methods with no specific permission. - """ - EXTERNAL = Iface #: let the json api know which methods are external - - - def __init__(self, core): - self.core = core - - - def _convertPyFile(self, p): - fdata = FileData(p['id'], p['url'], p['name'], p['plugin'], p['size'], - p['format_size'], p['status'], p['statusmsg'], - p['package'], p['error'], p['order']) - return fdata - - - def _convertConfigFormat(self, c): - sections = {} - for sectionName, sub in c.iteritems(): - section = ConfigSection(sectionName, sub['desc']) - items = [] - for key, data in sub.iteritems(): - if key in ("desc", "outline"): - continue - item = ConfigItem() - item.name = key - item.description = data['desc'] - item.value = str(data['value']) if not isinstance(data['value'], basestring) else data['value'] - item.type = data['type'] - items.append(item) - section.items = items - sections[sectionName] = section - if "outline" in sub: - section.outline = sub['outline'] - return sections - - - @permission(PERMS.SETTINGS) - def getConfigValue(self, category, option, section="core"): - """Retrieve config value. - - :param category: name of category, or plugin - :param option: config option - :param section: 'plugin' or 'core' - :return: config value as string - """ - if section == "core": - value = self.core.config[category][option] - else: - value = self.core.config.getPlugin(category, option) - return str(value) - - - @permission(PERMS.SETTINGS) - def setConfigValue(self, category, option, value, section="core"): - """Set new config value. - - :param section: - :param option: - :param value: new config value - :param section: 'plugin' or 'core - """ - self.core.addonManager.dispatchEvent("config-changed", category, option, value, section) - if section == "core": - self.core.config[category][option] = value - if option in ("limit_speed", "max_speed"): #: not so nice to update the limit - self.core.requestFactory.updateBucket() - elif section == "plugin": - self.core.config.setPlugin(category, option, value) - - - @permission(PERMS.SETTINGS) - def getConfig(self): - """Retrieves complete config of core. - - :return: list of `ConfigSection` - """ - return self._convertConfigFormat(self.core.config.config) - - - def getConfigDict(self): - """Retrieves complete config in dict format, not for RPC. - - :return: dict - """ - return self.core.config.config - - - @permission(PERMS.SETTINGS) - def getPluginConfig(self): - """Retrieves complete config for all plugins. - - :return: list of `ConfigSection` - """ - return self._convertConfigFormat(self.core.config.plugin) - - - def getPluginConfigDict(self): - """Plugin config as dict, not for RPC. - - :return: dict - """ - return self.core.config.plugin - - - @permission(PERMS.STATUS) - def pauseServer(self): - """Pause server: Tt wont start any new downloads, but nothing gets aborted.""" - self.core.threadManager.pause = True - - - @permission(PERMS.STATUS) - def unpauseServer(self): - """Unpause server: New Downloads will be started.""" - self.core.threadManager.pause = False - - - @permission(PERMS.STATUS) - def togglePause(self): - """Toggle pause state. - - :return: new pause state - """ - self.core.threadManager.pause ^= True - return self.core.threadManager.pause - - - @permission(PERMS.STATUS) - def toggleReconnect(self): - """Toggle reconnect activation. - - :return: new reconnect state - """ - self.core.config['reconnect']['activated'] ^= True - return self.core.config.get("reconnect", "activated") - - - @permission(PERMS.LIST) - def statusServer(self): - """Some general information about the current status of pyLoad. - - :return: `ServerStatus` - """ - serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), - self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, - not self.core.threadManager.pause and self.isTimeDownload(), - self.core.config.get("reconnect", "activated") and self.isTimeReconnect()) - for pyfile in [x.active for x in self.core.threadManager.threads if x.active and isinstance(x.active, PyFile)]: - serverStatus.speed += pyfile.getSpeed() #: bytes/s - return serverStatus - - - @permission(PERMS.STATUS) - def freeSpace(self): - """Available free space at download directory in bytes""" - return freeSpace(self.core.config.get("general", "download_folder")) - - - @permission(PERMS.ALL) - def getServerVersion(self): - """pyLoad Core version """ - return self.core.version - - - def kill(self): - """Clean way to quit pyLoad""" - self.core.do_kill = True - - - def restart(self): - """Restart pyload core""" - self.core.do_restart = True - - - @permission(PERMS.LOGS) - def getLog(self, offset=0): - """Returns most recent log entries. - - :param offset: line offset - :return: List of log entries - """ - filename = os.path.join(self.core.config.get("log", "log_folder"), 'log.txt') - try: - with open(filename, "r") as fh: - lines = fh.readlines() - if offset >= len(lines): - return [] - return lines[offset:] - except Exception: - return ['No log available'] - - - @permission(PERMS.STATUS) - def isTimeDownload(self): - """Checks if pyload will start new downloads according to time in config. - - :return: bool - """ - start = self.core.config.get("downloadTime", "start").split(":") - end = self.core.config.get("downloadTime", "end").split(":") - return compare_time(start, end) - - - @permission(PERMS.STATUS) - def isTimeReconnect(self): - """Checks if pyload will try to make a reconnect - - :return: bool - """ - start = self.core.config.get("reconnect", "startTime").split(":") - end = self.core.config.get("reconnect", "endTime").split(":") - return compare_time(start, end) and self.core.config.get("reconnect", "activated") - - - @permission(PERMS.LIST) - def statusDownloads(self): - """ Status off all currently running downloads. - - :return: list of `DownloadStatus` - """ - data = [] - for pyfile in self.core.threadManager.getActiveFiles(): - if not isinstance(pyfile, PyFile): - continue - data.append(DownloadInfo( - pyfile.id, pyfile.name, pyfile.getSpeed(), pyfile.getETA(), pyfile.formatETA(), - pyfile.getBytesLeft(), pyfile.getSize(), pyfile.formatSize(), pyfile.getPercent(), - pyfile.status, pyfile.getStatusName(), pyfile.formatWait(), - pyfile.waitUntil, pyfile.packageid, pyfile.package().name, pyfile.pluginname)) - return data - - - @permission(PERMS.ADD) - def addPackage(self, name, links, dest=Destination.Queue): - """Adds a package, with links to desired destination. - - :param name: name of the new package - :param links: list of urls - :param dest: `Destination` - :return: package id of the new package - """ - if self.core.config.get("general", "folder_per_package"): - folder = urlparse.urlparse(name).path.split("/")[-1] - else: - folder = "" - - folder = safe_filename(folder) - - pid = self.core.files.addPackage(name, folder, dest) - - self.core.files.addLinks(links, pid) - - self.core.log.info(_("Added package %(name)s containing %(count)d links") % {"name": name.decode('utf-8'), "count": len(links)}) - - self.core.files.save() - - return pid - - - @permission(PERMS.ADD) - def parseURLs(self, html=None, url=None): - """Parses html content or any arbitaty text for links and returns result of `checkURLs` - - :param html: html source - :return: - """ - urls = [] - if html: - urls += [x[0] for x in urlmatcher.findall(html)] - if url: - page = getURL(url) - urls += [x[0] for x in urlmatcher.findall(page)] - # remove duplicates - return self.checkURLs(set(urls)) - - - @permission(PERMS.ADD) - def checkURLs(self, urls): - """ Gets urls and returns pluginname mapped to list of matches urls. - - :param urls: - :return: {plugin: urls} - """ - data = self.core.pluginManager.parseUrls(urls) - plugins = {} - - for url, plugintype, pluginname in data: - try: - plugins[plugintype][pluginname].append(url) - except Exception: - plugins[plugintype][pluginname] = [url] - - return plugins - - - @permission(PERMS.ADD) - def checkOnlineStatus(self, urls): - """ initiates online status check - - :param urls: - :return: initial set of data as `OnlineCheck` instance containing the result id - """ - data = self.core.pluginManager.parseUrls(urls) - - rid = self.core.threadManager.createResultThread(data, False) - - tmp = [(url, (url, OnlineStatus(url, (plugintype, pluginname), "unknown", 3, 0))) for url, plugintype, pluginname in data] - data = parseNames(tmp) - result = {} - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - return OnlineCheck(rid, result) - - - @permission(PERMS.ADD) - def checkOnlineStatusContainer(self, urls, container, data): - """ checks online status of urls and a submited container file - - :param urls: list of urls - :param container: container file name - :param data: file content - :return: online check - """ - with open(os.path.join(self.core.config.get("general", "download_folder"), "tmp_" + container), "wb") as th: - th.write(str(data)) - return self.checkOnlineStatus(urls + [th.name]) - - - @permission(PERMS.ADD) - def pollResults(self, rid): - """ Polls the result available for ResultID - - :param rid: `ResultID` - :return: `OnlineCheck`, if rid is -1 then no more data available - """ - result = self.core.threadManager.getInfoResult(rid) - if "ALL_INFO_FETCHED" in result: - del result['ALL_INFO_FETCHED'] - return OnlineCheck(-1, result) - else: - return OnlineCheck(rid, result) - - - @permission(PERMS.ADD) - def generatePackages(self, links): - """ Parses links, generates packages names from urls - - :param links: list of urls - :return: package names mapped to urls - """ - return parseNames((x, x) for x in links) - - - @permission(PERMS.ADD) - def generateAndAddPackages(self, links, dest=Destination.Queue): - """Generates and add packages - - :param links: list of urls - :param dest: `Destination` - :return: list of package ids - """ - return [self.addPackage(name, urls, dest) for name, urls - in self.generatePackages(links).iteritems()] - - - @permission(PERMS.ADD) - def checkAndAddPackages(self, links, dest=Destination.Queue): - """Checks online status, retrieves names, and will add packages.\ - Because of this packages are not added immediatly, only for internal use. - - :param links: list of urls - :param dest: `Destination` - :return: None - """ - data = self.core.pluginManager.parseUrls(links) - self.core.threadManager.createResultThread(data, True) - - - @permission(PERMS.LIST) - def getPackageData(self, pid): - """Returns complete information about package, and included files. - - :param pid: package id - :return: `PackageData` with .links attribute - """ - data = self.core.files.getPackageData(int(pid)) - if not data: - raise PackageDoesNotExists(pid) - return PackageData(data['id'], data['name'], data['folder'], data['site'], data['password'], - data['queue'], data['order'], - links=[self._convertPyFile(x) for x in data['links'].itervalues()]) - - - @permission(PERMS.LIST) - def getPackageInfo(self, pid): - """Returns information about package, without detailed information about containing files - - :param pid: package id - :return: `PackageData` with .fid attribute - """ - data = self.core.files.getPackageData(int(pid)) - - if not data: - raise PackageDoesNotExists(pid) - return PackageData(data['id'], data['name'], data['folder'], data['site'], data['password'], - data['queue'], data['order'], - fids=[int(x) for x in data['links']]) - - - @permission(PERMS.LIST) - def getFileData(self, fid): - """Get complete information about a specific file. - - :param fid: file id - :return: `FileData` - """ - info = self.core.files.getFileData(int(fid)) - if not info: - raise FileDoesNotExists(fid) - return self._convertPyFile(info.values()[0]) - - - @permission(PERMS.DELETE) - def deleteFiles(self, fids): - """Deletes several file entries from pyload. - - :param fids: list of file ids - """ - for fid in fids: - self.core.files.deleteLink(int(fid)) - self.core.files.save() - - - @permission(PERMS.DELETE) - def deletePackages(self, pids): - """Deletes packages and containing links. - - :param pids: list of package ids - """ - for pid in pids: - self.core.files.deletePackage(int(pid)) - self.core.files.save() - - - @permission(PERMS.LIST) - def getQueue(self): - """Returns info about queue and packages, **not** about files, see `getQueueData` \ - or `getPackageData` instead. - - :return: list of `PackageInfo` - """ - return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], - pack['password'], pack['queue'], pack['order'], - pack['linksdone'], pack['sizedone'], pack['sizetotal'], - pack['linkstotal']) - for pack in self.core.files.getInfoData(Destination.Queue).itervalues()] - - - @permission(PERMS.LIST) - def getQueueData(self): - """Return complete data about everything in queue, this is very expensive use it sparely.\ - See `getQueue` for alternative. - - :return: list of `PackageData` - """ - return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], - pack['password'], pack['queue'], pack['order'], - pack['linksdone'], pack['sizedone'], pack['sizetotal'], - links=[self._convertPyFile(x) for x in pack['links'].itervalues()]) - for pack in self.core.files.getCompleteData(Destination.Queue).itervalues()] - - - @permission(PERMS.LIST) - def getCollector(self): - """same as `getQueue` for collector. - - :return: list of `PackageInfo` - """ - return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], - pack['password'], pack['queue'], pack['order'], - pack['linksdone'], pack['sizedone'], pack['sizetotal'], - pack['linkstotal']) - for pack in self.core.files.getInfoData(Destination.Collector).itervalues()] - - - @permission(PERMS.LIST) - def getCollectorData(self): - """same as `getQueueData` for collector. - - :return: list of `PackageInfo` - """ - return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], - pack['password'], pack['queue'], pack['order'], - pack['linksdone'], pack['sizedone'], pack['sizetotal'], - links=[self._convertPyFile(x) for x in pack['links'].itervalues()]) - for pack in self.core.files.getCompleteData(Destination.Collector).itervalues()] - - - @permission(PERMS.ADD) - def addFiles(self, pid, links): - """Adds files to specific package. - - :param pid: package id - :param links: list of urls - """ - self.core.files.addLinks(links, int(pid)) - self.core.log.info(_("Added %(count)d links to package #%(package)d ") % {"count": len(links), "package": pid}) - self.core.files.save() - - - @permission(PERMS.MODIFY) - def pushToQueue(self, pid): - """Moves package from Collector to Queue. - - :param pid: package id - """ - self.core.files.setPackageLocation(pid, Destination.Queue) - - - @permission(PERMS.MODIFY) - def pullFromQueue(self, pid): - """Moves package from Queue to Collector. - - :param pid: package id - """ - self.core.files.setPackageLocation(pid, Destination.Collector) - - - @permission(PERMS.MODIFY) - def restartPackage(self, pid): - """Restarts a package, resets every containing files. - - :param pid: package id - """ - self.core.files.restartPackage(int(pid)) - - - @permission(PERMS.MODIFY) - def restartFile(self, fid): - """Resets file status, so it will be downloaded again. - - :param fid: file id - """ - self.core.files.restartFile(int(fid)) - - - @permission(PERMS.MODIFY) - def recheckPackage(self, pid): - """Proofes online status of all files in a package, also a default action when package is added. - - :param pid: - :return: - """ - self.core.files.reCheckPackage(int(pid)) - - - @permission(PERMS.MODIFY) - def stopAllDownloads(self): - """Aborts all running downloads.""" - - pyfiles = self.core.files.cache.values() - for pyfile in pyfiles: - pyfile.abortDownload() - - - @permission(PERMS.MODIFY) - def stopDownloads(self, fids): - """Aborts specific downloads. - - :param fids: list of file ids - :return: - """ - pyfiles = self.core.files.cache.values() - for pyfile in pyfiles: - if pyfile.id in fids: - pyfile.abortDownload() - - - @permission(PERMS.MODIFY) - def setPackageName(self, pid, name): - """Renames a package. - - :param pid: package id - :param name: new package name - """ - pack = self.core.files.getPackage(pid) - pack.name = name - pack.sync() - - - @permission(PERMS.MODIFY) - def movePackage(self, destination, pid): - """Set a new package location. - - :param destination: `Destination` - :param pid: package id - """ - if destination in (0, 1): - self.core.files.setPackageLocation(pid, destination) - - - @permission(PERMS.MODIFY) - def moveFiles(self, fids, pid): - """Move multiple files to another package - - :param fids: list of file ids - :param pid: destination package - :return: - """ - # TODO: implement - pass - - - @permission(PERMS.ADD) - def uploadContainer(self, filename, data): - """Uploads and adds a container file to pyLoad. - - :param filename: filename, extension is important so it can correctly decrypted - :param data: file content - """ - with open(os.path.join(self.core.config.get("general", "download_folder"), "tmp_" + filename), "wb") as th: - th.write(str(data)) - self.addPackage(th.name, [th.name], Destination.Queue) - - - @permission(PERMS.MODIFY) - def orderPackage(self, pid, position): - """Gives a package a new position. - - :param pid: package id - :param position: - """ - self.core.files.reorderPackage(pid, position) - - - @permission(PERMS.MODIFY) - def orderFile(self, fid, position): - """Gives a new position to a file within its package. - - :param fid: file id - :param position: - """ - self.core.files.reorderFile(fid, position) - - - @permission(PERMS.MODIFY) - def setPackageData(self, pid, data): - """Allows to modify several package attributes. - - :param pid: package id - :param data: dict that maps attribute to desired value - """ - package = self.core.files.getPackage(pid) - if not package: - raise PackageDoesNotExists(pid) - for key, value in data.iteritems(): - if key == "id": - continue - setattr(package, key, value) - package.sync() - self.core.files.save() - - - @permission(PERMS.DELETE) - def deleteFinished(self): - """Deletes all finished files and completly finished packages. - - :return: list of deleted package ids - """ - return self.core.files.deleteFinishedLinks() - - - @permission(PERMS.MODIFY) - def restartFailed(self): - """Restarts all failed failes.""" - self.core.files.restartFailed() - - - @permission(PERMS.LIST) - def getPackageOrder(self, destination): - """Returns information about package order. - - :param destination: `Destination` - :return: dict mapping order to package id - """ - packs = self.core.files.getInfoData(destination) - order = {} - for pid in packs: - pack = self.core.files.getPackageData(int(pid)) - while pack['order'] in order.keys(): #: just in case - pack['order'] += 1 - order[pack['order']] = pack['id'] - return order - - - @permission(PERMS.LIST) - def getFileOrder(self, pid): - """Information about file order within package. - - :param pid: - :return: dict mapping order to file id - """ - rawdata = self.core.files.getPackageData(int(pid)) - order = {} - for id, pyfile in rawdata['links'].iteritems(): - while pyfile['order'] in order.keys(): #: just in case - pyfile['order'] += 1 - order[pyfile['order']] = pyfile['id'] - return order - - - @permission(PERMS.STATUS) - def isCaptchaWaiting(self): - """Indicates wether a captcha task is available - - :return: bool - """ - self.core.lastClientConnected = time.time() - task = self.core.captchaManager.getTask() - return not task is None - - - @permission(PERMS.STATUS) - def getCaptchaTask(self, exclusive=False): - """Returns a captcha task - - :param exclusive: unused - :return: `CaptchaTask` - """ - self.core.lastClientConnected = time.time() - task = self.core.captchaManager.getTask() - if task: - task.setWatingForUser(exclusive=exclusive) - data, type, result = task.getCaptcha() - ctask = CaptchaTask(int(task.id), base64.standard_b64encode(data), type, result) - return ctask - return CaptchaTask(-1) - - - @permission(PERMS.STATUS) - def getCaptchaTaskStatus(self, tid): - """Get information about captcha task - - :param tid: task id - :return: string - """ - self.core.lastClientConnected = time.time() - task = self.core.captchaManager.getTaskByID(tid) - return task.getStatus() if task else "" - - - @permission(PERMS.STATUS) - def setCaptchaResult(self, tid, result): - """Set result for a captcha task - - :param tid: task id - :param result: captcha result - """ - self.core.lastClientConnected = time.time() - task = self.core.captchaManager.getTaskByID(tid) - if task: - task.setResult(result) - self.core.captchaManager.removeTask(task) - - - @permission(PERMS.STATUS) - def getEvents(self, uuid): - """Lists occured events, may be affected to changes in future. - - :param uuid: - :return: list of `Events` - """ - events = self.core.pullManager.getEvents(uuid) - new_events = [] - - - def convDest(d): - return Destination.Queue if d == "queue" else Destination.Collector - - for e in events: - event = EventInfo() - event.eventname = e[0] - if e[0] in ("update", "remove", "insert"): - event.id = e[3] - event.type = ElementType.Package if e[2] == "pack" else ElementType.File - event.destination = convDest(e[1]) - elif e[0] == "order": - if e[1]: - event.id = e[1] - event.type = ElementType.Package if e[2] == "pack" else ElementType.File - event.destination = convDest(e[3]) - elif e[0] == "reload": - event.destination = convDest(e[1]) - new_events.append(event) - return new_events - - - @permission(PERMS.ACCOUNTS) - def getAccounts(self, refresh): - """Get information about all entered accounts. - - :param refresh: reload account info - :return: list of `AccountInfo` - """ - accs = self.core.accountManager.getAccountInfos(False, refresh) - for group in accs.values(): - accounts = [AccountInfo(acc['validuntil'], acc['login'], acc['options'], acc['valid'], - acc['trafficleft'], acc['maxtraffic'], acc['premium'], acc['type']) - for acc in group] - return accounts or list() - - - @permission(PERMS.ALL) - def getAccountTypes(self): - """All available account types. - - :return: list - """ - return self.core.accountManager.accounts.keys() - - - @permission(PERMS.ACCOUNTS) - def updateAccount(self, plugin, account, password=None, options={}): - """Changes pw/options for specific account.""" - self.core.accountManager.updateAccount(plugin, account, password, options) - - - @permission(PERMS.ACCOUNTS) - def removeAccount(self, plugin, account): - """Remove account from pyload. - - :param plugin: pluginname - :param account: accountname - """ - self.core.accountManager.removeAccount(plugin, account) - - - @permission(PERMS.ALL) - def login(self, username, password, remoteip=None): - """Login into pyLoad, this **must** be called when using rpc before any methods can be used. - - :param username: - :param password: - :param remoteip: Omit this argument, its only used internal - :return: bool indicating login was successful - """ - return bool(self.checkAuth(username, password, remoteip)) - - - def checkAuth(self, username, password, remoteip=None): - """Check authentication and returns details - - :param username: - :param password: - :param remoteip: - :return: dict with info, empty when login is incorrect - """ - if self.core.config.get("remote", "nolocalauth") and remoteip == "127.0.0.1": - return "local" - else: - return self.core.db.checkAuth(username, password) - - - def isAuthorized(self, func, userdata): - """checks if the user is authorized for specific method - - :param func: function name - :param userdata: dictionary of user data - :return: boolean - """ - if userdata == "local" or userdata['role'] == ROLE.ADMIN: - return True - elif func in permMap and has_permission(userdata['permission'], permMap[func]): - return True - else: - return False - - - @permission(PERMS.ALL) - def getUserData(self, username, password): - """similar to `checkAuth` but returns UserData thrift type """ - user = self.checkAuth(username, password) - if user: - return UserData(user['name'], user['email'], user['role'], user['permission'], user['template']) - else: - return UserData() - - - def getAllUserData(self): - """returns all known user and info""" - return dict((user, UserData(user, data['email'], data['role'], data['permission'], data['template'])) for user, data - in self.core.db.getAllUserData().iteritems()) - - - @permission(PERMS.STATUS) - def getServices(self): - """ A dict of available services, these can be defined by addon plugins. - - :return: dict with this style: {"plugin": {"method": "description"}} - """ - return dict((plugin, funcs) for plugin, funcs in self.core.addonManager.methods.iteritems()) - - - @permission(PERMS.STATUS) - def hasService(self, plugin, func): - """Checks wether a service is available. - - :param plugin: - :param func: - :return: bool - """ - cont = self.core.addonManager.methods - return plugin in cont and func in cont[plugin] - - - @permission(PERMS.STATUS) - def call(self, info): - """Calls a service (a method in addon plugin). - - :param info: `ServiceCall` - :return: result - :raises: ServiceDoesNotExists, when its not available - :raises: ServiceException, when a exception was raised - """ - plugin = info.plugin - func = info.func - args = info.arguments - parse = info.parseArguments - if not self.hasService(plugin, func): - raise ServiceDoesNotExists(plugin, func) - try: - ret = self.core.addonManager.callRPC(plugin, func, args, parse) - except Exception, e: - raise ServiceException(e.message) - - - @permission(PERMS.STATUS) - def getAllInfo(self): - """Returns all information stored by addon plugins. Values are always strings - - :return: {"plugin": {"name": value}} - """ - return self.core.addonManager.getAllInfo() - - - @permission(PERMS.STATUS) - def getInfoByPlugin(self, plugin): - """Returns information stored by a specific plugin. - - :param plugin: pluginname - :return: dict of attr names mapped to value {"name": value} - """ - return self.core.addonManager.getInfo(plugin) - - - def changePassword(self, user, oldpw, newpw): - """ changes password for specific user """ - return self.core.db.changePassword(user, oldpw, newpw) - - - def setUserPermission(self, user, perm, role): - self.core.db.setPermission(user, perm) - self.core.db.setRole(user, role) diff --git a/pyload/api/types.py b/pyload/api/types.py deleted file mode 100644 index 9381df3c7..000000000 --- a/pyload/api/types.py +++ /dev/null @@ -1,562 +0,0 @@ -# -*- coding: utf-8 -*- -# Autogenerated by pyload -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - -class BaseObject(object): - __slots__ = [] - - -class Destination(object): - Collector = 0 - Queue = 1 - - -class DownloadStatus(object): - Aborted = 9 - Custom = 11 - Decrypting = 10 - Downloading = 12 - Failed = 8 - Finished = 0 - Offline = 1 - Online = 2 - Processing = 13 - Queued = 3 - Skipped = 4 - Starting = 7 - TempOffline = 6 - Unknown = 14 - Waiting = 5 - - -class ElementType(object): - File = 1 - Package = 0 - - -class Input(object): - BOOL = 4 - CHOICE = 6 - CLICK = 5 - LIST = 8 - MULTIPLE = 7 - NONE = 0 - PASSWORD = 3 - TABLE = 9 - TEXT = 1 - TEXTBOX = 2 - - -class Output(object): - CAPTCHA = 1 - NOTIFICATION = 4 - QUESTION = 2 - - -class AccountInfo(BaseObject): - __slots__ = ['validuntil', 'login', 'options', 'valid', 'trafficleft', 'maxtraffic', 'premium', 'type'] - - - def __init__(self, validuntil=None, login=None, options=None, valid=None, trafficleft=None, maxtraffic=None, premium=None, type=None): - self.validuntil = validuntil - self.login = login - self.options = options - self.valid = valid - self.trafficleft = trafficleft - self.maxtraffic = maxtraffic - self.premium = premium - self.type = type - - -class CaptchaTask(BaseObject): - __slots__ = ['tid', 'data', 'type', 'resultType'] - - - def __init__(self, tid=None, data=None, type=None, resultType=None): - self.tid = tid - self.data = data - self.type = type - self.resultType = resultType - - -class ConfigItem(BaseObject): - __slots__ = ['name', 'description', 'value', 'type'] - - - def __init__(self, name=None, description=None, value=None, type=None): - self.name = name - self.description = description - self.value = value - self.type = type - - -class ConfigSection(BaseObject): - __slots__ = ['name', 'description', 'items', 'outline'] - - - def __init__(self, name=None, description=None, items=None, outline=None): - self.name = name - self.description = description - self.items = items - self.outline = outline - - -class DownloadInfo(BaseObject): - __slots__ = ['fid', 'name', 'speed', 'eta', 'format_eta', 'bleft', 'size', 'format_size', 'percent', 'status', 'statusmsg', 'format_wait', 'wait_until', 'packageID', 'packageName', 'plugin'] - - - def __init__(self, fid=None, name=None, speed=None, eta=None, format_eta=None, bleft=None, size=None, format_size=None, percent=None, status=None, statusmsg=None, format_wait=None, wait_until=None, packageID=None, packageName=None, plugin=None): - self.fid = fid - self.name = name - self.speed = speed - self.eta = eta - self.format_eta = format_eta - self.bleft = bleft - self.size = size - self.format_size = format_size - self.percent = percent - self.status = status - self.statusmsg = statusmsg - self.format_wait = format_wait - self.wait_until = wait_until - self.packageID = packageID - self.packageName = packageName - self.plugin = plugin - - -class EventInfo(BaseObject): - __slots__ = ['eventname', 'id', 'type', 'destination'] - - - def __init__(self, eventname=None, id=None, type=None, destination=None): - self.eventname = eventname - self.id = id - self.type = type - self.destination = destination - - -class FileData(BaseObject): - __slots__ = ['fid', 'url', 'name', 'plugin', 'size', 'format_size', 'status', 'statusmsg', 'packageID', 'error', 'order'] - - - def __init__(self, fid=None, url=None, name=None, plugin=None, size=None, format_size=None, status=None, statusmsg=None, packageID=None, error=None, order=None): - self.fid = fid - self.url = url - self.name = name - self.plugin = plugin - self.size = size - self.format_size = format_size - self.status = status - self.statusmsg = statusmsg - self.packageID = packageID - self.error = error - self.order = order - - -class FileDoesNotExists(Exception): - __slots__ = ['fid'] - - - def __init__(self, fid=None): - self.fid = fid - - -class InteractionTask(BaseObject): - __slots__ = ['iid', 'input', 'structure', 'preset', 'output', 'data', 'title', 'description', 'plugin'] - - - def __init__(self, iid=None, input=None, structure=None, preset=None, output=None, data=None, title=None, description=None, plugin=None): - self.iid = iid - self.input = input - self.structure = structure - self.preset = preset - self.output = output - self.data = data - self.title = title - self.description = description - self.plugin = plugin - - -class OnlineCheck(BaseObject): - __slots__ = ['rid', 'data'] - - - def __init__(self, rid=None, data=None): - self.rid = rid - self.data = data - - -class OnlineStatus(BaseObject): - __slots__ = ['name', 'plugin', 'packagename', 'status', 'size'] - - - def __init__(self, name=None, plugin=(None, None), packagename=None, status=None, size=None): - self.name = name - self.plugin = plugin - self.packagename = packagename - self.status = status - self.size = size - - -class PackageData(BaseObject): - __slots__ = ['pid', 'name', 'folder', 'site', 'password', 'dest', 'order', 'linksdone', 'sizedone', 'sizetotal', 'linkstotal', 'links', 'fids'] - - - def __init__(self, pid=None, name=None, folder=None, site=None, password=None, dest=None, order=None, linksdone=None, sizedone=None, sizetotal=None, linkstotal=None, links=None, fids=None): - self.pid = pid - self.name = name - self.folder = folder - self.site = site - self.password = password - self.dest = dest - self.order = order - self.linksdone = linksdone - self.sizedone = sizedone - self.sizetotal = sizetotal - self.linkstotal = linkstotal - self.links = links - self.fids = fids - - -class PackageDoesNotExists(Exception): - __slots__ = ['pid'] - - - def __init__(self, pid=None): - self.pid = pid - - -class ServerStatus(BaseObject): - __slots__ = ['pause', 'active', 'queue', 'total', 'speed', 'download', 'reconnect'] - - - def __init__(self, pause=None, active=None, queue=None, total=None, speed=None, download=None, reconnect=None): - self.pause = pause - self.active = active - self.queue = queue - self.total = total - self.speed = speed - self.download = download - self.reconnect = reconnect - - -class ServiceCall(BaseObject): - __slots__ = ['plugin', 'func', 'arguments', 'parseArguments'] - - - def __init__(self, plugin=None, func=None, arguments=None, parseArguments=None): - self.plugin = plugin - self.func = func - self.arguments = arguments - self.parseArguments = parseArguments - - -class ServiceDoesNotExists(Exception): - __slots__ = ['plugin', 'func'] - - - def __init__(self, plugin=None, func=None): - self.plugin = plugin - self.func = func - - -class ServiceException(Exception): - __slots__ = ['msg'] - - - def __init__(self, msg=None): - self.msg = msg - - -class UserData(BaseObject): - __slots__ = ['name', 'email', 'role', 'permission', 'templateName'] - - - def __init__(self, name=None, email=None, role=None, permission=None, templateName=None): - self.name = name - self.email = email - self.role = role - self.permission = permission - self.templateName = templateName - - -class Iface(object): - - def addFiles(self, pid, links): - pass - - - def addPackage(self, name, links, dest): - pass - - - def call(self, info): - pass - - - def checkOnlineStatus(self, urls): - pass - - - def checkOnlineStatusContainer(self, urls, filename, data): - pass - - - def checkURLs(self, urls): - pass - - - def deleteFiles(self, fids): - pass - - - def deleteFinished(self): - pass - - - def deletePackages(self, pids): - pass - - - def freeSpace(self): - pass - - - def generateAndAddPackages(self, links, dest): - pass - - - def generatePackages(self, links): - pass - - - def getAccountTypes(self): - pass - - - def getAccounts(self, refresh): - pass - - - def getAllInfo(self): - pass - - - def getAllUserData(self): - pass - - - def getCaptchaTask(self, exclusive): - pass - - - def getCaptchaTaskStatus(self, tid): - pass - - - def getCollector(self): - pass - - - def getCollectorData(self): - pass - - - def getConfig(self): - pass - - - def getConfigValue(self, category, option, section): - pass - - - def getEvents(self, uuid): - pass - - - def getFileData(self, fid): - pass - - - def getFileOrder(self, pid): - pass - - - def getInfoByPlugin(self, plugin): - pass - - - def getLog(self, offset): - pass - - - def getPackageData(self, pid): - pass - - - def getPackageInfo(self, pid): - pass - - - def getPackageOrder(self, destination): - pass - - - def getPluginConfig(self): - pass - - - def getQueue(self): - pass - - - def getQueueData(self): - pass - - - def getServerVersion(self): - pass - - - def getServices(self): - pass - - - def getUserData(self, username, password): - pass - - - def hasService(self, plugin, func): - pass - - - def isCaptchaWaiting(self): - pass - - - def isTimeDownload(self): - pass - - - def isTimeReconnect(self): - pass - - - def kill(self): - pass - - - def login(self, username, password): - pass - - - def moveFiles(self, fids, pid): - pass - - - def movePackage(self, destination, pid): - pass - - - def orderFile(self, fid, position): - pass - - - def orderPackage(self, pid, position): - pass - - - def parseURLs(self, html, url): - pass - - - def pauseServer(self): - pass - - - def pollResults(self, rid): - pass - - - def pullFromQueue(self, pid): - pass - - - def pushToQueue(self, pid): - pass - - - def recheckPackage(self, pid): - pass - - - def removeAccount(self, plugin, account): - pass - - - def restart(self): - pass - - - def restartFailed(self): - pass - - - def restartFile(self, fid): - pass - - - def restartPackage(self, pid): - pass - - - def setCaptchaResult(self, tid, result): - pass - - - def setConfigValue(self, category, option, value, section): - pass - - - def setPackageData(self, pid, data): - pass - - - def setPackageName(self, pid, name): - pass - - - def statusDownloads(self): - pass - - - def statusServer(self): - pass - - - def stopAllDownloads(self): - pass - - - def stopDownloads(self, fids): - pass - - - def togglePause(self): - pass - - - def toggleReconnect(self): - pass - - - def unpauseServer(self): - pass - - - def updateAccount(self, plugin, account, password, options): - pass - - - def uploadContainer(self, filename, data): - pass diff --git a/pyload/database/Backend.py b/pyload/database/Backend.py deleted file mode 100644 index 0fc961973..000000000 --- a/pyload/database/Backend.py +++ /dev/null @@ -1,329 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN, mkaay - -from __future__ import with_statement - -try: - from pysqlite2 import dbapi2 as sqlite3 -except Exception: - import sqlite3 - -import shutil -import threading -import traceback - -from Queue import Queue - -from pyload.utils import chmod - - -DB_VERSION = 4 - - -class style(object): - db = None - - - @classmethod - def setDB(cls, db): - cls.db = db - - - @classmethod - def inner(cls, f): - - - @staticmethod - def x(*args, **kwargs): - if cls.db: - return f(cls.db, *args, **kwargs) - - return x - - - @classmethod - def queue(cls, f): - - - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.queue(f, *args, **kwargs) - - return x - - - @classmethod - def async(cls, f): - - - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.async(f, *args, **kwargs) - return x - - -class DatabaseJob(object): - - def __init__(self, f, *args, **kwargs): - self.done = threading.Event() - - self.f = f - self.args = args - self.kwargs = kwargs - - self.result = None - self.exception = False - - # import inspect - # self.frame = inspect.currentframe() - - - def __repr__(self): - import os - - frame = self.frame.f_back - output = "" - for _i in xrange(5): - output += "\t%s:%s, %s\n" % (os.path.basename(frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name) - frame = frame.f_back - del frame - del self.frame - - return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result) - - - def processJob(self): - try: - self.result = self.f(*self.args, **self.kwargs) - except Exception, e: - traceback.print_exc() - try: - print "Database Error @", self.f.__name__, self.args[1:], self.kwargs, e - except Exception: - pass - - self.exception = e - finally: - self.done.set() - - - def wait(self): - self.done.wait() - - -class DatabaseBackend(threading.Thread): - subs = [] - - - def __init__(self, core): - threading.Thread.__init__(self) - self.setDaemon(True) - self.core = core - - self.jobs = Queue() - - self.setuplock = threading.Event() - - style.setDB(self) - - - def setup(self): - self.start() - self.setuplock.wait() - - - def run(self): - """main loop, which executes commands""" - convert = self._checkVersion() #: returns None or current version - - self.conn = sqlite3.connect("files.db") - os.chmod("files.db", 0600) - - self.c = self.conn.cursor() #: compatibility - - if convert is not None: - self._convertDB(convert) - - self._createTables() - self._migrateUser() - - self.conn.commit() - - self.setuplock.set() - - while True: - j = self.jobs.get() - if j == "quit": - self.c.close() - self.conn.close() - break - j.processJob() - - - @style.queue - def shutdown(self): - self.conn.commit() - self.jobs.put("quit") - - - def _checkVersion(self): - """ check db version and delete it if needed""" - if not os.path.exists("files.version"): - with open("files.version", "wb") as f: - f.write(str(DB_VERSION)) - return - - with open("files.version", "rb") as f: - v = int(f.read().strip() or 0) - - if v < DB_VERSION: - if v < 2: - try: - self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) - except Exception: - print "Filedatabase was deleted due to incompatible version." - reshutil.move("files.version") - shutil.move("files.db", "files.backup.db") - - with open("files.version", "wb") as f: - f.write(str(DB_VERSION)) - - return v - - - def _convertDB(self, v): - try: - getattr(self, "_convertV%i" % v)() - except Exception: - try: - self.core.log.error(_("Filedatabase could NOT be converted.")) - except Exception: - print "Filedatabase could NOT be converted." - - # convert scripts start --------------------------------------------------- - - - def _convertV2(self): - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') - try: - self.manager.core.log.info(_("Database was converted from v2 to v3.")) - except Exception: - print "Database was converted from v2 to v3." - self._convertV3() - - - def _convertV3(self): - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') - try: - self.manager.core.log.info(_("Database was converted from v3 to v4.")) - except Exception: - print "Database was converted from v3 to v4." - - # convert scripts end ----------------------------------------------------- - - - def _createTables(self): - """create tables for database""" - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') - self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)') - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') - - self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \ - SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\ - FROM packages p JOIN links l ON p.id = l.package LEFT OUTER JOIN\ - (SELECT p.id AS id, COUNT(*) AS linksdone, SUM(l.size) AS sizedone \ - FROM packages p JOIN links l ON p.id = l.package AND l.status IN (0, 4, 13) GROUP BY p.id) s ON s.id = p.id \ - GROUP BY p.id') - - # try to lower ids - self.c.execute('SELECT max(id) FROM LINKS') - fid = self.c.fetchone()[0] - fid = int(fid) if fid else 0 - self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links")) - - self.c.execute('SELECT max(id) FROM packages') - pid = self.c.fetchone()[0] - pid = int(pid) if pid else 0 - self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (pid, "packages")) - - self.c.execute('VACUUM') - - - def _migrateUser(self): - if os.path.exists("pyload.db"): - try: - self.core.log.info(_("Converting old Django DB")) - except Exception: - print "Converting old Django DB" - conn = sqlite3.connect('pyload.db') - c = conn.cursor() - c.execute("SELECT username, password, email FROM auth_user WHERE is_superuser") - users = [] - for r in c: - pw = r[1].split("$") - users.append((r[0], pw[1] + pw[2], r[2])) - c.close() - conn.close() - - self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users) - shutil.move("pyload.db", "pyload.old.db") - - - def createCursor(self): - return self.conn.cursor() - - - @style.async - def commit(self): - self.conn.commit() - - - @style.queue - def syncSave(self): - self.conn.commit() - - - @style.async - def rollback(self): - self.conn.rollback() - - - def async(self, f, *args, **kwargs): - args = (self,) + args - job = DatabaseJob(f, *args, **kwargs) - self.jobs.put(job) - - - def queue(self, f, *args, **kwargs): - args = (self,) + args - job = DatabaseJob(f, *args, **kwargs) - self.jobs.put(job) - job.wait() - return job.result - - - @classmethod - def registerSub(cls, klass): - cls.subs.append(klass) - - - @classmethod - def unregisterSub(cls, klass): - cls.subs.remove(klass) - - - def __getattr__(self, attr): - for sub in DatabaseBackend.subs: - if hasattr(sub, attr): - return getattr(sub, attr) diff --git a/pyload/database/File.py b/pyload/database/File.py deleted file mode 100644 index f28535ffa..000000000 --- a/pyload/database/File.py +++ /dev/null @@ -1,985 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN, mkaay - -import threading - -from pyload.Datatype import PyFile, PyPackage -from pyload.Database import DatabaseBackend, style -from pyload.manager.Event import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent -from pyload.utils import formatSize, lock - -try: - from pysqlite2 import dbapi2 as sqlite3 -except Exception: - import sqlite3 - - -class FileHandler(object): - """Handles all request made to obtain information, - modify status or other request for links or packages""" - - def __init__(self, core): - """Constructor""" - self.core = core - - # translations - self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), - _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), - _("downloading"), _("processing"), _("unknown")] - - self.cache = {} #: holds instances for files - self.packageCache = {} #: same for packages - #@TODO: purge the cache - - self.jobCache = {} - - self.lock = threading.RLock() #@TODO: should be a Lock w/o R - # self.lock._Verbose__verbose = True - - self.filecount = -1 #: if an invalid value is set get current value from db - self.queuecount = -1 #: number of package to be loaded - self.unchanged = False #: determines if any changes was made since last call - - self.db = self.core.db - - - def change(func): - - - def new(*args): - args[0].unchanged = False - args[0].filecount = -1 - args[0].queuecount = -1 - args[0].jobCache = {} - return func(*args) - - return new - - - #-------------------------------------------------------------------------- - - def save(self): - """saves all data to backend""" - self.db.commit() - - - #-------------------------------------------------------------------------- - - def syncSave(self): - """saves all data to backend and waits until all data are written""" - pyfiles = self.cache.values() - for pyfile in pyfiles: - pyfile.sync() - - pypacks = self.packageCache.values() - for pypack in pypacks: - pypack.sync() - - self.db.syncSave() - - - @lock - def getCompleteData(self, queue=1): - """gets a complete data representation""" - - data = self.db.getAllLinks(queue) - packs = self.db.getAllPackages(queue) - - data.update([(x.id, x.toDbDict()[x.id]) for x in self.cache.values()]) - - for x in self.packageCache.itervalues(): - if x.queue != queue or x.id not in packs: - continue - packs[x.id].update(x.toDict()[x.id]) - - for key, value in data.iteritems(): - if value['package'] in packs: - packs[value['package']]['links'][key] = value - - return packs - - - @lock - def getInfoData(self, queue=1): - """gets a data representation without links""" - - packs = self.db.getAllPackages(queue) - for x in self.packageCache.itervalues(): - if x.queue != queue or x.id not in packs: - continue - packs[x.id].update(x.toDict()[x.id]) - - return packs - - - @lock - @change - def addLinks(self, urls, package): - """adds links""" - - self.core.addonManager.dispatchEvent("links-added", urls, package) - - data = self.core.pluginManager.parseUrls(urls) - - self.db.addLinks(data, package) - self.core.threadManager.createInfoThread(data, package) - - #@TODO: change from reloadAll event to package update event - self.core.pullManager.addEvent(ReloadAllEvent("collector")) - - - #-------------------------------------------------------------------------- - - @lock - @change - def addPackage(self, name, folder, queue=0): - """adds a package, default to link collector""" - lastID = self.db.addPackage(name, folder, queue) - p = self.db.getPackage(lastID) - e = InsertEvent("pack", lastID, p.order, "collector" if not queue else "queue") - self.core.pullManager.addEvent(e) - return lastID - - - #-------------------------------------------------------------------------- - - @lock - @change - def deletePackage(self, id): - """delete package and all contained links""" - - p = self.getPackage(id) - if not p: - if id in self.packageCache: - del self.packageCache[id] - return - - oldorder = p.order - queue = p.queue - - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - - pyfiles = self.cache.values() - - for pyfile in pyfiles: - if pyfile.packageid == id: - pyfile.abortDownload() - pyfile.release() - - self.db.deletePackage(p) - self.core.pullManager.addEvent(e) - self.core.addonManager.dispatchEvent("package-deleted", id) - - if id in self.packageCache: - del self.packageCache[id] - - packs = self.packageCache.values() - for pack in packs: - if pack.queue == queue and pack.order > oldorder: - pack.order -= 1 - pack.notifyChange() - - - #-------------------------------------------------------------------------- - - @lock - @change - def deleteLink(self, id): - """deletes links""" - - f = self.getFile(id) - if not f: - return None - - pid = f.packageid - e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") - - oldorder = f.order - - if id in self.core.threadManager.processingIds(): - self.cache[id].abortDownload() - - if id in self.cache: - del self.cache[id] - - self.db.deleteLink(f) - - self.core.pullManager.addEvent(e) - - p = self.getPackage(pid) - if not len(p.getChildren()): - p.delete() - - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid == pid and pyfile.order > oldorder: - pyfile.order -= 1 - pyfile.notifyChange() - - - #-------------------------------------------------------------------------- - - def releaseLink(self, id): - """removes pyfile from cache""" - if id in self.cache: - del self.cache[id] - - - #-------------------------------------------------------------------------- - - def releasePackage(self, id): - """removes package from cache""" - if id in self.packageCache: - del self.packageCache[id] - - - #-------------------------------------------------------------------------- - - def updateLink(self, pyfile): - """updates link""" - self.db.updateLink(pyfile) - - e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") - self.core.pullManager.addEvent(e) - - - #-------------------------------------------------------------------------- - - def updatePackage(self, pypack): - """updates a package""" - self.db.updatePackage(pypack) - - e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") - self.core.pullManager.addEvent(e) - - - #-------------------------------------------------------------------------- - - def getPackage(self, id): - """return package instance""" - - if id in self.packageCache: - return self.packageCache[id] - else: - return self.db.getPackage(id) - - - #-------------------------------------------------------------------------- - - def getPackageData(self, id): - """returns dict with package information""" - pack = self.getPackage(id) - - if not pack: - return None - - pack = pack.toDict()[id] - - data = self.db.getPackageData(id) - - tmplist = [] - - cache = self.cache.values() - for x in cache: - if int(x.toDbDict()[x.id]['package']) == int(id): - tmplist.append((x.id, x.toDbDict()[x.id])) - data.update(tmplist) - - pack['links'] = data - - return pack - - - #-------------------------------------------------------------------------- - - def getFileData(self, id): - """returns dict with file information""" - if id in self.cache: - return self.cache[id].toDbDict() - - return self.db.getLinkData(id) - - - #-------------------------------------------------------------------------- - - def getFile(self, id): - """returns pyfile instance""" - if id in self.cache: - return self.cache[id] - else: - return self.db.getFile(id) - - - #-------------------------------------------------------------------------- - - @lock - def getJob(self, occ): - """get suitable job""" - - #@TODO: clean mess - #@TODO: improve selection of valid jobs - - if occ in self.jobCache: - if self.jobCache[occ]: - id = self.jobCache[occ].pop() - if id == "empty": - pyfile = None - self.jobCache[occ].append("empty") - else: - pyfile = self.getFile(id) - else: - jobs = self.db.getJob(occ) - jobs.reverse() - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - self.jobCache[occ].extend(jobs) - pyfile = self.getFile(self.jobCache[occ].pop()) - - else: - self.jobCache = {} #: better not caching to much - jobs = self.db.getJob(occ) - jobs.reverse() - self.jobCache[occ] = jobs - - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - pyfile = self.getFile(self.jobCache[occ].pop()) - - #@TODO: maybe the new job has to be approved... - - # pyfile = self.getFile(self.jobCache[occ].pop()) - return pyfile - - - @lock - def getDecryptJob(self): - """return job for decrypting""" - if "decrypt" in self.jobCache: - return None - - plugins = self.core.pluginManager.crypterPlugins.keys() + self.core.pluginManager.containerPlugins.keys() - plugins = str(tuple(plugins)) - - jobs = self.db.getPluginJob(plugins) - if jobs: - return self.getFile(jobs[0]) - else: - self.jobCache['decrypt'] = "empty" - return None - - - def getFileCount(self): - """returns number of files""" - - if self.filecount == -1: - self.filecount = self.db.filecount(1) - - return self.filecount - - - def getQueueCount(self, force=False): - """number of files that have to be processed""" - if self.queuecount == -1 or force: - self.queuecount = self.db.queuecount(1) - - return self.queuecount - - - def checkAllLinksFinished(self): - """checks if all files are finished and dispatch event""" - - if not self.getQueueCount(True): - self.core.addonManager.dispatchEvent("all_downloads-finished") - self.core.log.debug("All downloads finished") - return True - - return False - - - def checkAllLinksProcessed(self, fid): - """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" - - # reset count so statistic will update (this is called when dl was processed) - self.resetCount() - - if not self.db.processcount(1, fid): - self.core.addonManager.dispatchEvent("all_downloads-processed") - self.core.log.debug("All downloads processed") - return True - - return False - - - def resetCount(self): - self.queuecount = -1 - - - @lock - @change - def restartPackage(self, id): - """restart package""" - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid == id: - self.restartFile(pyfile.id) - - self.db.restartPackage(id) - - if id in self.packageCache: - self.packageCache[id].setFinished = False - - e = UpdateEvent("pack", id, "collector" if not self.getPackage(id).queue else "queue") - self.core.pullManager.addEvent(e) - - - @lock - @change - def restartFile(self, id): - """ restart file""" - if id in self.cache: - self.cache[id].status = 3 - self.cache[id].name = self.cache[id].url - self.cache[id].error = "" - self.cache[id].abortDownload() - - self.db.restartFile(id) - - e = UpdateEvent("file", id, "collector" if not self.getFile(id).package().queue else "queue") - self.core.pullManager.addEvent(e) - - - @lock - @change - def setPackageLocation(self, id, queue): - """push package to queue""" - - p = self.db.getPackage(id) - oldorder = p.order - - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - - self.db.clearPackageOrder(p) - - p = self.db.getPackage(id) - - p.queue = queue - self.db.updatePackage(p) - - self.db.reorderPackage(p, -1, True) - - packs = self.packageCache.values() - for pack in packs: - if pack.queue != queue and pack.order > oldorder: - pack.order -= 1 - pack.notifyChange() - - self.db.commit() - self.releasePackage(id) - p = self.getPackage(id) - - e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - - - @lock - @change - def reorderPackage(self, id, position): - p = self.getPackage(id) - - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - self.db.reorderPackage(p, position) - - packs = self.packageCache.values() - for pack in packs: - if pack.queue != p.queue or pack.order < 0 or pack == p: - continue - if p.order > position: - if position <= pack.order < p.order: - pack.order += 1 - pack.notifyChange() - elif p.order < position: - if position >= pack.order > p.order: - pack.order -= 1 - pack.notifyChange() - - p.order = position - self.db.commit() - - e = InsertEvent("pack", id, position, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - - - @lock - @change - def reorderFile(self, id, position): - f = self.getFileData(id) - f = f[id] - - e = RemoveEvent("file", id, "collector" if not self.getPackage(f['package']).queue else "queue") - self.core.pullManager.addEvent(e) - - self.db.reorderLink(f, position) - - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid != f['package'] or pyfile.order < 0: - continue - if f['order'] > position: - if position <= pyfile.order < f['order']: - pyfile.order += 1 - pyfile.notifyChange() - elif f['order'] < position: - if position >= pyfile.order > f['order']: - pyfile.order -= 1 - pyfile.notifyChange() - - if id in self.cache: - self.cache[id].order = position - - self.db.commit() - - e = InsertEvent("file", id, position, "collector" if not self.getPackage(f['package']).queue else "queue") - self.core.pullManager.addEvent(e) - - - @change - def updateFileInfo(self, data, pid): - """ updates file info (name, size, status, url)""" - ids = self.db.updateLinkInfo(data) - e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue") - self.core.pullManager.addEvent(e) - - - def checkPackageFinished(self, pyfile): - """ checks if package is finished and calls AddonManager """ - - ids = self.db.getUnfinished(pyfile.packageid) - if not ids or (pyfile.id in ids and len(ids) == 1): - if not pyfile.package().setFinished: - self.core.log.info(_("Package finished: %s") % pyfile.package().name) - self.core.addonManager.packageFinished(pyfile.package()) - pyfile.package().setFinished = True - - - def reCheckPackage(self, pid): - """ recheck links in package """ - data = self.db.getPackageData(pid) - - urls = [] - - for pyfile in data.itervalues(): - if pyfile['status'] not in (0, 12, 13): - urls.append((pyfile['url'], pyfile['plugin'])) - - self.core.threadManager.createInfoThread(urls, pid) - - - @lock - @change - def deleteFinishedLinks(self): - """ deletes finished links and packages, return deleted packages """ - - old_packs = self.getInfoData(0) - old_packs.update(self.getInfoData(1)) - - self.db.deleteFinished() - - new_packs = self.db.getAllPackages(0) - new_packs.update(self.db.getAllPackages(1)) - # get new packages only from db - - deleted = [id for id in old_packs.iterkeys() if id not in new_packs] - for id_deleted in deleted: - self.deletePackage(int(id_deleted)) - - return deleted - - - @lock - @change - def restartFailed(self): - """ restart all failed links """ - self.db.restartFailed() - - -class FileMethods(object): - - - @style.queue - def filecount(self, queue): - """returns number of files in queue""" - self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", - (queue,)) - return self.c.fetchone()[0] - - - @style.queue - def queuecount(self, queue): - """ number of files in queue not finished yet""" - self.c.execute( - "SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0, 4)", - (queue,)) - return self.c.fetchone()[0] - - - @style.queue - def processcount(self, queue, fid): - """ number of files which have to be proccessed """ - self.c.execute( - "SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2, 3, 5, 7, 12) AND l.id != ?", - (queue, str(fid))) - return self.c.fetchone()[0] - - - @style.inner - def _nextPackageOrder(self, queue=0): - self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) - max = self.c.fetchone()[0] - if max is not None: - return max + 1 - else: - return 0 - - - @style.inner - def _nextFileOrder(self, package): - self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) - max = self.c.fetchone()[0] - if max is not None: - return max + 1 - else: - return 0 - - - @style.queue - def addLink(self, url, name, plugin, package): - order = self._nextFileOrder(package) - self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', - (url, name, ".".join(plugintype, pluginname), package, order)) - return self.c.lastrowid - - - @style.queue - def addLinks(self, links, package): - """ links is a list of tupels (url, plugin)""" - order = self._nextFileOrder(package) - orders = [order + x for x in xrange(len(links))] - links = [(x[0], x[0], ".".join((x[1], x[2])), package, o) for x, o in zip(links, orders)] - self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) - - - @style.queue - def addPackage(self, name, folder, queue): - order = self._nextPackageOrder(queue) - self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', - (name, folder, queue, order)) - return self.c.lastrowid - - - @style.queue - def deletePackage(self, p): - self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) - self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', - (p.order, p.queue)) - - - @style.queue - def deleteLink(self, f): - self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) - self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', - (f.order, str(f.packageid))) - - - @style.queue - def getAllLinks(self, q): - """return information about all links in queue q - - q0 queue - q1 collector - - format: - - { - id: {'name': name, ... 'package': id }, ... - } - - """ - self.c.execute( - 'SELECT l.id, l.url, l.name, l.size, l.status, l.error, l.plugin, l.package, l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', - (q,)) - data = {} - for r in self.c: - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': tuple(r[6].split('.')), - 'package': r[7], - 'order': r[8], - } - - return data - - - @style.queue - def getAllPackages(self, q): - """return information about packages in queue q - (only useful in get all data) - - q0 queue - q1 collector - - format: - - { - id: {'name': name ... 'links': {}}, ... - } - """ - self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ - FROM packages p JOIN pstats s ON p.id = s.id \ - WHERE p.queue=? ORDER BY p.packageorder', str(q)) - - data = {} - for r in self.c: - data[r[0]] = { - 'id': r[0], - 'name': r[1], - 'folder': r[2], - 'site': r[3], - 'password': r[4], - 'queue': r[5], - 'order': r[6], - 'sizetotal': int(r[7]), - 'sizedone': r[8] if r[8] else 0, #: these can be None - 'linksdone': r[9] if r[9] else 0, - 'linkstotal': r[10], - 'links': {} - } - - return data - - - @style.queue - def getLinkData(self, id): - """get link information as dict""" - self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?', - (str(id),)) - data = {} - r = self.c.fetchone() - if not r: - return None - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': tuple(r[6].split('.')), - 'package': r[7], - 'order': r[8], - } - - return data - - - @style.queue - def getPackageData(self, id): - """get data about links for a package""" - self.c.execute( - 'SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE package=? ORDER BY linkorder', - (str(id),)) - - data = {} - for r in self.c: - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': tuple(r[6].split('.')), - 'package': r[7], - 'order': r[8], - } - - return data - - - @style.async - def updateLink(self, f): - self.c.execute('UPDATE links SET url=?, name=?, size=?, status=?, error=?, package=? WHERE id=?', - (f.url, f.name, f.size, f.status, str(f.error), str(f.packageid), str(f.id))) - - - @style.queue - def updatePackage(self, p): - self.c.execute('UPDATE packages SET name=?, folder=?, site=?, password=?, queue=? WHERE id=?', - (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) - - @style.queue - def updateLinkInfo(self, data): - """ data is list of tupels (name, size, status, url) """ - self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1, 2, 3, 14)', data) - ids = [] - self.c.execute('SELECT id FROM links WHERE url IN (\'%s\')' % "','".join([x[3] for x in data])) - for r in self.c: - ids.append(int(r[0])) - return ids - - - @style.queue - def reorderPackage(self, p, position, noMove=False): - if position == -1: - position = self._nextPackageOrder(p.queue) - if not noMove: - if p.order > position: - self.c.execute( - 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND queue=? AND packageorder >= 0', - (position, p.order, p.queue)) - elif p.order < position: - self.c.execute( - 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND queue=? AND packageorder >= 0', - (position, p.order, p.queue)) - - self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) - - - @style.queue - def reorderLink(self, f, position): - """ reorder link with f as dict for pyfile """ - if f['order'] > position: - self.c.execute('UPDATE links SET linkorder=linkorder+1 WHERE linkorder >= ? AND linkorder < ? AND package=?', - (position, f['order'], f['package'])) - elif f['order'] < position: - self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder <= ? AND linkorder > ? AND package=?', - (position, f['order'], f['package'])) - - self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f['id'])) - - - @style.queue - def clearPackageOrder(self, p): - self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', - (p.order, p.queue, str(p.id))) - - - @style.async - def restartFile(self, id): - self.c.execute('UPDATE links SET status=3, error="" WHERE id=?', (str(id),)) - - - @style.async - def restartPackage(self, id): - self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) - - - @style.queue - def getPackage(self, id): - """return package instance from id""" - self.c.execute("SELECT name, folder, site, password, queue, packageorder FROM packages WHERE id=?", (str(id),)) - r = self.c.fetchone() - if not r: - return None - return PyPackage(self.manager, id, *r) - - - #-------------------------------------------------------------------------- - - @style.queue - def getFile(self, id): - """return link instance from id""" - self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", - (str(id),)) - r = self.c.fetchone() - if not r: - return None - r = list(r) - r[5] = tuple(r[5].split('.')) - return PyFile(self.manager, id, *r) - - - @style.queue - def getJob(self, occ): - """return pyfile ids, which are suitable for download and dont use a occupied plugin""" - - #@TODO: improve this hardcoded method - pre = "('CCF', 'DLC', 'LinkList', 'RSDF', 'TXT')" #: plugins which are processed in collector - - cmd = "(" - for i, item in enumerate(occ): - if i: cmd += ", " - cmd += "'%s'" % item - - cmd += ")" - - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) - - self.c.execute(cmd) #: very bad! - - return [x[0] for x in self.c] - - - @style.queue - def getPluginJob(self, plugins): - """returns pyfile ids with suited plugins""" - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins - - self.c.execute(cmd) #: very bad! - - return [x[0] for x in self.c] - - - @style.queue - def getUnfinished(self, pid): - """return list of max length 3 ids with pyfiles in package not finished or processed""" - - self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) - return [r[0] for r in self.c] - - - @style.queue - def deleteFinished(self): - self.c.execute("DELETE FROM links WHERE status IN (0, 4)") - self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") - - - @style.queue - def restartFailed(self): - self.c.execute("UPDATE links SET status=3, error='' WHERE status IN (6, 8, 9)") - - - @style.queue - def findDuplicates(self, id, folder, filename): - """ checks if filename exists with different id and same package """ - self.c.execute( - "SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", - (folder, id, filename)) - return self.c.fetchone() - - - @style.queue - def purgeLinks(self): - self.c.execute("DELETE FROM links;") - self.c.execute("DELETE FROM packages;") - - -DatabaseBackend.registerSub(FileMethods) diff --git a/pyload/database/Storage.py b/pyload/database/Storage.py deleted file mode 100644 index a19f67606..000000000 --- a/pyload/database/Storage.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: mkaay - -from pyload.Database import style -from pyload.Database import DatabaseBackend - - -class StorageMethods(object): - - @style.queue - def setStorage(db, identifier, key, value): - db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) - if db.c.fetchone() is not None: - db.c.execute("UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key)) - else: - db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) - - - @style.queue - def getStorage(db, identifier, key=None): - if key is not None: - db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) - row = db.c.fetchone() - if row is not None: - return row[0] - else: - db.c.execute("SELECT key, value FROM storage WHERE identifier=?", (identifier,)) - return {row[0]: row[1] for row in db.c} - - - @style.queue - def delStorage(db, identifier, key): - db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) - - -DatabaseBackend.registerSub(StorageMethods) diff --git a/pyload/database/User.py b/pyload/database/User.py deleted file mode 100644 index dc60ce23a..000000000 --- a/pyload/database/User.py +++ /dev/null @@ -1,93 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: mkaay - -from hashlib import sha1 -import random - -from pyload.Database import DatabaseBackend, style - - -class UserMethods(object): - - - @style.queue - def checkAuth(db, user, password): - c = db.c - c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user,)) - r = c.fetchone() - if not r: - return {} - - salt = r[2][:5] - pw = r[2][5:] - h = sha1(salt + password) - if h.hexdigest() == pw: - return {"id": r[0], "name": r[1], "role": r[3], - "permission": r[4], "template": r[5], "email": r[6]} - else: - return {} - - - @style.queue - def addUser(db, user, password): - salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for _i in xrange(0, 5)]) - h = sha1(salt + password) - password = salt + h.hexdigest() - - c = db.c - c.execute('SELECT name FROM users WHERE name=?', (user,)) - if c.fetchone() is not None: - c.execute('UPDATE users SET password=? WHERE name=?', (password, user)) - else: - c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) - - - @style.queue - def changePassword(db, user, oldpw, newpw): - db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user,)) - r = db.c.fetchone() - if not r: - return False - - salt = r[2][:5] - pw = r[2][5:] - h = sha1(salt + oldpw) - if h.hexdigest() == pw: - salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for _i in xrange(0, 5)]) - h = sha1(salt + newpw) - password = salt + h.hexdigest() - - db.c.execute("UPDATE users SET password=? WHERE name=?", (password, user)) - return True - - return False - - - @style.async - def setPermission(db, user, perms): - db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) - - - @style.async - def setRole(db, user, role): - db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) - - - @style.queue - def listUsers(db): - db.c.execute('SELECT name FROM users') - return [row[0] for row in db.c] - - - @style.queue - def getAllUserData(db): - db.c.execute("SELECT name, permission, role, template, email FROM users") - return {r[0]: {"permission": r[1], "role": r[2], "template": r[3], "email": r[4]} for r in db.c} - - - @style.queue - def removeUser(db, user): - db.c.execute('DELETE FROM users WHERE name=?', (user,)) - - -DatabaseBackend.registerSub(UserMethods) diff --git a/pyload/database/__init__.py b/pyload/database/__init__.py deleted file mode 100644 index b4200a698..000000000 --- a/pyload/database/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- - -from pyload.Database.Backend import DatabaseBackend, style -from pyload.Database.File import FileHandler -from pyload.Database.Storage import StorageMethods -from pyload.Database.User import UserMethods diff --git a/pyload/datatype/File.py b/pyload/datatype/File.py deleted file mode 100644 index cb6989d98..000000000 --- a/pyload/datatype/File.py +++ /dev/null @@ -1,299 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN, mkaay - -import time -import threading - -from pyload.manager.Event import UpdateEvent -from pyload.utils import formatSize, lock - - -statusMap = { - "finished" : 0, - "offline" : 1, - "online" : 2, - "queued" : 3, - "skipped" : 4, - "waiting" : 5, - "temp. offline": 6, - "starting" : 7, - "failed" : 8, - "aborted" : 9, - "decrypting" : 10, - "custom" : 11, - "downloading" : 12, - "processing" : 13, - "unknown" : 14, -} - - -def setSize(self, value): - self._size = int(value) - - -class PyFile(object): - """ - Represents a file object at runtime - """ - __slots__ = ("m", "id", "url", "name", "size", "_size", "status", "plugintype", "pluginname", - "packageid", "error", "order", "lock", "plugin", "waitUntil", - "active", "abort", "statusname", "reconnected", "progress", - "maxprogress", "pluginmodule", "pluginclass") - - - def __init__(self, manager, id, url, name, size, status, error, plugin, package, order): - self.m = manager - - self.id = int(id) - self.url = url - self.name = name - self.size = size - self.status = status - self.plugintype, self.pluginname = plugin - self.packageid = package #: should not be used, use package() instead - self.error = error - self.order = order - # database information ends here - - self.lock = threading.RLock() - - self.plugin = None - # self.download = None - - self.waitUntil = 0 #: time.time() + time to wait - - # status attributes - self.active = False #: obsolete? - self.abort = False - self.reconnected = False - - self.statusname = None - - self.progress = 0 - self.maxprogress = 100 - - self.m.cache[int(id)] = self - - # will convert all sizes to ints - size = property(lambda self: self._size, setSize) - - - def __repr__(self): - return "PyFile %s: %s@%s" % (self.id, self.name, self.pluginname) - - - @lock - def initPlugin(self): - """ inits plugin instance """ - if not self.plugin: - self.pluginmodule = self.m.core.pluginManager.getPlugin(self.plugintype, self.pluginname) - self.pluginclass = getattr(self.pluginmodule, self.m.core.pluginManager.getPluginName(self.plugintype, self.pluginname)) - self.plugin = self.pluginclass(self) - - - @lock - def hasPlugin(self): - """Thread safe way to determine this file has initialized plugin attribute - - :return: - """ - return hasattr(self, "plugin") and self.plugin - - - def package(self): - """ return package instance""" - return self.m.getPackage(self.packageid) - - - def setStatus(self, status): - self.status = statusMap[status] - self.sync() #@TODO: needed aslong no better job approving exists - - - def setCustomStatus(self, msg, status="processing"): - self.statusname = msg - self.setStatus(status) - - - def getStatusName(self): - if self.status not in (13, 14) or not self.statusname: - return self.m.statusMsg[self.status] - else: - return self.statusname - - - def hasStatus(self, status): - return statusMap[status] == self.status - - - def sync(self): - """sync PyFile instance with database""" - self.m.updateLink(self) - - - @lock - def release(self): - """sync and remove from cache""" - # file has valid package - if self.packageid > 0: - self.sync() - - if hasattr(self, "plugin") and self.plugin: - self.plugin.clean() - del self.plugin - - self.m.releaseLink(self.id) - - - def delete(self): - """delete pyfile from database""" - self.m.deleteLink(self.id) - - - def toDict(self): - """return dict with all information for interface""" - return self.toDbDict() - - - def toDbDict(self): - """return data as dict for databse - - format: - - { - id: {'url': url, 'name': name ... } - } - - """ - return { - self.id: { - 'id': self.id, - 'url': self.url, - 'name': self.name, - 'plugin': self.pluginname, - 'size': self.getSize(), - 'format_size': self.formatSize(), - 'status': self.status, - 'statusmsg': self.getStatusName(), - 'package': self.packageid, - 'error': self.error, - 'order': self.order - } - } - - - def abortDownload(self): - """abort pyfile if possible""" - while self.id in self.m.core.threadManager.processingIds(): - self.abort = True - if self.plugin and self.plugin.req: - self.plugin.req.abortDownloads() - time.sleep(0.1) - - self.abort = False - if self.hasPlugin() and self.plugin.req: - self.plugin.req.abortDownloads() - - self.release() - - - def finishIfDone(self): - """set status to finish and release file if every thread is finished with it""" - - if self.id in self.m.core.threadManager.processingIds(): - return False - - self.setStatus("finished") - self.release() - self.m.checkAllLinksFinished() - return True - - - def checkIfProcessed(self): - self.m.checkAllLinksProcessed(self.id) - - - def formatWait(self): - """ formats and return wait time in humanreadable format """ - seconds = self.waitUntil - time.time() - - if seconds < 0: - return "00:00:00" - - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - - - def formatSize(self): - """ formats size to readable format """ - return formatSize(self.getSize()) - - - def formatETA(self): - """ formats eta to readable format """ - seconds = self.getETA() - - if seconds < 0: - return "00:00:00" - - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - - - def getSpeed(self): - """ calculates speed """ - try: - return self.plugin.req.speed - except Exception: - return 0 - - - def getETA(self): - """ gets established time of arrival""" - try: - return self.getBytesLeft() / self.getSpeed() - except Exception: - return 0 - - - def getBytesLeft(self): - """ gets bytes left """ - try: - return self.getSize() - self.plugin.req.arrived - except Exception: - return 0 - - - def getPercent(self): - """ get % of download """ - if self.status == 12: - try: - return self.plugin.req.percent - except Exception: - return 0 - else: - return self.progress - - - def getSize(self): - """ get size of download """ - try: - if self.plugin.req.size: - return self.plugin.req.size - else: - return self.size - except Exception: - return self.size - - - def notifyChange(self): - e = UpdateEvent("file", self.id, "collector" if not self.package().queue else "queue") - self.m.core.pullManager.addEvent(e) - - - def setProgress(self, value): - if not value == self.progress: - self.progress = value - self.notifyChange() diff --git a/pyload/datatype/Package.py b/pyload/datatype/Package.py deleted file mode 100644 index 5ba42f596..000000000 --- a/pyload/datatype/Package.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN, mkaay - -from pyload.manager.Event import UpdateEvent -from pyload.utils import safe_filename - - -class PyPackage(object): - """ - Represents a package object at runtime - """ - - def __init__(self, manager, id, name, folder, site, password, queue, order): - self.m = manager - self.m.packageCache[int(id)] = self - - self.id = int(id) - self.name = name - self._folder = folder - self.site = site - self.password = password - self.queue = queue - self.order = order - self.setFinished = False - - - @property - def folder(self): - return safe_filename(self._folder) - - - def toDict(self): - """ Returns a dictionary representation of the data. - - :return: dict: {id: { attr: value }} - """ - return { - self.id: { - 'id': self.id, - 'name': self.name, - 'folder': self.folder, - 'site': self.site, - 'password': self.password, - 'queue': self.queue, - 'order': self.order, - 'links': {} - } - } - - - def getChildren(self): - """get information about contained links""" - return self.m.getPackageData(self.id)["links"] - - - def sync(self): - """sync with db""" - self.m.updatePackage(self) - - - def release(self): - """sync and delete from cache""" - self.sync() - self.m.releasePackage(self.id) - - - def delete(self): - self.m.deletePackage(self.id) - - - def notifyChange(self): - e = UpdateEvent("pack", self.id, "collector" if not self.queue else "queue") - self.m.core.pullManager.addEvent(e) diff --git a/pyload/datatype/__init__.py b/pyload/datatype/__init__.py deleted file mode 100644 index 29f0d40aa..000000000 --- a/pyload/datatype/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- - -from pyload.Datatype.File import PyFile -from pyload.Datatype.Package import PyPackage diff --git a/pyload/plugin/addon/IRCInterface.py b/pyload/plugin/addon/IRCInterface.py index 1d7683bf4..eb015c362 100644 --- a/pyload/plugin/addon/IRCInterface.py +++ b/pyload/plugin/addon/IRCInterface.py @@ -11,7 +11,7 @@ import pycurl from select import select -from pyload.api import PackageDoesNotExists, FileDoesNotExists +from pyload.Api import PackageDoesNotExists, FileDoesNotExists from pyload.network.RequestFactory import getURL from pyload.plugin.Addon import Addon from pyload.utils import formatSize diff --git a/pyload/webui/app/api.py b/pyload/webui/app/api.py index 591b97437..35dbe9009 100644 --- a/pyload/webui/app/api.py +++ b/pyload/webui/app/api.py @@ -8,7 +8,7 @@ from itertools import chain from SafeEval import const_eval as literal_eval from bottle import route, request, response, HTTPError -from pyload.api import BaseObject +from pyload.Api import BaseObject from pyload.utils import json from pyload.webui import PYLOAD from pyload.webui.app.utils import toDict, set_session diff --git a/pyload/webui/app/utils.py b/pyload/webui/app/utils.py index 69682fe67..695162f91 100644 --- a/pyload/webui/app/utils.py +++ b/pyload/webui/app/utils.py @@ -5,7 +5,7 @@ import os from bottle import request, HTTPError, redirect, ServerAdapter -from pyload.api import has_permission, PERMS, ROLE +from pyload.Api import has_permission, PERMS, ROLE from pyload.webui import env, THEME -- cgit v1.2.3