From 6f48bf541b7b0eeb0c6968ca63125bf73e016643 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 12 May 2015 03:15:13 +0200 Subject: Fix some directory names --- pyload/Api/__init__.py | 1051 +++++++++++++++++++++++++++++++++++ pyload/Api/types.py | 562 +++++++++++++++++++ pyload/Core.py | 4 +- pyload/Database/Backend.py | 329 +++++++++++ pyload/Database/File.py | 985 ++++++++++++++++++++++++++++++++ pyload/Database/Storage.py | 36 ++ pyload/Database/User.py | 93 ++++ pyload/Database/__init__.py | 6 + pyload/Datatype/File.py | 299 ++++++++++ pyload/Datatype/Package.py | 73 +++ pyload/Datatype/__init__.py | 4 + pyload/Thread/Info.py | 2 +- pyload/Thread/Plugin.py | 2 +- pyload/Thread/Server.py | 2 +- pyload/api/__init__.py | 1051 ----------------------------------- pyload/api/types.py | 562 ------------------- pyload/database/Backend.py | 329 ----------- pyload/database/File.py | 985 -------------------------------- pyload/database/Storage.py | 36 -- pyload/database/User.py | 93 ---- pyload/database/__init__.py | 6 - pyload/datatype/File.py | 299 ---------- pyload/datatype/Package.py | 73 --- pyload/datatype/__init__.py | 4 - pyload/plugin/addon/IRCInterface.py | 2 +- pyload/webui/app/api.py | 2 +- pyload/webui/app/utils.py | 2 +- 27 files changed, 3446 insertions(+), 3446 deletions(-) create mode 100644 pyload/Api/__init__.py create mode 100644 pyload/Api/types.py create mode 100644 pyload/Database/Backend.py create mode 100644 pyload/Database/File.py create mode 100644 pyload/Database/Storage.py create mode 100644 pyload/Database/User.py create mode 100644 pyload/Database/__init__.py create mode 100644 pyload/Datatype/File.py create mode 100644 pyload/Datatype/Package.py create mode 100644 pyload/Datatype/__init__.py delete mode 100644 pyload/api/__init__.py delete mode 100644 pyload/api/types.py delete mode 100644 pyload/database/Backend.py delete mode 100644 pyload/database/File.py delete mode 100644 pyload/database/Storage.py delete mode 100644 pyload/database/User.py delete mode 100644 pyload/database/__init__.py delete mode 100644 pyload/datatype/File.py delete mode 100644 pyload/datatype/Package.py delete mode 100644 pyload/datatype/__init__.py (limited to 'pyload') diff --git a/pyload/Api/__init__.py b/pyload/Api/__init__.py new file mode 100644 index 000000000..44bae48bc --- /dev/null +++ b/pyload/Api/__init__.py @@ -0,0 +1,1051 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from __future__ import with_statement + +import base64 +import os +import re +import time +import urlparse + +from pyload.Datatype import PyFile +from pyload.utils.packagetools import parseNames +from pyload.network.RequestFactory import getURL +from pyload.remote import activated +from pyload.utils import compare_time, freeSpace, safe_filename + +if activated: + try: + from thrift.protocol import TBase + from pyload.remote.thriftbackend.thriftgen.pyload.ttypes import * + from pyload.remote.thriftbackend.thriftgen.pyload.Pyload import Iface + + BaseObject = TBase + + except ImportError: + from pyload.Api.types import * + + print "Thrift not imported" + +else: + from pyload.Api.types import * + +# contains function names mapped to their permissions +# unlisted functions are for admins only +permMap = {} + + +# decorator only called on init, never initialized, so has no effect on runtime +def permission(bits): + class _Dec(object): + + def __new__(cls, func, *args, **kwargs): + permMap[func.__name__] = bits + return func + return _Dec + + +urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&\[\]\|]*)", re.IGNORECASE) + + +class PERMS(object): + ALL = 0 #: requires no permission, but login + ADD = 1 #: can add packages + DELETE = 2 #: can delete packages + STATUS = 4 #: see and change server status + LIST = 16 #: see queue and collector + MODIFY = 32 #: moddify some attribute of downloads + DOWNLOAD = 64 #: can download from webinterface + SETTINGS = 128 #: can access settings + ACCOUNTS = 256 #: can access accounts + LOGS = 512 #: can see server logs + + +class ROLE(object): + ADMIN = 0 #: admin has all permissions implicit + USER = 1 + + +def has_permission(userperms, perms): + # bytewise or perms before if needed + return perms == (userperms & perms) + + +class Api(Iface): + """ + **pyLoads API** + + This is accessible either internal via core.api or via thrift backend. + + see Thrift specification file remote/thriftbackend/pyload.thrift\ + for information about data structures and what methods are usuable with rpc. + + Most methods requires specific permissions, please look at the source code if you need to know.\ + These can be configured via webinterface. + Admin user have all permissions, and are the only ones who can access the methods with no specific permission. + """ + EXTERNAL = Iface #: let the json api know which methods are external + + + def __init__(self, core): + self.core = core + + + def _convertPyFile(self, p): + fdata = FileData(p['id'], p['url'], p['name'], p['plugin'], p['size'], + p['format_size'], p['status'], p['statusmsg'], + p['package'], p['error'], p['order']) + return fdata + + + def _convertConfigFormat(self, c): + sections = {} + for sectionName, sub in c.iteritems(): + section = ConfigSection(sectionName, sub['desc']) + items = [] + for key, data in sub.iteritems(): + if key in ("desc", "outline"): + continue + item = ConfigItem() + item.name = key + item.description = data['desc'] + item.value = str(data['value']) if not isinstance(data['value'], basestring) else data['value'] + item.type = data['type'] + items.append(item) + section.items = items + sections[sectionName] = section + if "outline" in sub: + section.outline = sub['outline'] + return sections + + + @permission(PERMS.SETTINGS) + def getConfigValue(self, category, option, section="core"): + """Retrieve config value. + + :param category: name of category, or plugin + :param option: config option + :param section: 'plugin' or 'core' + :return: config value as string + """ + if section == "core": + value = self.core.config[category][option] + else: + value = self.core.config.getPlugin(category, option) + return str(value) + + + @permission(PERMS.SETTINGS) + def setConfigValue(self, category, option, value, section="core"): + """Set new config value. + + :param section: + :param option: + :param value: new config value + :param section: 'plugin' or 'core + """ + self.core.addonManager.dispatchEvent("config-changed", category, option, value, section) + if section == "core": + self.core.config[category][option] = value + if option in ("limit_speed", "max_speed"): #: not so nice to update the limit + self.core.requestFactory.updateBucket() + elif section == "plugin": + self.core.config.setPlugin(category, option, value) + + + @permission(PERMS.SETTINGS) + def getConfig(self): + """Retrieves complete config of core. + + :return: list of `ConfigSection` + """ + return self._convertConfigFormat(self.core.config.config) + + + def getConfigDict(self): + """Retrieves complete config in dict format, not for RPC. + + :return: dict + """ + return self.core.config.config + + + @permission(PERMS.SETTINGS) + def getPluginConfig(self): + """Retrieves complete config for all plugins. + + :return: list of `ConfigSection` + """ + return self._convertConfigFormat(self.core.config.plugin) + + + def getPluginConfigDict(self): + """Plugin config as dict, not for RPC. + + :return: dict + """ + return self.core.config.plugin + + + @permission(PERMS.STATUS) + def pauseServer(self): + """Pause server: Tt wont start any new downloads, but nothing gets aborted.""" + self.core.threadManager.pause = True + + + @permission(PERMS.STATUS) + def unpauseServer(self): + """Unpause server: New Downloads will be started.""" + self.core.threadManager.pause = False + + + @permission(PERMS.STATUS) + def togglePause(self): + """Toggle pause state. + + :return: new pause state + """ + self.core.threadManager.pause ^= True + return self.core.threadManager.pause + + + @permission(PERMS.STATUS) + def toggleReconnect(self): + """Toggle reconnect activation. + + :return: new reconnect state + """ + self.core.config['reconnect']['activated'] ^= True + return self.core.config.get("reconnect", "activated") + + + @permission(PERMS.LIST) + def statusServer(self): + """Some general information about the current status of pyLoad. + + :return: `ServerStatus` + """ + serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), + self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, + not self.core.threadManager.pause and self.isTimeDownload(), + self.core.config.get("reconnect", "activated") and self.isTimeReconnect()) + for pyfile in [x.active for x in self.core.threadManager.threads if x.active and isinstance(x.active, PyFile)]: + serverStatus.speed += pyfile.getSpeed() #: bytes/s + return serverStatus + + + @permission(PERMS.STATUS) + def freeSpace(self): + """Available free space at download directory in bytes""" + return freeSpace(self.core.config.get("general", "download_folder")) + + + @permission(PERMS.ALL) + def getServerVersion(self): + """pyLoad Core version """ + return self.core.version + + + def kill(self): + """Clean way to quit pyLoad""" + self.core.do_kill = True + + + def restart(self): + """Restart pyload core""" + self.core.do_restart = True + + + @permission(PERMS.LOGS) + def getLog(self, offset=0): + """Returns most recent log entries. + + :param offset: line offset + :return: List of log entries + """ + filename = os.path.join(self.core.config.get("log", "log_folder"), 'log.txt') + try: + with open(filename, "r") as fh: + lines = fh.readlines() + if offset >= len(lines): + return [] + return lines[offset:] + except Exception: + return ['No log available'] + + + @permission(PERMS.STATUS) + def isTimeDownload(self): + """Checks if pyload will start new downloads according to time in config. + + :return: bool + """ + start = self.core.config.get("downloadTime", "start").split(":") + end = self.core.config.get("downloadTime", "end").split(":") + return compare_time(start, end) + + + @permission(PERMS.STATUS) + def isTimeReconnect(self): + """Checks if pyload will try to make a reconnect + + :return: bool + """ + start = self.core.config.get("reconnect", "startTime").split(":") + end = self.core.config.get("reconnect", "endTime").split(":") + return compare_time(start, end) and self.core.config.get("reconnect", "activated") + + + @permission(PERMS.LIST) + def statusDownloads(self): + """ Status off all currently running downloads. + + :return: list of `DownloadStatus` + """ + data = [] + for pyfile in self.core.threadManager.getActiveFiles(): + if not isinstance(pyfile, PyFile): + continue + data.append(DownloadInfo( + pyfile.id, pyfile.name, pyfile.getSpeed(), pyfile.getETA(), pyfile.formatETA(), + pyfile.getBytesLeft(), pyfile.getSize(), pyfile.formatSize(), pyfile.getPercent(), + pyfile.status, pyfile.getStatusName(), pyfile.formatWait(), + pyfile.waitUntil, pyfile.packageid, pyfile.package().name, pyfile.pluginname)) + return data + + + @permission(PERMS.ADD) + def addPackage(self, name, links, dest=Destination.Queue): + """Adds a package, with links to desired destination. + + :param name: name of the new package + :param links: list of urls + :param dest: `Destination` + :return: package id of the new package + """ + if self.core.config.get("general", "folder_per_package"): + folder = urlparse.urlparse(name).path.split("/")[-1] + else: + folder = "" + + folder = safe_filename(folder) + + pid = self.core.files.addPackage(name, folder, dest) + + self.core.files.addLinks(links, pid) + + self.core.log.info(_("Added package %(name)s containing %(count)d links") % {"name": name.decode('utf-8'), "count": len(links)}) + + self.core.files.save() + + return pid + + + @permission(PERMS.ADD) + def parseURLs(self, html=None, url=None): + """Parses html content or any arbitaty text for links and returns result of `checkURLs` + + :param html: html source + :return: + """ + urls = [] + if html: + urls += [x[0] for x in urlmatcher.findall(html)] + if url: + page = getURL(url) + urls += [x[0] for x in urlmatcher.findall(page)] + # remove duplicates + return self.checkURLs(set(urls)) + + + @permission(PERMS.ADD) + def checkURLs(self, urls): + """ Gets urls and returns pluginname mapped to list of matches urls. + + :param urls: + :return: {plugin: urls} + """ + data = self.core.pluginManager.parseUrls(urls) + plugins = {} + + for url, plugintype, pluginname in data: + try: + plugins[plugintype][pluginname].append(url) + except Exception: + plugins[plugintype][pluginname] = [url] + + return plugins + + + @permission(PERMS.ADD) + def checkOnlineStatus(self, urls): + """ initiates online status check + + :param urls: + :return: initial set of data as `OnlineCheck` instance containing the result id + """ + data = self.core.pluginManager.parseUrls(urls) + + rid = self.core.threadManager.createResultThread(data, False) + + tmp = [(url, (url, OnlineStatus(url, (plugintype, pluginname), "unknown", 3, 0))) for url, plugintype, pluginname in data] + data = parseNames(tmp) + result = {} + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + return OnlineCheck(rid, result) + + + @permission(PERMS.ADD) + def checkOnlineStatusContainer(self, urls, container, data): + """ checks online status of urls and a submited container file + + :param urls: list of urls + :param container: container file name + :param data: file content + :return: online check + """ + with open(os.path.join(self.core.config.get("general", "download_folder"), "tmp_" + container), "wb") as th: + th.write(str(data)) + return self.checkOnlineStatus(urls + [th.name]) + + + @permission(PERMS.ADD) + def pollResults(self, rid): + """ Polls the result available for ResultID + + :param rid: `ResultID` + :return: `OnlineCheck`, if rid is -1 then no more data available + """ + result = self.core.threadManager.getInfoResult(rid) + if "ALL_INFO_FETCHED" in result: + del result['ALL_INFO_FETCHED'] + return OnlineCheck(-1, result) + else: + return OnlineCheck(rid, result) + + + @permission(PERMS.ADD) + def generatePackages(self, links): + """ Parses links, generates packages names from urls + + :param links: list of urls + :return: package names mapped to urls + """ + return parseNames((x, x) for x in links) + + + @permission(PERMS.ADD) + def generateAndAddPackages(self, links, dest=Destination.Queue): + """Generates and add packages + + :param links: list of urls + :param dest: `Destination` + :return: list of package ids + """ + return [self.addPackage(name, urls, dest) for name, urls + in self.generatePackages(links).iteritems()] + + + @permission(PERMS.ADD) + def checkAndAddPackages(self, links, dest=Destination.Queue): + """Checks online status, retrieves names, and will add packages.\ + Because of this packages are not added immediatly, only for internal use. + + :param links: list of urls + :param dest: `Destination` + :return: None + """ + data = self.core.pluginManager.parseUrls(links) + self.core.threadManager.createResultThread(data, True) + + + @permission(PERMS.LIST) + def getPackageData(self, pid): + """Returns complete information about package, and included files. + + :param pid: package id + :return: `PackageData` with .links attribute + """ + data = self.core.files.getPackageData(int(pid)) + if not data: + raise PackageDoesNotExists(pid) + return PackageData(data['id'], data['name'], data['folder'], data['site'], data['password'], + data['queue'], data['order'], + links=[self._convertPyFile(x) for x in data['links'].itervalues()]) + + + @permission(PERMS.LIST) + def getPackageInfo(self, pid): + """Returns information about package, without detailed information about containing files + + :param pid: package id + :return: `PackageData` with .fid attribute + """ + data = self.core.files.getPackageData(int(pid)) + + if not data: + raise PackageDoesNotExists(pid) + return PackageData(data['id'], data['name'], data['folder'], data['site'], data['password'], + data['queue'], data['order'], + fids=[int(x) for x in data['links']]) + + + @permission(PERMS.LIST) + def getFileData(self, fid): + """Get complete information about a specific file. + + :param fid: file id + :return: `FileData` + """ + info = self.core.files.getFileData(int(fid)) + if not info: + raise FileDoesNotExists(fid) + return self._convertPyFile(info.values()[0]) + + + @permission(PERMS.DELETE) + def deleteFiles(self, fids): + """Deletes several file entries from pyload. + + :param fids: list of file ids + """ + for fid in fids: + self.core.files.deleteLink(int(fid)) + self.core.files.save() + + + @permission(PERMS.DELETE) + def deletePackages(self, pids): + """Deletes packages and containing links. + + :param pids: list of package ids + """ + for pid in pids: + self.core.files.deletePackage(int(pid)) + self.core.files.save() + + + @permission(PERMS.LIST) + def getQueue(self): + """Returns info about queue and packages, **not** about files, see `getQueueData` \ + or `getPackageData` instead. + + :return: list of `PackageInfo` + """ + return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], + pack['password'], pack['queue'], pack['order'], + pack['linksdone'], pack['sizedone'], pack['sizetotal'], + pack['linkstotal']) + for pack in self.core.files.getInfoData(Destination.Queue).itervalues()] + + + @permission(PERMS.LIST) + def getQueueData(self): + """Return complete data about everything in queue, this is very expensive use it sparely.\ + See `getQueue` for alternative. + + :return: list of `PackageData` + """ + return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], + pack['password'], pack['queue'], pack['order'], + pack['linksdone'], pack['sizedone'], pack['sizetotal'], + links=[self._convertPyFile(x) for x in pack['links'].itervalues()]) + for pack in self.core.files.getCompleteData(Destination.Queue).itervalues()] + + + @permission(PERMS.LIST) + def getCollector(self): + """same as `getQueue` for collector. + + :return: list of `PackageInfo` + """ + return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], + pack['password'], pack['queue'], pack['order'], + pack['linksdone'], pack['sizedone'], pack['sizetotal'], + pack['linkstotal']) + for pack in self.core.files.getInfoData(Destination.Collector).itervalues()] + + + @permission(PERMS.LIST) + def getCollectorData(self): + """same as `getQueueData` for collector. + + :return: list of `PackageInfo` + """ + return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], + pack['password'], pack['queue'], pack['order'], + pack['linksdone'], pack['sizedone'], pack['sizetotal'], + links=[self._convertPyFile(x) for x in pack['links'].itervalues()]) + for pack in self.core.files.getCompleteData(Destination.Collector).itervalues()] + + + @permission(PERMS.ADD) + def addFiles(self, pid, links): + """Adds files to specific package. + + :param pid: package id + :param links: list of urls + """ + self.core.files.addLinks(links, int(pid)) + self.core.log.info(_("Added %(count)d links to package #%(package)d ") % {"count": len(links), "package": pid}) + self.core.files.save() + + + @permission(PERMS.MODIFY) + def pushToQueue(self, pid): + """Moves package from Collector to Queue. + + :param pid: package id + """ + self.core.files.setPackageLocation(pid, Destination.Queue) + + + @permission(PERMS.MODIFY) + def pullFromQueue(self, pid): + """Moves package from Queue to Collector. + + :param pid: package id + """ + self.core.files.setPackageLocation(pid, Destination.Collector) + + + @permission(PERMS.MODIFY) + def restartPackage(self, pid): + """Restarts a package, resets every containing files. + + :param pid: package id + """ + self.core.files.restartPackage(int(pid)) + + + @permission(PERMS.MODIFY) + def restartFile(self, fid): + """Resets file status, so it will be downloaded again. + + :param fid: file id + """ + self.core.files.restartFile(int(fid)) + + + @permission(PERMS.MODIFY) + def recheckPackage(self, pid): + """Proofes online status of all files in a package, also a default action when package is added. + + :param pid: + :return: + """ + self.core.files.reCheckPackage(int(pid)) + + + @permission(PERMS.MODIFY) + def stopAllDownloads(self): + """Aborts all running downloads.""" + + pyfiles = self.core.files.cache.values() + for pyfile in pyfiles: + pyfile.abortDownload() + + + @permission(PERMS.MODIFY) + def stopDownloads(self, fids): + """Aborts specific downloads. + + :param fids: list of file ids + :return: + """ + pyfiles = self.core.files.cache.values() + for pyfile in pyfiles: + if pyfile.id in fids: + pyfile.abortDownload() + + + @permission(PERMS.MODIFY) + def setPackageName(self, pid, name): + """Renames a package. + + :param pid: package id + :param name: new package name + """ + pack = self.core.files.getPackage(pid) + pack.name = name + pack.sync() + + + @permission(PERMS.MODIFY) + def movePackage(self, destination, pid): + """Set a new package location. + + :param destination: `Destination` + :param pid: package id + """ + if destination in (0, 1): + self.core.files.setPackageLocation(pid, destination) + + + @permission(PERMS.MODIFY) + def moveFiles(self, fids, pid): + """Move multiple files to another package + + :param fids: list of file ids + :param pid: destination package + :return: + """ + # TODO: implement + pass + + + @permission(PERMS.ADD) + def uploadContainer(self, filename, data): + """Uploads and adds a container file to pyLoad. + + :param filename: filename, extension is important so it can correctly decrypted + :param data: file content + """ + with open(os.path.join(self.core.config.get("general", "download_folder"), "tmp_" + filename), "wb") as th: + th.write(str(data)) + self.addPackage(th.name, [th.name], Destination.Queue) + + + @permission(PERMS.MODIFY) + def orderPackage(self, pid, position): + """Gives a package a new position. + + :param pid: package id + :param position: + """ + self.core.files.reorderPackage(pid, position) + + + @permission(PERMS.MODIFY) + def orderFile(self, fid, position): + """Gives a new position to a file within its package. + + :param fid: file id + :param position: + """ + self.core.files.reorderFile(fid, position) + + + @permission(PERMS.MODIFY) + def setPackageData(self, pid, data): + """Allows to modify several package attributes. + + :param pid: package id + :param data: dict that maps attribute to desired value + """ + package = self.core.files.getPackage(pid) + if not package: + raise PackageDoesNotExists(pid) + for key, value in data.iteritems(): + if key == "id": + continue + setattr(package, key, value) + package.sync() + self.core.files.save() + + + @permission(PERMS.DELETE) + def deleteFinished(self): + """Deletes all finished files and completly finished packages. + + :return: list of deleted package ids + """ + return self.core.files.deleteFinishedLinks() + + + @permission(PERMS.MODIFY) + def restartFailed(self): + """Restarts all failed failes.""" + self.core.files.restartFailed() + + + @permission(PERMS.LIST) + def getPackageOrder(self, destination): + """Returns information about package order. + + :param destination: `Destination` + :return: dict mapping order to package id + """ + packs = self.core.files.getInfoData(destination) + order = {} + for pid in packs: + pack = self.core.files.getPackageData(int(pid)) + while pack['order'] in order.keys(): #: just in case + pack['order'] += 1 + order[pack['order']] = pack['id'] + return order + + + @permission(PERMS.LIST) + def getFileOrder(self, pid): + """Information about file order within package. + + :param pid: + :return: dict mapping order to file id + """ + rawdata = self.core.files.getPackageData(int(pid)) + order = {} + for id, pyfile in rawdata['links'].iteritems(): + while pyfile['order'] in order.keys(): #: just in case + pyfile['order'] += 1 + order[pyfile['order']] = pyfile['id'] + return order + + + @permission(PERMS.STATUS) + def isCaptchaWaiting(self): + """Indicates wether a captcha task is available + + :return: bool + """ + self.core.lastClientConnected = time.time() + task = self.core.captchaManager.getTask() + return not task is None + + + @permission(PERMS.STATUS) + def getCaptchaTask(self, exclusive=False): + """Returns a captcha task + + :param exclusive: unused + :return: `CaptchaTask` + """ + self.core.lastClientConnected = time.time() + task = self.core.captchaManager.getTask() + if task: + task.setWatingForUser(exclusive=exclusive) + data, type, result = task.getCaptcha() + ctask = CaptchaTask(int(task.id), base64.standard_b64encode(data), type, result) + return ctask + return CaptchaTask(-1) + + + @permission(PERMS.STATUS) + def getCaptchaTaskStatus(self, tid): + """Get information about captcha task + + :param tid: task id + :return: string + """ + self.core.lastClientConnected = time.time() + task = self.core.captchaManager.getTaskByID(tid) + return task.getStatus() if task else "" + + + @permission(PERMS.STATUS) + def setCaptchaResult(self, tid, result): + """Set result for a captcha task + + :param tid: task id + :param result: captcha result + """ + self.core.lastClientConnected = time.time() + task = self.core.captchaManager.getTaskByID(tid) + if task: + task.setResult(result) + self.core.captchaManager.removeTask(task) + + + @permission(PERMS.STATUS) + def getEvents(self, uuid): + """Lists occured events, may be affected to changes in future. + + :param uuid: + :return: list of `Events` + """ + events = self.core.pullManager.getEvents(uuid) + new_events = [] + + + def convDest(d): + return Destination.Queue if d == "queue" else Destination.Collector + + for e in events: + event = EventInfo() + event.eventname = e[0] + if e[0] in ("update", "remove", "insert"): + event.id = e[3] + event.type = ElementType.Package if e[2] == "pack" else ElementType.File + event.destination = convDest(e[1]) + elif e[0] == "order": + if e[1]: + event.id = e[1] + event.type = ElementType.Package if e[2] == "pack" else ElementType.File + event.destination = convDest(e[3]) + elif e[0] == "reload": + event.destination = convDest(e[1]) + new_events.append(event) + return new_events + + + @permission(PERMS.ACCOUNTS) + def getAccounts(self, refresh): + """Get information about all entered accounts. + + :param refresh: reload account info + :return: list of `AccountInfo` + """ + accs = self.core.accountManager.getAccountInfos(False, refresh) + for group in accs.values(): + accounts = [AccountInfo(acc['validuntil'], acc['login'], acc['options'], acc['valid'], + acc['trafficleft'], acc['maxtraffic'], acc['premium'], acc['type']) + for acc in group] + return accounts or list() + + + @permission(PERMS.ALL) + def getAccountTypes(self): + """All available account types. + + :return: list + """ + return self.core.accountManager.accounts.keys() + + + @permission(PERMS.ACCOUNTS) + def updateAccount(self, plugin, account, password=None, options={}): + """Changes pw/options for specific account.""" + self.core.accountManager.updateAccount(plugin, account, password, options) + + + @permission(PERMS.ACCOUNTS) + def removeAccount(self, plugin, account): + """Remove account from pyload. + + :param plugin: pluginname + :param account: accountname + """ + self.core.accountManager.removeAccount(plugin, account) + + + @permission(PERMS.ALL) + def login(self, username, password, remoteip=None): + """Login into pyLoad, this **must** be called when using rpc before any methods can be used. + + :param username: + :param password: + :param remoteip: Omit this argument, its only used internal + :return: bool indicating login was successful + """ + return bool(self.checkAuth(username, password, remoteip)) + + + def checkAuth(self, username, password, remoteip=None): + """Check authentication and returns details + + :param username: + :param password: + :param remoteip: + :return: dict with info, empty when login is incorrect + """ + if self.core.config.get("remote", "nolocalauth") and remoteip == "127.0.0.1": + return "local" + else: + return self.core.db.checkAuth(username, password) + + + def isAuthorized(self, func, userdata): + """checks if the user is authorized for specific method + + :param func: function name + :param userdata: dictionary of user data + :return: boolean + """ + if userdata == "local" or userdata['role'] == ROLE.ADMIN: + return True + elif func in permMap and has_permission(userdata['permission'], permMap[func]): + return True + else: + return False + + + @permission(PERMS.ALL) + def getUserData(self, username, password): + """similar to `checkAuth` but returns UserData thrift type """ + user = self.checkAuth(username, password) + if user: + return UserData(user['name'], user['email'], user['role'], user['permission'], user['template']) + else: + return UserData() + + + def getAllUserData(self): + """returns all known user and info""" + return dict((user, UserData(user, data['email'], data['role'], data['permission'], data['template'])) for user, data + in self.core.db.getAllUserData().iteritems()) + + + @permission(PERMS.STATUS) + def getServices(self): + """ A dict of available services, these can be defined by addon plugins. + + :return: dict with this style: {"plugin": {"method": "description"}} + """ + return dict((plugin, funcs) for plugin, funcs in self.core.addonManager.methods.iteritems()) + + + @permission(PERMS.STATUS) + def hasService(self, plugin, func): + """Checks wether a service is available. + + :param plugin: + :param func: + :return: bool + """ + cont = self.core.addonManager.methods + return plugin in cont and func in cont[plugin] + + + @permission(PERMS.STATUS) + def call(self, info): + """Calls a service (a method in addon plugin). + + :param info: `ServiceCall` + :return: result + :raises: ServiceDoesNotExists, when its not available + :raises: ServiceException, when a exception was raised + """ + plugin = info.plugin + func = info.func + args = info.arguments + parse = info.parseArguments + if not self.hasService(plugin, func): + raise ServiceDoesNotExists(plugin, func) + try: + ret = self.core.addonManager.callRPC(plugin, func, args, parse) + except Exception, e: + raise ServiceException(e.message) + + + @permission(PERMS.STATUS) + def getAllInfo(self): + """Returns all information stored by addon plugins. Values are always strings + + :return: {"plugin": {"name": value}} + """ + return self.core.addonManager.getAllInfo() + + + @permission(PERMS.STATUS) + def getInfoByPlugin(self, plugin): + """Returns information stored by a specific plugin. + + :param plugin: pluginname + :return: dict of attr names mapped to value {"name": value} + """ + return self.core.addonManager.getInfo(plugin) + + + def changePassword(self, user, oldpw, newpw): + """ changes password for specific user """ + return self.core.db.changePassword(user, oldpw, newpw) + + + def setUserPermission(self, user, perm, role): + self.core.db.setPermission(user, perm) + self.core.db.setRole(user, role) diff --git a/pyload/Api/types.py b/pyload/Api/types.py new file mode 100644 index 000000000..9381df3c7 --- /dev/null +++ b/pyload/Api/types.py @@ -0,0 +1,562 @@ +# -*- coding: utf-8 -*- +# Autogenerated by pyload +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + +class BaseObject(object): + __slots__ = [] + + +class Destination(object): + Collector = 0 + Queue = 1 + + +class DownloadStatus(object): + Aborted = 9 + Custom = 11 + Decrypting = 10 + Downloading = 12 + Failed = 8 + Finished = 0 + Offline = 1 + Online = 2 + Processing = 13 + Queued = 3 + Skipped = 4 + Starting = 7 + TempOffline = 6 + Unknown = 14 + Waiting = 5 + + +class ElementType(object): + File = 1 + Package = 0 + + +class Input(object): + BOOL = 4 + CHOICE = 6 + CLICK = 5 + LIST = 8 + MULTIPLE = 7 + NONE = 0 + PASSWORD = 3 + TABLE = 9 + TEXT = 1 + TEXTBOX = 2 + + +class Output(object): + CAPTCHA = 1 + NOTIFICATION = 4 + QUESTION = 2 + + +class AccountInfo(BaseObject): + __slots__ = ['validuntil', 'login', 'options', 'valid', 'trafficleft', 'maxtraffic', 'premium', 'type'] + + + def __init__(self, validuntil=None, login=None, options=None, valid=None, trafficleft=None, maxtraffic=None, premium=None, type=None): + self.validuntil = validuntil + self.login = login + self.options = options + self.valid = valid + self.trafficleft = trafficleft + self.maxtraffic = maxtraffic + self.premium = premium + self.type = type + + +class CaptchaTask(BaseObject): + __slots__ = ['tid', 'data', 'type', 'resultType'] + + + def __init__(self, tid=None, data=None, type=None, resultType=None): + self.tid = tid + self.data = data + self.type = type + self.resultType = resultType + + +class ConfigItem(BaseObject): + __slots__ = ['name', 'description', 'value', 'type'] + + + def __init__(self, name=None, description=None, value=None, type=None): + self.name = name + self.description = description + self.value = value + self.type = type + + +class ConfigSection(BaseObject): + __slots__ = ['name', 'description', 'items', 'outline'] + + + def __init__(self, name=None, description=None, items=None, outline=None): + self.name = name + self.description = description + self.items = items + self.outline = outline + + +class DownloadInfo(BaseObject): + __slots__ = ['fid', 'name', 'speed', 'eta', 'format_eta', 'bleft', 'size', 'format_size', 'percent', 'status', 'statusmsg', 'format_wait', 'wait_until', 'packageID', 'packageName', 'plugin'] + + + def __init__(self, fid=None, name=None, speed=None, eta=None, format_eta=None, bleft=None, size=None, format_size=None, percent=None, status=None, statusmsg=None, format_wait=None, wait_until=None, packageID=None, packageName=None, plugin=None): + self.fid = fid + self.name = name + self.speed = speed + self.eta = eta + self.format_eta = format_eta + self.bleft = bleft + self.size = size + self.format_size = format_size + self.percent = percent + self.status = status + self.statusmsg = statusmsg + self.format_wait = format_wait + self.wait_until = wait_until + self.packageID = packageID + self.packageName = packageName + self.plugin = plugin + + +class EventInfo(BaseObject): + __slots__ = ['eventname', 'id', 'type', 'destination'] + + + def __init__(self, eventname=None, id=None, type=None, destination=None): + self.eventname = eventname + self.id = id + self.type = type + self.destination = destination + + +class FileData(BaseObject): + __slots__ = ['fid', 'url', 'name', 'plugin', 'size', 'format_size', 'status', 'statusmsg', 'packageID', 'error', 'order'] + + + def __init__(self, fid=None, url=None, name=None, plugin=None, size=None, format_size=None, status=None, statusmsg=None, packageID=None, error=None, order=None): + self.fid = fid + self.url = url + self.name = name + self.plugin = plugin + self.size = size + self.format_size = format_size + self.status = status + self.statusmsg = statusmsg + self.packageID = packageID + self.error = error + self.order = order + + +class FileDoesNotExists(Exception): + __slots__ = ['fid'] + + + def __init__(self, fid=None): + self.fid = fid + + +class InteractionTask(BaseObject): + __slots__ = ['iid', 'input', 'structure', 'preset', 'output', 'data', 'title', 'description', 'plugin'] + + + def __init__(self, iid=None, input=None, structure=None, preset=None, output=None, data=None, title=None, description=None, plugin=None): + self.iid = iid + self.input = input + self.structure = structure + self.preset = preset + self.output = output + self.data = data + self.title = title + self.description = description + self.plugin = plugin + + +class OnlineCheck(BaseObject): + __slots__ = ['rid', 'data'] + + + def __init__(self, rid=None, data=None): + self.rid = rid + self.data = data + + +class OnlineStatus(BaseObject): + __slots__ = ['name', 'plugin', 'packagename', 'status', 'size'] + + + def __init__(self, name=None, plugin=(None, None), packagename=None, status=None, size=None): + self.name = name + self.plugin = plugin + self.packagename = packagename + self.status = status + self.size = size + + +class PackageData(BaseObject): + __slots__ = ['pid', 'name', 'folder', 'site', 'password', 'dest', 'order', 'linksdone', 'sizedone', 'sizetotal', 'linkstotal', 'links', 'fids'] + + + def __init__(self, pid=None, name=None, folder=None, site=None, password=None, dest=None, order=None, linksdone=None, sizedone=None, sizetotal=None, linkstotal=None, links=None, fids=None): + self.pid = pid + self.name = name + self.folder = folder + self.site = site + self.password = password + self.dest = dest + self.order = order + self.linksdone = linksdone + self.sizedone = sizedone + self.sizetotal = sizetotal + self.linkstotal = linkstotal + self.links = links + self.fids = fids + + +class PackageDoesNotExists(Exception): + __slots__ = ['pid'] + + + def __init__(self, pid=None): + self.pid = pid + + +class ServerStatus(BaseObject): + __slots__ = ['pause', 'active', 'queue', 'total', 'speed', 'download', 'reconnect'] + + + def __init__(self, pause=None, active=None, queue=None, total=None, speed=None, download=None, reconnect=None): + self.pause = pause + self.active = active + self.queue = queue + self.total = total + self.speed = speed + self.download = download + self.reconnect = reconnect + + +class ServiceCall(BaseObject): + __slots__ = ['plugin', 'func', 'arguments', 'parseArguments'] + + + def __init__(self, plugin=None, func=None, arguments=None, parseArguments=None): + self.plugin = plugin + self.func = func + self.arguments = arguments + self.parseArguments = parseArguments + + +class ServiceDoesNotExists(Exception): + __slots__ = ['plugin', 'func'] + + + def __init__(self, plugin=None, func=None): + self.plugin = plugin + self.func = func + + +class ServiceException(Exception): + __slots__ = ['msg'] + + + def __init__(self, msg=None): + self.msg = msg + + +class UserData(BaseObject): + __slots__ = ['name', 'email', 'role', 'permission', 'templateName'] + + + def __init__(self, name=None, email=None, role=None, permission=None, templateName=None): + self.name = name + self.email = email + self.role = role + self.permission = permission + self.templateName = templateName + + +class Iface(object): + + def addFiles(self, pid, links): + pass + + + def addPackage(self, name, links, dest): + pass + + + def call(self, info): + pass + + + def checkOnlineStatus(self, urls): + pass + + + def checkOnlineStatusContainer(self, urls, filename, data): + pass + + + def checkURLs(self, urls): + pass + + + def deleteFiles(self, fids): + pass + + + def deleteFinished(self): + pass + + + def deletePackages(self, pids): + pass + + + def freeSpace(self): + pass + + + def generateAndAddPackages(self, links, dest): + pass + + + def generatePackages(self, links): + pass + + + def getAccountTypes(self): + pass + + + def getAccounts(self, refresh): + pass + + + def getAllInfo(self): + pass + + + def getAllUserData(self): + pass + + + def getCaptchaTask(self, exclusive): + pass + + + def getCaptchaTaskStatus(self, tid): + pass + + + def getCollector(self): + pass + + + def getCollectorData(self): + pass + + + def getConfig(self): + pass + + + def getConfigValue(self, category, option, section): + pass + + + def getEvents(self, uuid): + pass + + + def getFileData(self, fid): + pass + + + def getFileOrder(self, pid): + pass + + + def getInfoByPlugin(self, plugin): + pass + + + def getLog(self, offset): + pass + + + def getPackageData(self, pid): + pass + + + def getPackageInfo(self, pid): + pass + + + def getPackageOrder(self, destination): + pass + + + def getPluginConfig(self): + pass + + + def getQueue(self): + pass + + + def getQueueData(self): + pass + + + def getServerVersion(self): + pass + + + def getServices(self): + pass + + + def getUserData(self, username, password): + pass + + + def hasService(self, plugin, func): + pass + + + def isCaptchaWaiting(self): + pass + + + def isTimeDownload(self): + pass + + + def isTimeReconnect(self): + pass + + + def kill(self): + pass + + + def login(self, username, password): + pass + + + def moveFiles(self, fids, pid): + pass + + + def movePackage(self, destination, pid): + pass + + + def orderFile(self, fid, position): + pass + + + def orderPackage(self, pid, position): + pass + + + def parseURLs(self, html, url): + pass + + + def pauseServer(self): + pass + + + def pollResults(self, rid): + pass + + + def pullFromQueue(self, pid): + pass + + + def pushToQueue(self, pid): + pass + + + def recheckPackage(self, pid): + pass + + + def removeAccount(self, plugin, account): + pass + + + def restart(self): + pass + + + def restartFailed(self): + pass + + + def restartFile(self, fid): + pass + + + def restartPackage(self, pid): + pass + + + def setCaptchaResult(self, tid, result): + pass + + + def setConfigValue(self, category, option, value, section): + pass + + + def setPackageData(self, pid, data): + pass + + + def setPackageName(self, pid, name): + pass + + + def statusDownloads(self): + pass + + + def statusServer(self): + pass + + + def stopAllDownloads(self): + pass + + + def stopDownloads(self, fids): + pass + + + def togglePause(self): + pass + + + def toggleReconnect(self): + pass + + + def unpauseServer(self): + pass + + + def updateAccount(self, plugin, account, password, options): + pass + + + def uploadContainer(self, filename, data): + pass diff --git a/pyload/Core.py b/pyload/Core.py index eb6a968b4..545c1d63d 100755 --- a/pyload/Core.py +++ b/pyload/Core.py @@ -360,11 +360,11 @@ class Core(object): self.lastClientConnected = 0 # later imported because they would trigger api import, and remote value not set correctly - from pyload.api import Api + from pyload.Api import Api from pyload.manager.Addon import AddonManager from pyload.manager.Thread import ThreadManager - if pyload.api.activated != self.remote: + if pyload.Api.activated != self.remote: self.log.warning("Import error: API remote status not correct.") self.api = Api(self) diff --git a/pyload/Database/Backend.py b/pyload/Database/Backend.py new file mode 100644 index 000000000..0fc961973 --- /dev/null +++ b/pyload/Database/Backend.py @@ -0,0 +1,329 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +from __future__ import with_statement + +try: + from pysqlite2 import dbapi2 as sqlite3 +except Exception: + import sqlite3 + +import shutil +import threading +import traceback + +from Queue import Queue + +from pyload.utils import chmod + + +DB_VERSION = 4 + + +class style(object): + db = None + + + @classmethod + def setDB(cls, db): + cls.db = db + + + @classmethod + def inner(cls, f): + + + @staticmethod + def x(*args, **kwargs): + if cls.db: + return f(cls.db, *args, **kwargs) + + return x + + + @classmethod + def queue(cls, f): + + + @staticmethod + def x(*args, **kwargs): + if cls.db: + return cls.db.queue(f, *args, **kwargs) + + return x + + + @classmethod + def async(cls, f): + + + @staticmethod + def x(*args, **kwargs): + if cls.db: + return cls.db.async(f, *args, **kwargs) + return x + + +class DatabaseJob(object): + + def __init__(self, f, *args, **kwargs): + self.done = threading.Event() + + self.f = f + self.args = args + self.kwargs = kwargs + + self.result = None + self.exception = False + + # import inspect + # self.frame = inspect.currentframe() + + + def __repr__(self): + import os + + frame = self.frame.f_back + output = "" + for _i in xrange(5): + output += "\t%s:%s, %s\n" % (os.path.basename(frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name) + frame = frame.f_back + del frame + del self.frame + + return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result) + + + def processJob(self): + try: + self.result = self.f(*self.args, **self.kwargs) + except Exception, e: + traceback.print_exc() + try: + print "Database Error @", self.f.__name__, self.args[1:], self.kwargs, e + except Exception: + pass + + self.exception = e + finally: + self.done.set() + + + def wait(self): + self.done.wait() + + +class DatabaseBackend(threading.Thread): + subs = [] + + + def __init__(self, core): + threading.Thread.__init__(self) + self.setDaemon(True) + self.core = core + + self.jobs = Queue() + + self.setuplock = threading.Event() + + style.setDB(self) + + + def setup(self): + self.start() + self.setuplock.wait() + + + def run(self): + """main loop, which executes commands""" + convert = self._checkVersion() #: returns None or current version + + self.conn = sqlite3.connect("files.db") + os.chmod("files.db", 0600) + + self.c = self.conn.cursor() #: compatibility + + if convert is not None: + self._convertDB(convert) + + self._createTables() + self._migrateUser() + + self.conn.commit() + + self.setuplock.set() + + while True: + j = self.jobs.get() + if j == "quit": + self.c.close() + self.conn.close() + break + j.processJob() + + + @style.queue + def shutdown(self): + self.conn.commit() + self.jobs.put("quit") + + + def _checkVersion(self): + """ check db version and delete it if needed""" + if not os.path.exists("files.version"): + with open("files.version", "wb") as f: + f.write(str(DB_VERSION)) + return + + with open("files.version", "rb") as f: + v = int(f.read().strip() or 0) + + if v < DB_VERSION: + if v < 2: + try: + self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) + except Exception: + print "Filedatabase was deleted due to incompatible version." + reshutil.move("files.version") + shutil.move("files.db", "files.backup.db") + + with open("files.version", "wb") as f: + f.write(str(DB_VERSION)) + + return v + + + def _convertDB(self, v): + try: + getattr(self, "_convertV%i" % v)() + except Exception: + try: + self.core.log.error(_("Filedatabase could NOT be converted.")) + except Exception: + print "Filedatabase could NOT be converted." + + # convert scripts start --------------------------------------------------- + + + def _convertV2(self): + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + try: + self.manager.core.log.info(_("Database was converted from v2 to v3.")) + except Exception: + print "Database was converted from v2 to v3." + self._convertV3() + + + def _convertV3(self): + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + try: + self.manager.core.log.info(_("Database was converted from v3 to v4.")) + except Exception: + print "Database was converted from v3 to v4." + + # convert scripts end ----------------------------------------------------- + + + def _createTables(self): + """create tables for database""" + + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') + self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + + self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \ + SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\ + FROM packages p JOIN links l ON p.id = l.package LEFT OUTER JOIN\ + (SELECT p.id AS id, COUNT(*) AS linksdone, SUM(l.size) AS sizedone \ + FROM packages p JOIN links l ON p.id = l.package AND l.status IN (0, 4, 13) GROUP BY p.id) s ON s.id = p.id \ + GROUP BY p.id') + + # try to lower ids + self.c.execute('SELECT max(id) FROM LINKS') + fid = self.c.fetchone()[0] + fid = int(fid) if fid else 0 + self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links")) + + self.c.execute('SELECT max(id) FROM packages') + pid = self.c.fetchone()[0] + pid = int(pid) if pid else 0 + self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (pid, "packages")) + + self.c.execute('VACUUM') + + + def _migrateUser(self): + if os.path.exists("pyload.db"): + try: + self.core.log.info(_("Converting old Django DB")) + except Exception: + print "Converting old Django DB" + conn = sqlite3.connect('pyload.db') + c = conn.cursor() + c.execute("SELECT username, password, email FROM auth_user WHERE is_superuser") + users = [] + for r in c: + pw = r[1].split("$") + users.append((r[0], pw[1] + pw[2], r[2])) + c.close() + conn.close() + + self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users) + shutil.move("pyload.db", "pyload.old.db") + + + def createCursor(self): + return self.conn.cursor() + + + @style.async + def commit(self): + self.conn.commit() + + + @style.queue + def syncSave(self): + self.conn.commit() + + + @style.async + def rollback(self): + self.conn.rollback() + + + def async(self, f, *args, **kwargs): + args = (self,) + args + job = DatabaseJob(f, *args, **kwargs) + self.jobs.put(job) + + + def queue(self, f, *args, **kwargs): + args = (self,) + args + job = DatabaseJob(f, *args, **kwargs) + self.jobs.put(job) + job.wait() + return job.result + + + @classmethod + def registerSub(cls, klass): + cls.subs.append(klass) + + + @classmethod + def unregisterSub(cls, klass): + cls.subs.remove(klass) + + + def __getattr__(self, attr): + for sub in DatabaseBackend.subs: + if hasattr(sub, attr): + return getattr(sub, attr) diff --git a/pyload/Database/File.py b/pyload/Database/File.py new file mode 100644 index 000000000..f28535ffa --- /dev/null +++ b/pyload/Database/File.py @@ -0,0 +1,985 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +import threading + +from pyload.Datatype import PyFile, PyPackage +from pyload.Database import DatabaseBackend, style +from pyload.manager.Event import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent +from pyload.utils import formatSize, lock + +try: + from pysqlite2 import dbapi2 as sqlite3 +except Exception: + import sqlite3 + + +class FileHandler(object): + """Handles all request made to obtain information, + modify status or other request for links or packages""" + + def __init__(self, core): + """Constructor""" + self.core = core + + # translations + self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), + _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), + _("downloading"), _("processing"), _("unknown")] + + self.cache = {} #: holds instances for files + self.packageCache = {} #: same for packages + #@TODO: purge the cache + + self.jobCache = {} + + self.lock = threading.RLock() #@TODO: should be a Lock w/o R + # self.lock._Verbose__verbose = True + + self.filecount = -1 #: if an invalid value is set get current value from db + self.queuecount = -1 #: number of package to be loaded + self.unchanged = False #: determines if any changes was made since last call + + self.db = self.core.db + + + def change(func): + + + def new(*args): + args[0].unchanged = False + args[0].filecount = -1 + args[0].queuecount = -1 + args[0].jobCache = {} + return func(*args) + + return new + + + #-------------------------------------------------------------------------- + + def save(self): + """saves all data to backend""" + self.db.commit() + + + #-------------------------------------------------------------------------- + + def syncSave(self): + """saves all data to backend and waits until all data are written""" + pyfiles = self.cache.values() + for pyfile in pyfiles: + pyfile.sync() + + pypacks = self.packageCache.values() + for pypack in pypacks: + pypack.sync() + + self.db.syncSave() + + + @lock + def getCompleteData(self, queue=1): + """gets a complete data representation""" + + data = self.db.getAllLinks(queue) + packs = self.db.getAllPackages(queue) + + data.update([(x.id, x.toDbDict()[x.id]) for x in self.cache.values()]) + + for x in self.packageCache.itervalues(): + if x.queue != queue or x.id not in packs: + continue + packs[x.id].update(x.toDict()[x.id]) + + for key, value in data.iteritems(): + if value['package'] in packs: + packs[value['package']]['links'][key] = value + + return packs + + + @lock + def getInfoData(self, queue=1): + """gets a data representation without links""" + + packs = self.db.getAllPackages(queue) + for x in self.packageCache.itervalues(): + if x.queue != queue or x.id not in packs: + continue + packs[x.id].update(x.toDict()[x.id]) + + return packs + + + @lock + @change + def addLinks(self, urls, package): + """adds links""" + + self.core.addonManager.dispatchEvent("links-added", urls, package) + + data = self.core.pluginManager.parseUrls(urls) + + self.db.addLinks(data, package) + self.core.threadManager.createInfoThread(data, package) + + #@TODO: change from reloadAll event to package update event + self.core.pullManager.addEvent(ReloadAllEvent("collector")) + + + #-------------------------------------------------------------------------- + + @lock + @change + def addPackage(self, name, folder, queue=0): + """adds a package, default to link collector""" + lastID = self.db.addPackage(name, folder, queue) + p = self.db.getPackage(lastID) + e = InsertEvent("pack", lastID, p.order, "collector" if not queue else "queue") + self.core.pullManager.addEvent(e) + return lastID + + + #-------------------------------------------------------------------------- + + @lock + @change + def deletePackage(self, id): + """delete package and all contained links""" + + p = self.getPackage(id) + if not p: + if id in self.packageCache: + del self.packageCache[id] + return + + oldorder = p.order + queue = p.queue + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + + pyfiles = self.cache.values() + + for pyfile in pyfiles: + if pyfile.packageid == id: + pyfile.abortDownload() + pyfile.release() + + self.db.deletePackage(p) + self.core.pullManager.addEvent(e) + self.core.addonManager.dispatchEvent("package-deleted", id) + + if id in self.packageCache: + del self.packageCache[id] + + packs = self.packageCache.values() + for pack in packs: + if pack.queue == queue and pack.order > oldorder: + pack.order -= 1 + pack.notifyChange() + + + #-------------------------------------------------------------------------- + + @lock + @change + def deleteLink(self, id): + """deletes links""" + + f = self.getFile(id) + if not f: + return None + + pid = f.packageid + e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") + + oldorder = f.order + + if id in self.core.threadManager.processingIds(): + self.cache[id].abortDownload() + + if id in self.cache: + del self.cache[id] + + self.db.deleteLink(f) + + self.core.pullManager.addEvent(e) + + p = self.getPackage(pid) + if not len(p.getChildren()): + p.delete() + + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid == pid and pyfile.order > oldorder: + pyfile.order -= 1 + pyfile.notifyChange() + + + #-------------------------------------------------------------------------- + + def releaseLink(self, id): + """removes pyfile from cache""" + if id in self.cache: + del self.cache[id] + + + #-------------------------------------------------------------------------- + + def releasePackage(self, id): + """removes package from cache""" + if id in self.packageCache: + del self.packageCache[id] + + + #-------------------------------------------------------------------------- + + def updateLink(self, pyfile): + """updates link""" + self.db.updateLink(pyfile) + + e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") + self.core.pullManager.addEvent(e) + + + #-------------------------------------------------------------------------- + + def updatePackage(self, pypack): + """updates a package""" + self.db.updatePackage(pypack) + + e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") + self.core.pullManager.addEvent(e) + + + #-------------------------------------------------------------------------- + + def getPackage(self, id): + """return package instance""" + + if id in self.packageCache: + return self.packageCache[id] + else: + return self.db.getPackage(id) + + + #-------------------------------------------------------------------------- + + def getPackageData(self, id): + """returns dict with package information""" + pack = self.getPackage(id) + + if not pack: + return None + + pack = pack.toDict()[id] + + data = self.db.getPackageData(id) + + tmplist = [] + + cache = self.cache.values() + for x in cache: + if int(x.toDbDict()[x.id]['package']) == int(id): + tmplist.append((x.id, x.toDbDict()[x.id])) + data.update(tmplist) + + pack['links'] = data + + return pack + + + #-------------------------------------------------------------------------- + + def getFileData(self, id): + """returns dict with file information""" + if id in self.cache: + return self.cache[id].toDbDict() + + return self.db.getLinkData(id) + + + #-------------------------------------------------------------------------- + + def getFile(self, id): + """returns pyfile instance""" + if id in self.cache: + return self.cache[id] + else: + return self.db.getFile(id) + + + #-------------------------------------------------------------------------- + + @lock + def getJob(self, occ): + """get suitable job""" + + #@TODO: clean mess + #@TODO: improve selection of valid jobs + + if occ in self.jobCache: + if self.jobCache[occ]: + id = self.jobCache[occ].pop() + if id == "empty": + pyfile = None + self.jobCache[occ].append("empty") + else: + pyfile = self.getFile(id) + else: + jobs = self.db.getJob(occ) + jobs.reverse() + if not jobs: + self.jobCache[occ].append("empty") + pyfile = None + else: + self.jobCache[occ].extend(jobs) + pyfile = self.getFile(self.jobCache[occ].pop()) + + else: + self.jobCache = {} #: better not caching to much + jobs = self.db.getJob(occ) + jobs.reverse() + self.jobCache[occ] = jobs + + if not jobs: + self.jobCache[occ].append("empty") + pyfile = None + else: + pyfile = self.getFile(self.jobCache[occ].pop()) + + #@TODO: maybe the new job has to be approved... + + # pyfile = self.getFile(self.jobCache[occ].pop()) + return pyfile + + + @lock + def getDecryptJob(self): + """return job for decrypting""" + if "decrypt" in self.jobCache: + return None + + plugins = self.core.pluginManager.crypterPlugins.keys() + self.core.pluginManager.containerPlugins.keys() + plugins = str(tuple(plugins)) + + jobs = self.db.getPluginJob(plugins) + if jobs: + return self.getFile(jobs[0]) + else: + self.jobCache['decrypt'] = "empty" + return None + + + def getFileCount(self): + """returns number of files""" + + if self.filecount == -1: + self.filecount = self.db.filecount(1) + + return self.filecount + + + def getQueueCount(self, force=False): + """number of files that have to be processed""" + if self.queuecount == -1 or force: + self.queuecount = self.db.queuecount(1) + + return self.queuecount + + + def checkAllLinksFinished(self): + """checks if all files are finished and dispatch event""" + + if not self.getQueueCount(True): + self.core.addonManager.dispatchEvent("all_downloads-finished") + self.core.log.debug("All downloads finished") + return True + + return False + + + def checkAllLinksProcessed(self, fid): + """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" + + # reset count so statistic will update (this is called when dl was processed) + self.resetCount() + + if not self.db.processcount(1, fid): + self.core.addonManager.dispatchEvent("all_downloads-processed") + self.core.log.debug("All downloads processed") + return True + + return False + + + def resetCount(self): + self.queuecount = -1 + + + @lock + @change + def restartPackage(self, id): + """restart package""" + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid == id: + self.restartFile(pyfile.id) + + self.db.restartPackage(id) + + if id in self.packageCache: + self.packageCache[id].setFinished = False + + e = UpdateEvent("pack", id, "collector" if not self.getPackage(id).queue else "queue") + self.core.pullManager.addEvent(e) + + + @lock + @change + def restartFile(self, id): + """ restart file""" + if id in self.cache: + self.cache[id].status = 3 + self.cache[id].name = self.cache[id].url + self.cache[id].error = "" + self.cache[id].abortDownload() + + self.db.restartFile(id) + + e = UpdateEvent("file", id, "collector" if not self.getFile(id).package().queue else "queue") + self.core.pullManager.addEvent(e) + + + @lock + @change + def setPackageLocation(self, id, queue): + """push package to queue""" + + p = self.db.getPackage(id) + oldorder = p.order + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + self.db.clearPackageOrder(p) + + p = self.db.getPackage(id) + + p.queue = queue + self.db.updatePackage(p) + + self.db.reorderPackage(p, -1, True) + + packs = self.packageCache.values() + for pack in packs: + if pack.queue != queue and pack.order > oldorder: + pack.order -= 1 + pack.notifyChange() + + self.db.commit() + self.releasePackage(id) + p = self.getPackage(id) + + e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + + @lock + @change + def reorderPackage(self, id, position): + p = self.getPackage(id) + + e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + self.db.reorderPackage(p, position) + + packs = self.packageCache.values() + for pack in packs: + if pack.queue != p.queue or pack.order < 0 or pack == p: + continue + if p.order > position: + if position <= pack.order < p.order: + pack.order += 1 + pack.notifyChange() + elif p.order < position: + if position >= pack.order > p.order: + pack.order -= 1 + pack.notifyChange() + + p.order = position + self.db.commit() + + e = InsertEvent("pack", id, position, "collector" if not p.queue else "queue") + self.core.pullManager.addEvent(e) + + + @lock + @change + def reorderFile(self, id, position): + f = self.getFileData(id) + f = f[id] + + e = RemoveEvent("file", id, "collector" if not self.getPackage(f['package']).queue else "queue") + self.core.pullManager.addEvent(e) + + self.db.reorderLink(f, position) + + pyfiles = self.cache.values() + for pyfile in pyfiles: + if pyfile.packageid != f['package'] or pyfile.order < 0: + continue + if f['order'] > position: + if position <= pyfile.order < f['order']: + pyfile.order += 1 + pyfile.notifyChange() + elif f['order'] < position: + if position >= pyfile.order > f['order']: + pyfile.order -= 1 + pyfile.notifyChange() + + if id in self.cache: + self.cache[id].order = position + + self.db.commit() + + e = InsertEvent("file", id, position, "collector" if not self.getPackage(f['package']).queue else "queue") + self.core.pullManager.addEvent(e) + + + @change + def updateFileInfo(self, data, pid): + """ updates file info (name, size, status, url)""" + ids = self.db.updateLinkInfo(data) + e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue") + self.core.pullManager.addEvent(e) + + + def checkPackageFinished(self, pyfile): + """ checks if package is finished and calls AddonManager """ + + ids = self.db.getUnfinished(pyfile.packageid) + if not ids or (pyfile.id in ids and len(ids) == 1): + if not pyfile.package().setFinished: + self.core.log.info(_("Package finished: %s") % pyfile.package().name) + self.core.addonManager.packageFinished(pyfile.package()) + pyfile.package().setFinished = True + + + def reCheckPackage(self, pid): + """ recheck links in package """ + data = self.db.getPackageData(pid) + + urls = [] + + for pyfile in data.itervalues(): + if pyfile['status'] not in (0, 12, 13): + urls.append((pyfile['url'], pyfile['plugin'])) + + self.core.threadManager.createInfoThread(urls, pid) + + + @lock + @change + def deleteFinishedLinks(self): + """ deletes finished links and packages, return deleted packages """ + + old_packs = self.getInfoData(0) + old_packs.update(self.getInfoData(1)) + + self.db.deleteFinished() + + new_packs = self.db.getAllPackages(0) + new_packs.update(self.db.getAllPackages(1)) + # get new packages only from db + + deleted = [id for id in old_packs.iterkeys() if id not in new_packs] + for id_deleted in deleted: + self.deletePackage(int(id_deleted)) + + return deleted + + + @lock + @change + def restartFailed(self): + """ restart all failed links """ + self.db.restartFailed() + + +class FileMethods(object): + + + @style.queue + def filecount(self, queue): + """returns number of files in queue""" + self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", + (queue,)) + return self.c.fetchone()[0] + + + @style.queue + def queuecount(self, queue): + """ number of files in queue not finished yet""" + self.c.execute( + "SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0, 4)", + (queue,)) + return self.c.fetchone()[0] + + + @style.queue + def processcount(self, queue, fid): + """ number of files which have to be proccessed """ + self.c.execute( + "SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2, 3, 5, 7, 12) AND l.id != ?", + (queue, str(fid))) + return self.c.fetchone()[0] + + + @style.inner + def _nextPackageOrder(self, queue=0): + self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) + max = self.c.fetchone()[0] + if max is not None: + return max + 1 + else: + return 0 + + + @style.inner + def _nextFileOrder(self, package): + self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) + max = self.c.fetchone()[0] + if max is not None: + return max + 1 + else: + return 0 + + + @style.queue + def addLink(self, url, name, plugin, package): + order = self._nextFileOrder(package) + self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', + (url, name, ".".join(plugintype, pluginname), package, order)) + return self.c.lastrowid + + + @style.queue + def addLinks(self, links, package): + """ links is a list of tupels (url, plugin)""" + order = self._nextFileOrder(package) + orders = [order + x for x in xrange(len(links))] + links = [(x[0], x[0], ".".join((x[1], x[2])), package, o) for x, o in zip(links, orders)] + self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) + + + @style.queue + def addPackage(self, name, folder, queue): + order = self._nextPackageOrder(queue) + self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', + (name, folder, queue, order)) + return self.c.lastrowid + + + @style.queue + def deletePackage(self, p): + self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) + self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', + (p.order, p.queue)) + + + @style.queue + def deleteLink(self, f): + self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) + self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', + (f.order, str(f.packageid))) + + + @style.queue + def getAllLinks(self, q): + """return information about all links in queue q + + q0 queue + q1 collector + + format: + + { + id: {'name': name, ... 'package': id }, ... + } + + """ + self.c.execute( + 'SELECT l.id, l.url, l.name, l.size, l.status, l.error, l.plugin, l.package, l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', + (q,)) + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': tuple(r[6].split('.')), + 'package': r[7], + 'order': r[8], + } + + return data + + + @style.queue + def getAllPackages(self, q): + """return information about packages in queue q + (only useful in get all data) + + q0 queue + q1 collector + + format: + + { + id: {'name': name ... 'links': {}}, ... + } + """ + self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ + FROM packages p JOIN pstats s ON p.id = s.id \ + WHERE p.queue=? ORDER BY p.packageorder', str(q)) + + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'name': r[1], + 'folder': r[2], + 'site': r[3], + 'password': r[4], + 'queue': r[5], + 'order': r[6], + 'sizetotal': int(r[7]), + 'sizedone': r[8] if r[8] else 0, #: these can be None + 'linksdone': r[9] if r[9] else 0, + 'linkstotal': r[10], + 'links': {} + } + + return data + + + @style.queue + def getLinkData(self, id): + """get link information as dict""" + self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?', + (str(id),)) + data = {} + r = self.c.fetchone() + if not r: + return None + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': tuple(r[6].split('.')), + 'package': r[7], + 'order': r[8], + } + + return data + + + @style.queue + def getPackageData(self, id): + """get data about links for a package""" + self.c.execute( + 'SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE package=? ORDER BY linkorder', + (str(id),)) + + data = {} + for r in self.c: + data[r[0]] = { + 'id': r[0], + 'url': r[1], + 'name': r[2], + 'size': r[3], + 'format_size': formatSize(r[3]), + 'status': r[4], + 'statusmsg': self.manager.statusMsg[r[4]], + 'error': r[5], + 'plugin': tuple(r[6].split('.')), + 'package': r[7], + 'order': r[8], + } + + return data + + + @style.async + def updateLink(self, f): + self.c.execute('UPDATE links SET url=?, name=?, size=?, status=?, error=?, package=? WHERE id=?', + (f.url, f.name, f.size, f.status, str(f.error), str(f.packageid), str(f.id))) + + + @style.queue + def updatePackage(self, p): + self.c.execute('UPDATE packages SET name=?, folder=?, site=?, password=?, queue=? WHERE id=?', + (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) + + @style.queue + def updateLinkInfo(self, data): + """ data is list of tupels (name, size, status, url) """ + self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1, 2, 3, 14)', data) + ids = [] + self.c.execute('SELECT id FROM links WHERE url IN (\'%s\')' % "','".join([x[3] for x in data])) + for r in self.c: + ids.append(int(r[0])) + return ids + + + @style.queue + def reorderPackage(self, p, position, noMove=False): + if position == -1: + position = self._nextPackageOrder(p.queue) + if not noMove: + if p.order > position: + self.c.execute( + 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND queue=? AND packageorder >= 0', + (position, p.order, p.queue)) + elif p.order < position: + self.c.execute( + 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND queue=? AND packageorder >= 0', + (position, p.order, p.queue)) + + self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) + + + @style.queue + def reorderLink(self, f, position): + """ reorder link with f as dict for pyfile """ + if f['order'] > position: + self.c.execute('UPDATE links SET linkorder=linkorder+1 WHERE linkorder >= ? AND linkorder < ? AND package=?', + (position, f['order'], f['package'])) + elif f['order'] < position: + self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder <= ? AND linkorder > ? AND package=?', + (position, f['order'], f['package'])) + + self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f['id'])) + + + @style.queue + def clearPackageOrder(self, p): + self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', + (p.order, p.queue, str(p.id))) + + + @style.async + def restartFile(self, id): + self.c.execute('UPDATE links SET status=3, error="" WHERE id=?', (str(id),)) + + + @style.async + def restartPackage(self, id): + self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) + + + @style.queue + def getPackage(self, id): + """return package instance from id""" + self.c.execute("SELECT name, folder, site, password, queue, packageorder FROM packages WHERE id=?", (str(id),)) + r = self.c.fetchone() + if not r: + return None + return PyPackage(self.manager, id, *r) + + + #-------------------------------------------------------------------------- + + @style.queue + def getFile(self, id): + """return link instance from id""" + self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", + (str(id),)) + r = self.c.fetchone() + if not r: + return None + r = list(r) + r[5] = tuple(r[5].split('.')) + return PyFile(self.manager, id, *r) + + + @style.queue + def getJob(self, occ): + """return pyfile ids, which are suitable for download and dont use a occupied plugin""" + + #@TODO: improve this hardcoded method + pre = "('CCF', 'DLC', 'LinkList', 'RSDF', 'TXT')" #: plugins which are processed in collector + + cmd = "(" + for i, item in enumerate(occ): + if i: cmd += ", " + cmd += "'%s'" % item + + cmd += ")" + + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) + + self.c.execute(cmd) #: very bad! + + return [x[0] for x in self.c] + + + @style.queue + def getPluginJob(self, plugins): + """returns pyfile ids with suited plugins""" + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins + + self.c.execute(cmd) #: very bad! + + return [x[0] for x in self.c] + + + @style.queue + def getUnfinished(self, pid): + """return list of max length 3 ids with pyfiles in package not finished or processed""" + + self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) + return [r[0] for r in self.c] + + + @style.queue + def deleteFinished(self): + self.c.execute("DELETE FROM links WHERE status IN (0, 4)") + self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") + + + @style.queue + def restartFailed(self): + self.c.execute("UPDATE links SET status=3, error='' WHERE status IN (6, 8, 9)") + + + @style.queue + def findDuplicates(self, id, folder, filename): + """ checks if filename exists with different id and same package """ + self.c.execute( + "SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", + (folder, id, filename)) + return self.c.fetchone() + + + @style.queue + def purgeLinks(self): + self.c.execute("DELETE FROM links;") + self.c.execute("DELETE FROM packages;") + + +DatabaseBackend.registerSub(FileMethods) diff --git a/pyload/Database/Storage.py b/pyload/Database/Storage.py new file mode 100644 index 000000000..a19f67606 --- /dev/null +++ b/pyload/Database/Storage.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# @author: mkaay + +from pyload.Database import style +from pyload.Database import DatabaseBackend + + +class StorageMethods(object): + + @style.queue + def setStorage(db, identifier, key, value): + db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) + if db.c.fetchone() is not None: + db.c.execute("UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key)) + else: + db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) + + + @style.queue + def getStorage(db, identifier, key=None): + if key is not None: + db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) + row = db.c.fetchone() + if row is not None: + return row[0] + else: + db.c.execute("SELECT key, value FROM storage WHERE identifier=?", (identifier,)) + return {row[0]: row[1] for row in db.c} + + + @style.queue + def delStorage(db, identifier, key): + db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) + + +DatabaseBackend.registerSub(StorageMethods) diff --git a/pyload/Database/User.py b/pyload/Database/User.py new file mode 100644 index 000000000..dc60ce23a --- /dev/null +++ b/pyload/Database/User.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# @author: mkaay + +from hashlib import sha1 +import random + +from pyload.Database import DatabaseBackend, style + + +class UserMethods(object): + + + @style.queue + def checkAuth(db, user, password): + c = db.c + c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user,)) + r = c.fetchone() + if not r: + return {} + + salt = r[2][:5] + pw = r[2][5:] + h = sha1(salt + password) + if h.hexdigest() == pw: + return {"id": r[0], "name": r[1], "role": r[3], + "permission": r[4], "template": r[5], "email": r[6]} + else: + return {} + + + @style.queue + def addUser(db, user, password): + salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for _i in xrange(0, 5)]) + h = sha1(salt + password) + password = salt + h.hexdigest() + + c = db.c + c.execute('SELECT name FROM users WHERE name=?', (user,)) + if c.fetchone() is not None: + c.execute('UPDATE users SET password=? WHERE name=?', (password, user)) + else: + c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) + + + @style.queue + def changePassword(db, user, oldpw, newpw): + db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user,)) + r = db.c.fetchone() + if not r: + return False + + salt = r[2][:5] + pw = r[2][5:] + h = sha1(salt + oldpw) + if h.hexdigest() == pw: + salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for _i in xrange(0, 5)]) + h = sha1(salt + newpw) + password = salt + h.hexdigest() + + db.c.execute("UPDATE users SET password=? WHERE name=?", (password, user)) + return True + + return False + + + @style.async + def setPermission(db, user, perms): + db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) + + + @style.async + def setRole(db, user, role): + db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) + + + @style.queue + def listUsers(db): + db.c.execute('SELECT name FROM users') + return [row[0] for row in db.c] + + + @style.queue + def getAllUserData(db): + db.c.execute("SELECT name, permission, role, template, email FROM users") + return {r[0]: {"permission": r[1], "role": r[2], "template": r[3], "email": r[4]} for r in db.c} + + + @style.queue + def removeUser(db, user): + db.c.execute('DELETE FROM users WHERE name=?', (user,)) + + +DatabaseBackend.registerSub(UserMethods) diff --git a/pyload/Database/__init__.py b/pyload/Database/__init__.py new file mode 100644 index 000000000..b4200a698 --- /dev/null +++ b/pyload/Database/__init__.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- + +from pyload.Database.Backend import DatabaseBackend, style +from pyload.Database.File import FileHandler +from pyload.Database.Storage import StorageMethods +from pyload.Database.User import UserMethods diff --git a/pyload/Datatype/File.py b/pyload/Datatype/File.py new file mode 100644 index 000000000..cb6989d98 --- /dev/null +++ b/pyload/Datatype/File.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +import time +import threading + +from pyload.manager.Event import UpdateEvent +from pyload.utils import formatSize, lock + + +statusMap = { + "finished" : 0, + "offline" : 1, + "online" : 2, + "queued" : 3, + "skipped" : 4, + "waiting" : 5, + "temp. offline": 6, + "starting" : 7, + "failed" : 8, + "aborted" : 9, + "decrypting" : 10, + "custom" : 11, + "downloading" : 12, + "processing" : 13, + "unknown" : 14, +} + + +def setSize(self, value): + self._size = int(value) + + +class PyFile(object): + """ + Represents a file object at runtime + """ + __slots__ = ("m", "id", "url", "name", "size", "_size", "status", "plugintype", "pluginname", + "packageid", "error", "order", "lock", "plugin", "waitUntil", + "active", "abort", "statusname", "reconnected", "progress", + "maxprogress", "pluginmodule", "pluginclass") + + + def __init__(self, manager, id, url, name, size, status, error, plugin, package, order): + self.m = manager + + self.id = int(id) + self.url = url + self.name = name + self.size = size + self.status = status + self.plugintype, self.pluginname = plugin + self.packageid = package #: should not be used, use package() instead + self.error = error + self.order = order + # database information ends here + + self.lock = threading.RLock() + + self.plugin = None + # self.download = None + + self.waitUntil = 0 #: time.time() + time to wait + + # status attributes + self.active = False #: obsolete? + self.abort = False + self.reconnected = False + + self.statusname = None + + self.progress = 0 + self.maxprogress = 100 + + self.m.cache[int(id)] = self + + # will convert all sizes to ints + size = property(lambda self: self._size, setSize) + + + def __repr__(self): + return "PyFile %s: %s@%s" % (self.id, self.name, self.pluginname) + + + @lock + def initPlugin(self): + """ inits plugin instance """ + if not self.plugin: + self.pluginmodule = self.m.core.pluginManager.getPlugin(self.plugintype, self.pluginname) + self.pluginclass = getattr(self.pluginmodule, self.m.core.pluginManager.getPluginName(self.plugintype, self.pluginname)) + self.plugin = self.pluginclass(self) + + + @lock + def hasPlugin(self): + """Thread safe way to determine this file has initialized plugin attribute + + :return: + """ + return hasattr(self, "plugin") and self.plugin + + + def package(self): + """ return package instance""" + return self.m.getPackage(self.packageid) + + + def setStatus(self, status): + self.status = statusMap[status] + self.sync() #@TODO: needed aslong no better job approving exists + + + def setCustomStatus(self, msg, status="processing"): + self.statusname = msg + self.setStatus(status) + + + def getStatusName(self): + if self.status not in (13, 14) or not self.statusname: + return self.m.statusMsg[self.status] + else: + return self.statusname + + + def hasStatus(self, status): + return statusMap[status] == self.status + + + def sync(self): + """sync PyFile instance with database""" + self.m.updateLink(self) + + + @lock + def release(self): + """sync and remove from cache""" + # file has valid package + if self.packageid > 0: + self.sync() + + if hasattr(self, "plugin") and self.plugin: + self.plugin.clean() + del self.plugin + + self.m.releaseLink(self.id) + + + def delete(self): + """delete pyfile from database""" + self.m.deleteLink(self.id) + + + def toDict(self): + """return dict with all information for interface""" + return self.toDbDict() + + + def toDbDict(self): + """return data as dict for databse + + format: + + { + id: {'url': url, 'name': name ... } + } + + """ + return { + self.id: { + 'id': self.id, + 'url': self.url, + 'name': self.name, + 'plugin': self.pluginname, + 'size': self.getSize(), + 'format_size': self.formatSize(), + 'status': self.status, + 'statusmsg': self.getStatusName(), + 'package': self.packageid, + 'error': self.error, + 'order': self.order + } + } + + + def abortDownload(self): + """abort pyfile if possible""" + while self.id in self.m.core.threadManager.processingIds(): + self.abort = True + if self.plugin and self.plugin.req: + self.plugin.req.abortDownloads() + time.sleep(0.1) + + self.abort = False + if self.hasPlugin() and self.plugin.req: + self.plugin.req.abortDownloads() + + self.release() + + + def finishIfDone(self): + """set status to finish and release file if every thread is finished with it""" + + if self.id in self.m.core.threadManager.processingIds(): + return False + + self.setStatus("finished") + self.release() + self.m.checkAllLinksFinished() + return True + + + def checkIfProcessed(self): + self.m.checkAllLinksProcessed(self.id) + + + def formatWait(self): + """ formats and return wait time in humanreadable format """ + seconds = self.waitUntil - time.time() + + if seconds < 0: + return "00:00:00" + + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + + + def formatSize(self): + """ formats size to readable format """ + return formatSize(self.getSize()) + + + def formatETA(self): + """ formats eta to readable format """ + seconds = self.getETA() + + if seconds < 0: + return "00:00:00" + + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + + + def getSpeed(self): + """ calculates speed """ + try: + return self.plugin.req.speed + except Exception: + return 0 + + + def getETA(self): + """ gets established time of arrival""" + try: + return self.getBytesLeft() / self.getSpeed() + except Exception: + return 0 + + + def getBytesLeft(self): + """ gets bytes left """ + try: + return self.getSize() - self.plugin.req.arrived + except Exception: + return 0 + + + def getPercent(self): + """ get % of download """ + if self.status == 12: + try: + return self.plugin.req.percent + except Exception: + return 0 + else: + return self.progress + + + def getSize(self): + """ get size of download """ + try: + if self.plugin.req.size: + return self.plugin.req.size + else: + return self.size + except Exception: + return self.size + + + def notifyChange(self): + e = UpdateEvent("file", self.id, "collector" if not self.package().queue else "queue") + self.m.core.pullManager.addEvent(e) + + + def setProgress(self, value): + if not value == self.progress: + self.progress = value + self.notifyChange() diff --git a/pyload/Datatype/Package.py b/pyload/Datatype/Package.py new file mode 100644 index 000000000..5ba42f596 --- /dev/null +++ b/pyload/Datatype/Package.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN, mkaay + +from pyload.manager.Event import UpdateEvent +from pyload.utils import safe_filename + + +class PyPackage(object): + """ + Represents a package object at runtime + """ + + def __init__(self, manager, id, name, folder, site, password, queue, order): + self.m = manager + self.m.packageCache[int(id)] = self + + self.id = int(id) + self.name = name + self._folder = folder + self.site = site + self.password = password + self.queue = queue + self.order = order + self.setFinished = False + + + @property + def folder(self): + return safe_filename(self._folder) + + + def toDict(self): + """ Returns a dictionary representation of the data. + + :return: dict: {id: { attr: value }} + """ + return { + self.id: { + 'id': self.id, + 'name': self.name, + 'folder': self.folder, + 'site': self.site, + 'password': self.password, + 'queue': self.queue, + 'order': self.order, + 'links': {} + } + } + + + def getChildren(self): + """get information about contained links""" + return self.m.getPackageData(self.id)["links"] + + + def sync(self): + """sync with db""" + self.m.updatePackage(self) + + + def release(self): + """sync and delete from cache""" + self.sync() + self.m.releasePackage(self.id) + + + def delete(self): + self.m.deletePackage(self.id) + + + def notifyChange(self): + e = UpdateEvent("pack", self.id, "collector" if not self.queue else "queue") + self.m.core.pullManager.addEvent(e) diff --git a/pyload/Datatype/__init__.py b/pyload/Datatype/__init__.py new file mode 100644 index 000000000..29f0d40aa --- /dev/null +++ b/pyload/Datatype/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +from pyload.Datatype.File import PyFile +from pyload.Datatype.Package import PyPackage diff --git a/pyload/Thread/Info.py b/pyload/Thread/Info.py index 36779a962..c1960fa9a 100644 --- a/pyload/Thread/Info.py +++ b/pyload/Thread/Info.py @@ -11,7 +11,7 @@ from copy import copy from pprint import pformat from types import MethodType -from pyload.api import OnlineStatus +from pyload.Api import OnlineStatus from pyload.Datatype import PyFile from pyload.Thread.Plugin import PluginThread diff --git a/pyload/Thread/Plugin.py b/pyload/Thread/Plugin.py index c7350a735..0f1b14d26 100644 --- a/pyload/Thread/Plugin.py +++ b/pyload/Thread/Plugin.py @@ -14,7 +14,7 @@ from copy import copy from pprint import pformat from types import MethodType -from pyload.api import OnlineStatus +from pyload.Api import OnlineStatus from pyload.Datatype import PyFile from pyload.plugin.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload from pyload.utils.packagetools import parseNames diff --git a/pyload/Thread/Server.py b/pyload/Thread/Server.py index 6b6ef6989..a26ffe6a1 100644 --- a/pyload/Thread/Server.py +++ b/pyload/Thread/Server.py @@ -31,7 +31,7 @@ class WebServer(threading.Thread): def run(self): - import pyload.Webui as webinterface + import pyload.webui as webinterface global webinterface reset = False diff --git a/pyload/api/__init__.py b/pyload/api/__init__.py deleted file mode 100644 index 55c21572f..000000000 --- a/pyload/api/__init__.py +++ /dev/null @@ -1,1051 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -from __future__ import with_statement - -import base64 -import os -import re -import time -import urlparse - -from pyload.Datatype import PyFile -from pyload.utils.packagetools import parseNames -from pyload.network.RequestFactory import getURL -from pyload.remote import activated -from pyload.utils import compare_time, freeSpace, safe_filename - -if activated: - try: - from thrift.protocol import TBase - from pyload.remote.thriftbackend.thriftgen.pyload.ttypes import * - from pyload.remote.thriftbackend.thriftgen.pyload.Pyload import Iface - - BaseObject = TBase - - except ImportError: - from pyload.api.types import * - - print "Thrift not imported" - -else: - from pyload.api.types import * - -# contains function names mapped to their permissions -# unlisted functions are for admins only -permMap = {} - - -# decorator only called on init, never initialized, so has no effect on runtime -def permission(bits): - class _Dec(object): - - def __new__(cls, func, *args, **kwargs): - permMap[func.__name__] = bits - return func - return _Dec - - -urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&\[\]\|]*)", re.IGNORECASE) - - -class PERMS(object): - ALL = 0 #: requires no permission, but login - ADD = 1 #: can add packages - DELETE = 2 #: can delete packages - STATUS = 4 #: see and change server status - LIST = 16 #: see queue and collector - MODIFY = 32 #: moddify some attribute of downloads - DOWNLOAD = 64 #: can download from webinterface - SETTINGS = 128 #: can access settings - ACCOUNTS = 256 #: can access accounts - LOGS = 512 #: can see server logs - - -class ROLE(object): - ADMIN = 0 #: admin has all permissions implicit - USER = 1 - - -def has_permission(userperms, perms): - # bytewise or perms before if needed - return perms == (userperms & perms) - - -class Api(Iface): - """ - **pyLoads API** - - This is accessible either internal via core.api or via thrift backend. - - see Thrift specification file remote/thriftbackend/pyload.thrift\ - for information about data structures and what methods are usuable with rpc. - - Most methods requires specific permissions, please look at the source code if you need to know.\ - These can be configured via webinterface. - Admin user have all permissions, and are the only ones who can access the methods with no specific permission. - """ - EXTERNAL = Iface #: let the json api know which methods are external - - - def __init__(self, core): - self.core = core - - - def _convertPyFile(self, p): - fdata = FileData(p['id'], p['url'], p['name'], p['plugin'], p['size'], - p['format_size'], p['status'], p['statusmsg'], - p['package'], p['error'], p['order']) - return fdata - - - def _convertConfigFormat(self, c): - sections = {} - for sectionName, sub in c.iteritems(): - section = ConfigSection(sectionName, sub['desc']) - items = [] - for key, data in sub.iteritems(): - if key in ("desc", "outline"): - continue - item = ConfigItem() - item.name = key - item.description = data['desc'] - item.value = str(data['value']) if not isinstance(data['value'], basestring) else data['value'] - item.type = data['type'] - items.append(item) - section.items = items - sections[sectionName] = section - if "outline" in sub: - section.outline = sub['outline'] - return sections - - - @permission(PERMS.SETTINGS) - def getConfigValue(self, category, option, section="core"): - """Retrieve config value. - - :param category: name of category, or plugin - :param option: config option - :param section: 'plugin' or 'core' - :return: config value as string - """ - if section == "core": - value = self.core.config[category][option] - else: - value = self.core.config.getPlugin(category, option) - return str(value) - - - @permission(PERMS.SETTINGS) - def setConfigValue(self, category, option, value, section="core"): - """Set new config value. - - :param section: - :param option: - :param value: new config value - :param section: 'plugin' or 'core - """ - self.core.addonManager.dispatchEvent("config-changed", category, option, value, section) - if section == "core": - self.core.config[category][option] = value - if option in ("limit_speed", "max_speed"): #: not so nice to update the limit - self.core.requestFactory.updateBucket() - elif section == "plugin": - self.core.config.setPlugin(category, option, value) - - - @permission(PERMS.SETTINGS) - def getConfig(self): - """Retrieves complete config of core. - - :return: list of `ConfigSection` - """ - return self._convertConfigFormat(self.core.config.config) - - - def getConfigDict(self): - """Retrieves complete config in dict format, not for RPC. - - :return: dict - """ - return self.core.config.config - - - @permission(PERMS.SETTINGS) - def getPluginConfig(self): - """Retrieves complete config for all plugins. - - :return: list of `ConfigSection` - """ - return self._convertConfigFormat(self.core.config.plugin) - - - def getPluginConfigDict(self): - """Plugin config as dict, not for RPC. - - :return: dict - """ - return self.core.config.plugin - - - @permission(PERMS.STATUS) - def pauseServer(self): - """Pause server: Tt wont start any new downloads, but nothing gets aborted.""" - self.core.threadManager.pause = True - - - @permission(PERMS.STATUS) - def unpauseServer(self): - """Unpause server: New Downloads will be started.""" - self.core.threadManager.pause = False - - - @permission(PERMS.STATUS) - def togglePause(self): - """Toggle pause state. - - :return: new pause state - """ - self.core.threadManager.pause ^= True - return self.core.threadManager.pause - - - @permission(PERMS.STATUS) - def toggleReconnect(self): - """Toggle reconnect activation. - - :return: new reconnect state - """ - self.core.config['reconnect']['activated'] ^= True - return self.core.config.get("reconnect", "activated") - - - @permission(PERMS.LIST) - def statusServer(self): - """Some general information about the current status of pyLoad. - - :return: `ServerStatus` - """ - serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), - self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, - not self.core.threadManager.pause and self.isTimeDownload(), - self.core.config.get("reconnect", "activated") and self.isTimeReconnect()) - for pyfile in [x.active for x in self.core.threadManager.threads if x.active and isinstance(x.active, PyFile)]: - serverStatus.speed += pyfile.getSpeed() #: bytes/s - return serverStatus - - - @permission(PERMS.STATUS) - def freeSpace(self): - """Available free space at download directory in bytes""" - return freeSpace(self.core.config.get("general", "download_folder")) - - - @permission(PERMS.ALL) - def getServerVersion(self): - """pyLoad Core version """ - return self.core.version - - - def kill(self): - """Clean way to quit pyLoad""" - self.core.do_kill = True - - - def restart(self): - """Restart pyload core""" - self.core.do_restart = True - - - @permission(PERMS.LOGS) - def getLog(self, offset=0): - """Returns most recent log entries. - - :param offset: line offset - :return: List of log entries - """ - filename = os.path.join(self.core.config.get("log", "log_folder"), 'log.txt') - try: - with open(filename, "r") as fh: - lines = fh.readlines() - if offset >= len(lines): - return [] - return lines[offset:] - except Exception: - return ['No log available'] - - - @permission(PERMS.STATUS) - def isTimeDownload(self): - """Checks if pyload will start new downloads according to time in config. - - :return: bool - """ - start = self.core.config.get("downloadTime", "start").split(":") - end = self.core.config.get("downloadTime", "end").split(":") - return compare_time(start, end) - - - @permission(PERMS.STATUS) - def isTimeReconnect(self): - """Checks if pyload will try to make a reconnect - - :return: bool - """ - start = self.core.config.get("reconnect", "startTime").split(":") - end = self.core.config.get("reconnect", "endTime").split(":") - return compare_time(start, end) and self.core.config.get("reconnect", "activated") - - - @permission(PERMS.LIST) - def statusDownloads(self): - """ Status off all currently running downloads. - - :return: list of `DownloadStatus` - """ - data = [] - for pyfile in self.core.threadManager.getActiveFiles(): - if not isinstance(pyfile, PyFile): - continue - data.append(DownloadInfo( - pyfile.id, pyfile.name, pyfile.getSpeed(), pyfile.getETA(), pyfile.formatETA(), - pyfile.getBytesLeft(), pyfile.getSize(), pyfile.formatSize(), pyfile.getPercent(), - pyfile.status, pyfile.getStatusName(), pyfile.formatWait(), - pyfile.waitUntil, pyfile.packageid, pyfile.package().name, pyfile.pluginname)) - return data - - - @permission(PERMS.ADD) - def addPackage(self, name, links, dest=Destination.Queue): - """Adds a package, with links to desired destination. - - :param name: name of the new package - :param links: list of urls - :param dest: `Destination` - :return: package id of the new package - """ - if self.core.config.get("general", "folder_per_package"): - folder = urlparse.urlparse(name).path.split("/")[-1] - else: - folder = "" - - folder = safe_filename(folder) - - pid = self.core.files.addPackage(name, folder, dest) - - self.core.files.addLinks(links, pid) - - self.core.log.info(_("Added package %(name)s containing %(count)d links") % {"name": name.decode('utf-8'), "count": len(links)}) - - self.core.files.save() - - return pid - - - @permission(PERMS.ADD) - def parseURLs(self, html=None, url=None): - """Parses html content or any arbitaty text for links and returns result of `checkURLs` - - :param html: html source - :return: - """ - urls = [] - if html: - urls += [x[0] for x in urlmatcher.findall(html)] - if url: - page = getURL(url) - urls += [x[0] for x in urlmatcher.findall(page)] - # remove duplicates - return self.checkURLs(set(urls)) - - - @permission(PERMS.ADD) - def checkURLs(self, urls): - """ Gets urls and returns pluginname mapped to list of matches urls. - - :param urls: - :return: {plugin: urls} - """ - data = self.core.pluginManager.parseUrls(urls) - plugins = {} - - for url, plugintype, pluginname in data: - try: - plugins[plugintype][pluginname].append(url) - except Exception: - plugins[plugintype][pluginname] = [url] - - return plugins - - - @permission(PERMS.ADD) - def checkOnlineStatus(self, urls): - """ initiates online status check - - :param urls: - :return: initial set of data as `OnlineCheck` instance containing the result id - """ - data = self.core.pluginManager.parseUrls(urls) - - rid = self.core.threadManager.createResultThread(data, False) - - tmp = [(url, (url, OnlineStatus(url, (plugintype, pluginname), "unknown", 3, 0))) for url, plugintype, pluginname in data] - data = parseNames(tmp) - result = {} - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - return OnlineCheck(rid, result) - - - @permission(PERMS.ADD) - def checkOnlineStatusContainer(self, urls, container, data): - """ checks online status of urls and a submited container file - - :param urls: list of urls - :param container: container file name - :param data: file content - :return: online check - """ - with open(os.path.join(self.core.config.get("general", "download_folder"), "tmp_" + container), "wb") as th: - th.write(str(data)) - return self.checkOnlineStatus(urls + [th.name]) - - - @permission(PERMS.ADD) - def pollResults(self, rid): - """ Polls the result available for ResultID - - :param rid: `ResultID` - :return: `OnlineCheck`, if rid is -1 then no more data available - """ - result = self.core.threadManager.getInfoResult(rid) - if "ALL_INFO_FETCHED" in result: - del result['ALL_INFO_FETCHED'] - return OnlineCheck(-1, result) - else: - return OnlineCheck(rid, result) - - - @permission(PERMS.ADD) - def generatePackages(self, links): - """ Parses links, generates packages names from urls - - :param links: list of urls - :return: package names mapped to urls - """ - return parseNames((x, x) for x in links) - - - @permission(PERMS.ADD) - def generateAndAddPackages(self, links, dest=Destination.Queue): - """Generates and add packages - - :param links: list of urls - :param dest: `Destination` - :return: list of package ids - """ - return [self.addPackage(name, urls, dest) for name, urls - in self.generatePackages(links).iteritems()] - - - @permission(PERMS.ADD) - def checkAndAddPackages(self, links, dest=Destination.Queue): - """Checks online status, retrieves names, and will add packages.\ - Because of this packages are not added immediatly, only for internal use. - - :param links: list of urls - :param dest: `Destination` - :return: None - """ - data = self.core.pluginManager.parseUrls(links) - self.core.threadManager.createResultThread(data, True) - - - @permission(PERMS.LIST) - def getPackageData(self, pid): - """Returns complete information about package, and included files. - - :param pid: package id - :return: `PackageData` with .links attribute - """ - data = self.core.files.getPackageData(int(pid)) - if not data: - raise PackageDoesNotExists(pid) - return PackageData(data['id'], data['name'], data['folder'], data['site'], data['password'], - data['queue'], data['order'], - links=[self._convertPyFile(x) for x in data['links'].itervalues()]) - - - @permission(PERMS.LIST) - def getPackageInfo(self, pid): - """Returns information about package, without detailed information about containing files - - :param pid: package id - :return: `PackageData` with .fid attribute - """ - data = self.core.files.getPackageData(int(pid)) - - if not data: - raise PackageDoesNotExists(pid) - return PackageData(data['id'], data['name'], data['folder'], data['site'], data['password'], - data['queue'], data['order'], - fids=[int(x) for x in data['links']]) - - - @permission(PERMS.LIST) - def getFileData(self, fid): - """Get complete information about a specific file. - - :param fid: file id - :return: `FileData` - """ - info = self.core.files.getFileData(int(fid)) - if not info: - raise FileDoesNotExists(fid) - return self._convertPyFile(info.values()[0]) - - - @permission(PERMS.DELETE) - def deleteFiles(self, fids): - """Deletes several file entries from pyload. - - :param fids: list of file ids - """ - for fid in fids: - self.core.files.deleteLink(int(fid)) - self.core.files.save() - - - @permission(PERMS.DELETE) - def deletePackages(self, pids): - """Deletes packages and containing links. - - :param pids: list of package ids - """ - for pid in pids: - self.core.files.deletePackage(int(pid)) - self.core.files.save() - - - @permission(PERMS.LIST) - def getQueue(self): - """Returns info about queue and packages, **not** about files, see `getQueueData` \ - or `getPackageData` instead. - - :return: list of `PackageInfo` - """ - return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], - pack['password'], pack['queue'], pack['order'], - pack['linksdone'], pack['sizedone'], pack['sizetotal'], - pack['linkstotal']) - for pack in self.core.files.getInfoData(Destination.Queue).itervalues()] - - - @permission(PERMS.LIST) - def getQueueData(self): - """Return complete data about everything in queue, this is very expensive use it sparely.\ - See `getQueue` for alternative. - - :return: list of `PackageData` - """ - return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], - pack['password'], pack['queue'], pack['order'], - pack['linksdone'], pack['sizedone'], pack['sizetotal'], - links=[self._convertPyFile(x) for x in pack['links'].itervalues()]) - for pack in self.core.files.getCompleteData(Destination.Queue).itervalues()] - - - @permission(PERMS.LIST) - def getCollector(self): - """same as `getQueue` for collector. - - :return: list of `PackageInfo` - """ - return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], - pack['password'], pack['queue'], pack['order'], - pack['linksdone'], pack['sizedone'], pack['sizetotal'], - pack['linkstotal']) - for pack in self.core.files.getInfoData(Destination.Collector).itervalues()] - - - @permission(PERMS.LIST) - def getCollectorData(self): - """same as `getQueueData` for collector. - - :return: list of `PackageInfo` - """ - return [PackageData(pack['id'], pack['name'], pack['folder'], pack['site'], - pack['password'], pack['queue'], pack['order'], - pack['linksdone'], pack['sizedone'], pack['sizetotal'], - links=[self._convertPyFile(x) for x in pack['links'].itervalues()]) - for pack in self.core.files.getCompleteData(Destination.Collector).itervalues()] - - - @permission(PERMS.ADD) - def addFiles(self, pid, links): - """Adds files to specific package. - - :param pid: package id - :param links: list of urls - """ - self.core.files.addLinks(links, int(pid)) - self.core.log.info(_("Added %(count)d links to package #%(package)d ") % {"count": len(links), "package": pid}) - self.core.files.save() - - - @permission(PERMS.MODIFY) - def pushToQueue(self, pid): - """Moves package from Collector to Queue. - - :param pid: package id - """ - self.core.files.setPackageLocation(pid, Destination.Queue) - - - @permission(PERMS.MODIFY) - def pullFromQueue(self, pid): - """Moves package from Queue to Collector. - - :param pid: package id - """ - self.core.files.setPackageLocation(pid, Destination.Collector) - - - @permission(PERMS.MODIFY) - def restartPackage(self, pid): - """Restarts a package, resets every containing files. - - :param pid: package id - """ - self.core.files.restartPackage(int(pid)) - - - @permission(PERMS.MODIFY) - def restartFile(self, fid): - """Resets file status, so it will be downloaded again. - - :param fid: file id - """ - self.core.files.restartFile(int(fid)) - - - @permission(PERMS.MODIFY) - def recheckPackage(self, pid): - """Proofes online status of all files in a package, also a default action when package is added. - - :param pid: - :return: - """ - self.core.files.reCheckPackage(int(pid)) - - - @permission(PERMS.MODIFY) - def stopAllDownloads(self): - """Aborts all running downloads.""" - - pyfiles = self.core.files.cache.values() - for pyfile in pyfiles: - pyfile.abortDownload() - - - @permission(PERMS.MODIFY) - def stopDownloads(self, fids): - """Aborts specific downloads. - - :param fids: list of file ids - :return: - """ - pyfiles = self.core.files.cache.values() - for pyfile in pyfiles: - if pyfile.id in fids: - pyfile.abortDownload() - - - @permission(PERMS.MODIFY) - def setPackageName(self, pid, name): - """Renames a package. - - :param pid: package id - :param name: new package name - """ - pack = self.core.files.getPackage(pid) - pack.name = name - pack.sync() - - - @permission(PERMS.MODIFY) - def movePackage(self, destination, pid): - """Set a new package location. - - :param destination: `Destination` - :param pid: package id - """ - if destination in (0, 1): - self.core.files.setPackageLocation(pid, destination) - - - @permission(PERMS.MODIFY) - def moveFiles(self, fids, pid): - """Move multiple files to another package - - :param fids: list of file ids - :param pid: destination package - :return: - """ - # TODO: implement - pass - - - @permission(PERMS.ADD) - def uploadContainer(self, filename, data): - """Uploads and adds a container file to pyLoad. - - :param filename: filename, extension is important so it can correctly decrypted - :param data: file content - """ - with open(os.path.join(self.core.config.get("general", "download_folder"), "tmp_" + filename), "wb") as th: - th.write(str(data)) - self.addPackage(th.name, [th.name], Destination.Queue) - - - @permission(PERMS.MODIFY) - def orderPackage(self, pid, position): - """Gives a package a new position. - - :param pid: package id - :param position: - """ - self.core.files.reorderPackage(pid, position) - - - @permission(PERMS.MODIFY) - def orderFile(self, fid, position): - """Gives a new position to a file within its package. - - :param fid: file id - :param position: - """ - self.core.files.reorderFile(fid, position) - - - @permission(PERMS.MODIFY) - def setPackageData(self, pid, data): - """Allows to modify several package attributes. - - :param pid: package id - :param data: dict that maps attribute to desired value - """ - package = self.core.files.getPackage(pid) - if not package: - raise PackageDoesNotExists(pid) - for key, value in data.iteritems(): - if key == "id": - continue - setattr(package, key, value) - package.sync() - self.core.files.save() - - - @permission(PERMS.DELETE) - def deleteFinished(self): - """Deletes all finished files and completly finished packages. - - :return: list of deleted package ids - """ - return self.core.files.deleteFinishedLinks() - - - @permission(PERMS.MODIFY) - def restartFailed(self): - """Restarts all failed failes.""" - self.core.files.restartFailed() - - - @permission(PERMS.LIST) - def getPackageOrder(self, destination): - """Returns information about package order. - - :param destination: `Destination` - :return: dict mapping order to package id - """ - packs = self.core.files.getInfoData(destination) - order = {} - for pid in packs: - pack = self.core.files.getPackageData(int(pid)) - while pack['order'] in order.keys(): #: just in case - pack['order'] += 1 - order[pack['order']] = pack['id'] - return order - - - @permission(PERMS.LIST) - def getFileOrder(self, pid): - """Information about file order within package. - - :param pid: - :return: dict mapping order to file id - """ - rawdata = self.core.files.getPackageData(int(pid)) - order = {} - for id, pyfile in rawdata['links'].iteritems(): - while pyfile['order'] in order.keys(): #: just in case - pyfile['order'] += 1 - order[pyfile['order']] = pyfile['id'] - return order - - - @permission(PERMS.STATUS) - def isCaptchaWaiting(self): - """Indicates wether a captcha task is available - - :return: bool - """ - self.core.lastClientConnected = time.time() - task = self.core.captchaManager.getTask() - return not task is None - - - @permission(PERMS.STATUS) - def getCaptchaTask(self, exclusive=False): - """Returns a captcha task - - :param exclusive: unused - :return: `CaptchaTask` - """ - self.core.lastClientConnected = time.time() - task = self.core.captchaManager.getTask() - if task: - task.setWatingForUser(exclusive=exclusive) - data, type, result = task.getCaptcha() - ctask = CaptchaTask(int(task.id), base64.standard_b64encode(data), type, result) - return ctask - return CaptchaTask(-1) - - - @permission(PERMS.STATUS) - def getCaptchaTaskStatus(self, tid): - """Get information about captcha task - - :param tid: task id - :return: string - """ - self.core.lastClientConnected = time.time() - task = self.core.captchaManager.getTaskByID(tid) - return task.getStatus() if task else "" - - - @permission(PERMS.STATUS) - def setCaptchaResult(self, tid, result): - """Set result for a captcha task - - :param tid: task id - :param result: captcha result - """ - self.core.lastClientConnected = time.time() - task = self.core.captchaManager.getTaskByID(tid) - if task: - task.setResult(result) - self.core.captchaManager.removeTask(task) - - - @permission(PERMS.STATUS) - def getEvents(self, uuid): - """Lists occured events, may be affected to changes in future. - - :param uuid: - :return: list of `Events` - """ - events = self.core.pullManager.getEvents(uuid) - new_events = [] - - - def convDest(d): - return Destination.Queue if d == "queue" else Destination.Collector - - for e in events: - event = EventInfo() - event.eventname = e[0] - if e[0] in ("update", "remove", "insert"): - event.id = e[3] - event.type = ElementType.Package if e[2] == "pack" else ElementType.File - event.destination = convDest(e[1]) - elif e[0] == "order": - if e[1]: - event.id = e[1] - event.type = ElementType.Package if e[2] == "pack" else ElementType.File - event.destination = convDest(e[3]) - elif e[0] == "reload": - event.destination = convDest(e[1]) - new_events.append(event) - return new_events - - - @permission(PERMS.ACCOUNTS) - def getAccounts(self, refresh): - """Get information about all entered accounts. - - :param refresh: reload account info - :return: list of `AccountInfo` - """ - accs = self.core.accountManager.getAccountInfos(False, refresh) - for group in accs.values(): - accounts = [AccountInfo(acc['validuntil'], acc['login'], acc['options'], acc['valid'], - acc['trafficleft'], acc['maxtraffic'], acc['premium'], acc['type']) - for acc in group] - return accounts or list() - - - @permission(PERMS.ALL) - def getAccountTypes(self): - """All available account types. - - :return: list - """ - return self.core.accountManager.accounts.keys() - - - @permission(PERMS.ACCOUNTS) - def updateAccount(self, plugin, account, password=None, options={}): - """Changes pw/options for specific account.""" - self.core.accountManager.updateAccount(plugin, account, password, options) - - - @permission(PERMS.ACCOUNTS) - def removeAccount(self, plugin, account): - """Remove account from pyload. - - :param plugin: pluginname - :param account: accountname - """ - self.core.accountManager.removeAccount(plugin, account) - - - @permission(PERMS.ALL) - def login(self, username, password, remoteip=None): - """Login into pyLoad, this **must** be called when using rpc before any methods can be used. - - :param username: - :param password: - :param remoteip: Omit this argument, its only used internal - :return: bool indicating login was successful - """ - return bool(self.checkAuth(username, password, remoteip)) - - - def checkAuth(self, username, password, remoteip=None): - """Check authentication and returns details - - :param username: - :param password: - :param remoteip: - :return: dict with info, empty when login is incorrect - """ - if self.core.config.get("remote", "nolocalauth") and remoteip == "127.0.0.1": - return "local" - else: - return self.core.db.checkAuth(username, password) - - - def isAuthorized(self, func, userdata): - """checks if the user is authorized for specific method - - :param func: function name - :param userdata: dictionary of user data - :return: boolean - """ - if userdata == "local" or userdata['role'] == ROLE.ADMIN: - return True - elif func in permMap and has_permission(userdata['permission'], permMap[func]): - return True - else: - return False - - - @permission(PERMS.ALL) - def getUserData(self, username, password): - """similar to `checkAuth` but returns UserData thrift type """ - user = self.checkAuth(username, password) - if user: - return UserData(user['name'], user['email'], user['role'], user['permission'], user['template']) - else: - return UserData() - - - def getAllUserData(self): - """returns all known user and info""" - return dict((user, UserData(user, data['email'], data['role'], data['permission'], data['template'])) for user, data - in self.core.db.getAllUserData().iteritems()) - - - @permission(PERMS.STATUS) - def getServices(self): - """ A dict of available services, these can be defined by addon plugins. - - :return: dict with this style: {"plugin": {"method": "description"}} - """ - return dict((plugin, funcs) for plugin, funcs in self.core.addonManager.methods.iteritems()) - - - @permission(PERMS.STATUS) - def hasService(self, plugin, func): - """Checks wether a service is available. - - :param plugin: - :param func: - :return: bool - """ - cont = self.core.addonManager.methods - return plugin in cont and func in cont[plugin] - - - @permission(PERMS.STATUS) - def call(self, info): - """Calls a service (a method in addon plugin). - - :param info: `ServiceCall` - :return: result - :raises: ServiceDoesNotExists, when its not available - :raises: ServiceException, when a exception was raised - """ - plugin = info.plugin - func = info.func - args = info.arguments - parse = info.parseArguments - if not self.hasService(plugin, func): - raise ServiceDoesNotExists(plugin, func) - try: - ret = self.core.addonManager.callRPC(plugin, func, args, parse) - except Exception, e: - raise ServiceException(e.message) - - - @permission(PERMS.STATUS) - def getAllInfo(self): - """Returns all information stored by addon plugins. Values are always strings - - :return: {"plugin": {"name": value}} - """ - return self.core.addonManager.getAllInfo() - - - @permission(PERMS.STATUS) - def getInfoByPlugin(self, plugin): - """Returns information stored by a specific plugin. - - :param plugin: pluginname - :return: dict of attr names mapped to value {"name": value} - """ - return self.core.addonManager.getInfo(plugin) - - - def changePassword(self, user, oldpw, newpw): - """ changes password for specific user """ - return self.core.db.changePassword(user, oldpw, newpw) - - - def setUserPermission(self, user, perm, role): - self.core.db.setPermission(user, perm) - self.core.db.setRole(user, role) diff --git a/pyload/api/types.py b/pyload/api/types.py deleted file mode 100644 index 9381df3c7..000000000 --- a/pyload/api/types.py +++ /dev/null @@ -1,562 +0,0 @@ -# -*- coding: utf-8 -*- -# Autogenerated by pyload -# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING - -class BaseObject(object): - __slots__ = [] - - -class Destination(object): - Collector = 0 - Queue = 1 - - -class DownloadStatus(object): - Aborted = 9 - Custom = 11 - Decrypting = 10 - Downloading = 12 - Failed = 8 - Finished = 0 - Offline = 1 - Online = 2 - Processing = 13 - Queued = 3 - Skipped = 4 - Starting = 7 - TempOffline = 6 - Unknown = 14 - Waiting = 5 - - -class ElementType(object): - File = 1 - Package = 0 - - -class Input(object): - BOOL = 4 - CHOICE = 6 - CLICK = 5 - LIST = 8 - MULTIPLE = 7 - NONE = 0 - PASSWORD = 3 - TABLE = 9 - TEXT = 1 - TEXTBOX = 2 - - -class Output(object): - CAPTCHA = 1 - NOTIFICATION = 4 - QUESTION = 2 - - -class AccountInfo(BaseObject): - __slots__ = ['validuntil', 'login', 'options', 'valid', 'trafficleft', 'maxtraffic', 'premium', 'type'] - - - def __init__(self, validuntil=None, login=None, options=None, valid=None, trafficleft=None, maxtraffic=None, premium=None, type=None): - self.validuntil = validuntil - self.login = login - self.options = options - self.valid = valid - self.trafficleft = trafficleft - self.maxtraffic = maxtraffic - self.premium = premium - self.type = type - - -class CaptchaTask(BaseObject): - __slots__ = ['tid', 'data', 'type', 'resultType'] - - - def __init__(self, tid=None, data=None, type=None, resultType=None): - self.tid = tid - self.data = data - self.type = type - self.resultType = resultType - - -class ConfigItem(BaseObject): - __slots__ = ['name', 'description', 'value', 'type'] - - - def __init__(self, name=None, description=None, value=None, type=None): - self.name = name - self.description = description - self.value = value - self.type = type - - -class ConfigSection(BaseObject): - __slots__ = ['name', 'description', 'items', 'outline'] - - - def __init__(self, name=None, description=None, items=None, outline=None): - self.name = name - self.description = description - self.items = items - self.outline = outline - - -class DownloadInfo(BaseObject): - __slots__ = ['fid', 'name', 'speed', 'eta', 'format_eta', 'bleft', 'size', 'format_size', 'percent', 'status', 'statusmsg', 'format_wait', 'wait_until', 'packageID', 'packageName', 'plugin'] - - - def __init__(self, fid=None, name=None, speed=None, eta=None, format_eta=None, bleft=None, size=None, format_size=None, percent=None, status=None, statusmsg=None, format_wait=None, wait_until=None, packageID=None, packageName=None, plugin=None): - self.fid = fid - self.name = name - self.speed = speed - self.eta = eta - self.format_eta = format_eta - self.bleft = bleft - self.size = size - self.format_size = format_size - self.percent = percent - self.status = status - self.statusmsg = statusmsg - self.format_wait = format_wait - self.wait_until = wait_until - self.packageID = packageID - self.packageName = packageName - self.plugin = plugin - - -class EventInfo(BaseObject): - __slots__ = ['eventname', 'id', 'type', 'destination'] - - - def __init__(self, eventname=None, id=None, type=None, destination=None): - self.eventname = eventname - self.id = id - self.type = type - self.destination = destination - - -class FileData(BaseObject): - __slots__ = ['fid', 'url', 'name', 'plugin', 'size', 'format_size', 'status', 'statusmsg', 'packageID', 'error', 'order'] - - - def __init__(self, fid=None, url=None, name=None, plugin=None, size=None, format_size=None, status=None, statusmsg=None, packageID=None, error=None, order=None): - self.fid = fid - self.url = url - self.name = name - self.plugin = plugin - self.size = size - self.format_size = format_size - self.status = status - self.statusmsg = statusmsg - self.packageID = packageID - self.error = error - self.order = order - - -class FileDoesNotExists(Exception): - __slots__ = ['fid'] - - - def __init__(self, fid=None): - self.fid = fid - - -class InteractionTask(BaseObject): - __slots__ = ['iid', 'input', 'structure', 'preset', 'output', 'data', 'title', 'description', 'plugin'] - - - def __init__(self, iid=None, input=None, structure=None, preset=None, output=None, data=None, title=None, description=None, plugin=None): - self.iid = iid - self.input = input - self.structure = structure - self.preset = preset - self.output = output - self.data = data - self.title = title - self.description = description - self.plugin = plugin - - -class OnlineCheck(BaseObject): - __slots__ = ['rid', 'data'] - - - def __init__(self, rid=None, data=None): - self.rid = rid - self.data = data - - -class OnlineStatus(BaseObject): - __slots__ = ['name', 'plugin', 'packagename', 'status', 'size'] - - - def __init__(self, name=None, plugin=(None, None), packagename=None, status=None, size=None): - self.name = name - self.plugin = plugin - self.packagename = packagename - self.status = status - self.size = size - - -class PackageData(BaseObject): - __slots__ = ['pid', 'name', 'folder', 'site', 'password', 'dest', 'order', 'linksdone', 'sizedone', 'sizetotal', 'linkstotal', 'links', 'fids'] - - - def __init__(self, pid=None, name=None, folder=None, site=None, password=None, dest=None, order=None, linksdone=None, sizedone=None, sizetotal=None, linkstotal=None, links=None, fids=None): - self.pid = pid - self.name = name - self.folder = folder - self.site = site - self.password = password - self.dest = dest - self.order = order - self.linksdone = linksdone - self.sizedone = sizedone - self.sizetotal = sizetotal - self.linkstotal = linkstotal - self.links = links - self.fids = fids - - -class PackageDoesNotExists(Exception): - __slots__ = ['pid'] - - - def __init__(self, pid=None): - self.pid = pid - - -class ServerStatus(BaseObject): - __slots__ = ['pause', 'active', 'queue', 'total', 'speed', 'download', 'reconnect'] - - - def __init__(self, pause=None, active=None, queue=None, total=None, speed=None, download=None, reconnect=None): - self.pause = pause - self.active = active - self.queue = queue - self.total = total - self.speed = speed - self.download = download - self.reconnect = reconnect - - -class ServiceCall(BaseObject): - __slots__ = ['plugin', 'func', 'arguments', 'parseArguments'] - - - def __init__(self, plugin=None, func=None, arguments=None, parseArguments=None): - self.plugin = plugin - self.func = func - self.arguments = arguments - self.parseArguments = parseArguments - - -class ServiceDoesNotExists(Exception): - __slots__ = ['plugin', 'func'] - - - def __init__(self, plugin=None, func=None): - self.plugin = plugin - self.func = func - - -class ServiceException(Exception): - __slots__ = ['msg'] - - - def __init__(self, msg=None): - self.msg = msg - - -class UserData(BaseObject): - __slots__ = ['name', 'email', 'role', 'permission', 'templateName'] - - - def __init__(self, name=None, email=None, role=None, permission=None, templateName=None): - self.name = name - self.email = email - self.role = role - self.permission = permission - self.templateName = templateName - - -class Iface(object): - - def addFiles(self, pid, links): - pass - - - def addPackage(self, name, links, dest): - pass - - - def call(self, info): - pass - - - def checkOnlineStatus(self, urls): - pass - - - def checkOnlineStatusContainer(self, urls, filename, data): - pass - - - def checkURLs(self, urls): - pass - - - def deleteFiles(self, fids): - pass - - - def deleteFinished(self): - pass - - - def deletePackages(self, pids): - pass - - - def freeSpace(self): - pass - - - def generateAndAddPackages(self, links, dest): - pass - - - def generatePackages(self, links): - pass - - - def getAccountTypes(self): - pass - - - def getAccounts(self, refresh): - pass - - - def getAllInfo(self): - pass - - - def getAllUserData(self): - pass - - - def getCaptchaTask(self, exclusive): - pass - - - def getCaptchaTaskStatus(self, tid): - pass - - - def getCollector(self): - pass - - - def getCollectorData(self): - pass - - - def getConfig(self): - pass - - - def getConfigValue(self, category, option, section): - pass - - - def getEvents(self, uuid): - pass - - - def getFileData(self, fid): - pass - - - def getFileOrder(self, pid): - pass - - - def getInfoByPlugin(self, plugin): - pass - - - def getLog(self, offset): - pass - - - def getPackageData(self, pid): - pass - - - def getPackageInfo(self, pid): - pass - - - def getPackageOrder(self, destination): - pass - - - def getPluginConfig(self): - pass - - - def getQueue(self): - pass - - - def getQueueData(self): - pass - - - def getServerVersion(self): - pass - - - def getServices(self): - pass - - - def getUserData(self, username, password): - pass - - - def hasService(self, plugin, func): - pass - - - def isCaptchaWaiting(self): - pass - - - def isTimeDownload(self): - pass - - - def isTimeReconnect(self): - pass - - - def kill(self): - pass - - - def login(self, username, password): - pass - - - def moveFiles(self, fids, pid): - pass - - - def movePackage(self, destination, pid): - pass - - - def orderFile(self, fid, position): - pass - - - def orderPackage(self, pid, position): - pass - - - def parseURLs(self, html, url): - pass - - - def pauseServer(self): - pass - - - def pollResults(self, rid): - pass - - - def pullFromQueue(self, pid): - pass - - - def pushToQueue(self, pid): - pass - - - def recheckPackage(self, pid): - pass - - - def removeAccount(self, plugin, account): - pass - - - def restart(self): - pass - - - def restartFailed(self): - pass - - - def restartFile(self, fid): - pass - - - def restartPackage(self, pid): - pass - - - def setCaptchaResult(self, tid, result): - pass - - - def setConfigValue(self, category, option, value, section): - pass - - - def setPackageData(self, pid, data): - pass - - - def setPackageName(self, pid, name): - pass - - - def statusDownloads(self): - pass - - - def statusServer(self): - pass - - - def stopAllDownloads(self): - pass - - - def stopDownloads(self, fids): - pass - - - def togglePause(self): - pass - - - def toggleReconnect(self): - pass - - - def unpauseServer(self): - pass - - - def updateAccount(self, plugin, account, password, options): - pass - - - def uploadContainer(self, filename, data): - pass diff --git a/pyload/database/Backend.py b/pyload/database/Backend.py deleted file mode 100644 index 0fc961973..000000000 --- a/pyload/database/Backend.py +++ /dev/null @@ -1,329 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN, mkaay - -from __future__ import with_statement - -try: - from pysqlite2 import dbapi2 as sqlite3 -except Exception: - import sqlite3 - -import shutil -import threading -import traceback - -from Queue import Queue - -from pyload.utils import chmod - - -DB_VERSION = 4 - - -class style(object): - db = None - - - @classmethod - def setDB(cls, db): - cls.db = db - - - @classmethod - def inner(cls, f): - - - @staticmethod - def x(*args, **kwargs): - if cls.db: - return f(cls.db, *args, **kwargs) - - return x - - - @classmethod - def queue(cls, f): - - - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.queue(f, *args, **kwargs) - - return x - - - @classmethod - def async(cls, f): - - - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.async(f, *args, **kwargs) - return x - - -class DatabaseJob(object): - - def __init__(self, f, *args, **kwargs): - self.done = threading.Event() - - self.f = f - self.args = args - self.kwargs = kwargs - - self.result = None - self.exception = False - - # import inspect - # self.frame = inspect.currentframe() - - - def __repr__(self): - import os - - frame = self.frame.f_back - output = "" - for _i in xrange(5): - output += "\t%s:%s, %s\n" % (os.path.basename(frame.f_code.co_filename), frame.f_lineno, frame.f_code.co_name) - frame = frame.f_back - del frame - del self.frame - - return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result) - - - def processJob(self): - try: - self.result = self.f(*self.args, **self.kwargs) - except Exception, e: - traceback.print_exc() - try: - print "Database Error @", self.f.__name__, self.args[1:], self.kwargs, e - except Exception: - pass - - self.exception = e - finally: - self.done.set() - - - def wait(self): - self.done.wait() - - -class DatabaseBackend(threading.Thread): - subs = [] - - - def __init__(self, core): - threading.Thread.__init__(self) - self.setDaemon(True) - self.core = core - - self.jobs = Queue() - - self.setuplock = threading.Event() - - style.setDB(self) - - - def setup(self): - self.start() - self.setuplock.wait() - - - def run(self): - """main loop, which executes commands""" - convert = self._checkVersion() #: returns None or current version - - self.conn = sqlite3.connect("files.db") - os.chmod("files.db", 0600) - - self.c = self.conn.cursor() #: compatibility - - if convert is not None: - self._convertDB(convert) - - self._createTables() - self._migrateUser() - - self.conn.commit() - - self.setuplock.set() - - while True: - j = self.jobs.get() - if j == "quit": - self.c.close() - self.conn.close() - break - j.processJob() - - - @style.queue - def shutdown(self): - self.conn.commit() - self.jobs.put("quit") - - - def _checkVersion(self): - """ check db version and delete it if needed""" - if not os.path.exists("files.version"): - with open("files.version", "wb") as f: - f.write(str(DB_VERSION)) - return - - with open("files.version", "rb") as f: - v = int(f.read().strip() or 0) - - if v < DB_VERSION: - if v < 2: - try: - self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) - except Exception: - print "Filedatabase was deleted due to incompatible version." - reshutil.move("files.version") - shutil.move("files.db", "files.backup.db") - - with open("files.version", "wb") as f: - f.write(str(DB_VERSION)) - - return v - - - def _convertDB(self, v): - try: - getattr(self, "_convertV%i" % v)() - except Exception: - try: - self.core.log.error(_("Filedatabase could NOT be converted.")) - except Exception: - print "Filedatabase could NOT be converted." - - # convert scripts start --------------------------------------------------- - - - def _convertV2(self): - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') - try: - self.manager.core.log.info(_("Database was converted from v2 to v3.")) - except Exception: - print "Database was converted from v2 to v3." - self._convertV3() - - - def _convertV3(self): - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') - try: - self.manager.core.log.info(_("Database was converted from v3 to v4.")) - except Exception: - print "Database was converted from v3 to v4." - - # convert scripts end ----------------------------------------------------- - - - def _createTables(self): - """create tables for database""" - - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') - self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)') - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') - self.c.execute( - 'CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') - - self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \ - SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\ - FROM packages p JOIN links l ON p.id = l.package LEFT OUTER JOIN\ - (SELECT p.id AS id, COUNT(*) AS linksdone, SUM(l.size) AS sizedone \ - FROM packages p JOIN links l ON p.id = l.package AND l.status IN (0, 4, 13) GROUP BY p.id) s ON s.id = p.id \ - GROUP BY p.id') - - # try to lower ids - self.c.execute('SELECT max(id) FROM LINKS') - fid = self.c.fetchone()[0] - fid = int(fid) if fid else 0 - self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links")) - - self.c.execute('SELECT max(id) FROM packages') - pid = self.c.fetchone()[0] - pid = int(pid) if pid else 0 - self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (pid, "packages")) - - self.c.execute('VACUUM') - - - def _migrateUser(self): - if os.path.exists("pyload.db"): - try: - self.core.log.info(_("Converting old Django DB")) - except Exception: - print "Converting old Django DB" - conn = sqlite3.connect('pyload.db') - c = conn.cursor() - c.execute("SELECT username, password, email FROM auth_user WHERE is_superuser") - users = [] - for r in c: - pw = r[1].split("$") - users.append((r[0], pw[1] + pw[2], r[2])) - c.close() - conn.close() - - self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users) - shutil.move("pyload.db", "pyload.old.db") - - - def createCursor(self): - return self.conn.cursor() - - - @style.async - def commit(self): - self.conn.commit() - - - @style.queue - def syncSave(self): - self.conn.commit() - - - @style.async - def rollback(self): - self.conn.rollback() - - - def async(self, f, *args, **kwargs): - args = (self,) + args - job = DatabaseJob(f, *args, **kwargs) - self.jobs.put(job) - - - def queue(self, f, *args, **kwargs): - args = (self,) + args - job = DatabaseJob(f, *args, **kwargs) - self.jobs.put(job) - job.wait() - return job.result - - - @classmethod - def registerSub(cls, klass): - cls.subs.append(klass) - - - @classmethod - def unregisterSub(cls, klass): - cls.subs.remove(klass) - - - def __getattr__(self, attr): - for sub in DatabaseBackend.subs: - if hasattr(sub, attr): - return getattr(sub, attr) diff --git a/pyload/database/File.py b/pyload/database/File.py deleted file mode 100644 index f28535ffa..000000000 --- a/pyload/database/File.py +++ /dev/null @@ -1,985 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN, mkaay - -import threading - -from pyload.Datatype import PyFile, PyPackage -from pyload.Database import DatabaseBackend, style -from pyload.manager.Event import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent -from pyload.utils import formatSize, lock - -try: - from pysqlite2 import dbapi2 as sqlite3 -except Exception: - import sqlite3 - - -class FileHandler(object): - """Handles all request made to obtain information, - modify status or other request for links or packages""" - - def __init__(self, core): - """Constructor""" - self.core = core - - # translations - self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), - _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), - _("downloading"), _("processing"), _("unknown")] - - self.cache = {} #: holds instances for files - self.packageCache = {} #: same for packages - #@TODO: purge the cache - - self.jobCache = {} - - self.lock = threading.RLock() #@TODO: should be a Lock w/o R - # self.lock._Verbose__verbose = True - - self.filecount = -1 #: if an invalid value is set get current value from db - self.queuecount = -1 #: number of package to be loaded - self.unchanged = False #: determines if any changes was made since last call - - self.db = self.core.db - - - def change(func): - - - def new(*args): - args[0].unchanged = False - args[0].filecount = -1 - args[0].queuecount = -1 - args[0].jobCache = {} - return func(*args) - - return new - - - #-------------------------------------------------------------------------- - - def save(self): - """saves all data to backend""" - self.db.commit() - - - #-------------------------------------------------------------------------- - - def syncSave(self): - """saves all data to backend and waits until all data are written""" - pyfiles = self.cache.values() - for pyfile in pyfiles: - pyfile.sync() - - pypacks = self.packageCache.values() - for pypack in pypacks: - pypack.sync() - - self.db.syncSave() - - - @lock - def getCompleteData(self, queue=1): - """gets a complete data representation""" - - data = self.db.getAllLinks(queue) - packs = self.db.getAllPackages(queue) - - data.update([(x.id, x.toDbDict()[x.id]) for x in self.cache.values()]) - - for x in self.packageCache.itervalues(): - if x.queue != queue or x.id not in packs: - continue - packs[x.id].update(x.toDict()[x.id]) - - for key, value in data.iteritems(): - if value['package'] in packs: - packs[value['package']]['links'][key] = value - - return packs - - - @lock - def getInfoData(self, queue=1): - """gets a data representation without links""" - - packs = self.db.getAllPackages(queue) - for x in self.packageCache.itervalues(): - if x.queue != queue or x.id not in packs: - continue - packs[x.id].update(x.toDict()[x.id]) - - return packs - - - @lock - @change - def addLinks(self, urls, package): - """adds links""" - - self.core.addonManager.dispatchEvent("links-added", urls, package) - - data = self.core.pluginManager.parseUrls(urls) - - self.db.addLinks(data, package) - self.core.threadManager.createInfoThread(data, package) - - #@TODO: change from reloadAll event to package update event - self.core.pullManager.addEvent(ReloadAllEvent("collector")) - - - #-------------------------------------------------------------------------- - - @lock - @change - def addPackage(self, name, folder, queue=0): - """adds a package, default to link collector""" - lastID = self.db.addPackage(name, folder, queue) - p = self.db.getPackage(lastID) - e = InsertEvent("pack", lastID, p.order, "collector" if not queue else "queue") - self.core.pullManager.addEvent(e) - return lastID - - - #-------------------------------------------------------------------------- - - @lock - @change - def deletePackage(self, id): - """delete package and all contained links""" - - p = self.getPackage(id) - if not p: - if id in self.packageCache: - del self.packageCache[id] - return - - oldorder = p.order - queue = p.queue - - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - - pyfiles = self.cache.values() - - for pyfile in pyfiles: - if pyfile.packageid == id: - pyfile.abortDownload() - pyfile.release() - - self.db.deletePackage(p) - self.core.pullManager.addEvent(e) - self.core.addonManager.dispatchEvent("package-deleted", id) - - if id in self.packageCache: - del self.packageCache[id] - - packs = self.packageCache.values() - for pack in packs: - if pack.queue == queue and pack.order > oldorder: - pack.order -= 1 - pack.notifyChange() - - - #-------------------------------------------------------------------------- - - @lock - @change - def deleteLink(self, id): - """deletes links""" - - f = self.getFile(id) - if not f: - return None - - pid = f.packageid - e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") - - oldorder = f.order - - if id in self.core.threadManager.processingIds(): - self.cache[id].abortDownload() - - if id in self.cache: - del self.cache[id] - - self.db.deleteLink(f) - - self.core.pullManager.addEvent(e) - - p = self.getPackage(pid) - if not len(p.getChildren()): - p.delete() - - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid == pid and pyfile.order > oldorder: - pyfile.order -= 1 - pyfile.notifyChange() - - - #-------------------------------------------------------------------------- - - def releaseLink(self, id): - """removes pyfile from cache""" - if id in self.cache: - del self.cache[id] - - - #-------------------------------------------------------------------------- - - def releasePackage(self, id): - """removes package from cache""" - if id in self.packageCache: - del self.packageCache[id] - - - #-------------------------------------------------------------------------- - - def updateLink(self, pyfile): - """updates link""" - self.db.updateLink(pyfile) - - e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") - self.core.pullManager.addEvent(e) - - - #-------------------------------------------------------------------------- - - def updatePackage(self, pypack): - """updates a package""" - self.db.updatePackage(pypack) - - e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") - self.core.pullManager.addEvent(e) - - - #-------------------------------------------------------------------------- - - def getPackage(self, id): - """return package instance""" - - if id in self.packageCache: - return self.packageCache[id] - else: - return self.db.getPackage(id) - - - #-------------------------------------------------------------------------- - - def getPackageData(self, id): - """returns dict with package information""" - pack = self.getPackage(id) - - if not pack: - return None - - pack = pack.toDict()[id] - - data = self.db.getPackageData(id) - - tmplist = [] - - cache = self.cache.values() - for x in cache: - if int(x.toDbDict()[x.id]['package']) == int(id): - tmplist.append((x.id, x.toDbDict()[x.id])) - data.update(tmplist) - - pack['links'] = data - - return pack - - - #-------------------------------------------------------------------------- - - def getFileData(self, id): - """returns dict with file information""" - if id in self.cache: - return self.cache[id].toDbDict() - - return self.db.getLinkData(id) - - - #-------------------------------------------------------------------------- - - def getFile(self, id): - """returns pyfile instance""" - if id in self.cache: - return self.cache[id] - else: - return self.db.getFile(id) - - - #-------------------------------------------------------------------------- - - @lock - def getJob(self, occ): - """get suitable job""" - - #@TODO: clean mess - #@TODO: improve selection of valid jobs - - if occ in self.jobCache: - if self.jobCache[occ]: - id = self.jobCache[occ].pop() - if id == "empty": - pyfile = None - self.jobCache[occ].append("empty") - else: - pyfile = self.getFile(id) - else: - jobs = self.db.getJob(occ) - jobs.reverse() - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - self.jobCache[occ].extend(jobs) - pyfile = self.getFile(self.jobCache[occ].pop()) - - else: - self.jobCache = {} #: better not caching to much - jobs = self.db.getJob(occ) - jobs.reverse() - self.jobCache[occ] = jobs - - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - pyfile = self.getFile(self.jobCache[occ].pop()) - - #@TODO: maybe the new job has to be approved... - - # pyfile = self.getFile(self.jobCache[occ].pop()) - return pyfile - - - @lock - def getDecryptJob(self): - """return job for decrypting""" - if "decrypt" in self.jobCache: - return None - - plugins = self.core.pluginManager.crypterPlugins.keys() + self.core.pluginManager.containerPlugins.keys() - plugins = str(tuple(plugins)) - - jobs = self.db.getPluginJob(plugins) - if jobs: - return self.getFile(jobs[0]) - else: - self.jobCache['decrypt'] = "empty" - return None - - - def getFileCount(self): - """returns number of files""" - - if self.filecount == -1: - self.filecount = self.db.filecount(1) - - return self.filecount - - - def getQueueCount(self, force=False): - """number of files that have to be processed""" - if self.queuecount == -1 or force: - self.queuecount = self.db.queuecount(1) - - return self.queuecount - - - def checkAllLinksFinished(self): - """checks if all files are finished and dispatch event""" - - if not self.getQueueCount(True): - self.core.addonManager.dispatchEvent("all_downloads-finished") - self.core.log.debug("All downloads finished") - return True - - return False - - - def checkAllLinksProcessed(self, fid): - """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" - - # reset count so statistic will update (this is called when dl was processed) - self.resetCount() - - if not self.db.processcount(1, fid): - self.core.addonManager.dispatchEvent("all_downloads-processed") - self.core.log.debug("All downloads processed") - return True - - return False - - - def resetCount(self): - self.queuecount = -1 - - - @lock - @change - def restartPackage(self, id): - """restart package""" - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid == id: - self.restartFile(pyfile.id) - - self.db.restartPackage(id) - - if id in self.packageCache: - self.packageCache[id].setFinished = False - - e = UpdateEvent("pack", id, "collector" if not self.getPackage(id).queue else "queue") - self.core.pullManager.addEvent(e) - - - @lock - @change - def restartFile(self, id): - """ restart file""" - if id in self.cache: - self.cache[id].status = 3 - self.cache[id].name = self.cache[id].url - self.cache[id].error = "" - self.cache[id].abortDownload() - - self.db.restartFile(id) - - e = UpdateEvent("file", id, "collector" if not self.getFile(id).package().queue else "queue") - self.core.pullManager.addEvent(e) - - - @lock - @change - def setPackageLocation(self, id, queue): - """push package to queue""" - - p = self.db.getPackage(id) - oldorder = p.order - - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - - self.db.clearPackageOrder(p) - - p = self.db.getPackage(id) - - p.queue = queue - self.db.updatePackage(p) - - self.db.reorderPackage(p, -1, True) - - packs = self.packageCache.values() - for pack in packs: - if pack.queue != queue and pack.order > oldorder: - pack.order -= 1 - pack.notifyChange() - - self.db.commit() - self.releasePackage(id) - p = self.getPackage(id) - - e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - - - @lock - @change - def reorderPackage(self, id, position): - p = self.getPackage(id) - - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - self.db.reorderPackage(p, position) - - packs = self.packageCache.values() - for pack in packs: - if pack.queue != p.queue or pack.order < 0 or pack == p: - continue - if p.order > position: - if position <= pack.order < p.order: - pack.order += 1 - pack.notifyChange() - elif p.order < position: - if position >= pack.order > p.order: - pack.order -= 1 - pack.notifyChange() - - p.order = position - self.db.commit() - - e = InsertEvent("pack", id, position, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - - - @lock - @change - def reorderFile(self, id, position): - f = self.getFileData(id) - f = f[id] - - e = RemoveEvent("file", id, "collector" if not self.getPackage(f['package']).queue else "queue") - self.core.pullManager.addEvent(e) - - self.db.reorderLink(f, position) - - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid != f['package'] or pyfile.order < 0: - continue - if f['order'] > position: - if position <= pyfile.order < f['order']: - pyfile.order += 1 - pyfile.notifyChange() - elif f['order'] < position: - if position >= pyfile.order > f['order']: - pyfile.order -= 1 - pyfile.notifyChange() - - if id in self.cache: - self.cache[id].order = position - - self.db.commit() - - e = InsertEvent("file", id, position, "collector" if not self.getPackage(f['package']).queue else "queue") - self.core.pullManager.addEvent(e) - - - @change - def updateFileInfo(self, data, pid): - """ updates file info (name, size, status, url)""" - ids = self.db.updateLinkInfo(data) - e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue") - self.core.pullManager.addEvent(e) - - - def checkPackageFinished(self, pyfile): - """ checks if package is finished and calls AddonManager """ - - ids = self.db.getUnfinished(pyfile.packageid) - if not ids or (pyfile.id in ids and len(ids) == 1): - if not pyfile.package().setFinished: - self.core.log.info(_("Package finished: %s") % pyfile.package().name) - self.core.addonManager.packageFinished(pyfile.package()) - pyfile.package().setFinished = True - - - def reCheckPackage(self, pid): - """ recheck links in package """ - data = self.db.getPackageData(pid) - - urls = [] - - for pyfile in data.itervalues(): - if pyfile['status'] not in (0, 12, 13): - urls.append((pyfile['url'], pyfile['plugin'])) - - self.core.threadManager.createInfoThread(urls, pid) - - - @lock - @change - def deleteFinishedLinks(self): - """ deletes finished links and packages, return deleted packages """ - - old_packs = self.getInfoData(0) - old_packs.update(self.getInfoData(1)) - - self.db.deleteFinished() - - new_packs = self.db.getAllPackages(0) - new_packs.update(self.db.getAllPackages(1)) - # get new packages only from db - - deleted = [id for id in old_packs.iterkeys() if id not in new_packs] - for id_deleted in deleted: - self.deletePackage(int(id_deleted)) - - return deleted - - - @lock - @change - def restartFailed(self): - """ restart all failed links """ - self.db.restartFailed() - - -class FileMethods(object): - - - @style.queue - def filecount(self, queue): - """returns number of files in queue""" - self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", - (queue,)) - return self.c.fetchone()[0] - - - @style.queue - def queuecount(self, queue): - """ number of files in queue not finished yet""" - self.c.execute( - "SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0, 4)", - (queue,)) - return self.c.fetchone()[0] - - - @style.queue - def processcount(self, queue, fid): - """ number of files which have to be proccessed """ - self.c.execute( - "SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2, 3, 5, 7, 12) AND l.id != ?", - (queue, str(fid))) - return self.c.fetchone()[0] - - - @style.inner - def _nextPackageOrder(self, queue=0): - self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) - max = self.c.fetchone()[0] - if max is not None: - return max + 1 - else: - return 0 - - - @style.inner - def _nextFileOrder(self, package): - self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) - max = self.c.fetchone()[0] - if max is not None: - return max + 1 - else: - return 0 - - - @style.queue - def addLink(self, url, name, plugin, package): - order = self._nextFileOrder(package) - self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', - (url, name, ".".join(plugintype, pluginname), package, order)) - return self.c.lastrowid - - - @style.queue - def addLinks(self, links, package): - """ links is a list of tupels (url, plugin)""" - order = self._nextFileOrder(package) - orders = [order + x for x in xrange(len(links))] - links = [(x[0], x[0], ".".join((x[1], x[2])), package, o) for x, o in zip(links, orders)] - self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) - - - @style.queue - def addPackage(self, name, folder, queue): - order = self._nextPackageOrder(queue) - self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', - (name, folder, queue, order)) - return self.c.lastrowid - - - @style.queue - def deletePackage(self, p): - self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) - self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', - (p.order, p.queue)) - - - @style.queue - def deleteLink(self, f): - self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) - self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', - (f.order, str(f.packageid))) - - - @style.queue - def getAllLinks(self, q): - """return information about all links in queue q - - q0 queue - q1 collector - - format: - - { - id: {'name': name, ... 'package': id }, ... - } - - """ - self.c.execute( - 'SELECT l.id, l.url, l.name, l.size, l.status, l.error, l.plugin, l.package, l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', - (q,)) - data = {} - for r in self.c: - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': tuple(r[6].split('.')), - 'package': r[7], - 'order': r[8], - } - - return data - - - @style.queue - def getAllPackages(self, q): - """return information about packages in queue q - (only useful in get all data) - - q0 queue - q1 collector - - format: - - { - id: {'name': name ... 'links': {}}, ... - } - """ - self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ - FROM packages p JOIN pstats s ON p.id = s.id \ - WHERE p.queue=? ORDER BY p.packageorder', str(q)) - - data = {} - for r in self.c: - data[r[0]] = { - 'id': r[0], - 'name': r[1], - 'folder': r[2], - 'site': r[3], - 'password': r[4], - 'queue': r[5], - 'order': r[6], - 'sizetotal': int(r[7]), - 'sizedone': r[8] if r[8] else 0, #: these can be None - 'linksdone': r[9] if r[9] else 0, - 'linkstotal': r[10], - 'links': {} - } - - return data - - - @style.queue - def getLinkData(self, id): - """get link information as dict""" - self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?', - (str(id),)) - data = {} - r = self.c.fetchone() - if not r: - return None - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': tuple(r[6].split('.')), - 'package': r[7], - 'order': r[8], - } - - return data - - - @style.queue - def getPackageData(self, id): - """get data about links for a package""" - self.c.execute( - 'SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE package=? ORDER BY linkorder', - (str(id),)) - - data = {} - for r in self.c: - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': tuple(r[6].split('.')), - 'package': r[7], - 'order': r[8], - } - - return data - - - @style.async - def updateLink(self, f): - self.c.execute('UPDATE links SET url=?, name=?, size=?, status=?, error=?, package=? WHERE id=?', - (f.url, f.name, f.size, f.status, str(f.error), str(f.packageid), str(f.id))) - - - @style.queue - def updatePackage(self, p): - self.c.execute('UPDATE packages SET name=?, folder=?, site=?, password=?, queue=? WHERE id=?', - (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) - - @style.queue - def updateLinkInfo(self, data): - """ data is list of tupels (name, size, status, url) """ - self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1, 2, 3, 14)', data) - ids = [] - self.c.execute('SELECT id FROM links WHERE url IN (\'%s\')' % "','".join([x[3] for x in data])) - for r in self.c: - ids.append(int(r[0])) - return ids - - - @style.queue - def reorderPackage(self, p, position, noMove=False): - if position == -1: - position = self._nextPackageOrder(p.queue) - if not noMove: - if p.order > position: - self.c.execute( - 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND queue=? AND packageorder >= 0', - (position, p.order, p.queue)) - elif p.order < position: - self.c.execute( - 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND queue=? AND packageorder >= 0', - (position, p.order, p.queue)) - - self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) - - - @style.queue - def reorderLink(self, f, position): - """ reorder link with f as dict for pyfile """ - if f['order'] > position: - self.c.execute('UPDATE links SET linkorder=linkorder+1 WHERE linkorder >= ? AND linkorder < ? AND package=?', - (position, f['order'], f['package'])) - elif f['order'] < position: - self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder <= ? AND linkorder > ? AND package=?', - (position, f['order'], f['package'])) - - self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f['id'])) - - - @style.queue - def clearPackageOrder(self, p): - self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', - (p.order, p.queue, str(p.id))) - - - @style.async - def restartFile(self, id): - self.c.execute('UPDATE links SET status=3, error="" WHERE id=?', (str(id),)) - - - @style.async - def restartPackage(self, id): - self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) - - - @style.queue - def getPackage(self, id): - """return package instance from id""" - self.c.execute("SELECT name, folder, site, password, queue, packageorder FROM packages WHERE id=?", (str(id),)) - r = self.c.fetchone() - if not r: - return None - return PyPackage(self.manager, id, *r) - - - #-------------------------------------------------------------------------- - - @style.queue - def getFile(self, id): - """return link instance from id""" - self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", - (str(id),)) - r = self.c.fetchone() - if not r: - return None - r = list(r) - r[5] = tuple(r[5].split('.')) - return PyFile(self.manager, id, *r) - - - @style.queue - def getJob(self, occ): - """return pyfile ids, which are suitable for download and dont use a occupied plugin""" - - #@TODO: improve this hardcoded method - pre = "('CCF', 'DLC', 'LinkList', 'RSDF', 'TXT')" #: plugins which are processed in collector - - cmd = "(" - for i, item in enumerate(occ): - if i: cmd += ", " - cmd += "'%s'" % item - - cmd += ")" - - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) - - self.c.execute(cmd) #: very bad! - - return [x[0] for x in self.c] - - - @style.queue - def getPluginJob(self, plugins): - """returns pyfile ids with suited plugins""" - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins - - self.c.execute(cmd) #: very bad! - - return [x[0] for x in self.c] - - - @style.queue - def getUnfinished(self, pid): - """return list of max length 3 ids with pyfiles in package not finished or processed""" - - self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) - return [r[0] for r in self.c] - - - @style.queue - def deleteFinished(self): - self.c.execute("DELETE FROM links WHERE status IN (0, 4)") - self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") - - - @style.queue - def restartFailed(self): - self.c.execute("UPDATE links SET status=3, error='' WHERE status IN (6, 8, 9)") - - - @style.queue - def findDuplicates(self, id, folder, filename): - """ checks if filename exists with different id and same package """ - self.c.execute( - "SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", - (folder, id, filename)) - return self.c.fetchone() - - - @style.queue - def purgeLinks(self): - self.c.execute("DELETE FROM links;") - self.c.execute("DELETE FROM packages;") - - -DatabaseBackend.registerSub(FileMethods) diff --git a/pyload/database/Storage.py b/pyload/database/Storage.py deleted file mode 100644 index a19f67606..000000000 --- a/pyload/database/Storage.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: mkaay - -from pyload.Database import style -from pyload.Database import DatabaseBackend - - -class StorageMethods(object): - - @style.queue - def setStorage(db, identifier, key, value): - db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) - if db.c.fetchone() is not None: - db.c.execute("UPDATE storage SET value=? WHERE identifier=? AND key=?", (value, identifier, key)) - else: - db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) - - - @style.queue - def getStorage(db, identifier, key=None): - if key is not None: - db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) - row = db.c.fetchone() - if row is not None: - return row[0] - else: - db.c.execute("SELECT key, value FROM storage WHERE identifier=?", (identifier,)) - return {row[0]: row[1] for row in db.c} - - - @style.queue - def delStorage(db, identifier, key): - db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) - - -DatabaseBackend.registerSub(StorageMethods) diff --git a/pyload/database/User.py b/pyload/database/User.py deleted file mode 100644 index dc60ce23a..000000000 --- a/pyload/database/User.py +++ /dev/null @@ -1,93 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: mkaay - -from hashlib import sha1 -import random - -from pyload.Database import DatabaseBackend, style - - -class UserMethods(object): - - - @style.queue - def checkAuth(db, user, password): - c = db.c - c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user,)) - r = c.fetchone() - if not r: - return {} - - salt = r[2][:5] - pw = r[2][5:] - h = sha1(salt + password) - if h.hexdigest() == pw: - return {"id": r[0], "name": r[1], "role": r[3], - "permission": r[4], "template": r[5], "email": r[6]} - else: - return {} - - - @style.queue - def addUser(db, user, password): - salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for _i in xrange(0, 5)]) - h = sha1(salt + password) - password = salt + h.hexdigest() - - c = db.c - c.execute('SELECT name FROM users WHERE name=?', (user,)) - if c.fetchone() is not None: - c.execute('UPDATE users SET password=? WHERE name=?', (password, user)) - else: - c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) - - - @style.queue - def changePassword(db, user, oldpw, newpw): - db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user,)) - r = db.c.fetchone() - if not r: - return False - - salt = r[2][:5] - pw = r[2][5:] - h = sha1(salt + oldpw) - if h.hexdigest() == pw: - salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for _i in xrange(0, 5)]) - h = sha1(salt + newpw) - password = salt + h.hexdigest() - - db.c.execute("UPDATE users SET password=? WHERE name=?", (password, user)) - return True - - return False - - - @style.async - def setPermission(db, user, perms): - db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) - - - @style.async - def setRole(db, user, role): - db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) - - - @style.queue - def listUsers(db): - db.c.execute('SELECT name FROM users') - return [row[0] for row in db.c] - - - @style.queue - def getAllUserData(db): - db.c.execute("SELECT name, permission, role, template, email FROM users") - return {r[0]: {"permission": r[1], "role": r[2], "template": r[3], "email": r[4]} for r in db.c} - - - @style.queue - def removeUser(db, user): - db.c.execute('DELETE FROM users WHERE name=?', (user,)) - - -DatabaseBackend.registerSub(UserMethods) diff --git a/pyload/database/__init__.py b/pyload/database/__init__.py deleted file mode 100644 index b4200a698..000000000 --- a/pyload/database/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# -*- coding: utf-8 -*- - -from pyload.Database.Backend import DatabaseBackend, style -from pyload.Database.File import FileHandler -from pyload.Database.Storage import StorageMethods -from pyload.Database.User import UserMethods diff --git a/pyload/datatype/File.py b/pyload/datatype/File.py deleted file mode 100644 index cb6989d98..000000000 --- a/pyload/datatype/File.py +++ /dev/null @@ -1,299 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN, mkaay - -import time -import threading - -from pyload.manager.Event import UpdateEvent -from pyload.utils import formatSize, lock - - -statusMap = { - "finished" : 0, - "offline" : 1, - "online" : 2, - "queued" : 3, - "skipped" : 4, - "waiting" : 5, - "temp. offline": 6, - "starting" : 7, - "failed" : 8, - "aborted" : 9, - "decrypting" : 10, - "custom" : 11, - "downloading" : 12, - "processing" : 13, - "unknown" : 14, -} - - -def setSize(self, value): - self._size = int(value) - - -class PyFile(object): - """ - Represents a file object at runtime - """ - __slots__ = ("m", "id", "url", "name", "size", "_size", "status", "plugintype", "pluginname", - "packageid", "error", "order", "lock", "plugin", "waitUntil", - "active", "abort", "statusname", "reconnected", "progress", - "maxprogress", "pluginmodule", "pluginclass") - - - def __init__(self, manager, id, url, name, size, status, error, plugin, package, order): - self.m = manager - - self.id = int(id) - self.url = url - self.name = name - self.size = size - self.status = status - self.plugintype, self.pluginname = plugin - self.packageid = package #: should not be used, use package() instead - self.error = error - self.order = order - # database information ends here - - self.lock = threading.RLock() - - self.plugin = None - # self.download = None - - self.waitUntil = 0 #: time.time() + time to wait - - # status attributes - self.active = False #: obsolete? - self.abort = False - self.reconnected = False - - self.statusname = None - - self.progress = 0 - self.maxprogress = 100 - - self.m.cache[int(id)] = self - - # will convert all sizes to ints - size = property(lambda self: self._size, setSize) - - - def __repr__(self): - return "PyFile %s: %s@%s" % (self.id, self.name, self.pluginname) - - - @lock - def initPlugin(self): - """ inits plugin instance """ - if not self.plugin: - self.pluginmodule = self.m.core.pluginManager.getPlugin(self.plugintype, self.pluginname) - self.pluginclass = getattr(self.pluginmodule, self.m.core.pluginManager.getPluginName(self.plugintype, self.pluginname)) - self.plugin = self.pluginclass(self) - - - @lock - def hasPlugin(self): - """Thread safe way to determine this file has initialized plugin attribute - - :return: - """ - return hasattr(self, "plugin") and self.plugin - - - def package(self): - """ return package instance""" - return self.m.getPackage(self.packageid) - - - def setStatus(self, status): - self.status = statusMap[status] - self.sync() #@TODO: needed aslong no better job approving exists - - - def setCustomStatus(self, msg, status="processing"): - self.statusname = msg - self.setStatus(status) - - - def getStatusName(self): - if self.status not in (13, 14) or not self.statusname: - return self.m.statusMsg[self.status] - else: - return self.statusname - - - def hasStatus(self, status): - return statusMap[status] == self.status - - - def sync(self): - """sync PyFile instance with database""" - self.m.updateLink(self) - - - @lock - def release(self): - """sync and remove from cache""" - # file has valid package - if self.packageid > 0: - self.sync() - - if hasattr(self, "plugin") and self.plugin: - self.plugin.clean() - del self.plugin - - self.m.releaseLink(self.id) - - - def delete(self): - """delete pyfile from database""" - self.m.deleteLink(self.id) - - - def toDict(self): - """return dict with all information for interface""" - return self.toDbDict() - - - def toDbDict(self): - """return data as dict for databse - - format: - - { - id: {'url': url, 'name': name ... } - } - - """ - return { - self.id: { - 'id': self.id, - 'url': self.url, - 'name': self.name, - 'plugin': self.pluginname, - 'size': self.getSize(), - 'format_size': self.formatSize(), - 'status': self.status, - 'statusmsg': self.getStatusName(), - 'package': self.packageid, - 'error': self.error, - 'order': self.order - } - } - - - def abortDownload(self): - """abort pyfile if possible""" - while self.id in self.m.core.threadManager.processingIds(): - self.abort = True - if self.plugin and self.plugin.req: - self.plugin.req.abortDownloads() - time.sleep(0.1) - - self.abort = False - if self.hasPlugin() and self.plugin.req: - self.plugin.req.abortDownloads() - - self.release() - - - def finishIfDone(self): - """set status to finish and release file if every thread is finished with it""" - - if self.id in self.m.core.threadManager.processingIds(): - return False - - self.setStatus("finished") - self.release() - self.m.checkAllLinksFinished() - return True - - - def checkIfProcessed(self): - self.m.checkAllLinksProcessed(self.id) - - - def formatWait(self): - """ formats and return wait time in humanreadable format """ - seconds = self.waitUntil - time.time() - - if seconds < 0: - return "00:00:00" - - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - - - def formatSize(self): - """ formats size to readable format """ - return formatSize(self.getSize()) - - - def formatETA(self): - """ formats eta to readable format """ - seconds = self.getETA() - - if seconds < 0: - return "00:00:00" - - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - - - def getSpeed(self): - """ calculates speed """ - try: - return self.plugin.req.speed - except Exception: - return 0 - - - def getETA(self): - """ gets established time of arrival""" - try: - return self.getBytesLeft() / self.getSpeed() - except Exception: - return 0 - - - def getBytesLeft(self): - """ gets bytes left """ - try: - return self.getSize() - self.plugin.req.arrived - except Exception: - return 0 - - - def getPercent(self): - """ get % of download """ - if self.status == 12: - try: - return self.plugin.req.percent - except Exception: - return 0 - else: - return self.progress - - - def getSize(self): - """ get size of download """ - try: - if self.plugin.req.size: - return self.plugin.req.size - else: - return self.size - except Exception: - return self.size - - - def notifyChange(self): - e = UpdateEvent("file", self.id, "collector" if not self.package().queue else "queue") - self.m.core.pullManager.addEvent(e) - - - def setProgress(self, value): - if not value == self.progress: - self.progress = value - self.notifyChange() diff --git a/pyload/datatype/Package.py b/pyload/datatype/Package.py deleted file mode 100644 index 5ba42f596..000000000 --- a/pyload/datatype/Package.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN, mkaay - -from pyload.manager.Event import UpdateEvent -from pyload.utils import safe_filename - - -class PyPackage(object): - """ - Represents a package object at runtime - """ - - def __init__(self, manager, id, name, folder, site, password, queue, order): - self.m = manager - self.m.packageCache[int(id)] = self - - self.id = int(id) - self.name = name - self._folder = folder - self.site = site - self.password = password - self.queue = queue - self.order = order - self.setFinished = False - - - @property - def folder(self): - return safe_filename(self._folder) - - - def toDict(self): - """ Returns a dictionary representation of the data. - - :return: dict: {id: { attr: value }} - """ - return { - self.id: { - 'id': self.id, - 'name': self.name, - 'folder': self.folder, - 'site': self.site, - 'password': self.password, - 'queue': self.queue, - 'order': self.order, - 'links': {} - } - } - - - def getChildren(self): - """get information about contained links""" - return self.m.getPackageData(self.id)["links"] - - - def sync(self): - """sync with db""" - self.m.updatePackage(self) - - - def release(self): - """sync and delete from cache""" - self.sync() - self.m.releasePackage(self.id) - - - def delete(self): - self.m.deletePackage(self.id) - - - def notifyChange(self): - e = UpdateEvent("pack", self.id, "collector" if not self.queue else "queue") - self.m.core.pullManager.addEvent(e) diff --git a/pyload/datatype/__init__.py b/pyload/datatype/__init__.py deleted file mode 100644 index 29f0d40aa..000000000 --- a/pyload/datatype/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- - -from pyload.Datatype.File import PyFile -from pyload.Datatype.Package import PyPackage diff --git a/pyload/plugin/addon/IRCInterface.py b/pyload/plugin/addon/IRCInterface.py index 1d7683bf4..eb015c362 100644 --- a/pyload/plugin/addon/IRCInterface.py +++ b/pyload/plugin/addon/IRCInterface.py @@ -11,7 +11,7 @@ import pycurl from select import select -from pyload.api import PackageDoesNotExists, FileDoesNotExists +from pyload.Api import PackageDoesNotExists, FileDoesNotExists from pyload.network.RequestFactory import getURL from pyload.plugin.Addon import Addon from pyload.utils import formatSize diff --git a/pyload/webui/app/api.py b/pyload/webui/app/api.py index 591b97437..35dbe9009 100644 --- a/pyload/webui/app/api.py +++ b/pyload/webui/app/api.py @@ -8,7 +8,7 @@ from itertools import chain from SafeEval import const_eval as literal_eval from bottle import route, request, response, HTTPError -from pyload.api import BaseObject +from pyload.Api import BaseObject from pyload.utils import json from pyload.webui import PYLOAD from pyload.webui.app.utils import toDict, set_session diff --git a/pyload/webui/app/utils.py b/pyload/webui/app/utils.py index 69682fe67..695162f91 100644 --- a/pyload/webui/app/utils.py +++ b/pyload/webui/app/utils.py @@ -5,7 +5,7 @@ import os from bottle import request, HTTPError, redirect, ServerAdapter -from pyload.api import has_permission, PERMS, ROLE +from pyload.Api import has_permission, PERMS, ROLE from pyload.webui import env, THEME -- cgit v1.2.3