From 91115fd577f20704ef7f2e74c4527ffbb0730a09 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Sep 2014 01:08:03 +0200 Subject: Restructure pyload file tree (1) --- pyload/Api.py | 1030 -------------------------- pyload/CaptchaManager.py | 158 ---- pyload/ConfigParser.py | 373 ---------- pyload/Core.py | 50 +- pyload/HookManager.py | 305 -------- pyload/PullEvents.py | 120 --- pyload/PyFile.py | 284 ------- pyload/PyPackage.py | 79 -- pyload/Scheduler.py | 141 ---- pyload/ThreadManager.py | 317 -------- pyload/api/__init__.py | 1030 ++++++++++++++++++++++++++ pyload/cli/Cli.py | 6 +- pyload/cli/ManageFiles.py | 2 +- pyload/common/APIExerciser.py | 157 ---- pyload/common/ImportDebugger.py | 19 - pyload/common/JsEngine.py | 155 ---- pyload/common/__init__.py | 0 pyload/common/json_layer.py | 12 - pyload/common/packagetools.py | 136 ---- pyload/common/pavement.py | 412 ----------- pyload/common/pylgettext.py | 60 -- pyload/common/test_api.py | 20 - pyload/common/test_json.py | 48 -- pyload/config/Parser.py | 373 ++++++++++ pyload/config/Setup.py | 538 ++++++++++++++ pyload/database/FileDatabase.py | 6 +- pyload/datatypes/PyFile.py | 284 +++++++ pyload/datatypes/PyPackage.py | 79 ++ pyload/datatypes/__init__.py | 0 pyload/debug.py | 94 --- pyload/forwarder.py | 61 -- pyload/manager/AccountManager.py | 173 +++++ pyload/manager/CaptchaManager.py | 158 ++++ pyload/manager/HookManager.py | 305 ++++++++ pyload/manager/PluginManager.py | 356 +++++++++ pyload/manager/RemoteManager.py | 91 +++ pyload/manager/ThreadManager.py | 317 ++++++++ pyload/manager/__init__.py | 0 pyload/manager/event/PullEvents.py | 120 +++ pyload/manager/event/Scheduler.py | 141 ++++ pyload/manager/event/__init__.py | 0 pyload/manager/thread/PluginThread.py | 675 +++++++++++++++++ pyload/manager/thread/ServerThread.py | 108 +++ pyload/manager/thread/__init__.py | 0 pyload/plugins/AccountManager.py | 173 ----- pyload/plugins/PluginManager.py | 356 --------- pyload/plugins/accounts/BayfilesCom.py | 2 +- pyload/plugins/accounts/FastixRu.py | 2 +- pyload/plugins/accounts/FilecloudIo.py | 2 +- pyload/plugins/accounts/FileserveCom.py | 2 +- pyload/plugins/accounts/FourSharedCom.py | 2 +- pyload/plugins/accounts/FreeWayMe.py | 2 +- pyload/plugins/accounts/LetitbitNet.py | 2 +- pyload/plugins/accounts/LinksnappyCom.py | 2 +- pyload/plugins/accounts/MegaDebridEu.py | 2 +- pyload/plugins/accounts/MultiDebridCom.py | 2 +- pyload/plugins/accounts/OboomCom.py | 2 +- pyload/plugins/accounts/OverLoadMe.py | 2 +- pyload/plugins/accounts/PremiumizeMe.py | 2 +- pyload/plugins/accounts/RPNetBiz.py | 2 +- pyload/plugins/accounts/RapidgatorNet.py | 2 +- pyload/plugins/accounts/SimplyPremiumCom.py | 2 +- pyload/plugins/accounts/UnrestrictLi.py | 2 +- pyload/plugins/crypter/DailymotionBatch.py | 2 +- pyload/plugins/crypter/GooGl.py | 2 +- pyload/plugins/crypter/LinkSaveIn.py | 6 +- pyload/plugins/crypter/MediafireComFolder.py | 2 +- pyload/plugins/crypter/MultiuploadCom.py | 2 +- pyload/plugins/crypter/OneKhDe.py | 4 +- pyload/plugins/crypter/SafelinkingNet.py | 2 +- pyload/plugins/crypter/SerienjunkiesOrg.py | 12 +- pyload/plugins/crypter/TurbobitNetFolder.py | 2 +- pyload/plugins/crypter/YoutubeBatch.py | 2 +- pyload/plugins/hooks/DeathByCaptcha.py | 2 +- pyload/plugins/hooks/FastixRu.py | 2 +- pyload/plugins/hooks/IRCInterface.py | 2 +- pyload/plugins/hooks/LinksnappyCom.py | 2 +- pyload/plugins/hooks/MegaDebridEu.py | 2 +- pyload/plugins/hooks/MultiDebridCom.py | 2 +- pyload/plugins/hooks/PremiumizeMe.py | 2 +- pyload/plugins/hooks/RPNetBiz.py | 2 +- pyload/plugins/hooks/SimplyPremiumCom.py | 2 +- pyload/plugins/hooks/UnSkipOnFail.py | 2 +- pyload/plugins/hooks/UnrestrictLi.py | 2 +- pyload/plugins/hoster/AlldebridCom.py | 2 +- pyload/plugins/hoster/BayfilesCom.py | 2 +- pyload/plugins/hoster/DailymotionCom.py | 4 +- pyload/plugins/hoster/DlFreeFr.py | 2 +- pyload/plugins/hoster/ExtabitCom.py | 2 +- pyload/plugins/hoster/FastixRu.py | 2 +- pyload/plugins/hoster/FilecloudIo.py | 2 +- pyload/plugins/hoster/FilepostCom.py | 2 +- pyload/plugins/hoster/FileserveCom.py | 2 +- pyload/plugins/hoster/IfileIt.py | 2 +- pyload/plugins/hoster/LetitbitNet.py | 2 +- pyload/plugins/hoster/LinksnappyCom.py | 2 +- pyload/plugins/hoster/MegaDebridEu.py | 2 +- pyload/plugins/hoster/MegaNz.py | 2 +- pyload/plugins/hoster/MegacrypterCom.py | 2 +- pyload/plugins/hoster/MultiDebridCom.py | 2 +- pyload/plugins/hoster/MyvideoDe.py | 4 +- pyload/plugins/hoster/OboomCom.py | 2 +- pyload/plugins/hoster/OverLoadMe.py | 2 +- pyload/plugins/hoster/PremiumizeMe.py | 2 +- pyload/plugins/hoster/RPNetBiz.py | 2 +- pyload/plugins/hoster/RapidgatorNet.py | 2 +- pyload/plugins/hoster/RealdebridCom.py | 2 +- pyload/plugins/hoster/RedtubeCom.py | 4 +- pyload/plugins/hoster/UlozTo.py | 2 +- pyload/plugins/hoster/UnrestrictLi.py | 2 +- pyload/plugins/hoster/UploadingCom.py | 2 +- pyload/plugins/hoster/XHamsterCom.py | 2 +- pyload/plugins/hoster/YibaishiwuCom.py | 2 +- pyload/remote/ClickAndLoadBackend.py | 2 +- pyload/remote/RemoteManager.py | 91 --- pyload/remote/SocketBackend.py | 2 +- pyload/remote/ThriftBackend.py | 2 +- pyload/setup.py | 538 -------------- pyload/threads/PluginThread.py | 675 ----------------- pyload/threads/ServerThread.py | 108 --- pyload/threads/__init__.py | 0 pyload/unescape.py | 3 - pyload/utils.py | 227 ------ pyload/utils/JsEngine.py | 155 ++++ pyload/utils/__init__.py | 227 ++++++ pyload/utils/packagetools.py | 136 ++++ pyload/utils/pylgettext.py | 60 ++ pyload/webui/__init__.py | 6 +- pyload/webui/app/api.py | 4 +- pyload/webui/app/utils.py | 2 +- 130 files changed, 5439 insertions(+), 6277 deletions(-) delete mode 100644 pyload/Api.py delete mode 100644 pyload/CaptchaManager.py delete mode 100644 pyload/ConfigParser.py delete mode 100644 pyload/HookManager.py delete mode 100644 pyload/PullEvents.py delete mode 100644 pyload/PyFile.py delete mode 100644 pyload/PyPackage.py delete mode 100644 pyload/Scheduler.py delete mode 100644 pyload/ThreadManager.py create mode 100644 pyload/api/__init__.py delete mode 100644 pyload/common/APIExerciser.py delete mode 100644 pyload/common/ImportDebugger.py delete mode 100644 pyload/common/JsEngine.py delete mode 100644 pyload/common/__init__.py delete mode 100644 pyload/common/json_layer.py delete mode 100644 pyload/common/packagetools.py delete mode 100644 pyload/common/pavement.py delete mode 100644 pyload/common/pylgettext.py delete mode 100644 pyload/common/test_api.py delete mode 100644 pyload/common/test_json.py create mode 100644 pyload/config/Parser.py create mode 100644 pyload/config/Setup.py create mode 100644 pyload/datatypes/PyFile.py create mode 100644 pyload/datatypes/PyPackage.py create mode 100644 pyload/datatypes/__init__.py delete mode 100644 pyload/debug.py delete mode 100644 pyload/forwarder.py create mode 100644 pyload/manager/AccountManager.py create mode 100644 pyload/manager/CaptchaManager.py create mode 100644 pyload/manager/HookManager.py create mode 100644 pyload/manager/PluginManager.py create mode 100644 pyload/manager/RemoteManager.py create mode 100644 pyload/manager/ThreadManager.py create mode 100644 pyload/manager/__init__.py create mode 100644 pyload/manager/event/PullEvents.py create mode 100644 pyload/manager/event/Scheduler.py create mode 100644 pyload/manager/event/__init__.py create mode 100644 pyload/manager/thread/PluginThread.py create mode 100644 pyload/manager/thread/ServerThread.py create mode 100644 pyload/manager/thread/__init__.py delete mode 100644 pyload/plugins/AccountManager.py delete mode 100644 pyload/plugins/PluginManager.py delete mode 100644 pyload/remote/RemoteManager.py delete mode 100644 pyload/setup.py delete mode 100644 pyload/threads/PluginThread.py delete mode 100644 pyload/threads/ServerThread.py delete mode 100644 pyload/threads/__init__.py delete mode 100644 pyload/unescape.py delete mode 100644 pyload/utils.py create mode 100644 pyload/utils/JsEngine.py create mode 100644 pyload/utils/__init__.py create mode 100644 pyload/utils/packagetools.py create mode 100644 pyload/utils/pylgettext.py (limited to 'pyload') diff --git a/pyload/Api.py b/pyload/Api.py deleted file mode 100644 index 066d490ec..000000000 --- a/pyload/Api.py +++ /dev/null @@ -1,1030 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from base64 import standard_b64encode -from os.path import join -from time import time -import re - -from PyFile import PyFile -from utils import freeSpace, compare_time -from common.packagetools import parseNames -from network.RequestFactory import getURL -from remote import activated - -if activated: - try: - from remote.thriftbackend.thriftgen.pyload.ttypes import * - from remote.thriftbackend.thriftgen.pyload.Pyload import Iface - BaseObject = TBase - except ImportError: - print "Thrift not imported" - from remote.socketbackend.ttypes import * -else: - from remote.socketbackend.ttypes import * - -# contains function names mapped to their permissions -# unlisted functions are for admins only -permMap = {} - -# decorator only called on init, never initialized, so has no effect on runtime -def permission(bits): - class _Dec(object): - def __new__(cls, func, *args, **kwargs): - permMap[func.__name__] = bits - return func - - return _Dec - - -urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&]*)", re.IGNORECASE) - -class PERMS: - ALL = 0 # requires no permission, but login - ADD = 1 # can add packages - DELETE = 2 # can delete packages - STATUS = 4 # see and change server status - LIST = 16 # see queue and collector - MODIFY = 32 # moddify some attribute of downloads - DOWNLOAD = 64 # can download from webinterface - SETTINGS = 128 # can access settings - ACCOUNTS = 256 # can access accounts - LOGS = 512 # can see server logs - -class ROLE: - ADMIN = 0 #admin has all permissions implicit - USER = 1 - -def has_permission(userperms, perms): - # bytewise or perms before if needed - return perms == (userperms & perms) - - -class Api(Iface): - """ - **pyLoads API** - - This is accessible either internal via core.api or via thrift backend. - - see Thrift specification file remote/thriftbackend/pyload.thrift\ - for information about data structures and what methods are usuable with rpc. - - Most methods requires specific permissions, please look at the source code if you need to know.\ - These can be configured via webinterface. - Admin user have all permissions, and are the only ones who can access the methods with no specific permission. - """ - - EXTERNAL = Iface # let the json api know which methods are external - - def __init__(self, core): - self.core = core - - def _convertPyFile(self, p): - f = FileData(p["id"], p["url"], p["name"], p["plugin"], p["size"], - p["format_size"], p["status"], p["statusmsg"], - p["package"], p["error"], p["order"]) - return f - - def _convertConfigFormat(self, c): - sections = {} - for sectionName, sub in c.iteritems(): - section = ConfigSection(sectionName, sub["desc"]) - items = [] - for key, data in sub.iteritems(): - if key in ("desc", "outline"): - continue - item = ConfigItem() - item.name = key - item.description = data["desc"] - item.value = str(data["value"]) if not isinstance(data["value"], basestring) else data["value"] - item.type = data["type"] - items.append(item) - section.items = items - sections[sectionName] = section - if "outline" in sub: - section.outline = sub["outline"] - return sections - - @permission(PERMS.SETTINGS) - def getConfigValue(self, category, option, section="core"): - """Retrieve config value. - - :param category: name of category, or plugin - :param option: config option - :param section: 'plugin' or 'core' - :return: config value as string - """ - if section == "core": - value = self.core.config[category][option] - else: - value = self.core.config.getPlugin(category, option) - - return str(value) if not isinstance(value, basestring) else value - - @permission(PERMS.SETTINGS) - def setConfigValue(self, category, option, value, section="core"): - """Set new config value. - - :param category: - :param option: - :param value: new config value - :param section: 'plugin' or 'core - """ - self.core.hookManager.dispatchEvent("configChanged", category, option, value, section) - - if section == "core": - self.core.config[category][option] = value - - if option in ("limit_speed", "max_speed"): #not so nice to update the limit - self.core.requestFactory.updateBucket() - - elif section == "plugin": - self.core.config.setPlugin(category, option, value) - - @permission(PERMS.SETTINGS) - def getConfig(self): - """Retrieves complete config of core. - - :return: list of `ConfigSection` - """ - return self._convertConfigFormat(self.core.config.config) - - def getConfigDict(self): - """Retrieves complete config in dict format, not for RPC. - - :return: dict - """ - return self.core.config.config - - @permission(PERMS.SETTINGS) - def getPluginConfig(self): - """Retrieves complete config for all plugins. - - :return: list of `ConfigSection` - """ - return self._convertConfigFormat(self.core.config.plugin) - - def getPluginConfigDict(self): - """Plugin config as dict, not for RPC. - - :return: dict - """ - return self.core.config.plugin - - - @permission(PERMS.STATUS) - def pauseServer(self): - """Pause server: Tt wont start any new downloads, but nothing gets aborted.""" - self.core.threadManager.pause = True - - @permission(PERMS.STATUS) - def unpauseServer(self): - """Unpause server: New Downloads will be started.""" - self.core.threadManager.pause = False - - @permission(PERMS.STATUS) - def togglePause(self): - """Toggle pause state. - - :return: new pause state - """ - self.core.threadManager.pause ^= True - return self.core.threadManager.pause - - @permission(PERMS.STATUS) - def toggleReconnect(self): - """Toggle reconnect activation. - - :return: new reconnect state - """ - self.core.config["reconnect"]["activated"] ^= True - return self.core.config["reconnect"]["activated"] - - @permission(PERMS.LIST) - def statusServer(self): - """Some general information about the current status of pyLoad. - - :return: `ServerStatus` - """ - serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), - self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, - not self.core.threadManager.pause and self.isTimeDownload(), - self.core.config['reconnect']['activated'] and self.isTimeReconnect()) - - for pyfile in [x.active for x in self.core.threadManager.threads if x.active and isinstance(x.active, PyFile)]: - serverStatus.speed += pyfile.getSpeed() #bytes/s - - return serverStatus - - @permission(PERMS.STATUS) - def freeSpace(self): - """Available free space at download directory in bytes""" - return freeSpace(self.core.config["general"]["download_folder"]) - - @permission(PERMS.ALL) - def getServerVersion(self): - """pyLoad Core version """ - return self.core.version - - def kill(self): - """Clean way to quit pyLoad""" - self.core.do_kill = True - - def restart(self): - """Restart pyload core""" - self.core.do_restart = True - - @permission(PERMS.LOGS) - def getLog(self, offset=0): - """Returns most recent log entries. - - :param offset: line offset - :return: List of log entries - """ - filename = join(self.core.config['log']['log_folder'], 'log.txt') - try: - fh = open(filename, "r") - lines = fh.readlines() - fh.close() - if offset >= len(lines): - return [] - return lines[offset:] - except: - return ['No log available'] - - @permission(PERMS.STATUS) - def isTimeDownload(self): - """Checks if pyload will start new downloads according to time in config. - - :return: bool - """ - start = self.core.config['downloadTime']['start'].split(":") - end = self.core.config['downloadTime']['end'].split(":") - return compare_time(start, end) - - @permission(PERMS.STATUS) - def isTimeReconnect(self): - """Checks if pyload will try to make a reconnect - - :return: bool - """ - start = self.core.config['reconnect']['startTime'].split(":") - end = self.core.config['reconnect']['endTime'].split(":") - return compare_time(start, end) and self.core.config["reconnect"]["activated"] - - @permission(PERMS.LIST) - def statusDownloads(self): - """ Status off all currently running downloads. - - :return: list of `DownloadStatus` - """ - data = [] - for pyfile in self.core.threadManager.getActiveFiles(): - if not isinstance(pyfile, PyFile): - continue - - data.append(DownloadInfo( - pyfile.id, pyfile.name, pyfile.getSpeed(), pyfile.getETA(), pyfile.formatETA(), - pyfile.getBytesLeft(), pyfile.getSize(), pyfile.formatSize(), pyfile.getPercent(), - pyfile.status, pyfile.getStatusName(), pyfile.formatWait(), - pyfile.waitUntil, pyfile.packageid, pyfile.package().name, pyfile.pluginname)) - - return data - - @permission(PERMS.ADD) - def addPackage(self, name, links, dest=Destination.Queue): - """Adds a package, with links to desired destination. - - :param name: name of the new package - :param links: list of urls - :param dest: `Destination` - :return: package id of the new package - """ - if self.core.config['general']['folder_per_package']: - folder = name - else: - folder = "" - - folder = folder.replace("http://", "").replace(":", "").replace("/", "_").replace("\\", "_") - - pid = self.core.files.addPackage(name, folder, dest) - - self.core.files.addLinks(links, pid) - - self.core.log.info(_("Added package %(name)s containing %(count)d links") % {"name": name, "count": len(links)}) - - self.core.files.save() - - return pid - - @permission(PERMS.ADD) - def parseURLs(self, html=None, url=None): - """Parses html content or any arbitaty text for links and returns result of `checkURLs` - - :param html: html source - :return: - """ - urls = [] - - if html: - urls += [x[0] for x in urlmatcher.findall(html)] - - if url: - page = getURL(url) - urls += [x[0] for x in urlmatcher.findall(page)] - - # remove duplicates - return self.checkURLs(set(urls)) - - - @permission(PERMS.ADD) - def checkURLs(self, urls): - """ Gets urls and returns pluginname mapped to list of matches urls. - - :param urls: - :return: {plugin: urls} - """ - data = self.core.pluginManager.parseUrls(urls) - plugins = {} - - for url, plugin in data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - return plugins - - @permission(PERMS.ADD) - def checkOnlineStatus(self, urls): - """ initiates online status check - - :param urls: - :return: initial set of data as `OnlineCheck` instance containing the result id - """ - data = self.core.pluginManager.parseUrls(urls) - - rid = self.core.threadManager.createResultThread(data, False) - - tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] - data = parseNames(tmp) - result = {} - - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - return OnlineCheck(rid, result) - - @permission(PERMS.ADD) - def checkOnlineStatusContainer(self, urls, container, data): - """ checks online status of urls and a submited container file - - :param urls: list of urls - :param container: container file name - :param data: file content - :return: online check - """ - th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb") - th.write(str(data)) - th.close() - - return self.checkOnlineStatus(urls + [th.name]) - - @permission(PERMS.ADD) - def pollResults(self, rid): - """ Polls the result available for ResultID - - :param rid: `ResultID` - :return: `OnlineCheck`, if rid is -1 then no more data available - """ - result = self.core.threadManager.getInfoResult(rid) - - if "ALL_INFO_FETCHED" in result: - del result["ALL_INFO_FETCHED"] - return OnlineCheck(-1, result) - else: - return OnlineCheck(rid, result) - - - @permission(PERMS.ADD) - def generatePackages(self, links): - """ Parses links, generates packages names from urls - - :param links: list of urls - :return: package names mapped to urls - """ - result = parseNames((x, x) for x in links) - return result - - @permission(PERMS.ADD) - def generateAndAddPackages(self, links, dest=Destination.Queue): - """Generates and add packages - - :param links: list of urls - :param dest: `Destination` - :return: list of package ids - """ - return [self.addPackage(name, urls, dest) for name, urls - in self.generatePackages(links).iteritems()] - - @permission(PERMS.ADD) - def checkAndAddPackages(self, links, dest=Destination.Queue): - """Checks online status, retrieves names, and will add packages.\ - Because of this packages are not added immediatly, only for internal use. - - :param links: list of urls - :param dest: `Destination` - :return: None - """ - data = self.core.pluginManager.parseUrls(links) - self.core.threadManager.createResultThread(data, True) - - - @permission(PERMS.LIST) - def getPackageData(self, pid): - """Returns complete information about package, and included files. - - :param pid: package id - :return: `PackageData` with .links attribute - """ - data = self.core.files.getPackageData(int(pid)) - - if not data: - raise PackageDoesNotExists(pid) - - pdata = PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], - data["queue"], data["order"], - links=[self._convertPyFile(x) for x in data["links"].itervalues()]) - - return pdata - - @permission(PERMS.LIST) - def getPackageInfo(self, pid): - """Returns information about package, without detailed information about containing files - - :param pid: package id - :return: `PackageData` with .fid attribute - """ - data = self.core.files.getPackageData(int(pid)) - - if not data: - raise PackageDoesNotExists(pid) - - pdata = PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], - data["queue"], data["order"], - fids=[int(x) for x in data["links"]]) - - return pdata - - @permission(PERMS.LIST) - def getFileData(self, fid): - """Get complete information about a specific file. - - :param fid: file id - :return: `FileData` - """ - info = self.core.files.getFileData(int(fid)) - if not info: - raise FileDoesNotExists(fid) - - fdata = self._convertPyFile(info.values()[0]) - return fdata - - @permission(PERMS.DELETE) - def deleteFiles(self, fids): - """Deletes several file entries from pyload. - - :param fids: list of file ids - """ - for id in fids: - self.core.files.deleteLink(int(id)) - - self.core.files.save() - - @permission(PERMS.DELETE) - def deletePackages(self, pids): - """Deletes packages and containing links. - - :param pids: list of package ids - """ - for id in pids: - self.core.files.deletePackage(int(id)) - - self.core.files.save() - - @permission(PERMS.LIST) - def getQueue(self): - """Returns info about queue and packages, **not** about files, see `getQueueData` \ - or `getPackageData` instead. - - :return: list of `PackageInfo` - """ - return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - pack["linkstotal"]) - for pack in self.core.files.getInfoData(Destination.Queue).itervalues()] - - @permission(PERMS.LIST) - def getQueueData(self): - """Return complete data about everything in queue, this is very expensive use it sparely.\ - See `getQueue` for alternative. - - :return: list of `PackageData` - """ - return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) - for pack in self.core.files.getCompleteData(Destination.Queue).itervalues()] - - @permission(PERMS.LIST) - def getCollector(self): - """same as `getQueue` for collector. - - :return: list of `PackageInfo` - """ - return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - pack["linkstotal"]) - for pack in self.core.files.getInfoData(Destination.Collector).itervalues()] - - @permission(PERMS.LIST) - def getCollectorData(self): - """same as `getQueueData` for collector. - - :return: list of `PackageInfo` - """ - return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) - for pack in self.core.files.getCompleteData(Destination.Collector).itervalues()] - - - @permission(PERMS.ADD) - def addFiles(self, pid, links): - """Adds files to specific package. - - :param pid: package id - :param links: list of urls - """ - self.core.files.addLinks(links, int(pid)) - - self.core.log.info(_("Added %(count)d links to package #%(package)d ") % {"count": len(links), "package": pid}) - self.core.files.save() - - @permission(PERMS.MODIFY) - def pushToQueue(self, pid): - """Moves package from Collector to Queue. - - :param pid: package id - """ - self.core.files.setPackageLocation(pid, Destination.Queue) - - @permission(PERMS.MODIFY) - def pullFromQueue(self, pid): - """Moves package from Queue to Collector. - - :param pid: package id - """ - self.core.files.setPackageLocation(pid, Destination.Collector) - - @permission(PERMS.MODIFY) - def restartPackage(self, pid): - """Restarts a package, resets every containing files. - - :param pid: package id - """ - self.core.files.restartPackage(int(pid)) - - @permission(PERMS.MODIFY) - def restartFile(self, fid): - """Resets file status, so it will be downloaded again. - - :param fid: file id - """ - self.core.files.restartFile(int(fid)) - - @permission(PERMS.MODIFY) - def recheckPackage(self, pid): - """Proofes online status of all files in a package, also a default action when package is added. - - :param pid: - :return: - """ - self.core.files.reCheckPackage(int(pid)) - - @permission(PERMS.MODIFY) - def stopAllDownloads(self): - """Aborts all running downloads.""" - - pyfiles = self.core.files.cache.values() - for pyfile in pyfiles: - pyfile.abortDownload() - - @permission(PERMS.MODIFY) - def stopDownloads(self, fids): - """Aborts specific downloads. - - :param fids: list of file ids - :return: - """ - pyfiles = self.core.files.cache.values() - - for pyfile in pyfiles: - if pyfile.id in fids: - pyfile.abortDownload() - - @permission(PERMS.MODIFY) - def setPackageName(self, pid, name): - """Renames a package. - - :param pid: package id - :param name: new package name - """ - pack = self.core.files.getPackage(pid) - pack.name = name - pack.sync() - - @permission(PERMS.MODIFY) - def movePackage(self, destination, pid): - """Set a new package location. - - :param destination: `Destination` - :param pid: package id - """ - if destination not in (0, 1): return - self.core.files.setPackageLocation(pid, destination) - - @permission(PERMS.MODIFY) - def moveFiles(self, fids, pid): - """Move multiple files to another package - - :param fids: list of file ids - :param pid: destination package - :return: - """ - #TODO: implement - pass - - - @permission(PERMS.ADD) - def uploadContainer(self, filename, data): - """Uploads and adds a container file to pyLoad. - - :param filename: filename, extension is important so it can correctly decrypted - :param data: file content - """ - th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb") - th.write(str(data)) - th.close() - - self.addPackage(th.name, [th.name], Destination.Queue) - - @permission(PERMS.MODIFY) - def orderPackage(self, pid, position): - """Gives a package a new position. - - :param pid: package id - :param position: - """ - self.core.files.reorderPackage(pid, position) - - @permission(PERMS.MODIFY) - def orderFile(self, fid, position): - """Gives a new position to a file within its package. - - :param fid: file id - :param position: - """ - self.core.files.reorderFile(fid, position) - - @permission(PERMS.MODIFY) - def setPackageData(self, pid, data): - """Allows to modify several package attributes. - - :param pid: package id - :param data: dict that maps attribute to desired value - """ - p = self.core.files.getPackage(pid) - if not p: raise PackageDoesNotExists(pid) - - for key, value in data.iteritems(): - if key == "id": continue - setattr(p, key, value) - - p.sync() - self.core.files.save() - - @permission(PERMS.DELETE) - def deleteFinished(self): - """Deletes all finished files and completly finished packages. - - :return: list of deleted package ids - """ - return self.core.files.deleteFinishedLinks() - - @permission(PERMS.MODIFY) - def restartFailed(self): - """Restarts all failed failes.""" - self.core.files.restartFailed() - - @permission(PERMS.LIST) - def getPackageOrder(self, destination): - """Returns information about package order. - - :param destination: `Destination` - :return: dict mapping order to package id - """ - - packs = self.core.files.getInfoData(destination) - order = {} - - for pid in packs: - pack = self.core.files.getPackageData(int(pid)) - while pack["order"] in order.keys(): #just in case - pack["order"] += 1 - order[pack["order"]] = pack["id"] - return order - - @permission(PERMS.LIST) - def getFileOrder(self, pid): - """Information about file order within package. - - :param pid: - :return: dict mapping order to file id - """ - rawData = self.core.files.getPackageData(int(pid)) - order = {} - for id, pyfile in rawData["links"].iteritems(): - while pyfile["order"] in order.keys(): #just in case - pyfile["order"] += 1 - order[pyfile["order"]] = pyfile["id"] - return order - - - @permission(PERMS.STATUS) - def isCaptchaWaiting(self): - """Indicates wether a captcha task is available - - :return: bool - """ - self.core.lastClientConnected = time() - task = self.core.captchaManager.getTask() - return not task is None - - @permission(PERMS.STATUS) - def getCaptchaTask(self, exclusive=False): - """Returns a captcha task - - :param exclusive: unused - :return: `CaptchaTask` - """ - self.core.lastClientConnected = time() - task = self.core.captchaManager.getTask() - if task: - task.setWatingForUser(exclusive=exclusive) - data, type, result = task.getCaptcha() - t = CaptchaTask(int(task.id), standard_b64encode(data), type, result) - return t - else: - return CaptchaTask(-1) - - @permission(PERMS.STATUS) - def getCaptchaTaskStatus(self, tid): - """Get information about captcha task - - :param tid: task id - :return: string - """ - self.core.lastClientConnected = time() - t = self.core.captchaManager.getTaskByID(tid) - return t.getStatus() if t else "" - - @permission(PERMS.STATUS) - def setCaptchaResult(self, tid, result): - """Set result for a captcha task - - :param tid: task id - :param result: captcha result - """ - self.core.lastClientConnected = time() - task = self.core.captchaManager.getTaskByID(tid) - if task: - task.setResult(result) - self.core.captchaManager.removeTask(task) - - - @permission(PERMS.STATUS) - def getEvents(self, uuid): - """Lists occured events, may be affected to changes in future. - - :param uuid: - :return: list of `Events` - """ - events = self.core.pullManager.getEvents(uuid) - newEvents = [] - - def convDest(d): - return Destination.Queue if d == "queue" else Destination.Collector - - for e in events: - event = EventInfo() - event.eventname = e[0] - if e[0] in ("update", "remove", "insert"): - event.id = e[3] - event.type = ElementType.Package if e[2] == "pack" else ElementType.File - event.destination = convDest(e[1]) - elif e[0] == "order": - if e[1]: - event.id = e[1] - event.type = ElementType.Package if e[2] == "pack" else ElementType.File - event.destination = convDest(e[3]) - elif e[0] == "reload": - event.destination = convDest(e[1]) - newEvents.append(event) - return newEvents - - @permission(PERMS.ACCOUNTS) - def getAccounts(self, refresh): - """Get information about all entered accounts. - - :param refresh: reload account info - :return: list of `AccountInfo` - """ - accs = self.core.accountManager.getAccountInfos(False, refresh) - accounts = [] - for group in accs.values(): - accounts.extend([AccountInfo(acc["validuntil"], acc["login"], acc["options"], acc["valid"], - acc["trafficleft"], acc["maxtraffic"], acc["premium"], acc["type"]) - for acc in group]) - return accounts - - @permission(PERMS.ALL) - def getAccountTypes(self): - """All available account types. - - :return: list - """ - return self.core.accountManager.accounts.keys() - - @permission(PERMS.ACCOUNTS) - def updateAccount(self, plugin, account, password=None, options={}): - """Changes pw/options for specific account.""" - self.core.accountManager.updateAccount(plugin, account, password, options) - - @permission(PERMS.ACCOUNTS) - def removeAccount(self, plugin, account): - """Remove account from pyload. - - :param plugin: pluginname - :param account: accountname - """ - self.core.accountManager.removeAccount(plugin, account) - - @permission(PERMS.ALL) - def login(self, username, password, remoteip=None): - """Login into pyLoad, this **must** be called when using rpc before any methods can be used. - - :param username: - :param password: - :param remoteip: Omit this argument, its only used internal - :return: bool indicating login was successful - """ - return True if self.checkAuth(username, password, remoteip) else False - - def checkAuth(self, username, password, remoteip=None): - """Check authentication and returns details - - :param username: - :param password: - :param remoteip: - :return: dict with info, empty when login is incorrect - """ - if self.core.config["remote"]["nolocalauth"] and remoteip == "127.0.0.1": - return "local" - else: - return self.core.db.checkAuth(username, password) - - def isAuthorized(self, func, userdata): - """checks if the user is authorized for specific method - - :param func: function name - :param userdata: dictionary of user data - :return: boolean - """ - if userdata == "local" or userdata["role"] == ROLE.ADMIN: - return True - elif func in permMap and has_permission(userdata["permission"], permMap[func]): - return True - else: - return False - - - @permission(PERMS.ALL) - def getUserData(self, username, password): - """similar to `checkAuth` but returns UserData thrift type """ - user = self.checkAuth(username, password) - if user: - return UserData(user["name"], user["email"], user["role"], user["permission"], user["template"]) - else: - return UserData() - - - def getAllUserData(self): - """returns all known user and info""" - res = {} - for user, data in self.core.db.getAllUserData().iteritems(): - res[user] = UserData(user, data["email"], data["role"], data["permission"], data["template"]) - - return res - - @permission(PERMS.STATUS) - def getServices(self): - """ A dict of available services, these can be defined by hook plugins. - - :return: dict with this style: {"plugin": {"method": "description"}} - """ - data = {} - for plugin, funcs in self.core.hookManager.methods.iteritems(): - data[plugin] = funcs - - return data - - @permission(PERMS.STATUS) - def hasService(self, plugin, func): - """Checks wether a service is available. - - :param plugin: - :param func: - :return: bool - """ - cont = self.core.hookManager.methods - return plugin in cont and func in cont[plugin] - - @permission(PERMS.STATUS) - def call(self, info): - """Calls a service (a method in hook plugin). - - :param info: `ServiceCall` - :return: result - :raises: ServiceDoesNotExists, when its not available - :raises: ServiceException, when a exception was raised - """ - plugin = info.plugin - func = info.func - args = info.arguments - parse = info.parseArguments - - if not self.hasService(plugin, func): - raise ServiceDoesNotExists(plugin, func) - - try: - ret = self.core.hookManager.callRPC(plugin, func, args, parse) - return str(ret) - except Exception, e: - raise ServiceException(e.message) - - @permission(PERMS.STATUS) - def getAllInfo(self): - """Returns all information stored by hook plugins. Values are always strings - - :return: {"plugin": {"name": value}} - """ - return self.core.hookManager.getAllInfo() - - @permission(PERMS.STATUS) - def getInfoByPlugin(self, plugin): - """Returns information stored by a specific plugin. - - :param plugin: pluginname - :return: dict of attr names mapped to value {"name": value} - """ - return self.core.hookManager.getInfo(plugin) - - def changePassword(self, user, oldpw, newpw): - """ changes password for specific user """ - return self.core.db.changePassword(user, oldpw, newpw) - - def setUserPermission(self, user, permission, role): - self.core.db.setPermission(user, permission) - self.core.db.setRole(user, role) diff --git a/pyload/CaptchaManager.py b/pyload/CaptchaManager.py deleted file mode 100644 index 0ba876ae8..000000000 --- a/pyload/CaptchaManager.py +++ /dev/null @@ -1,158 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay, RaNaN -""" - -from time import time -from traceback import print_exc -from threading import Lock - -class CaptchaManager: - def __init__(self, core): - self.lock = Lock() - self.core = core - self.tasks = [] #task store, for outgoing tasks only - - self.ids = 0 #only for internal purpose - - def newTask(self, img, format, file, result_type): - task = CaptchaTask(self.ids, img, format, file, result_type) - self.ids += 1 - return task - - def removeTask(self, task): - self.lock.acquire() - if task in self.tasks: - self.tasks.remove(task) - self.lock.release() - - def getTask(self): - self.lock.acquire() - for task in self.tasks: - if task.status in ("waiting", "shared-user"): - self.lock.release() - return task - self.lock.release() - return None - - def getTaskByID(self, tid): - self.lock.acquire() - for task in self.tasks: - if task.id == str(tid): #task ids are strings - self.lock.release() - return task - self.lock.release() - return None - - def handleCaptcha(self, task): - cli = self.core.isClientConnected() - - if cli: #client connected -> should solve the captcha - task.setWaiting(50) #wait 50 sec for response - - for plugin in self.core.hookManager.activePlugins(): - try: - plugin.newCaptchaTask(task) - except: - if self.core.debug: - print_exc() - - if task.handler or cli: #the captcha was handled - self.tasks.append(task) - return True - - task.error = _("No Client connected for captcha decrypting") - - return False - - -class CaptchaTask: - def __init__(self, id, img, format, file, result_type='textual'): - self.id = str(id) - self.captchaImg = img - self.captchaFormat = format - self.captchaFile = file - self.captchaResultType = result_type - self.handler = [] #the hook plugins that will take care of the solution - self.result = None - self.waitUntil = None - self.error = None #error message - - self.status = "init" - self.data = {} #handler can store data here - - def getCaptcha(self): - return self.captchaImg, self.captchaFormat, self.captchaResultType - - def setResult(self, text): - if self.isTextual(): - self.result = text - if self.isPositional(): - try: - parts = text.split(',') - self.result = (int(parts[0]), int(parts[1])) - except: - self.result = None - - def getResult(self): - try: - res = self.result.encode("utf8", "replace") - except: - res = self.result - - return res - - def getStatus(self): - return self.status - - def setWaiting(self, sec): - """ let the captcha wait secs for the solution """ - self.waitUntil = max(time() + sec, self.waitUntil) - self.status = "waiting" - - def isWaiting(self): - if self.result or self.error or time() > self.waitUntil: - return False - - return True - - def isTextual(self): - """ returns if text is written on the captcha """ - return self.captchaResultType == 'textual' - - def isPositional(self): - """ returns if user have to click a specific region on the captcha """ - return self.captchaResultType == 'positional' - - def setWatingForUser(self, exclusive): - if exclusive: - self.status = "user" - else: - self.status = "shared-user" - - def timedOut(self): - return time() > self.waitUntil - - def invalid(self): - """ indicates the captcha was not correct """ - [x.captchaInvalid(self) for x in self.handler] - - def correct(self): - [x.captchaCorrect(self) for x in self.handler] - - def __str__(self): - return "" % self.id diff --git a/pyload/ConfigParser.py b/pyload/ConfigParser.py deleted file mode 100644 index 64ce6b10e..000000000 --- a/pyload/ConfigParser.py +++ /dev/null @@ -1,373 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import with_statement -from time import sleep -from os.path import exists, join -from shutil import copy - -from traceback import print_exc -from utils import chmod - -# ignore these plugin configs, mainly because plugins were wiped out -IGNORE = ( - "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('hooks', 'UnRar'), - 'EasyShareCom', 'FlyshareCz' - ) - -CONF_VERSION = 1 - -class ConfigParser: - """ - holds and manage the configuration - - current dict layout: - - { - - section: { - option: { - value: - type: - desc: - } - desc: - - } - - """ - - - def __init__(self): - """Constructor""" - self.config = {} # the config values - self.plugin = {} # the config for plugins - self.oldRemoteData = {} - - self.pluginCB = None # callback when plugin config value is changed - - self.checkVersion() - - self.readConfig() - - self.deleteOldPlugins() - - - def checkVersion(self, n=0): - """determines if config need to be copied""" - try: - if not exists("pyload.conf"): - copy(join(pypath, "pyload", "config", "default.conf"), "pyload.conf") - - if not exists("plugin.conf"): - f = open("plugin.conf", "wb") - f.write("version: " + str(CONF_VERSION)) - f.close() - - f = open("pyload.conf", "rb") - v = f.readline() - f.close() - v = v[v.find(":") + 1:].strip() - - if not v or int(v) < CONF_VERSION: - copy(join(pypath, "pyload", "config", "default.conf"), "pyload.conf") - print "Old version of config was replaced" - - f = open("plugin.conf", "rb") - v = f.readline() - f.close() - v = v[v.find(":") + 1:].strip() - - if not v or int(v) < CONF_VERSION: - f = open("plugin.conf", "wb") - f.write("version: " + str(CONF_VERSION)) - f.close() - print "Old version of plugin-config replaced" - except: - if n < 3: - sleep(0.3) - self.checkVersion(n + 1) - else: - raise - - def readConfig(self): - """reads the config file""" - - self.config = self.parseConfig(join(pypath, "pyload", "config", "default.conf")) - self.plugin = self.parseConfig("plugin.conf") - - try: - homeconf = self.parseConfig("pyload.conf") - if "username" in homeconf["remote"]: - if "password" in homeconf["remote"]: - self.oldRemoteData = {"username": homeconf["remote"]["username"]["value"], - "password": homeconf["remote"]["username"]["value"]} - del homeconf["remote"]["password"] - del homeconf["remote"]["username"] - self.updateValues(homeconf, self.config) - - except Exception, e: - print "Config Warning" - print_exc() - - - def parseConfig(self, config): - """parses a given configfile""" - - f = open(config) - - config = f.read() - - config = config.splitlines()[1:] - - conf = {} - - section, option, value, typ, desc = "", "", "", "", "" - - listmode = False - - for line in config: - comment = line.rfind("#") - if line.find(":", comment) < 0 > line.find("=", comment) and comment > 0 and line[comment - 1].isspace(): - line = line.rpartition("#") # removes comments - if line[1]: - line = line[0] - else: - line = line[2] - - line = line.strip() - - try: - if line == "": - continue - elif line.endswith(":"): - section, none, desc = line[:-1].partition('-') - section = section.strip() - desc = desc.replace('"', "").strip() - conf[section] = {"desc": desc} - else: - if listmode: - if line.endswith("]"): - listmode = False - line = line.replace("]", "") - - value += [self.cast(typ, x.strip()) for x in line.split(",") if x] - - if not listmode: - conf[section][option] = {"desc": desc, - "type": typ, - "value": value} - - - else: - content, none, value = line.partition("=") - - content, none, desc = content.partition(":") - - desc = desc.replace('"', "").strip() - - typ, none, option = content.strip().rpartition(" ") - - value = value.strip() - - if value.startswith("["): - if value.endswith("]"): - listmode = False - value = value[:-1] - else: - listmode = True - - value = [self.cast(typ, x.strip()) for x in value[1:].split(",") if x] - else: - value = self.cast(typ, value) - - if not listmode: - conf[section][option] = {"desc": desc, - "type": typ, - "value": value} - - except Exception, e: - print "Config Warning" - print_exc() - - f.close() - return conf - - - def updateValues(self, config, dest): - """sets the config values from a parsed config file to values in destination""" - - for section in config.iterkeys(): - if section in dest: - for option in config[section].iterkeys(): - if option in ("desc", "outline"): continue - - if option in dest[section]: - dest[section][option]["value"] = config[section][option]["value"] - - #else: - # dest[section][option] = config[section][option] - - - #else: - # dest[section] = config[section] - - def saveConfig(self, config, filename): - """saves config to filename""" - with open(filename, "wb") as f: - chmod(filename, 0600) - f.write("version: %i \n" % CONF_VERSION) - for section in config.iterkeys(): - f.write('\n%s - "%s":\n' % (section, config[section]["desc"])) - - for option, data in config[section].iteritems(): - if option in ("desc", "outline"): continue - - if isinstance(data["value"], list): - value = "[ \n" - for x in data["value"]: - value += "\t\t" + str(x) + ",\n" - value += "\t\t]\n" - else: - if type(data["value"]) in (str, unicode): - value = data["value"] + "\n" - else: - value = str(data["value"]) + "\n" - try: - f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value)) - except UnicodeEncodeError: - f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value.encode("utf8"))) - - def cast(self, typ, value): - """cast value to given format""" - if type(value) not in (str, unicode): - return value - - elif typ == "int": - return int(value) - elif typ == "bool": - return True if value.lower() in ("1", "true", "on", "an", "yes") else False - elif typ == "time": - if not value: value = "0:00" - if not ":" in value: value += ":00" - return value - elif typ in ("str", "file", "folder"): - try: - return value.encode("utf8") - except: - return value - else: - return value - - - def save(self): - """saves the configs to disk""" - - self.saveConfig(self.config, "pyload.conf") - self.saveConfig(self.plugin, "plugin.conf") - - - def __getitem__(self, section): - """provides dictonary like access: c['section']['option']""" - return Section(self, section) - - - def get(self, section, option): - """get value""" - val = self.config[section][option]["value"] - try: - if type(val) in (str, unicode): - return val.decode("utf8") - else: - return val - except: - return val - - def set(self, section, option, value): - """set value""" - - value = self.cast(self.config[section][option]["type"], value) - - self.config[section][option]["value"] = value - self.save() - - def getPlugin(self, plugin, option): - """gets a value for a plugin""" - val = self.plugin[plugin][option]["value"] - try: - if type(val) in (str, unicode): - return val.decode("utf8") - else: - return val - except: - return val - - def setPlugin(self, plugin, option, value): - """sets a value for a plugin""" - - value = self.cast(self.plugin[plugin][option]["type"], value) - - if self.pluginCB: self.pluginCB(plugin, option, value) - - self.plugin[plugin][option]["value"] = value - self.save() - - def getMetaData(self, section, option): - """ get all config data for an option """ - return self.config[section][option] - - def addPluginConfig(self, name, config, outline=""): - """adds config options with tuples (name, type, desc, default)""" - if name not in self.plugin: - conf = {"desc": name, - "outline": outline} - self.plugin[name] = conf - else: - conf = self.plugin[name] - conf["outline"] = outline - - for item in config: - if item[0] in conf: - conf[item[0]]["type"] = item[1] - conf[item[0]]["desc"] = item[2] - else: - conf[item[0]] = { - "desc": item[2], - "type": item[1], - "value": self.cast(item[1], item[3]) - } - - values = [x[0] for x in config] + ["desc", "outline"] - #delete old values - for item in conf.keys(): - if item not in values: - del conf[item] - - def deleteConfig(self, name): - """Removes a plugin config""" - if name in self.plugin: - del self.plugin[name] - - - def deleteOldPlugins(self): - """ remove old plugins from config """ - - for name in IGNORE: - if name in self.plugin: - del self.plugin[name] - - -class Section: - """provides dictionary like access for configparser""" - - def __init__(self, parser, section): - """Constructor""" - self.parser = parser - self.section = section - - def __getitem__(self, item): - """getitem""" - return self.parser.get(self.section, item) - - def __setitem__(self, item, value): - """setitem""" - self.parser.set(self.section, item, value) diff --git a/pyload/Core.py b/pyload/Core.py index 5f506b980..3e78a5bea 100644 --- a/pyload/Core.py +++ b/pyload/Core.py @@ -25,7 +25,7 @@ CURRENT_VERSION = '0.4.10' import __builtin__ from getopt import getopt, GetoptError -import pyload.common.pylgettext as gettext +import pyload.utils.pylgettext as gettext from imp import find_module import logging import logging.handlers @@ -40,17 +40,17 @@ from time import time, sleep from traceback import print_exc from pyload import InitHomeDir -from pyload.plugins.AccountManager import AccountManager -from pyload.CaptchaManager import CaptchaManager -from pyload.ConfigParser import ConfigParser -from pyload.plugins.PluginManager import PluginManager -from pyload.PullEvents import PullManager +from pyload.manager.AccountManager import AccountManager +from pyload.manager.CaptchaManager import CaptchaManager +from pyload.config.Parser import ConfigParser +from pyload.manager.PluginManager import PluginManager +from pyload.manager.event.PullEvents import PullManager from pyload.network.RequestFactory import RequestFactory -from pyload.threads.ServerThread import WebServer -from pyload.Scheduler import Scheduler -from pyload.common.JsEngine import JsEngine +from pyload.manager.thread.ServerThread import WebServer +from pyload.manager.event.Scheduler import Scheduler +from pyload.utils.JsEngine import JsEngine from pyload import remote -from pyload.remote.RemoteManager import RemoteManager +from pyload.manager.RemoteManager import RemoteManager from pyload.database import DatabaseBackend, FileHandler from pyload.utils import freeSpace, formatSize, get_console_encoding @@ -99,21 +99,21 @@ class Core(object): elif option in ("-d", "--debug"): self.doDebug = True elif option in ("-u", "--user"): - from pyload.setup import Setup + from pyload.config.Setup import SetupAssistant as Setup self.config = ConfigParser() s = Setup(pypath, self.config) s.set_user() exit() elif option in ("-s", "--setup"): - from pyload.setup import Setup + from pyload.config.Setup import SetupAssistant as Setup self.config = ConfigParser() s = Setup(pypath, self.config) s.start() exit() elif option == "--changedir": - from pyload.setup import Setup + from pyload.config.Setup import SetupAssistant as Setup self.config = ConfigParser() s = Setup(pypath, self.config) @@ -263,7 +263,7 @@ class Core(object): self.version = CURRENT_VERSION if not exists("pyload.conf"): - from pyload.setup import Setup + from pyload.config.Setup import SetupAssistant as Setup print "This is your first start, running configuration assistent now." self.config = ConfigParser() @@ -376,14 +376,14 @@ class Core(object): self.lastClientConnected = 0 # later imported because they would trigger api import, and remote value not set correctly - from pyload import Api - from pyload.HookManager import HookManager - from pyload.ThreadManager import ThreadManager + from pyload import api + from pyload.manager.HookManager import HookManager + from pyload.manager.ThreadManager import ThreadManager - if Api.activated != self.remote: + if api.activated != self.remote: self.log.warning("Import error: API remote status not correct.") - self.api = Api.Api(self) + self.api = api.Api(self) self.scheduler = Scheduler(self) @@ -439,18 +439,6 @@ class Core(object): self.log.info(_("pyLoad is up and running")) - #test api -# from pyload.common.APIExerciser import startApiExerciser -# startApiExerciser(self, 3) - - #some memory stats -# from guppy import hpy -# hp=hpy() -# import objgraph -# objgraph.show_most_common_types(limit=20) -# import memdebug -# memdebug.start(8002) - locals().clear() while True: diff --git a/pyload/HookManager.py b/pyload/HookManager.py deleted file mode 100644 index 7545b4d60..000000000 --- a/pyload/HookManager.py +++ /dev/null @@ -1,305 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN, mkaay - @interface-version: 0.1 -""" -import __builtin__ - -import traceback -from thread import start_new_thread -from threading import RLock - -from types import MethodType - -from pyload.threads.PluginThread import HookThread -from pyload.plugins.PluginManager import literal_eval -from utils import lock - -class HookManager: - """Manages hooks, delegates and handles Events. - - Every plugin can define events, \ - but some very usefull events are called by the Core. - Contrary to overwriting hook methods you can use event listener, - which provides additional entry point in the control flow. - Only do very short tasks or use threads. - - **Known Events:** - Most hook methods exists as events. These are the additional known events. - - ===================== ============== ================================== - Name Arguments Description - ===================== ============== ================================== - downloadPreparing fid A download was just queued and will be prepared now. - downloadStarts fid A plugin will immediately starts the download afterwards. - linksAdded links, pid Someone just added links, you are able to modify the links. - allDownloadsProcessed Every link was handled, pyload would idle afterwards. - allDownloadsFinished Every download in queue is finished. - unrarFinished folder, fname An Unrar job finished - configChanged The config was changed via the api. - pluginConfigChanged The plugin config changed, due to api or internal process. - ===================== ============== ================================== - - | Notes: - | allDownloadsProcessed is *always* called before allDownloadsFinished. - | configChanged is *always* called before pluginConfigChanged. - - - """ - - def __init__(self, core): - self.core = core - self.config = self.core.config - - __builtin__.hookManager = self #needed to let hooks register themself - - self.log = self.core.log - self.plugins = [] - self.pluginMap = {} - self.methods = {} #dict of names and list of methods usable by rpc - - self.events = {} # contains events - - #registering callback for config event - self.config.pluginCB = MethodType(self.dispatchEvent, "pluginConfigChanged", basestring) - - self.addEvent("pluginConfigChanged", self.manageHooks) - - self.lock = RLock() - self.createIndex() - - def try_catch(func): - def new(*args): - try: - return func(*args) - except Exception, e: - args[0].log.error(_("Error executing hooks: %s") % str(e)) - if args[0].core.debug: - traceback.print_exc() - - return new - - - def addRPC(self, plugin, func, doc): - plugin = plugin.rpartition(".")[2] - doc = doc.strip() if doc else "" - - if plugin in self.methods: - self.methods[plugin][func] = doc - else: - self.methods[plugin] = {func: doc} - - def callRPC(self, plugin, func, args, parse): - if not args: args = tuple() - if parse: - args = tuple([literal_eval(x) for x in args]) - - plugin = self.pluginMap[plugin] - f = getattr(plugin, func) - return f(*args) - - - def createIndex(self): - plugins = [] - - active = [] - deactive = [] - - for pluginname in self.core.pluginManager.hookPlugins: - try: - #hookClass = getattr(plugin, plugin.__name__) - - if self.config.getPlugin(pluginname, "activated"): - pluginClass = self.core.pluginManager.loadClass("hooks", pluginname) - if not pluginClass: continue - - plugin = pluginClass(self.core, self) - plugins.append(plugin) - self.pluginMap[pluginClass.__name__] = plugin - if plugin.isActivated(): - active.append(pluginClass.__name__) - else: - deactive.append(pluginname) - - - except: - self.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) - if self.core.debug: - traceback.print_exc() - - self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active))) - self.log.info(_("Deactivate plugins: %s") % ", ".join(sorted(deactive))) - - self.plugins = plugins - - def manageHooks(self, plugin, name, value): - if name == "activated" and value: - self.activateHook(plugin) - elif name == "activated" and not value: - self.deactivateHook(plugin) - - def activateHook(self, plugin): - - #check if already loaded - for inst in self.plugins: - if inst.__name__ == plugin: - return - - pluginClass = self.core.pluginManager.loadClass("hooks", plugin) - - if not pluginClass: return - - self.log.debug("Plugin loaded: %s" % plugin) - - plugin = pluginClass(self.core, self) - self.plugins.append(plugin) - self.pluginMap[pluginClass.__name__] = plugin - - # call core Ready - start_new_thread(plugin.coreReady, tuple()) - - def deactivateHook(self, plugin): - - hook = None - for inst in self.plugins: - if inst.__name__ == plugin: - hook = inst - - if not hook: return - - self.log.debug("Plugin unloaded: %s" % plugin) - - hook.unload() - - #remove periodic call - self.log.debug("Removed callback %s" % self.core.scheduler.removeJob(hook.cb)) - self.plugins.remove(hook) - del self.pluginMap[hook.__name__] - - - @try_catch - def coreReady(self): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.coreReady() - - self.dispatchEvent("coreReady") - - @try_catch - def coreExiting(self): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.coreExiting() - - self.dispatchEvent("coreExiting") - - @lock - def downloadPreparing(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.downloadPreparing(pyfile) - - self.dispatchEvent("downloadPreparing", pyfile) - - @lock - def downloadFinished(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.downloadFinished(pyfile) - - self.dispatchEvent("downloadFinished", pyfile) - - @lock - @try_catch - def downloadFailed(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.downloadFailed(pyfile) - - self.dispatchEvent("downloadFailed", pyfile) - - @lock - def packageFinished(self, package): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.packageFinished(package) - - self.dispatchEvent("packageFinished", package) - - @lock - def beforeReconnecting(self, ip): - for plugin in self.plugins: - plugin.beforeReconnecting(ip) - - self.dispatchEvent("beforeReconnecting", ip) - - @lock - def afterReconnecting(self, ip): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.afterReconnecting(ip) - - self.dispatchEvent("afterReconnecting", ip) - - def startThread(self, function, *args, **kwargs): - t = HookThread(self.core.threadManager, function, args, kwargs) - - def activePlugins(self): - """ returns all active plugins """ - return [x for x in self.plugins if x.isActivated()] - - def getAllInfo(self): - """returns info stored by hook plugins""" - info = {} - for name, plugin in self.pluginMap.iteritems(): - if plugin.info: - #copy and convert so str - info[name] = dict([(x, str(y) if not isinstance(y, basestring) else y) for x, y in plugin.info.iteritems()]) - return info - - - def getInfo(self, plugin): - info = {} - if plugin in self.pluginMap and self.pluginMap[plugin].info: - info = dict([(x, str(y) if not isinstance(y, basestring) else y) - for x, y in self.pluginMap[plugin].info.iteritems()]) - - return info - - def addEvent(self, event, func): - """Adds an event listener for event name""" - if event in self.events: - self.events[event].append(func) - else: - self.events[event] = [func] - - def removeEvent(self, event, func): - """removes previously added event listener""" - if event in self.events: - self.events[event].remove(func) - - def dispatchEvent(self, event, *args): - """dispatches event with args""" - if event in self.events: - for f in self.events[event]: - try: - f(*args) - except Exception, e: - self.log.warning("Error calling event handler %s: %s, %s, %s" - % (event, f, args, str(e))) - if self.core.debug: - traceback.print_exc() diff --git a/pyload/PullEvents.py b/pyload/PullEvents.py deleted file mode 100644 index 0739b4ec8..000000000 --- a/pyload/PullEvents.py +++ /dev/null @@ -1,120 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from time import time -from pyload.utils import uniqify - -class PullManager: - def __init__(self, core): - self.core = core - self.clients = [] - - def newClient(self, uuid): - self.clients.append(Client(uuid)) - - def clean(self): - for n, client in enumerate(self.clients): - if client.lastActive + 30 < time(): - del self.clients[n] - - def getEvents(self, uuid): - events = [] - validUuid = False - for client in self.clients: - if client.uuid == uuid: - client.lastActive = time() - validUuid = True - while client.newEvents(): - events.append(client.popEvent().toList()) - break - if not validUuid: - self.newClient(uuid) - events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] - return uniqify(events) - - def addEvent(self, event): - for client in self.clients: - client.addEvent(event) - -class Client: - def __init__(self, uuid): - self.uuid = uuid - self.lastActive = time() - self.events = [] - - def newEvents(self): - return len(self.events) > 0 - - def popEvent(self): - if not len(self.events): - return None - return self.events.pop(0) - - def addEvent(self, event): - self.events.append(event) - -class UpdateEvent: - def __init__(self, itype, iid, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.destination = destination - - def toList(self): - return ["update", self.destination, self.type, self.id] - -class RemoveEvent: - def __init__(self, itype, iid, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.destination = destination - - def toList(self): - return ["remove", self.destination, self.type, self.id] - -class InsertEvent: - def __init__(self, itype, iid, after, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.after = after - self.destination = destination - - def toList(self): - return ["insert", self.destination, self.type, self.id, self.after] - -class ReloadAllEvent: - def __init__(self, destination): - assert destination == "queue" or destination == "collector" - self.destination = destination - - def toList(self): - return ["reload", self.destination] - -class AccountUpdateEvent: - def toList(self): - return ["account"] - -class ConfigUpdateEvent: - def toList(self): - return ["config"] diff --git a/pyload/PyFile.py b/pyload/PyFile.py deleted file mode 100644 index c4ce71570..000000000 --- a/pyload/PyFile.py +++ /dev/null @@ -1,284 +0,0 @@ -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN - @author: mkaay -""" - -from pyload.PullEvents import UpdateEvent -from pyload.utils import formatSize, lock - -from time import sleep, time - -from threading import RLock - -statusMap = { - "finished": 0, - "offline": 1, - "online": 2, - "queued": 3, - "skipped": 4, - "waiting": 5, - "temp. offline": 6, - "starting": 7, - "failed": 8, - "aborted": 9, - "decrypting": 10, - "custom": 11, - "downloading": 12, - "processing": 13, - "unknown": 14, -} - - -def setSize(self, value): - self._size = int(value) - -class PyFile(object): - """ - Represents a file object at runtime - """ - __slots__ = ("m", "id", "url", "name", "size", "_size", "status", "pluginname", "packageid", - "error", "order", "lock", "plugin", "waitUntil", "active", "abort", "statusname", - "reconnected", "progress", "maxprogress", "pluginmodule", "pluginclass") - - def __init__(self, manager, id, url, name, size, status, error, pluginname, package, order): - self.m = manager - - self.id = int(id) - self.url = url - self.name = name - self.size = size - self.status = status - self.pluginname = pluginname - self.packageid = package #should not be used, use package() instead - self.error = error - self.order = order - # database information ends here - - self.lock = RLock() - - self.plugin = None - #self.download = None - - self.waitUntil = 0 # time() + time to wait - - # status attributes - self.active = False #obsolete? - self.abort = False - self.reconnected = False - - self.statusname = None - - self.progress = 0 - self.maxprogress = 100 - - self.m.cache[int(id)] = self - - - # will convert all sizes to ints - size = property(lambda self: self._size, setSize) - - def __repr__(self): - return "PyFile %s: %s@%s" % (self.id, self.name, self.pluginname) - - @lock - def initPlugin(self): - """ inits plugin instance """ - if not self.plugin: - self.pluginmodule = self.m.core.pluginManager.getPlugin(self.pluginname) - self.pluginclass = getattr(self.pluginmodule, self.m.core.pluginManager.getPluginName(self.pluginname)) - self.plugin = self.pluginclass(self) - - @lock - def hasPlugin(self): - """Thread safe way to determine this file has initialized plugin attribute - - :return: - """ - return hasattr(self, "plugin") and self.plugin - - def package(self): - """ return package instance""" - return self.m.getPackage(self.packageid) - - def setStatus(self, status): - self.status = statusMap[status] - self.sync() #@TODO needed aslong no better job approving exists - - def setCustomStatus(self, msg, status="processing"): - self.statusname = msg - self.setStatus(status) - - def getStatusName(self): - if self.status not in (13, 14) or not self.statusname: - return self.m.statusMsg[self.status] - else: - return self.statusname - - def hasStatus(self, status): - return statusMap[status] == self.status - - def sync(self): - """sync PyFile instance with database""" - self.m.updateLink(self) - - @lock - def release(self): - """sync and remove from cache""" - # file has valid package - if self.packageid > 0: - self.sync() - - if hasattr(self, "plugin") and self.plugin: - self.plugin.clean() - del self.plugin - - self.m.releaseLink(self.id) - - def delete(self): - """delete pyfile from database""" - self.m.deleteLink(self.id) - - def toDict(self): - """return dict with all information for interface""" - return self.toDbDict() - - def toDbDict(self): - """return data as dict for databse - - format: - - { - id: {'url': url, 'name': name ... } - } - - """ - return { - self.id: { - 'id': self.id, - 'url': self.url, - 'name': self.name, - 'plugin': self.pluginname, - 'size': self.getSize(), - 'format_size': self.formatSize(), - 'status': self.status, - 'statusmsg': self.getStatusName(), - 'package': self.packageid, - 'error': self.error, - 'order': self.order - } - } - - def abortDownload(self): - """abort pyfile if possible""" - while self.id in self.m.core.threadManager.processingIds(): - self.abort = True - if self.plugin and self.plugin.req: - self.plugin.req.abortDownloads() - sleep(0.1) - - self.abort = False - if self.hasPlugin() and self.plugin.req: - self.plugin.req.abortDownloads() - - self.release() - - def finishIfDone(self): - """set status to finish and release file if every thread is finished with it""" - - if self.id in self.m.core.threadManager.processingIds(): - return False - - self.setStatus("finished") - self.release() - self.m.checkAllLinksFinished() - return True - - def checkIfProcessed(self): - self.m.checkAllLinksProcessed(self.id) - - def formatWait(self): - """ formats and return wait time in humanreadable format """ - seconds = self.waitUntil - time() - - if seconds < 0: return "00:00:00" - - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - - def formatSize(self): - """ formats size to readable format """ - return formatSize(self.getSize()) - - def formatETA(self): - """ formats eta to readable format """ - seconds = self.getETA() - - if seconds < 0: return "00:00:00" - - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - - def getSpeed(self): - """ calculates speed """ - try: - return self.plugin.req.speed - except: - return 0 - - def getETA(self): - """ gets established time of arrival""" - try: - return self.getBytesLeft() / self.getSpeed() - except: - return 0 - - def getBytesLeft(self): - """ gets bytes left """ - try: - return self.getSize() - self.plugin.req.arrived - except: - return 0 - - def getPercent(self): - """ get % of download """ - if self.status == 12: - try: - return self.plugin.req.percent - except: - return 0 - else: - return self.progress - - def getSize(self): - """ get size of download """ - try: - if self.plugin.req.size: - return self.plugin.req.size - else: - return self.size - except: - return self.size - - def notifyChange(self): - e = UpdateEvent("file", self.id, "collector" if not self.package().queue else "queue") - self.m.core.pullManager.addEvent(e) - - def setProgress(self, value): - if not value == self.progress: - self.progress = value - self.notifyChange() diff --git a/pyload/PyPackage.py b/pyload/PyPackage.py deleted file mode 100644 index 3961d8a70..000000000 --- a/pyload/PyPackage.py +++ /dev/null @@ -1,79 +0,0 @@ -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN - @author: mkaay -""" - -from pyload.PullEvents import UpdateEvent -from pyload.utils import safe_filename - -class PyPackage: - """ - Represents a package object at runtime - """ - def __init__(self, manager, id, name, folder, site, password, queue, order): - self.m = manager - self.m.packageCache[int(id)] = self - - self.id = int(id) - self.name = name - self._folder = folder - self.site = site - self.password = password - self.queue = queue - self.order = order - self.setFinished = False - - @property - def folder(self): - return safe_filename(self._folder) - - def toDict(self): - """ Returns a dictionary representation of the data. - - :return: dict: {id: { attr: value }} - """ - return { - self.id: { - 'id': self.id, - 'name': self.name, - 'folder': self.folder, - 'site': self.site, - 'password': self.password, - 'queue': self.queue, - 'order': self.order, - 'links': {} - } - } - - def getChildren(self): - """get information about contained links""" - return self.m.getPackageData(self.id)["links"] - - def sync(self): - """sync with db""" - self.m.updatePackage(self) - - def release(self): - """sync and delete from cache""" - self.sync() - self.m.releasePackage(self.id) - - def delete(self): - self.m.deletePackage(self.id) - - def notifyChange(self): - e = UpdateEvent("pack", self.id, "collector" if not self.queue else "queue") - self.m.core.pullManager.addEvent(e) diff --git a/pyload/Scheduler.py b/pyload/Scheduler.py deleted file mode 100644 index 71b5f96af..000000000 --- a/pyload/Scheduler.py +++ /dev/null @@ -1,141 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from time import time -from heapq import heappop, heappush -from thread import start_new_thread -from threading import Lock - -class AlreadyCalled(Exception): - pass - - -class Deferred: - def __init__(self): - self.call = [] - self.result = () - - def addCallback(self, f, *cargs, **ckwargs): - self.call.append((f, cargs, ckwargs)) - - def callback(self, *args, **kwargs): - if self.result: - raise AlreadyCalled - self.result = (args, kwargs) - for f, cargs, ckwargs in self.call: - args += tuple(cargs) - kwargs.update(ckwargs) - f(*args ** kwargs) - - -class Scheduler: - def __init__(self, core): - self.core = core - - self.queue = PriorityQueue() - - def addJob(self, t, call, args=[], kwargs={}, threaded=True): - d = Deferred() - t += time() - j = Job(t, call, args, kwargs, d, threaded) - self.queue.put((t, j)) - return d - - - def removeJob(self, d): - """ - :param d: defered object - :return: if job was deleted - """ - index = -1 - - for i, j in enumerate(self.queue): - if j[1].deferred == d: - index = i - - if index >= 0: - del self.queue[index] - return True - - return False - - def work(self): - while True: - t, j = self.queue.get() - if not j: - break - else: - if t <= time(): - j.start() - else: - self.queue.put((t, j)) - break - - -class Job: - def __init__(self, time, call, args=[], kwargs={}, deferred=None, threaded=True): - self.time = float(time) - self.call = call - self.args = args - self.kwargs = kwargs - self.deferred = deferred - self.threaded = threaded - - def run(self): - ret = self.call(*self.args, **self.kwargs) - if self.deferred is None: - return - else: - self.deferred.callback(ret) - - def start(self): - if self.threaded: - start_new_thread(self.run, ()) - else: - self.run() - - -class PriorityQueue: - """ a non blocking priority queue """ - - def __init__(self): - self.queue = [] - self.lock = Lock() - - def __iter__(self): - return iter(self.queue) - - def __delitem__(self, key): - del self.queue[key] - - def put(self, element): - self.lock.acquire() - heappush(self.queue, element) - self.lock.release() - - def get(self): - """ return element or None """ - self.lock.acquire() - try: - el = heappop(self.queue) - return el - except IndexError: - return None, None - finally: - self.lock.release() diff --git a/pyload/ThreadManager.py b/pyload/ThreadManager.py deleted file mode 100644 index d9a6e1b8c..000000000 --- a/pyload/ThreadManager.py +++ /dev/null @@ -1,317 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from os.path import exists, join -import re -from subprocess import Popen -from threading import Event, Lock -from time import sleep, time -from traceback import print_exc -from random import choice - -import pycurl - -from pyload.threads import PluginThread -from pyload.PyFile import PyFile -from pyload.network.RequestFactory import getURL -from pyload.utils import freeSpace, lock - - -class ThreadManager: - """manages the download threads, assign jobs, reconnect etc""" - - - def __init__(self, core): - """Constructor""" - self.core = core - self.log = core.log - - self.threads = [] # thread list - self.localThreads = [] #hook+decrypter threads - - self.pause = True - - self.reconnecting = Event() - self.reconnecting.clear() - self.downloaded = 0 #number of files downloaded since last cleanup - - self.lock = Lock() - - # some operations require to fetch url info from hoster, so we caching them so it wont be done twice - # contains a timestamp and will be purged after timeout - self.infoCache = {} - - # pool of ids for online check - self.resultIDs = 0 - - # threads which are fetching hoster results - self.infoResults = {} - #timeout for cache purge - self.timestamp = 0 - - pycurl.global_init(pycurl.GLOBAL_DEFAULT) - - for i in range(0, self.core.config.get("download", "max_downloads")): - self.createThread() - - - def createThread(self): - """create a download thread""" - - thread = PluginThread.DownloadThread(self) - self.threads.append(thread) - - def createInfoThread(self, data, pid): - """ - start a thread whichs fetches online status and other infos - data = [ .. () .. ] - """ - self.timestamp = time() + 5 * 60 - - PluginThread.InfoThread(self, data, pid) - - @lock - def createResultThread(self, data, add=False): - """ creates a thread to fetch online status, returns result id """ - self.timestamp = time() + 5 * 60 - - rid = self.resultIDs - self.resultIDs += 1 - - PluginThread.InfoThread(self, data, rid=rid, add=add) - - return rid - - - @lock - def getInfoResult(self, rid): - """returns result and clears it""" - self.timestamp = time() + 5 * 60 - - if rid in self.infoResults: - data = self.infoResults[rid] - self.infoResults[rid] = {} - return data - else: - return {} - - @lock - def setInfoResults(self, rid, result): - self.infoResults[rid].update(result) - - def getActiveFiles(self): - active = [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)] - - for t in self.localThreads: - active.extend(t.getActiveFiles()) - - return active - - def processingIds(self): - """get a id list of all pyfiles processed""" - return [x.id for x in self.getActiveFiles()] - - - def work(self): - """run all task which have to be done (this is for repetivive call by core)""" - try: - self.tryReconnect() - except Exception, e: - self.log.error(_("Reconnect Failed: %s") % str(e) ) - self.reconnecting.clear() - if self.core.debug: - print_exc() - self.checkThreadCount() - - try: - self.assignJob() - except Exception, e: - self.log.warning("Assign job error", e) - if self.core.debug: - print_exc() - - sleep(0.5) - self.assignJob() - #it may be failed non critical so we try it again - - if (self.infoCache or self.infoResults) and self.timestamp < time(): - self.infoCache.clear() - self.infoResults.clear() - self.log.debug("Cleared Result cache") - - #-------------------------------------------------------------------------- - def tryReconnect(self): - """checks if reconnect needed""" - - if not (self.core.config["reconnect"]["activated"] and self.core.api.isTimeReconnect()): - return False - - active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active] - - if not (0 < active.count(True) == len(active)): - return False - - if not exists(self.core.config['reconnect']['method']): - if exists(join(pypath, self.core.config['reconnect']['method'])): - self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method']) - else: - self.core.config["reconnect"]["activated"] = False - self.log.warning(_("Reconnect script not found!")) - return - - self.reconnecting.set() - - #Do reconnect - self.log.info(_("Starting reconnect")) - - while [x.active.plugin.waiting for x in self.threads if x.active].count(True) != 0: - sleep(0.25) - - ip = self.getIP() - - self.core.hookManager.beforeReconnecting(ip) - - self.log.debug("Old IP: %s" % ip) - - try: - reconn = Popen(self.core.config['reconnect']['method'], bufsize=-1, shell=True)#, stdout=subprocess.PIPE) - except: - self.log.warning(_("Failed executing reconnect script!")) - self.core.config["reconnect"]["activated"] = False - self.reconnecting.clear() - if self.core.debug: - print_exc() - return - - reconn.wait() - sleep(1) - ip = self.getIP() - self.core.hookManager.afterReconnecting(ip) - - self.log.info(_("Reconnected, new IP: %s") % ip) - - self.reconnecting.clear() - - def getIP(self): - """retrieve current ip""" - services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"), - ("http://checkip.dyndns.org/",".*Current IP Address: (\S+).*")] - - ip = "" - for i in range(10): - try: - sv = choice(services) - ip = getURL(sv[0]) - ip = re.match(sv[1], ip).group(1) - break - except: - ip = "" - sleep(1) - - return ip - - #-------------------------------------------------------------------------- - def checkThreadCount(self): - """checks if there are need for increasing or reducing thread count""" - - if len(self.threads) == self.core.config.get("download", "max_downloads"): - return True - elif len(self.threads) < self.core.config.get("download", "max_downloads"): - self.createThread() - else: - free = [x for x in self.threads if not x.active] - if free: - free[0].put("quit") - - - def cleanPycurl(self): - """ make a global curl cleanup (currently ununused) """ - if self.processingIds(): - return False - pycurl.global_cleanup() - pycurl.global_init(pycurl.GLOBAL_DEFAULT) - self.downloaded = 0 - self.log.debug("Cleaned up pycurl") - return True - - #-------------------------------------------------------------------------- - def assignJob(self): - """assing a job to a thread if possible""" - - if self.pause or not self.core.api.isTimeDownload(): return - - #if self.downloaded > 20: - # if not self.cleanPyCurl(): return - - free = [x for x in self.threads if not x.active] - - inuse = set([(x.active.pluginname, self.getLimit(x)) for x in self.threads if x.active and x.active.hasPlugin() and x.active.plugin.account]) - inuse = map(lambda x: (x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) ,inuse) - onlimit = [x[0] for x in inuse if x[1] > 0 and x[2] >= x[1]] - - occ = [x.active.pluginname for x in self.threads if x.active and x.active.hasPlugin() and not x.active.plugin.multiDL] + onlimit - - occ.sort() - occ = tuple(set(occ)) - job = self.core.files.getJob(occ) - if job: - try: - job.initPlugin() - except Exception, e: - self.log.critical(str(e)) - print_exc() - job.setStatus("failed") - job.error = str(e) - job.release() - return - - if job.plugin.__type__ == "hoster": - spaceLeft = freeSpace(self.core.config["general"]["download_folder"]) / 1024 / 1024 - if spaceLeft < self.core.config["general"]["min_free_space"]: - self.log.warning(_("Not enough space left on device")) - self.pause = True - - if free and not self.pause: - thread = free[0] - #self.downloaded += 1 - - thread.put(job) - else: - #put job back - if occ not in self.core.files.jobCache: - self.core.files.jobCache[occ] = [] - self.core.files.jobCache[occ].append(job.id) - - #check for decrypt jobs - job = self.core.files.getDecryptJob() - if job: - job.initPlugin() - thread = PluginThread.DecrypterThread(self, job) - - - else: - thread = PluginThread.DecrypterThread(self, job) - - def getLimit(self, thread): - limit = thread.active.plugin.account.getAccountData(thread.active.plugin.user)["options"].get("limitDL", ["0"])[0] - return int(limit) - - def cleanup(self): - """do global cleanup, should be called when finished with pycurl""" - pycurl.global_cleanup() diff --git a/pyload/api/__init__.py b/pyload/api/__init__.py new file mode 100644 index 000000000..3da1e13e1 --- /dev/null +++ b/pyload/api/__init__.py @@ -0,0 +1,1030 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from base64 import standard_b64encode +from os.path import join +from time import time +import re + +from pyload.datatypes.PyFile import PyFile +from utils import freeSpace, compare_time +from pyload.utils.packagetools import parseNames +from network.RequestFactory import getURL +from remote import activated + +if activated: + try: + from remote.thriftbackend.thriftgen.pyload.ttypes import * + from remote.thriftbackend.thriftgen.pyload.Pyload import Iface + BaseObject = TBase + except ImportError: + print "Thrift not imported" + from remote.socketbackend.ttypes import * +else: + from remote.socketbackend.ttypes import * + +# contains function names mapped to their permissions +# unlisted functions are for admins only +permMap = {} + +# decorator only called on init, never initialized, so has no effect on runtime +def permission(bits): + class _Dec(object): + def __new__(cls, func, *args, **kwargs): + permMap[func.__name__] = bits + return func + + return _Dec + + +urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&]*)", re.IGNORECASE) + +class PERMS: + ALL = 0 # requires no permission, but login + ADD = 1 # can add packages + DELETE = 2 # can delete packages + STATUS = 4 # see and change server status + LIST = 16 # see queue and collector + MODIFY = 32 # moddify some attribute of downloads + DOWNLOAD = 64 # can download from webinterface + SETTINGS = 128 # can access settings + ACCOUNTS = 256 # can access accounts + LOGS = 512 # can see server logs + +class ROLE: + ADMIN = 0 #admin has all permissions implicit + USER = 1 + +def has_permission(userperms, perms): + # bytewise or perms before if needed + return perms == (userperms & perms) + + +class Api(Iface): + """ + **pyLoads API** + + This is accessible either internal via core.api or via thrift backend. + + see Thrift specification file remote/thriftbackend/pyload.thrift\ + for information about data structures and what methods are usuable with rpc. + + Most methods requires specific permissions, please look at the source code if you need to know.\ + These can be configured via webinterface. + Admin user have all permissions, and are the only ones who can access the methods with no specific permission. + """ + + EXTERNAL = Iface # let the json api know which methods are external + + def __init__(self, core): + self.core = core + + def _convertPyFile(self, p): + f = FileData(p["id"], p["url"], p["name"], p["plugin"], p["size"], + p["format_size"], p["status"], p["statusmsg"], + p["package"], p["error"], p["order"]) + return f + + def _convertConfigFormat(self, c): + sections = {} + for sectionName, sub in c.iteritems(): + section = ConfigSection(sectionName, sub["desc"]) + items = [] + for key, data in sub.iteritems(): + if key in ("desc", "outline"): + continue + item = ConfigItem() + item.name = key + item.description = data["desc"] + item.value = str(data["value"]) if not isinstance(data["value"], basestring) else data["value"] + item.type = data["type"] + items.append(item) + section.items = items + sections[sectionName] = section + if "outline" in sub: + section.outline = sub["outline"] + return sections + + @permission(PERMS.SETTINGS) + def getConfigValue(self, category, option, section="core"): + """Retrieve config value. + + :param category: name of category, or plugin + :param option: config option + :param section: 'plugin' or 'core' + :return: config value as string + """ + if section == "core": + value = self.core.config[category][option] + else: + value = self.core.config.getPlugin(category, option) + + return str(value) if not isinstance(value, basestring) else value + + @permission(PERMS.SETTINGS) + def setConfigValue(self, category, option, value, section="core"): + """Set new config value. + + :param category: + :param option: + :param value: new config value + :param section: 'plugin' or 'core + """ + self.core.hookManager.dispatchEvent("configChanged", category, option, value, section) + + if section == "core": + self.core.config[category][option] = value + + if option in ("limit_speed", "max_speed"): #not so nice to update the limit + self.core.requestFactory.updateBucket() + + elif section == "plugin": + self.core.config.setPlugin(category, option, value) + + @permission(PERMS.SETTINGS) + def getConfig(self): + """Retrieves complete config of core. + + :return: list of `ConfigSection` + """ + return self._convertConfigFormat(self.core.config.config) + + def getConfigDict(self): + """Retrieves complete config in dict format, not for RPC. + + :return: dict + """ + return self.core.config.config + + @permission(PERMS.SETTINGS) + def getPluginConfig(self): + """Retrieves complete config for all plugins. + + :return: list of `ConfigSection` + """ + return self._convertConfigFormat(self.core.config.plugin) + + def getPluginConfigDict(self): + """Plugin config as dict, not for RPC. + + :return: dict + """ + return self.core.config.plugin + + + @permission(PERMS.STATUS) + def pauseServer(self): + """Pause server: Tt wont start any new downloads, but nothing gets aborted.""" + self.core.threadManager.pause = True + + @permission(PERMS.STATUS) + def unpauseServer(self): + """Unpause server: New Downloads will be started.""" + self.core.threadManager.pause = False + + @permission(PERMS.STATUS) + def togglePause(self): + """Toggle pause state. + + :return: new pause state + """ + self.core.threadManager.pause ^= True + return self.core.threadManager.pause + + @permission(PERMS.STATUS) + def toggleReconnect(self): + """Toggle reconnect activation. + + :return: new reconnect state + """ + self.core.config["reconnect"]["activated"] ^= True + return self.core.config["reconnect"]["activated"] + + @permission(PERMS.LIST) + def statusServer(self): + """Some general information about the current status of pyLoad. + + :return: `ServerStatus` + """ + serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), + self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, + not self.core.threadManager.pause and self.isTimeDownload(), + self.core.config['reconnect']['activated'] and self.isTimeReconnect()) + + for pyfile in [x.active for x in self.core.threadManager.threads if x.active and isinstance(x.active, PyFile)]: + serverStatus.speed += pyfile.getSpeed() #bytes/s + + return serverStatus + + @permission(PERMS.STATUS) + def freeSpace(self): + """Available free space at download directory in bytes""" + return freeSpace(self.core.config["general"]["download_folder"]) + + @permission(PERMS.ALL) + def getServerVersion(self): + """pyLoad Core version """ + return self.core.version + + def kill(self): + """Clean way to quit pyLoad""" + self.core.do_kill = True + + def restart(self): + """Restart pyload core""" + self.core.do_restart = True + + @permission(PERMS.LOGS) + def getLog(self, offset=0): + """Returns most recent log entries. + + :param offset: line offset + :return: List of log entries + """ + filename = join(self.core.config['log']['log_folder'], 'log.txt') + try: + fh = open(filename, "r") + lines = fh.readlines() + fh.close() + if offset >= len(lines): + return [] + return lines[offset:] + except: + return ['No log available'] + + @permission(PERMS.STATUS) + def isTimeDownload(self): + """Checks if pyload will start new downloads according to time in config. + + :return: bool + """ + start = self.core.config['downloadTime']['start'].split(":") + end = self.core.config['downloadTime']['end'].split(":") + return compare_time(start, end) + + @permission(PERMS.STATUS) + def isTimeReconnect(self): + """Checks if pyload will try to make a reconnect + + :return: bool + """ + start = self.core.config['reconnect']['startTime'].split(":") + end = self.core.config['reconnect']['endTime'].split(":") + return compare_time(start, end) and self.core.config["reconnect"]["activated"] + + @permission(PERMS.LIST) + def statusDownloads(self): + """ Status off all currently running downloads. + + :return: list of `DownloadStatus` + """ + data = [] + for pyfile in self.core.threadManager.getActiveFiles(): + if not isinstance(pyfile, PyFile): + continue + + data.append(DownloadInfo( + pyfile.id, pyfile.name, pyfile.getSpeed(), pyfile.getETA(), pyfile.formatETA(), + pyfile.getBytesLeft(), pyfile.getSize(), pyfile.formatSize(), pyfile.getPercent(), + pyfile.status, pyfile.getStatusName(), pyfile.formatWait(), + pyfile.waitUntil, pyfile.packageid, pyfile.package().name, pyfile.pluginname)) + + return data + + @permission(PERMS.ADD) + def addPackage(self, name, links, dest=Destination.Queue): + """Adds a package, with links to desired destination. + + :param name: name of the new package + :param links: list of urls + :param dest: `Destination` + :return: package id of the new package + """ + if self.core.config['general']['folder_per_package']: + folder = name + else: + folder = "" + + folder = folder.replace("http://", "").replace(":", "").replace("/", "_").replace("\\", "_") + + pid = self.core.files.addPackage(name, folder, dest) + + self.core.files.addLinks(links, pid) + + self.core.log.info(_("Added package %(name)s containing %(count)d links") % {"name": name, "count": len(links)}) + + self.core.files.save() + + return pid + + @permission(PERMS.ADD) + def parseURLs(self, html=None, url=None): + """Parses html content or any arbitaty text for links and returns result of `checkURLs` + + :param html: html source + :return: + """ + urls = [] + + if html: + urls += [x[0] for x in urlmatcher.findall(html)] + + if url: + page = getURL(url) + urls += [x[0] for x in urlmatcher.findall(page)] + + # remove duplicates + return self.checkURLs(set(urls)) + + + @permission(PERMS.ADD) + def checkURLs(self, urls): + """ Gets urls and returns pluginname mapped to list of matches urls. + + :param urls: + :return: {plugin: urls} + """ + data = self.core.pluginManager.parseUrls(urls) + plugins = {} + + for url, plugin in data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + return plugins + + @permission(PERMS.ADD) + def checkOnlineStatus(self, urls): + """ initiates online status check + + :param urls: + :return: initial set of data as `OnlineCheck` instance containing the result id + """ + data = self.core.pluginManager.parseUrls(urls) + + rid = self.core.threadManager.createResultThread(data, False) + + tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] + data = parseNames(tmp) + result = {} + + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + return OnlineCheck(rid, result) + + @permission(PERMS.ADD) + def checkOnlineStatusContainer(self, urls, container, data): + """ checks online status of urls and a submited container file + + :param urls: list of urls + :param container: container file name + :param data: file content + :return: online check + """ + th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb") + th.write(str(data)) + th.close() + + return self.checkOnlineStatus(urls + [th.name]) + + @permission(PERMS.ADD) + def pollResults(self, rid): + """ Polls the result available for ResultID + + :param rid: `ResultID` + :return: `OnlineCheck`, if rid is -1 then no more data available + """ + result = self.core.threadManager.getInfoResult(rid) + + if "ALL_INFO_FETCHED" in result: + del result["ALL_INFO_FETCHED"] + return OnlineCheck(-1, result) + else: + return OnlineCheck(rid, result) + + + @permission(PERMS.ADD) + def generatePackages(self, links): + """ Parses links, generates packages names from urls + + :param links: list of urls + :return: package names mapped to urls + """ + result = parseNames((x, x) for x in links) + return result + + @permission(PERMS.ADD) + def generateAndAddPackages(self, links, dest=Destination.Queue): + """Generates and add packages + + :param links: list of urls + :param dest: `Destination` + :return: list of package ids + """ + return [self.addPackage(name, urls, dest) for name, urls + in self.generatePackages(links).iteritems()] + + @permission(PERMS.ADD) + def checkAndAddPackages(self, links, dest=Destination.Queue): + """Checks online status, retrieves names, and will add packages.\ + Because of this packages are not added immediatly, only for internal use. + + :param links: list of urls + :param dest: `Destination` + :return: None + """ + data = self.core.pluginManager.parseUrls(links) + self.core.threadManager.createResultThread(data, True) + + + @permission(PERMS.LIST) + def getPackageData(self, pid): + """Returns complete information about package, and included files. + + :param pid: package id + :return: `PackageData` with .links attribute + """ + data = self.core.files.getPackageData(int(pid)) + + if not data: + raise PackageDoesNotExists(pid) + + pdata = PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], + data["queue"], data["order"], + links=[self._convertPyFile(x) for x in data["links"].itervalues()]) + + return pdata + + @permission(PERMS.LIST) + def getPackageInfo(self, pid): + """Returns information about package, without detailed information about containing files + + :param pid: package id + :return: `PackageData` with .fid attribute + """ + data = self.core.files.getPackageData(int(pid)) + + if not data: + raise PackageDoesNotExists(pid) + + pdata = PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], + data["queue"], data["order"], + fids=[int(x) for x in data["links"]]) + + return pdata + + @permission(PERMS.LIST) + def getFileData(self, fid): + """Get complete information about a specific file. + + :param fid: file id + :return: `FileData` + """ + info = self.core.files.getFileData(int(fid)) + if not info: + raise FileDoesNotExists(fid) + + fdata = self._convertPyFile(info.values()[0]) + return fdata + + @permission(PERMS.DELETE) + def deleteFiles(self, fids): + """Deletes several file entries from pyload. + + :param fids: list of file ids + """ + for id in fids: + self.core.files.deleteLink(int(id)) + + self.core.files.save() + + @permission(PERMS.DELETE) + def deletePackages(self, pids): + """Deletes packages and containing links. + + :param pids: list of package ids + """ + for id in pids: + self.core.files.deletePackage(int(id)) + + self.core.files.save() + + @permission(PERMS.LIST) + def getQueue(self): + """Returns info about queue and packages, **not** about files, see `getQueueData` \ + or `getPackageData` instead. + + :return: list of `PackageInfo` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + pack["linkstotal"]) + for pack in self.core.files.getInfoData(Destination.Queue).itervalues()] + + @permission(PERMS.LIST) + def getQueueData(self): + """Return complete data about everything in queue, this is very expensive use it sparely.\ + See `getQueue` for alternative. + + :return: list of `PackageData` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) + for pack in self.core.files.getCompleteData(Destination.Queue).itervalues()] + + @permission(PERMS.LIST) + def getCollector(self): + """same as `getQueue` for collector. + + :return: list of `PackageInfo` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + pack["linkstotal"]) + for pack in self.core.files.getInfoData(Destination.Collector).itervalues()] + + @permission(PERMS.LIST) + def getCollectorData(self): + """same as `getQueueData` for collector. + + :return: list of `PackageInfo` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) + for pack in self.core.files.getCompleteData(Destination.Collector).itervalues()] + + + @permission(PERMS.ADD) + def addFiles(self, pid, links): + """Adds files to specific package. + + :param pid: package id + :param links: list of urls + """ + self.core.files.addLinks(links, int(pid)) + + self.core.log.info(_("Added %(count)d links to package #%(package)d ") % {"count": len(links), "package": pid}) + self.core.files.save() + + @permission(PERMS.MODIFY) + def pushToQueue(self, pid): + """Moves package from Collector to Queue. + + :param pid: package id + """ + self.core.files.setPackageLocation(pid, Destination.Queue) + + @permission(PERMS.MODIFY) + def pullFromQueue(self, pid): + """Moves package from Queue to Collector. + + :param pid: package id + """ + self.core.files.setPackageLocation(pid, Destination.Collector) + + @permission(PERMS.MODIFY) + def restartPackage(self, pid): + """Restarts a package, resets every containing files. + + :param pid: package id + """ + self.core.files.restartPackage(int(pid)) + + @permission(PERMS.MODIFY) + def restartFile(self, fid): + """Resets file status, so it will be downloaded again. + + :param fid: file id + """ + self.core.files.restartFile(int(fid)) + + @permission(PERMS.MODIFY) + def recheckPackage(self, pid): + """Proofes online status of all files in a package, also a default action when package is added. + + :param pid: + :return: + """ + self.core.files.reCheckPackage(int(pid)) + + @permission(PERMS.MODIFY) + def stopAllDownloads(self): + """Aborts all running downloads.""" + + pyfiles = self.core.files.cache.values() + for pyfile in pyfiles: + pyfile.abortDownload() + + @permission(PERMS.MODIFY) + def stopDownloads(self, fids): + """Aborts specific downloads. + + :param fids: list of file ids + :return: + """ + pyfiles = self.core.files.cache.values() + + for pyfile in pyfiles: + if pyfile.id in fids: + pyfile.abortDownload() + + @permission(PERMS.MODIFY) + def setPackageName(self, pid, name): + """Renames a package. + + :param pid: package id + :param name: new package name + """ + pack = self.core.files.getPackage(pid) + pack.name = name + pack.sync() + + @permission(PERMS.MODIFY) + def movePackage(self, destination, pid): + """Set a new package location. + + :param destination: `Destination` + :param pid: package id + """ + if destination not in (0, 1): return + self.core.files.setPackageLocation(pid, destination) + + @permission(PERMS.MODIFY) + def moveFiles(self, fids, pid): + """Move multiple files to another package + + :param fids: list of file ids + :param pid: destination package + :return: + """ + #TODO: implement + pass + + + @permission(PERMS.ADD) + def uploadContainer(self, filename, data): + """Uploads and adds a container file to pyLoad. + + :param filename: filename, extension is important so it can correctly decrypted + :param data: file content + """ + th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb") + th.write(str(data)) + th.close() + + self.addPackage(th.name, [th.name], Destination.Queue) + + @permission(PERMS.MODIFY) + def orderPackage(self, pid, position): + """Gives a package a new position. + + :param pid: package id + :param position: + """ + self.core.files.reorderPackage(pid, position) + + @permission(PERMS.MODIFY) + def orderFile(self, fid, position): + """Gives a new position to a file within its package. + + :param fid: file id + :param position: + """ + self.core.files.reorderFile(fid, position) + + @permission(PERMS.MODIFY) + def setPackageData(self, pid, data): + """Allows to modify several package attributes. + + :param pid: package id + :param data: dict that maps attribute to desired value + """ + p = self.core.files.getPackage(pid) + if not p: raise PackageDoesNotExists(pid) + + for key, value in data.iteritems(): + if key == "id": continue + setattr(p, key, value) + + p.sync() + self.core.files.save() + + @permission(PERMS.DELETE) + def deleteFinished(self): + """Deletes all finished files and completly finished packages. + + :return: list of deleted package ids + """ + return self.core.files.deleteFinishedLinks() + + @permission(PERMS.MODIFY) + def restartFailed(self): + """Restarts all failed failes.""" + self.core.files.restartFailed() + + @permission(PERMS.LIST) + def getPackageOrder(self, destination): + """Returns information about package order. + + :param destination: `Destination` + :return: dict mapping order to package id + """ + + packs = self.core.files.getInfoData(destination) + order = {} + + for pid in packs: + pack = self.core.files.getPackageData(int(pid)) + while pack["order"] in order.keys(): #just in case + pack["order"] += 1 + order[pack["order"]] = pack["id"] + return order + + @permission(PERMS.LIST) + def getFileOrder(self, pid): + """Information about file order within package. + + :param pid: + :return: dict mapping order to file id + """ + rawData = self.core.files.getPackageData(int(pid)) + order = {} + for id, pyfile in rawData["links"].iteritems(): + while pyfile["order"] in order.keys(): #just in case + pyfile["order"] += 1 + order[pyfile["order"]] = pyfile["id"] + return order + + + @permission(PERMS.STATUS) + def isCaptchaWaiting(self): + """Indicates wether a captcha task is available + + :return: bool + """ + self.core.lastClientConnected = time() + task = self.core.captchaManager.getTask() + return not task is None + + @permission(PERMS.STATUS) + def getCaptchaTask(self, exclusive=False): + """Returns a captcha task + + :param exclusive: unused + :return: `CaptchaTask` + """ + self.core.lastClientConnected = time() + task = self.core.captchaManager.getTask() + if task: + task.setWatingForUser(exclusive=exclusive) + data, type, result = task.getCaptcha() + t = CaptchaTask(int(task.id), standard_b64encode(data), type, result) + return t + else: + return CaptchaTask(-1) + + @permission(PERMS.STATUS) + def getCaptchaTaskStatus(self, tid): + """Get information about captcha task + + :param tid: task id + :return: string + """ + self.core.lastClientConnected = time() + t = self.core.captchaManager.getTaskByID(tid) + return t.getStatus() if t else "" + + @permission(PERMS.STATUS) + def setCaptchaResult(self, tid, result): + """Set result for a captcha task + + :param tid: task id + :param result: captcha result + """ + self.core.lastClientConnected = time() + task = self.core.captchaManager.getTaskByID(tid) + if task: + task.setResult(result) + self.core.captchaManager.removeTask(task) + + + @permission(PERMS.STATUS) + def getEvents(self, uuid): + """Lists occured events, may be affected to changes in future. + + :param uuid: + :return: list of `Events` + """ + events = self.core.pullManager.getEvents(uuid) + newEvents = [] + + def convDest(d): + return Destination.Queue if d == "queue" else Destination.Collector + + for e in events: + event = EventInfo() + event.eventname = e[0] + if e[0] in ("update", "remove", "insert"): + event.id = e[3] + event.type = ElementType.Package if e[2] == "pack" else ElementType.File + event.destination = convDest(e[1]) + elif e[0] == "order": + if e[1]: + event.id = e[1] + event.type = ElementType.Package if e[2] == "pack" else ElementType.File + event.destination = convDest(e[3]) + elif e[0] == "reload": + event.destination = convDest(e[1]) + newEvents.append(event) + return newEvents + + @permission(PERMS.ACCOUNTS) + def getAccounts(self, refresh): + """Get information about all entered accounts. + + :param refresh: reload account info + :return: list of `AccountInfo` + """ + accs = self.core.accountManager.getAccountInfos(False, refresh) + accounts = [] + for group in accs.values(): + accounts.extend([AccountInfo(acc["validuntil"], acc["login"], acc["options"], acc["valid"], + acc["trafficleft"], acc["maxtraffic"], acc["premium"], acc["type"]) + for acc in group]) + return accounts + + @permission(PERMS.ALL) + def getAccountTypes(self): + """All available account types. + + :return: list + """ + return self.core.accountManager.accounts.keys() + + @permission(PERMS.ACCOUNTS) + def updateAccount(self, plugin, account, password=None, options={}): + """Changes pw/options for specific account.""" + self.core.accountManager.updateAccount(plugin, account, password, options) + + @permission(PERMS.ACCOUNTS) + def removeAccount(self, plugin, account): + """Remove account from pyload. + + :param plugin: pluginname + :param account: accountname + """ + self.core.accountManager.removeAccount(plugin, account) + + @permission(PERMS.ALL) + def login(self, username, password, remoteip=None): + """Login into pyLoad, this **must** be called when using rpc before any methods can be used. + + :param username: + :param password: + :param remoteip: Omit this argument, its only used internal + :return: bool indicating login was successful + """ + return True if self.checkAuth(username, password, remoteip) else False + + def checkAuth(self, username, password, remoteip=None): + """Check authentication and returns details + + :param username: + :param password: + :param remoteip: + :return: dict with info, empty when login is incorrect + """ + if self.core.config["remote"]["nolocalauth"] and remoteip == "127.0.0.1": + return "local" + else: + return self.core.db.checkAuth(username, password) + + def isAuthorized(self, func, userdata): + """checks if the user is authorized for specific method + + :param func: function name + :param userdata: dictionary of user data + :return: boolean + """ + if userdata == "local" or userdata["role"] == ROLE.ADMIN: + return True + elif func in permMap and has_permission(userdata["permission"], permMap[func]): + return True + else: + return False + + + @permission(PERMS.ALL) + def getUserData(self, username, password): + """similar to `checkAuth` but returns UserData thrift type """ + user = self.checkAuth(username, password) + if user: + return UserData(user["name"], user["email"], user["role"], user["permission"], user["template"]) + else: + return UserData() + + + def getAllUserData(self): + """returns all known user and info""" + res = {} + for user, data in self.core.db.getAllUserData().iteritems(): + res[user] = UserData(user, data["email"], data["role"], data["permission"], data["template"]) + + return res + + @permission(PERMS.STATUS) + def getServices(self): + """ A dict of available services, these can be defined by hook plugins. + + :return: dict with this style: {"plugin": {"method": "description"}} + """ + data = {} + for plugin, funcs in self.core.hookManager.methods.iteritems(): + data[plugin] = funcs + + return data + + @permission(PERMS.STATUS) + def hasService(self, plugin, func): + """Checks wether a service is available. + + :param plugin: + :param func: + :return: bool + """ + cont = self.core.hookManager.methods + return plugin in cont and func in cont[plugin] + + @permission(PERMS.STATUS) + def call(self, info): + """Calls a service (a method in hook plugin). + + :param info: `ServiceCall` + :return: result + :raises: ServiceDoesNotExists, when its not available + :raises: ServiceException, when a exception was raised + """ + plugin = info.plugin + func = info.func + args = info.arguments + parse = info.parseArguments + + if not self.hasService(plugin, func): + raise ServiceDoesNotExists(plugin, func) + + try: + ret = self.core.hookManager.callRPC(plugin, func, args, parse) + return str(ret) + except Exception, e: + raise ServiceException(e.message) + + @permission(PERMS.STATUS) + def getAllInfo(self): + """Returns all information stored by hook plugins. Values are always strings + + :return: {"plugin": {"name": value}} + """ + return self.core.hookManager.getAllInfo() + + @permission(PERMS.STATUS) + def getInfoByPlugin(self, plugin): + """Returns information stored by a specific plugin. + + :param plugin: pluginname + :return: dict of attr names mapped to value {"name": value} + """ + return self.core.hookManager.getInfo(plugin) + + def changePassword(self, user, oldpw, newpw): + """ changes password for specific user """ + return self.core.db.changePassword(user, oldpw, newpw) + + def setUserPermission(self, user, permission, role): + self.core.db.setPermission(user, permission) + self.core.db.setRole(user, role) diff --git a/pyload/cli/Cli.py b/pyload/cli/Cli.py index 447829ef7..f05e98b1a 100644 --- a/pyload/cli/Cli.py +++ b/pyload/cli/Cli.py @@ -19,7 +19,7 @@ from __future__ import with_statement from getopt import GetoptError, getopt -import pyload.common.pylgettext as gettext +import pyload.utils.pylgettext as gettext import os from os import _exit from os.path import join, exists, abspath, basename @@ -29,7 +29,7 @@ from threading import Thread, Lock from time import sleep from traceback import print_exc -import ConfigParser +from pyload.config.Parser import ConfigParser from codecs import getwriter @@ -44,7 +44,7 @@ from pyload import InitHomeDir from pyload.cli.printer import * from pyload.cli import AddPackage, ManageFiles -from pyload.Api import Destination +from pyload.api import Destination from pyload.utils import formatSize, decode from pyload.remote.thriftbackend.ThriftClient import ThriftClient, NoConnection, NoSSL, WrongLogin, ConnectionClosed from Getch import Getch diff --git a/pyload/cli/ManageFiles.py b/pyload/cli/ManageFiles.py index fba96b990..6901e3731 100644 --- a/pyload/cli/ManageFiles.py +++ b/pyload/cli/ManageFiles.py @@ -23,7 +23,7 @@ from time import time from Handler import Handler from printer import * -from pyload.Api import Destination, PackageData +from pyload.api import Destination, PackageData class ManageFiles(Handler): """ possibility to manage queue/collector """ diff --git a/pyload/common/APIExerciser.py b/pyload/common/APIExerciser.py deleted file mode 100644 index 886c72a4a..000000000 --- a/pyload/common/APIExerciser.py +++ /dev/null @@ -1,157 +0,0 @@ -# -*- coding: utf-8 -*- - -import string -from threading import Thread -from random import choice, random, sample, randint -from time import time, sleep -from math import floor -import gc - -from traceback import print_exc, format_exc - -from pyload.remote.thriftbackend.ThriftClient import ThriftClient, Destination - -def createURLs(): - """ create some urls, some may fail """ - urls = [] - for x in range(0, randint(20, 100)): - name = "DEBUG_API" - if randint(0, 5) == 5: - name = "" #this link will fail - - urls.append(name + "".join(sample(string.ascii_letters, randint(10, 20)))) - - return urls - -AVOID = (0, 3, 8) - -idPool = 0 -sumCalled = 0 - - -def startApiExerciser(core, n): - for i in range(n): - APIExerciser(core).start() - -class APIExerciser(Thread): - - - def __init__(self, core, thrift=False, user=None, pw=None): - global idPool - - Thread.__init__(self) - self.setDaemon(True) - self.core = core - self.count = 0 #number of methods - self.time = time() - - if thrift: - self.api = ThriftClient(user=user, password=pw) - else: - self.api = core.api - - - self.id = idPool - - idPool += 1 - - #self.start() - - def run(self): - - self.core.log.info("API Excerciser started %d" % self.id) - - out = open("error.log", "ab") - #core errors are not logged of course - out.write("\n" + "Starting\n") - out.flush() - - while True: - try: - self.testAPI() - except Exception: - self.core.log.error("Excerciser %d throw an execption" % self.id) - print_exc() - out.write(format_exc() + 2 * "\n") - out.flush() - - if not self.count % 100: - self.core.log.info("Exerciser %d tested %d api calls" % (self.id, self.count)) - if not self.count % 1000: - out.flush() - - if not sumCalled % 1000: #not thread safe - self.core.log.info("Exercisers tested %d api calls" % sumCalled) - persec = sumCalled / (time() - self.time) - self.core.log.info("Approx. %.2f calls per second." % persec) - self.core.log.info("Approx. %.2f ms per call." % (1000 / persec)) - self.core.log.info("Collected garbage: %d" % gc.collect()) - - - #sleep(random() / 500) - - def testAPI(self): - global sumCalled - - m = ["statusDownloads", "statusServer", "addPackage", "getPackageData", "getFileData", "deleteFiles", - "deletePackages", "getQueue", "getCollector", "getQueueData", "getCollectorData", "isCaptchaWaiting", - "getCaptchaTask", "stopAllDownloads", "getAllInfo", "getServices" , "getAccounts", "getAllUserData"] - - method = choice(m) - #print "Testing:", method - - if hasattr(self, method): - res = getattr(self, method)() - else: - res = getattr(self.api, method)() - - self.count += 1 - sumCalled += 1 - - #print res - - def addPackage(self): - name = "".join(sample(string.ascii_letters, 10)) - urls = createURLs() - - self.api.addPackage(name, urls, choice([Destination.Queue, Destination.Collector])) - - - def deleteFiles(self): - info = self.api.getQueueData() - if not info: return - - pack = choice(info) - fids = pack.links - - if len(fids): - fids = [f.fid for f in sample(fids, randint(1, max(len(fids) / 2, 1)))] - self.api.deleteFiles(fids) - - - def deletePackages(self): - info = choice([self.api.getQueue(), self.api.getCollector()]) - if not info: return - - pids = [p.pid for p in info] - if len(pids): - pids = sample(pids, randint(1, max(floor(len(pids) / 2.5), 1))) - self.api.deletePackages(pids) - - def getFileData(self): - info = self.api.getQueueData() - if info: - p = choice(info) - if p.links: - self.api.getFileData(choice(p.links).fid) - - def getPackageData(self): - info = self.api.getQueue() - if info: - self.api.getPackageData(choice(info).pid) - - def getAccounts(self): - self.api.getAccounts(False) - - def getCaptchaTask(self): - self.api.getCaptchaTask(False) diff --git a/pyload/common/ImportDebugger.py b/pyload/common/ImportDebugger.py deleted file mode 100644 index ae3aef629..000000000 --- a/pyload/common/ImportDebugger.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys - -class ImportDebugger(object): - - def __init__(self): - self.imported = {} - - def find_module(self, name, path=None): - - if name not in self.imported: - self.imported[name] = 0 - - self.imported[name] += 1 - - print name, path - -sys.meta_path.append(ImportDebugger()) diff --git a/pyload/common/JsEngine.py b/pyload/common/JsEngine.py deleted file mode 100644 index 46789f64d..000000000 --- a/pyload/common/JsEngine.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from imp import find_module -from os.path import join, exists -from urllib import quote - - -ENGINE = "" - -DEBUG = False -JS = False -PYV8 = False -RHINO = False - - -if not ENGINE: - try: - import subprocess - - subprocess.Popen(["js", "-v"], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - p = subprocess.Popen(["js", "-e", "print(23+19)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - #integrity check - if out.strip() == "42": - ENGINE = "js" - JS = True - except: - pass - -if not ENGINE or DEBUG: - try: - find_module("PyV8") - ENGINE = "pyv8" - PYV8 = True - except: - pass - -if not ENGINE or DEBUG: - try: - path = "" #path where to find rhino - - if exists("/usr/share/java/js.jar"): - path = "/usr/share/java/js.jar" - elif exists("js.jar"): - path = "js.jar" - elif exists(join(pypath, "js.jar")): #may raises an exception, but js.jar wasnt found anyway - path = join(pypath, "js.jar") - - if not path: - raise Exception - - import subprocess - - p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", "print(23+19)"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - #integrity check - if out.strip() == "42": - ENGINE = "rhino" - RHINO = True - except: - pass - -class JsEngine: - def __init__(self): - self.engine = ENGINE - self.init = False - - def __nonzero__(self): - return False if not ENGINE else True - - def eval(self, script): - if not self.init: - if ENGINE == "pyv8" or (DEBUG and PYV8): - import PyV8 - global PyV8 - - self.init = True - - if type(script) == unicode: - script = script.encode("utf8") - - if not ENGINE: - raise Exception("No JS Engine") - - if not DEBUG: - if ENGINE == "pyv8": - return self.eval_pyv8(script) - elif ENGINE == "js": - return self.eval_js(script) - elif ENGINE == "rhino": - return self.eval_rhino(script) - else: - results = [] - if PYV8: - res = self.eval_pyv8(script) - print "PyV8:", res - results.append(res) - if JS: - res = self.eval_js(script) - print "JS:", res - results.append(res) - if RHINO: - res = self.eval_rhino(script) - print "Rhino:", res - results.append(res) - - warning = False - for x in results: - for y in results: - if x != y: - warning = True - - if warning: print "### WARNING ###: Different results" - - return results[0] - - def eval_pyv8(self, script): - rt = PyV8.JSContext() - rt.enter() - return rt.eval(script) - - def eval_js(self, script): - script = "print(eval(unescape('%s')))" % quote(script) - p = subprocess.Popen(["js", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) - out, err = p.communicate() - res = out.strip() - return res - - def eval_rhino(self, script): - script = "print(eval(unescape('%s')))" % quote(script) - p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", script], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) - out, err = p.communicate() - res = out.strip() - return res.decode("utf8").encode("ISO-8859-1") - - def error(self): - return _("No js engine detected, please install either Spidermonkey, ossp-js, pyv8 or rhino") diff --git a/pyload/common/__init__.py b/pyload/common/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pyload/common/json_layer.py b/pyload/common/json_layer.py deleted file mode 100644 index bb3937cdc..000000000 --- a/pyload/common/json_layer.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- - -# abstraction layer for json operations - -try: # since python 2.6 - import json - from json import loads as json_loads - from json import dumps as json_dumps -except ImportError: #use system simplejson if available - import simplejson as json - from simplejson import loads as json_loads - from simplejson import dumps as json_dumps diff --git a/pyload/common/packagetools.py b/pyload/common/packagetools.py deleted file mode 100644 index d5ab4d182..000000000 --- a/pyload/common/packagetools.py +++ /dev/null @@ -1,136 +0,0 @@ -# JDownloader/src/jd/controlling/LinkGrabberPackager.java - -import re -from urlparse import urlparse - -def matchFirst(string, *args): - """ matches against list of regexp and returns first match""" - for patternlist in args: - for pattern in patternlist: - r = pattern.search(string) - if r is not None: - name = r.group(1) - return name - - return string - - -def parseNames(files): - """ Generates packages names from name, data lists - - :param files: list of (name, data) - :return: packagenames mapt to data lists (eg. urls) - """ - packs = {} - - endings = "\\.(3gp|7zip|7z|abr|ac3|aiff|aifc|aif|ai|au|avi|bin|bz2|cbr|cbz|ccf|cue|cvd|chm|dta|deb|divx|djvu|dlc|dmg|doc|docx|dot|eps|exe|ff|flv|f4v|gsd|gif|gz|iwd|iso|ipsw|java|jar|jpg|jpeg|jdeatme|load|mws|mw|m4v|m4a|mkv|mp2|mp3|mp4|mov|movie|mpeg|mpe|mpg|msi|msu|msp|nfo|npk|oga|ogg|ogv|otrkey|pkg|png|pdf|pptx|ppt|pps|ppz|pot|psd|qt|rmvb|rm|rar|ram|ra|rev|rnd|r\\d+|rpm|run|rsdf|rtf|sh(!?tml)|srt|snd|sfv|swf|tar|tif|tiff|ts|txt|viv|vivo|vob|wav|wmv|xla|xls|xpi|zeno|zip|z\\d+|_[_a-z]{2}|\\d+$)" - - rarPats = [re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I), - re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I), - re.compile("(.*)\\.rar$", re.I), - re.compile("(.*)\\.r\\d+$", re.I), - re.compile("(.*)(\\.|_|-)\\d+$", re.I)] - - zipPats = [re.compile("(.*)\\.zip$", re.I), - re.compile("(.*)\\.z\\d+$", re.I), - re.compile("(?is).*\\.7z\\.[\\d]+$", re.I), - re.compile("(.*)\\.a.$", re.I)] - - ffsjPats = [re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)"), - re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I)] - - iszPats = [re.compile("(.*)\\.isz$", re.I), - re.compile("(.*)\\.i\\d{2}$", re.I)] - - pat1 = re.compile("(\\.?CD\\d+)", re.I) - pat2 = re.compile("(\\.?part\\d+)", re.I) - - pat3 = re.compile("(.+)[\\.\\-_]+$") - pat4 = re.compile("(.+)\\.\\d+\\.xtm$") - - for file, url in files: - patternMatch = False - - if file is None: - continue - - # remove trailing / - name = file.rstrip('/') - - # extract last path part .. if there is a path - split = name.rsplit("/", 1) - if len(split) > 1: - name = split.pop(1) - - #check if a already existing package may be ok for this file - # found = False - # for pack in packs: - # if pack in file: - # packs[pack].append(url) - # found = True - # break - # - # if found: continue - - # unrar pattern, 7zip/zip and hjmerge pattern, isz pattern, FFSJ pattern - before = name - name = matchFirst(name, rarPats, zipPats, iszPats, ffsjPats) - if before != name: - patternMatch = True - - # xtremsplit pattern - r = pat4.search(name) - if r is not None: - name = r.group(1) - - # remove part and cd pattern - r = pat1.search(name) - if r is not None: - name = name.replace(r.group(0), "") - patternMatch = True - - r = pat2.search(name) - if r is not None: - name = name.replace(r.group(0), "") - patternMatch = True - - # additional checks if extension pattern matched - if patternMatch: - # remove extension - index = name.rfind(".") - if index <= 0: - index = name.rfind("_") - if index > 0: - length = len(name) - index - if length <= 4: - name = name[:-length] - - # remove endings like . _ - - r = pat3.search(name) - if r is not None: - name = r.group(1) - - # replace . and _ with space - name = name.replace(".", " ") - name = name.replace("_", " ") - - name = name.strip() - else: - name = "" - - # fallback: package by hoster - if not name: - name = urlparse(file).hostname - if name: name = name.replace("www.", "") - - # fallback : default name - if not name: - name = "unknown" - - # build mapping - if name in packs: - packs[name].append(url) - else: - packs[name] = [url] - - return packs diff --git a/pyload/common/pavement.py b/pyload/common/pavement.py deleted file mode 100644 index 9b2dc98b3..000000000 --- a/pyload/common/pavement.py +++ /dev/null @@ -1,412 +0,0 @@ -# -*- coding: utf-8 -*- - -from paver.easy import * -from paver.setuputils import setup -from paver.doctools import cog - -import os -import sys -import shutil -import re -from glob import glob -from tempfile import mkdtemp -from urllib import urlretrieve -from subprocess import call, Popen, PIPE -from zipfile import ZipFile - -PROJECT_DIR = path(__file__).dirname() -sys.path.append(PROJECT_DIR) - -options = environment.options -path("pyload").mkdir() - -extradeps = [] -if sys.version_info <= (2, 5): - extradeps += 'simplejson' - -setup( - name="pyload", - version="0.4.10", - description='Fast, lightweight and full featured download manager.', - long_description=open(PROJECT_DIR / "README.md").read(), - keywords = ("pyload", "download-manager", "one-click-hoster", "download"), - url="http://pyload.org", - download_url='http://pyload.org/download', - license='GPL v3', - author="pyLoad Team", - author_email="support@pyload.org", - platforms = ('Any',), - #package_dir={'pyload': "src"}, - packages=["pyload"], - #package_data=find_package_data(), - #data_files=[], - include_package_data=True, - exclude_package_data={'pyload': ["docs*", "scripts*", "tests*"]}, #exluced from build but not from sdist - # 'bottle >= 0.10.0' not in list, because its small and contain little modifications - install_requires=['thrift >= 0.8.0', 'jinja2', 'pycurl', 'Beaker', 'BeautifulSoup >= 3.2, < 3.3'] + extradeps, - extras_require={ - 'SSL': ["pyOpenSSL"], - 'DLC': ['pycrypto'], - 'lightweight webserver': ['bjoern'], - 'RSS plugins': ['feedparser'], - }, - #setup_requires=["setuptools_hg"], - entry_points={ - 'console_scripts': [ - 'pyLoadCore = pyLoadCore:main', - 'pyLoadCli = pyLoadCli:main' - ]}, - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Topic :: Internet :: WWW/HTTP", - "Environment :: Console", - "Environment :: Web Environment", - "Intended Audience :: End Users/Desktop", - "License :: OSI Approved :: GNU General Public License (GPL)", - "Operating System :: OS Independent", - "Programming Language :: Python :: 2" - ] -) - -options( - sphinx=Bunch( - builddir="_build", - sourcedir="" - ), - get_source=Bunch( - src="https://bitbucket.org/spoob/pyload/get/tip.zip", - rev=None, - clean=False - ), - thrift=Bunch( - path="../thrift/trunk/compiler/cpp/thrift", - gen="" - ), - virtualenv=Bunch( - dir="env", - python="python2", - virtual="virtualenv2", - ), - cog=Bunch( - pattern="*.py", - ) -) - -# xgettext args -xargs = ["--language=Python", "--add-comments=L10N", - "--from-code=utf-8", "--copyright-holder=pyLoad Team", "--package-name=pyLoad", - "--package-version=%s" % options.version, "--msgid-bugs-address='bugs@pyload.org'"] - -@task -@needs('cog') -def html(): - """Build html documentation""" - module = path("docs") / "pyload" - pyload.rmtree() - call_task('paver.doctools.html') - - -@task -@cmdopts([ - ('src=', 's', 'Url to source'), - ('rev=', 'r', "HG revision"), - ("clean", 'c', 'Delete old source folder') -]) -def get_source(options): - """ Downloads pyload source from bitbucket tip or given rev""" - if options.rev: options.url = "https://bitbucket.org/spoob/pyload/get/%s.zip" % options.rev - - pyload = path("pyload") - - if len(pyload.listdir()) and not options.clean: - return - elif pyload.exists(): - pyload.rmtree() - - urlretrieve(options.src, "pyload_src.zip") - zip = ZipFile("pyload_src.zip") - zip.extractall() - path("pyload_src.zip").remove() - - folder = [x for x in path(".").dirs() if x.name.startswith("spoob-pyload-")][0] - folder.move(pyload) - - change_mode(pyload, 0644) - change_mode(pyload, 0755, folder=True) - - for file in pyload.files(): - if file.name.endswith(".py"): - file.chmod(0755) - - (pyload / ".hgtags").remove() - (pyload / ".gitignore").remove() - #(pyload / "docs").rmtree() - - f = open(pyload / "__init__.py", "wb") - f.close() - - #options.setup.packages = find_packages() - #options.setup.package_data = find_package_data() - - -@task -@needs('clean', 'generate_setup', 'minilib', 'get_source', 'setuptools.command.sdist') -def sdist(): - """ Build source code package with distutils """ - - -@task -@cmdopts([ - ('path=', 'p', 'Thrift path'), - ('gen=', 'g', "Extra --gen option") -]) -def thrift(options): - """ Generate Thrift stubs """ - - print "add import for TApplicationException manually as long it is not fixed" - - outdir = path("pyload") / "remote" / "thriftbackend" - (outdir / "gen-py").rmtree() - - cmd = [options.thrift.path, "-strict", "-o", outdir, "--gen", "py:slots, dynamic", outdir / "pyload.thrift"] - - if options.gen: - cmd.insert(len(cmd) - 1, "--gen") - cmd.insert(len(cmd) - 1, options.gen) - - print "running", cmd - - p = Popen(cmd) - p.communicate() - - (outdir / "thriftgen").rmtree() - (outdir / "gen-py").move(outdir / "thriftgen") - - #create light ttypes - from pyload.remote.socketbackend.create_ttypes import main - main() - -@task -def compile_js(): - """ Compile .coffee files to javascript""" - - root = path("pyload") / "web" / "media" / "js" - for f in root.glob("*.coffee"): - print "generate", f - coffee = Popen(["coffee", "-cbs"], stdin=open(f, "rb"), stdout=PIPE) - yui = Popen(["yuicompressor", "--type", "js"], stdin=coffee.stdout, stdout=PIPE) - coffee.stdout.close() - content = yui.communicate()[0] - with open(root / f.name.replace(".coffee", ".js"), "wb") as js: - js.write("{% autoescape true %}\n") - js.write(content) - js.write("\n{% endautoescape %}") - - -@task -def generate_locale(): - """ Generates localization files """ - - EXCLUDE = ["BeautifulSoup.py", "pyload/cli", "web/locale", "web/ajax", "web/cnl", "web/pyload", - "setup.py"] - makepot("core", path("pyload"), EXCLUDE, "./pyload.py\n") - - makepot("cli", path("pyload") / "cli", [], includes="./pyload-cli.py\n") - makepot("setup", "", [], includes="./pyload/setup.py\n") - - EXCLUDE = ["ServerThread.py", "web/media/default"] - - # strings from js files - strings = set() - - for fi in path("pyload/web").walkfiles(): - if not fi.name.endswith(".js") and not fi.endswith(".coffee"): continue - with open(fi, "rb") as c: - content = c.read() - - strings.update(re.findall(r"_\s*\(\s*\"([^\"]+)", content)) - strings.update(re.findall(r"_\s*\(\s*\'([^\']+)", content)) - - trans = path("pyload") / "web" / "translations.js" - - with open(trans, "wb") as js: - for s in strings: - js.write('_("%s")\n' % s) - - makepot("django", path("pyload/web"), EXCLUDE, "./%s\n" % trans.relpath(), [".py", ".html"], ["--language=Python"]) - - trans.remove() - - path("includes.txt").remove() - - print "Locale generated" - - -@task -@cmdopts([ - ('key=', 'k', 'api key') -]) -def upload_translations(options): - """ Uploads the locale files to translation server """ - tmp = path(mkdtemp()) - - shutil.copy('locale/crowdin.yaml', tmp) - os.mkdir(tmp / 'pyLoad') - for f in glob('locale/*.pot'): - if os.path.isfile(f): - shutil.copy(f, tmp / 'pyLoad') - - config = tmp / 'crowdin.yaml' - content = open(config, 'rb').read() - content = content.format(key=options.key, tmp=tmp) - f = open(config, 'wb') - f.write(content) - f.close() - - call(['crowdin-cli', '-c', config, 'upload', 'source']) - - shutil.rmtree(tmp) - - print "Translations uploaded" - - -@task -@cmdopts([ - ('key=', 'k', 'api key') -]) -def download_translations(options): - """ Downloads the translated files from translation server """ - tmp = path(mkdtemp()) - - shutil.copy('locale/crowdin.yaml', tmp) - os.mkdir(tmp / 'pyLoad') - for f in glob('locale/*.pot'): - if os.path.isfile(f): - shutil.copy(f, tmp / 'pyLoad') - - config = tmp / 'crowdin.yaml' - content = open(config, 'rb').read() - content = content.format(key=options.key, tmp=tmp) - f = open(config, 'wb') - f.write(content) - f.close() - - call(['crowdin-cli', '-c', config, 'download']) - - for language in (tmp / 'pyLoad').listdir(): - if not language.isdir(): - continue - - target = path('locale') / language.basename() - print "Copy language %s" % target - if target.exists(): - shutil.rmtree(target) - - shutil.copytree(language, target) - - shutil.rmtree(tmp) - - -@task -def compile_translations(): - """ Compile PO files to MO """ - for language in path('locale').listdir(): - if not language.isdir(): - continue - - for f in glob(language / 'LC_MESSAGES' / '*.po'): - print "Compiling %s" % f - call(['msgfmt', '-o', f.replace('.po', '.mo'), f]) - - -@task -def tests(): - call(["nosetests2"]) - -@task -def virtualenv(options): - """Setup virtual environment""" - if path(options.dir).exists(): - return - - call([options.virtual, "--no-site-packages", "--python", options.python, options.dir]) - print "$ source %s/bin/activate" % options.dir - - -@task -def clean_env(): - """Deletes the virtual environment""" - env = path(options.virtualenv.dir) - if env.exists(): - env.rmtree() - - -@task -@needs('generate_setup', 'minilib', 'get_source', 'virtualenv') -def env_install(): - """Install pyLoad into the virtualenv""" - venv = options.virtualenv - call([path(venv.dir) / "bin" / "easy_install", "."]) - - -@task -def clean(): - """Cleans build directories""" - path("build").rmtree() - path("dist").rmtree() - - -#helper functions - -def walk_trans(path, EXCLUDE, endings=[".py"]): - result = "" - - for f in path.walkfiles(): - if [True for x in EXCLUDE if x in f.dirname().relpath()]: continue - if f.name in EXCLUDE: continue - - for e in endings: - if f.name.endswith(e): - result += "./%s\n" % f.relpath() - break - - return result - - -def makepot(domain, p, excludes=[], includes="", endings=[".py"], xxargs=[]): - print "Generate %s.pot" % domain - - f = open("includes.txt", "wb") - if includes: - f.write(includes) - - if p: - f.write(walk_trans(path(p), excludes, endings)) - - f.close() - - call(["xgettext", "--files-from=includes.txt", "--default-domain=%s" % domain] + xargs + xxargs) - - # replace charset und move file - with open("%s.po" % domain, "rb") as f: - content = f.read() - - path("%s.po" % domain).remove() - content = content.replace("charset=CHARSET", "charset=UTF-8") - - with open("locale/%s.pot" % domain, "wb") as f: - f.write(content) - - -def change_owner(dir, uid, gid): - for p in dir.walk(): - p.chown(uid, gid) - - -def change_mode(dir, mode, folder=False): - for p in dir.walk(): - if folder and p.isdir(): - p.chmod(mode) - elif p.isfile() and not folder: - p.chmod(mode) diff --git a/pyload/common/pylgettext.py b/pyload/common/pylgettext.py deleted file mode 100644 index cab631cf4..000000000 --- a/pyload/common/pylgettext.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- - -from gettext import * - -_searchdirs = None - -origfind = find - -def setpaths(pathlist): - global _searchdirs - if isinstance(pathlist, list): - _searchdirs = pathlist - else: - _searchdirs = list(pathlist) - - -def addpath(path): - global _searchdirs - if _searchdirs is None: - _searchdirs = list(path) - else: - if path not in _searchdirs: - _searchdirs.append(path) - - -def delpath(path): - global _searchdirs - if _searchdirs is not None: - if path in _searchdirs: - _searchdirs.remove(path) - - -def clearpath(): - global _searchdirs - if _searchdirs is not None: - _searchdirs = None - - -def find(domain, localedir=None, languages=None, all=False): - if _searchdirs is None: - return origfind(domain, localedir, languages, all) - searches = [localedir] + _searchdirs - results = list() - for dir in searches: - res = origfind(domain, dir, languages, all) - if all is False: - results.append(res) - else: - results.extend(res) - if all is False: - results = filter(lambda x: x is not None, results) - if len(results) == 0: - return None - else: - return results[0] - else: - return results - -#Is there a smarter/cleaner pythonic way for this? -translation.func_globals['find'] = find diff --git a/pyload/common/test_api.py b/pyload/common/test_api.py deleted file mode 100644 index 4efaa35d6..000000000 --- a/pyload/common/test_api.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- - -from pyload.common import APIExerciser -from nose.tools import nottest - - -class TestApi: - - def __init__(self): - self.api = APIExerciser.APIExerciser(None, True, "TestUser", "pwhere") - - def test_login(self): - assert self.api.api.login("crapp", "wrong pw") is False - - #takes really long, only test when needed - @nottest - def test_random(self): - - for i in range(0, 100): - self.api.testAPI() diff --git a/pyload/common/test_json.py b/pyload/common/test_json.py deleted file mode 100644 index 320a42d4f..000000000 --- a/pyload/common/test_json.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- - -from urllib import urlencode -from urllib2 import urlopen, HTTPError -from json import loads - -from logging import log - -url = "http://localhost:8001/api/%s" - -class TestJson: - - def call(self, name, post=None): - if not post: post = {} - post["session"] = self.key - u = urlopen(url % name, data=urlencode(post)) - return loads(u.read()) - - def setUp(self): - u = urlopen(url % "login", data=urlencode({"username": "TestUser", "password": "pwhere"})) - self.key = loads(u.read()) - assert self.key is not False - - def test_wronglogin(self): - u = urlopen(url % "login", data=urlencode({"username": "crap", "password": "wrongpw"})) - assert loads(u.read()) is False - - def test_access(self): - try: - urlopen(url % "getServerVersion") - except HTTPError, e: - assert e.code == 403 - else: - assert False - - def test_status(self): - ret = self.call("statusServer") - log(1, str(ret)) - assert "pause" in ret - assert "queue" in ret - - def test_unknown_method(self): - try: - self.call("notExisting") - except HTTPError, e: - assert e.code == 404 - else: - assert False diff --git a/pyload/config/Parser.py b/pyload/config/Parser.py new file mode 100644 index 000000000..64ce6b10e --- /dev/null +++ b/pyload/config/Parser.py @@ -0,0 +1,373 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement +from time import sleep +from os.path import exists, join +from shutil import copy + +from traceback import print_exc +from utils import chmod + +# ignore these plugin configs, mainly because plugins were wiped out +IGNORE = ( + "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('hooks', 'UnRar'), + 'EasyShareCom', 'FlyshareCz' + ) + +CONF_VERSION = 1 + +class ConfigParser: + """ + holds and manage the configuration + + current dict layout: + + { + + section: { + option: { + value: + type: + desc: + } + desc: + + } + + """ + + + def __init__(self): + """Constructor""" + self.config = {} # the config values + self.plugin = {} # the config for plugins + self.oldRemoteData = {} + + self.pluginCB = None # callback when plugin config value is changed + + self.checkVersion() + + self.readConfig() + + self.deleteOldPlugins() + + + def checkVersion(self, n=0): + """determines if config need to be copied""" + try: + if not exists("pyload.conf"): + copy(join(pypath, "pyload", "config", "default.conf"), "pyload.conf") + + if not exists("plugin.conf"): + f = open("plugin.conf", "wb") + f.write("version: " + str(CONF_VERSION)) + f.close() + + f = open("pyload.conf", "rb") + v = f.readline() + f.close() + v = v[v.find(":") + 1:].strip() + + if not v or int(v) < CONF_VERSION: + copy(join(pypath, "pyload", "config", "default.conf"), "pyload.conf") + print "Old version of config was replaced" + + f = open("plugin.conf", "rb") + v = f.readline() + f.close() + v = v[v.find(":") + 1:].strip() + + if not v or int(v) < CONF_VERSION: + f = open("plugin.conf", "wb") + f.write("version: " + str(CONF_VERSION)) + f.close() + print "Old version of plugin-config replaced" + except: + if n < 3: + sleep(0.3) + self.checkVersion(n + 1) + else: + raise + + def readConfig(self): + """reads the config file""" + + self.config = self.parseConfig(join(pypath, "pyload", "config", "default.conf")) + self.plugin = self.parseConfig("plugin.conf") + + try: + homeconf = self.parseConfig("pyload.conf") + if "username" in homeconf["remote"]: + if "password" in homeconf["remote"]: + self.oldRemoteData = {"username": homeconf["remote"]["username"]["value"], + "password": homeconf["remote"]["username"]["value"]} + del homeconf["remote"]["password"] + del homeconf["remote"]["username"] + self.updateValues(homeconf, self.config) + + except Exception, e: + print "Config Warning" + print_exc() + + + def parseConfig(self, config): + """parses a given configfile""" + + f = open(config) + + config = f.read() + + config = config.splitlines()[1:] + + conf = {} + + section, option, value, typ, desc = "", "", "", "", "" + + listmode = False + + for line in config: + comment = line.rfind("#") + if line.find(":", comment) < 0 > line.find("=", comment) and comment > 0 and line[comment - 1].isspace(): + line = line.rpartition("#") # removes comments + if line[1]: + line = line[0] + else: + line = line[2] + + line = line.strip() + + try: + if line == "": + continue + elif line.endswith(":"): + section, none, desc = line[:-1].partition('-') + section = section.strip() + desc = desc.replace('"', "").strip() + conf[section] = {"desc": desc} + else: + if listmode: + if line.endswith("]"): + listmode = False + line = line.replace("]", "") + + value += [self.cast(typ, x.strip()) for x in line.split(",") if x] + + if not listmode: + conf[section][option] = {"desc": desc, + "type": typ, + "value": value} + + + else: + content, none, value = line.partition("=") + + content, none, desc = content.partition(":") + + desc = desc.replace('"', "").strip() + + typ, none, option = content.strip().rpartition(" ") + + value = value.strip() + + if value.startswith("["): + if value.endswith("]"): + listmode = False + value = value[:-1] + else: + listmode = True + + value = [self.cast(typ, x.strip()) for x in value[1:].split(",") if x] + else: + value = self.cast(typ, value) + + if not listmode: + conf[section][option] = {"desc": desc, + "type": typ, + "value": value} + + except Exception, e: + print "Config Warning" + print_exc() + + f.close() + return conf + + + def updateValues(self, config, dest): + """sets the config values from a parsed config file to values in destination""" + + for section in config.iterkeys(): + if section in dest: + for option in config[section].iterkeys(): + if option in ("desc", "outline"): continue + + if option in dest[section]: + dest[section][option]["value"] = config[section][option]["value"] + + #else: + # dest[section][option] = config[section][option] + + + #else: + # dest[section] = config[section] + + def saveConfig(self, config, filename): + """saves config to filename""" + with open(filename, "wb") as f: + chmod(filename, 0600) + f.write("version: %i \n" % CONF_VERSION) + for section in config.iterkeys(): + f.write('\n%s - "%s":\n' % (section, config[section]["desc"])) + + for option, data in config[section].iteritems(): + if option in ("desc", "outline"): continue + + if isinstance(data["value"], list): + value = "[ \n" + for x in data["value"]: + value += "\t\t" + str(x) + ",\n" + value += "\t\t]\n" + else: + if type(data["value"]) in (str, unicode): + value = data["value"] + "\n" + else: + value = str(data["value"]) + "\n" + try: + f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value)) + except UnicodeEncodeError: + f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value.encode("utf8"))) + + def cast(self, typ, value): + """cast value to given format""" + if type(value) not in (str, unicode): + return value + + elif typ == "int": + return int(value) + elif typ == "bool": + return True if value.lower() in ("1", "true", "on", "an", "yes") else False + elif typ == "time": + if not value: value = "0:00" + if not ":" in value: value += ":00" + return value + elif typ in ("str", "file", "folder"): + try: + return value.encode("utf8") + except: + return value + else: + return value + + + def save(self): + """saves the configs to disk""" + + self.saveConfig(self.config, "pyload.conf") + self.saveConfig(self.plugin, "plugin.conf") + + + def __getitem__(self, section): + """provides dictonary like access: c['section']['option']""" + return Section(self, section) + + + def get(self, section, option): + """get value""" + val = self.config[section][option]["value"] + try: + if type(val) in (str, unicode): + return val.decode("utf8") + else: + return val + except: + return val + + def set(self, section, option, value): + """set value""" + + value = self.cast(self.config[section][option]["type"], value) + + self.config[section][option]["value"] = value + self.save() + + def getPlugin(self, plugin, option): + """gets a value for a plugin""" + val = self.plugin[plugin][option]["value"] + try: + if type(val) in (str, unicode): + return val.decode("utf8") + else: + return val + except: + return val + + def setPlugin(self, plugin, option, value): + """sets a value for a plugin""" + + value = self.cast(self.plugin[plugin][option]["type"], value) + + if self.pluginCB: self.pluginCB(plugin, option, value) + + self.plugin[plugin][option]["value"] = value + self.save() + + def getMetaData(self, section, option): + """ get all config data for an option """ + return self.config[section][option] + + def addPluginConfig(self, name, config, outline=""): + """adds config options with tuples (name, type, desc, default)""" + if name not in self.plugin: + conf = {"desc": name, + "outline": outline} + self.plugin[name] = conf + else: + conf = self.plugin[name] + conf["outline"] = outline + + for item in config: + if item[0] in conf: + conf[item[0]]["type"] = item[1] + conf[item[0]]["desc"] = item[2] + else: + conf[item[0]] = { + "desc": item[2], + "type": item[1], + "value": self.cast(item[1], item[3]) + } + + values = [x[0] for x in config] + ["desc", "outline"] + #delete old values + for item in conf.keys(): + if item not in values: + del conf[item] + + def deleteConfig(self, name): + """Removes a plugin config""" + if name in self.plugin: + del self.plugin[name] + + + def deleteOldPlugins(self): + """ remove old plugins from config """ + + for name in IGNORE: + if name in self.plugin: + del self.plugin[name] + + +class Section: + """provides dictionary like access for configparser""" + + def __init__(self, parser, section): + """Constructor""" + self.parser = parser + self.section = section + + def __getitem__(self, item): + """getitem""" + return self.parser.get(self.section, item) + + def __setitem__(self, item, value): + """setitem""" + self.parser.set(self.section, item, value) diff --git a/pyload/config/Setup.py b/pyload/config/Setup.py new file mode 100644 index 000000000..63c6188c4 --- /dev/null +++ b/pyload/config/Setup.py @@ -0,0 +1,538 @@ +# -*- coding: utf-8 -*- + +import os +import sys + +import pyload.utils.pylgettext as gettext + +from getpass import getpass +from os import makedirs +from os.path import abspath, dirname, exists, join +from subprocess import PIPE, call + +from pyload.utils import get_console_encoding, versiontuple + + +class SetupAssistant: + """ pyLoads initial setup configuration assistant """ + + def __init__(self, path, config): + self.path = path + self.config = config + self.stdin_encoding = get_console_encoding(sys.stdin.encoding) + + + def start(self): + langs = self.config.getMetaData("general", "language")["type"].split(";") + lang = self.ask(u"Choose setup language", "en", langs) + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("setup", join(self.path, "locale"), languages=[lang, "en"], fallback=True) + translation.install(True) + + #Input shorthand for yes + self.yes = _("y") + #Input shorthand for no + self.no = _("n") + + # print + # print _("Would you like to configure pyLoad via Webinterface?") + # print _("You need a Browser and a connection to this PC for it.") + # viaweb = self.ask(_("Start initial webinterface for configuration?"), "y", bool=True) + # if viaweb: + # try: + # from pyload.manager.thread import ServerThread + # ServerThread.setup = self + # import pyload.webui as webinterface + # webinterface.run_simple() + # return False + # except Exception, e: + # print "Setup failed with this error: ", e + # print "Falling back to commandline setup." + + print + print + print _("Welcome to the pyLoad Configuration Assistant.") + print _("It will check your system and make a basic setup in order to run pyLoad.") + print + print _("The value in brackets [] always is the default value,") + print _("in case you don't want to change it or you are unsure what to choose, just hit enter.") + print _( + "Don't forget: You can always rerun this assistant with --setup or -s parameter, when you start pyload.py .") + print _("If you have any problems with this assistant hit STRG-C,") + print _("to abort and don't let him start with pyload.py automatically anymore.") + print + print + raw_input(_("When you are ready for system check, hit enter.")) + print + print + + basic, ssl, captcha, web, js = self.system_check() + print + print + + if not basic: + print _("You need pycurl, sqlite and python 2.5, 2.6 or 2.7 to run pyLoad.") + print _("Please correct this and re-run pyLoad.") + print + print _("Setup will now close.") + print + print + raw_input(_("Press Enter to exit.")) + return False + + raw_input(_("System check finished, hit enter to see your status report.")) + print + print + print _("## Status ##") + print + + avail = [] + if self.check_module("Crypto"): + avail.append(_("container decrypting")) + if ssl: + avail.append(_("ssl connection")) + if captcha: + avail.append(_("automatic captcha decryption")) + if web: + avail.append(_("webinterface")) + if js: + avail.append(_("extended Click'N'Load")) + + string = "" + + for av in avail: + string += ", " + av + + print _("AVAILABLE FEATURES:") + string[1:] + print + + if len(avail) < 5: + print _("MISSING FEATURES: ") + + if not self.check_module("Crypto"): + print _("- no py-crypto available") + print _("You need this if you want to decrypt container files.") + print + + if not ssl: + print _("- no SSL available") + print _("This is needed if you want to establish a secure connection to core or webinterface.") + print _("If you only want to access locally to pyLoad ssl is not usefull.") + print + + if not captcha: + print _("- no Captcha Recognition available") + print _("Only needed for some hosters and as freeuser.") + print + + if not js: + print _("- no JavaScript engine found") + print _("You will need this for some Click'N'Load links. Install Spidermonkey, ossp-js, pyv8 or rhino") + print + + print + print _("You can abort the setup now and fix some dependicies if you want.") + + print + con = self.ask(_("Continue with setup?"), self.yes, bool=True) + + if not con: + return False + + print + print + print _("CURRENT CONFIG PATH: %s") % abspath("") + print + print _("NOTE: If you use pyLoad on a server or the home partition lives on an iternal flash it may be a good idea to change it.") + path = self.ask(_("Do you want to change the config path?"), self.no, bool=True) + if path: + print + self.conf_path() + #calls exit when changed + + print + print _("Do you want to configure login data and basic settings?") + print _("This is recommend for first run.") + con = self.ask(_("Make basic setup?"), self.yes, bool=True) + + if con: + print + print + self.conf_basic() + + if ssl: + print + print _("Do you want to configure ssl?") + ssl = self.ask(_("Configure ssl?"), self.no, bool=True) + if ssl: + print + print + self.conf_ssl() + + if web: + print + print _("Do you want to configure webinterface?") + web = self.ask(_("Configure webinterface?"), self.yes, bool=True) + if web: + print + print + self.conf_web() + + print + print + print _("Setup finished successfully!") + print + print + raw_input(_("Hit enter to exit and restart pyLoad.")) + return True + + + def system_check(self): + """ make a systemcheck and return the results""" + + print _("## System Information ##") + print + print _("Platform: %s") % sys.platform + print _("Operating System: %s") % os.name + print _("Python: %s") % sys.version.replace("\n", "") + print + print + + print _("## System Check ##") + print + + if sys.version_info[:2] > (2, 7): + print _("Your python version is to new, Please use Python 2.6/2.7") + python = False + elif sys.version_info[:2] < (2, 5): + print _("Your python version is to old, Please use at least Python 2.5") + python = False + else: + print _("Python Version: OK") + python = True + + curl = self.check_module("pycurl") + self.print_dep("pycurl", curl) + + sqlite = self.check_module("sqlite3") + self.print_dep("sqlite3", sqlite) + + basic = python and curl and sqlite + + print + + crypto = self.check_module("Crypto") + self.print_dep("pycrypto", crypto) + + ssl = self.check_module("OpenSSL") + self.print_dep("py-OpenSSL", ssl) + + print + + pil = self.check_module("PIL.Image") + self.print_dep("PIL/Pillow", pil) + + if os.name == "nt": + tesser = self.check_prog([join(pypath, "tesseract", "tesseract.exe"), "-v"]) + else: + tesser = self.check_prog(["tesseract", "-v"]) + + self.print_dep("tesseract", tesser) + + captcha = pil and tesser + + print + + try: + import jinja2 + + v = jinja2.__version__ + if v and versiontuple(v) < (2, 5, 0): + jinja = False + else: + jinja = True + except: + jinja = False + + jinja = self.print_dep("jinja2", jinja) + + beaker = self.check_module("beaker") + self.print_dep("beaker", beaker) + + bjoern = self.check_module("bjoern") + self.print_dep("bjoern", bjoern) + + web = sqlite and beaker + + from pyload.utils import JsEngine + js = True if JsEngine.ENGINE else False + self.print_dep(_("JS engine"), js) + + if not jinja: + print + print + print _("WARNING: Your installed jinja2 version %s seems too old.") % jinja2.__version__ + print _("You can safely continue but if the webinterface is not working,") + print _("please upgrade or uninstall it, because pyLoad self-includes jinja2 libary.") + + return basic, ssl, captcha, web, js + + + def conf_basic(self): + print _("## Basic Setup ##") + + print + print _("The following logindata is valid for CLI and webinterface.") + + from pyload.database import DatabaseBackend + + db = DatabaseBackend(None) + db.setup() + print _("NOTE: Consider a password of 10 or more symbols if you expect to access from outside your local network (ex. internet).") + print + username = self.ask(_("Username"), "User") + password = self.ask("", "", password=True) + db.addUser(username, password) + db.shutdown() + + print + print _("External clients (GUI, CLI or other) need remote access to work over the network.") + print _("However, if you only want to use the webinterface you may disable it to save ram.") + self.config["remote"]["activated"] = self.ask(_("Enable remote access"), self.no, bool=True) + + print + langs = self.config.getMetaData("general", "language") + self.config["general"]["language"] = self.ask(_("Choose pyLoad language"), "en", langs["type"].split(";")) + + print + self.config["general"]["download_folder"] = self.ask(_("Download folder"), "Downloads") + print + self.config["download"]["max_downloads"] = self.ask(_("Max parallel downloads"), "3") + print + reconnect = self.ask(_("Use Reconnect?"), self.no, bool=True) + self.config["reconnect"]["activated"] = reconnect + if reconnect: + self.config["reconnect"]["method"] = self.ask(_("Reconnect script location"), "./reconnect.sh") + + + def conf_web(self): + print _("## Webinterface Setup ##") + + print + self.config["webinterface"]["activated"] = self.ask(_("Activate webinterface?"), self.yes, bool=True) + print + print _("Listen address, if you use 127.0.0.1 or localhost, the webinterface will only accessible locally.") + self.config["webinterface"]["host"] = self.ask(_("Address"), "0.0.0.0") + self.config["webinterface"]["port"] = self.ask(_("Port"), "8000") + print + print _("pyLoad offers several server backends, now following a short explanation.") + print "- builtin:", _("Default server; best choice if you plan to use pyLoad just for you.") + print "- threaded:", _("Support SSL connection and can serve simultaneously more client flawlessly.") + print "- fastcgi:", _( + "Can be used by apache, lighttpd, etc.; needs to be properly configured before.") + if os.name != "nt": + print "- lightweight:", _("Very fast alternative to builtin; requires libev and bjoern packages.") + + print + print _("NOTE: In some rare cases the builtin server is not working, if you notice problems with the webinterface") + print _("come back here and change the builtin server to the threaded one here.") + + if os.name == "nt": + servers = ["builtin", "threaded", "fastcgi"] + default = "threaded" + else: + servers = ["builtin", "threaded", "fastcgi", "lightweight"] + default = "lightweight" if self.check_module("bjoern") else "builtin" + + self.config["webinterface"]["server"] = self.ask(_("Server"), default, servers) + + + def conf_ssl(self): + print _("## SSL Setup ##") + print + print _("Execute these commands from pyLoad config folder to make ssl certificates:") + print + print "openssl genrsa -out ssl.key 1024" + print "openssl req -new -key ssl.key -out ssl.csr" + print "openssl req -days 36500 -x509 -key ssl.key -in ssl.csr > ssl.crt " + print + print _("If you're done and everything went fine, you can activate ssl now.") + + self.config["ssl"]["activated"] = self.ask(_("Activate SSL?"), self.yes, bool=True) + + + def set_user(self): + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("setup", join(self.path, "locale"), + languages=[self.config["general"]["language"], "en"], fallback=True) + translation.install(True) + + from pyload.database import DatabaseBackend + + db = DatabaseBackend(None) + db.setup() + + noaction = True + try: + while True: + print _("Select action") + print _("1 - Create/Edit user") + print _("2 - List users") + print _("3 - Remove user") + print _("4 - Quit") + action = raw_input("[1]/2/3/4: ") + if not action in ("1", "2", "3", "4"): + continue + elif action == "1": + print + username = self.ask(_("Username"), "User") + password = self.ask("", "", password=True) + db.addUser(username, password) + noaction = False + elif action == "2": + print + print _("Users") + print "-----" + users = db.listUsers() + noaction = False + for user in users: + print user + print "-----" + print + elif action == "3": + print + username = self.ask(_("Username"), "") + if username: + db.removeUser(username) + noaction = False + elif action == "4": + break + finally: + if not noaction: + db.shutdown() + + + def conf_path(self, trans=False): + if trans: + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("setup", join(self.path, "locale"), + languages=[self.config["general"]["language"], "en"], fallback=True) + translation.install(True) + + print _("Setting new config path, current configuration will not be transfered!") + path = self.ask(_("CONFIG PATH"), abspath("")) + try: + path = join(pypath, path) + if not exists(path): + makedirs(path) + f = open(join(pypath, "pyload", "config", "configdir"), "wb") + f.write(path) + f.close() + print + print + print _("pyLoad config path changed, setup will now close!") + print + print + raw_input(_("Press Enter to exit.")) + sys.exit() + except Exception, e: + print _("Setting config path failed: %s") % str(e) + + + def print_dep(self, name, value): + """Print Status of dependency""" + if value: + print _("%s: OK") % name + else: + print _("%s: MISSING") % name + + + def check_module(self, module): + try: + __import__(module) + return True + except: + return False + + + def check_prog(self, command): + pipe = PIPE + try: + call(command, stdout=pipe, stderr=pipe) + return True + except: + return False + + + def ask(self, qst, default, answers=[], bool=False, password=False): + """produce one line to asking for input""" + if answers: + info = "(" + + for i, answer in enumerate(answers): + info += (", " if i != 0 else "") + str((answer == default and "[%s]" % answer) or answer) + + info += ")" + elif bool: + if default == self.yes: + info = "([%s]/%s)" % (self.yes, self.no) + else: + info = "(%s/[%s])" % (self.yes, self.no) + else: + info = "[%s]" % default + + if password: + p1 = True + p2 = False + pwlen = 8 + while p1 != p2: + # getpass(_("Password: ")) will crash on systems with broken locales (Win, NAS) + sys.stdout.write(_("Password: ")) + p1 = getpass("") + + if len(p1) < pwlen: + print _("Password too short! Use at least %s symbols." % pwlen) + continue + elif not p1.isalnum(): + print _("Password must be alphanumeric.") + continue + + sys.stdout.write(_("Password (again): ")) + p2 = getpass("") + + if p1 == p2: + return p1 + else: + print _("Passwords did not match.") + + while True: + try: + input = raw_input(qst + " %s: " % info) + except KeyboardInterrupt: + print "\nSetup interrupted" + sys.exit() + + input = input.decode(self.stdin_encoding) + + if input.strip() == "": + input = default + + if bool: + # yes, true, t are inputs for booleans with value true + if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]: + return True + # no, false, f are inputs for booleans with value false + elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]: + return False + else: + print _("Invalid Input") + continue + + if not answers: + return input + + else: + if input in answers: + return input + else: + print _("Invalid Input") diff --git a/pyload/database/FileDatabase.py b/pyload/database/FileDatabase.py index fb30735bb..54e5450e8 100644 --- a/pyload/database/FileDatabase.py +++ b/pyload/database/FileDatabase.py @@ -21,9 +21,9 @@ from threading import RLock from time import time from pyload.utils import formatSize, lock -from pyload.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent -from pyload.PyPackage import PyPackage -from pyload.PyFile import PyFile +from pyload.manager.event.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent +from pyload.datatypes.PyPackage import PyPackage +from pyload.datatypes.PyFile import PyFile from pyload.database import style, DatabaseBackend try: diff --git a/pyload/datatypes/PyFile.py b/pyload/datatypes/PyFile.py new file mode 100644 index 000000000..be3129681 --- /dev/null +++ b/pyload/datatypes/PyFile.py @@ -0,0 +1,284 @@ +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN + @author: mkaay +""" + +from pyload.manager.event.PullEvents import UpdateEvent +from pyload.utils import formatSize, lock + +from time import sleep, time + +from threading import RLock + +statusMap = { + "finished": 0, + "offline": 1, + "online": 2, + "queued": 3, + "skipped": 4, + "waiting": 5, + "temp. offline": 6, + "starting": 7, + "failed": 8, + "aborted": 9, + "decrypting": 10, + "custom": 11, + "downloading": 12, + "processing": 13, + "unknown": 14, +} + + +def setSize(self, value): + self._size = int(value) + +class PyFile(object): + """ + Represents a file object at runtime + """ + __slots__ = ("m", "id", "url", "name", "size", "_size", "status", "pluginname", "packageid", + "error", "order", "lock", "plugin", "waitUntil", "active", "abort", "statusname", + "reconnected", "progress", "maxprogress", "pluginmodule", "pluginclass") + + def __init__(self, manager, id, url, name, size, status, error, pluginname, package, order): + self.m = manager + + self.id = int(id) + self.url = url + self.name = name + self.size = size + self.status = status + self.pluginname = pluginname + self.packageid = package #should not be used, use package() instead + self.error = error + self.order = order + # database information ends here + + self.lock = RLock() + + self.plugin = None + #self.download = None + + self.waitUntil = 0 # time() + time to wait + + # status attributes + self.active = False #obsolete? + self.abort = False + self.reconnected = False + + self.statusname = None + + self.progress = 0 + self.maxprogress = 100 + + self.m.cache[int(id)] = self + + + # will convert all sizes to ints + size = property(lambda self: self._size, setSize) + + def __repr__(self): + return "PyFile %s: %s@%s" % (self.id, self.name, self.pluginname) + + @lock + def initPlugin(self): + """ inits plugin instance """ + if not self.plugin: + self.pluginmodule = self.m.core.pluginManager.getPlugin(self.pluginname) + self.pluginclass = getattr(self.pluginmodule, self.m.core.pluginManager.getPluginName(self.pluginname)) + self.plugin = self.pluginclass(self) + + @lock + def hasPlugin(self): + """Thread safe way to determine this file has initialized plugin attribute + + :return: + """ + return hasattr(self, "plugin") and self.plugin + + def package(self): + """ return package instance""" + return self.m.getPackage(self.packageid) + + def setStatus(self, status): + self.status = statusMap[status] + self.sync() #@TODO needed aslong no better job approving exists + + def setCustomStatus(self, msg, status="processing"): + self.statusname = msg + self.setStatus(status) + + def getStatusName(self): + if self.status not in (13, 14) or not self.statusname: + return self.m.statusMsg[self.status] + else: + return self.statusname + + def hasStatus(self, status): + return statusMap[status] == self.status + + def sync(self): + """sync PyFile instance with database""" + self.m.updateLink(self) + + @lock + def release(self): + """sync and remove from cache""" + # file has valid package + if self.packageid > 0: + self.sync() + + if hasattr(self, "plugin") and self.plugin: + self.plugin.clean() + del self.plugin + + self.m.releaseLink(self.id) + + def delete(self): + """delete pyfile from database""" + self.m.deleteLink(self.id) + + def toDict(self): + """return dict with all information for interface""" + return self.toDbDict() + + def toDbDict(self): + """return data as dict for databse + + format: + + { + id: {'url': url, 'name': name ... } + } + + """ + return { + self.id: { + 'id': self.id, + 'url': self.url, + 'name': self.name, + 'plugin': self.pluginname, + 'size': self.getSize(), + 'format_size': self.formatSize(), + 'status': self.status, + 'statusmsg': self.getStatusName(), + 'package': self.packageid, + 'error': self.error, + 'order': self.order + } + } + + def abortDownload(self): + """abort pyfile if possible""" + while self.id in self.m.core.threadManager.processingIds(): + self.abort = True + if self.plugin and self.plugin.req: + self.plugin.req.abortDownloads() + sleep(0.1) + + self.abort = False + if self.hasPlugin() and self.plugin.req: + self.plugin.req.abortDownloads() + + self.release() + + def finishIfDone(self): + """set status to finish and release file if every thread is finished with it""" + + if self.id in self.m.core.threadManager.processingIds(): + return False + + self.setStatus("finished") + self.release() + self.m.checkAllLinksFinished() + return True + + def checkIfProcessed(self): + self.m.checkAllLinksProcessed(self.id) + + def formatWait(self): + """ formats and return wait time in humanreadable format """ + seconds = self.waitUntil - time() + + if seconds < 0: return "00:00:00" + + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + + def formatSize(self): + """ formats size to readable format """ + return formatSize(self.getSize()) + + def formatETA(self): + """ formats eta to readable format """ + seconds = self.getETA() + + if seconds < 0: return "00:00:00" + + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + + def getSpeed(self): + """ calculates speed """ + try: + return self.plugin.req.speed + except: + return 0 + + def getETA(self): + """ gets established time of arrival""" + try: + return self.getBytesLeft() / self.getSpeed() + except: + return 0 + + def getBytesLeft(self): + """ gets bytes left """ + try: + return self.getSize() - self.plugin.req.arrived + except: + return 0 + + def getPercent(self): + """ get % of download """ + if self.status == 12: + try: + return self.plugin.req.percent + except: + return 0 + else: + return self.progress + + def getSize(self): + """ get size of download """ + try: + if self.plugin.req.size: + return self.plugin.req.size + else: + return self.size + except: + return self.size + + def notifyChange(self): + e = UpdateEvent("file", self.id, "collector" if not self.package().queue else "queue") + self.m.core.pullManager.addEvent(e) + + def setProgress(self, value): + if not value == self.progress: + self.progress = value + self.notifyChange() diff --git a/pyload/datatypes/PyPackage.py b/pyload/datatypes/PyPackage.py new file mode 100644 index 000000000..c8d3e6096 --- /dev/null +++ b/pyload/datatypes/PyPackage.py @@ -0,0 +1,79 @@ +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN + @author: mkaay +""" + +from pyload.manager.event.PullEvents import UpdateEvent +from pyload.utils import safe_filename + +class PyPackage: + """ + Represents a package object at runtime + """ + def __init__(self, manager, id, name, folder, site, password, queue, order): + self.m = manager + self.m.packageCache[int(id)] = self + + self.id = int(id) + self.name = name + self._folder = folder + self.site = site + self.password = password + self.queue = queue + self.order = order + self.setFinished = False + + @property + def folder(self): + return safe_filename(self._folder) + + def toDict(self): + """ Returns a dictionary representation of the data. + + :return: dict: {id: { attr: value }} + """ + return { + self.id: { + 'id': self.id, + 'name': self.name, + 'folder': self.folder, + 'site': self.site, + 'password': self.password, + 'queue': self.queue, + 'order': self.order, + 'links': {} + } + } + + def getChildren(self): + """get information about contained links""" + return self.m.getPackageData(self.id)["links"] + + def sync(self): + """sync with db""" + self.m.updatePackage(self) + + def release(self): + """sync and delete from cache""" + self.sync() + self.m.releasePackage(self.id) + + def delete(self): + self.m.deletePackage(self.id) + + def notifyChange(self): + e = UpdateEvent("pack", self.id, "collector" if not self.queue else "queue") + self.m.core.pullManager.addEvent(e) diff --git a/pyload/datatypes/__init__.py b/pyload/datatypes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/debug.py b/pyload/debug.py deleted file mode 100644 index 54a159e78..000000000 --- a/pyload/debug.py +++ /dev/null @@ -1,94 +0,0 @@ -#coding:utf-8 - -import re -import InitHomeDir -from os import listdir - -class Wrapper(object): - pass - -def filter_info(line): - if "object at 0x" in line: - return False - elif " at line " in line: - return False - elif " . - - @author: RaNaN -""" - -from sys import argv -from sys import exit - -import socket -import thread - -from traceback import print_exc - -class Forwarder: - - def __init__(self, extip, extport=9666): - print "Start portforwarding to %s:%s" % (extip, extport) - proxy(extip, extport, 9666) - - -def proxy(*settings): - while True: - server(*settings) - -def server(*settings): - try: - dock_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - dock_socket.bind(("127.0.0.1", settings[2])) - dock_socket.listen(5) - while True: - client_socket = dock_socket.accept()[0] - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.connect((settings[0], settings[1])) - thread.start_new_thread(forward, (client_socket, server_socket)) - thread.start_new_thread(forward, (server_socket, client_socket)) - except Exception: - print_exc() - -def forward(source, destination): - string = ' ' - while string: - string = source.recv(1024) - if string: - destination.sendall(string) - else: - #source.shutdown(socket.SHUT_RD) - destination.shutdown(socket.SHUT_WR) diff --git a/pyload/manager/AccountManager.py b/pyload/manager/AccountManager.py new file mode 100644 index 000000000..d1958f4b6 --- /dev/null +++ b/pyload/manager/AccountManager.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- + +from os.path import exists +from shutil import copy + +from threading import Lock + +from pyload.manager.event.PullEvents import AccountUpdateEvent +from pyload.utils import chmod, lock + +ACC_VERSION = 1 + + +class AccountManager: + """manages all accounts""" + + #-------------------------------------------------------------------------- + def __init__(self, core): + """Constructor""" + + self.core = core + self.lock = Lock() + + self.initPlugins() + self.saveAccounts() # save to add categories to conf + + def initPlugins(self): + self.accounts = {} # key = ( plugin ) + self.plugins = {} + + self.initAccountPlugins() + self.loadAccounts() + + def getAccountPlugin(self, plugin): + """get account instance for plugin or None if anonymous""" + if plugin in self.accounts: + if plugin not in self.plugins: + try: + self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin]) + except TypeError: # The account class no longer exists (blacklisted plugin). Skipping the account to avoid crash + return None + + return self.plugins[plugin] + else: + return None + + def getAccountPlugins(self): + """ get all account instances""" + + plugins = [] + for plugin in self.accounts.keys(): + plugins.append(self.getAccountPlugin(plugin)) + + return plugins + + #-------------------------------------------------------------------------- + def loadAccounts(self): + """loads all accounts available""" + + if not exists("accounts.conf"): + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION)) + f.close() + + f = open("accounts.conf", "rb") + content = f.readlines() + version = content[0].split(":")[1].strip() if content else "" + f.close() + + if not version or int(version) < ACC_VERSION: + copy("accounts.conf", "accounts.backup") + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION)) + f.close() + self.core.log.warning(_("Account settings deleted, due to new config format.")) + return + + plugin = "" + name = "" + + for line in content[1:]: + line = line.strip() + + if not line: continue + if line.startswith("#"): continue + if line.startswith("version"): continue + + if line.endswith(":") and line.count(":") == 1: + plugin = line[:-1] + self.accounts[plugin] = {} + + elif line.startswith("@"): + try: + option = line[1:].split() + self.accounts[plugin][name]['options'][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) + except: + pass + + elif ":" in line: + name, sep, pw = line.partition(":") + self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} + + #-------------------------------------------------------------------------- + def saveAccounts(self): + """save all account information""" + + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION) + "\n") + + for plugin, accounts in self.accounts.iteritems(): + f.write("\n") + f.write(plugin+":\n") + + for name,data in accounts.iteritems(): + f.write("\n\t%s:%s\n" % (name,data['password']) ) + if data['options']: + for option, values in data['options'].iteritems(): + f.write("\t@%s %s\n" % (option, " ".join(values))) + + f.close() + chmod(f.name, 0600) + + #-------------------------------------------------------------------------- + def initAccountPlugins(self): + """init names""" + for name in self.core.pluginManager.getAccountPlugins(): + self.accounts[name] = {} + + @lock + def updateAccount(self, plugin , user, password=None, options={}): + """add or update account""" + if plugin in self.accounts: + p = self.getAccountPlugin(plugin) + updated = p.updateAccounts(user, password, options) + #since accounts is a ref in plugin self.accounts doesnt need to be updated here + + self.saveAccounts() + if updated: p.scheduleRefresh(user, force=False) + + @lock + def removeAccount(self, plugin, user): + """remove account""" + + if plugin in self.accounts: + p = self.getAccountPlugin(plugin) + p.removeAccount(user) + + self.saveAccounts() + + @lock + def getAccountInfos(self, force=True, refresh=False): + data = {} + + if refresh: + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + force = False + + for p in self.accounts.keys(): + if self.accounts[p]: + p = self.getAccountPlugin(p) + if p: + data[p.__name__] = p.getAllAccounts(force) + else: # When an account has been skipped, p is None + data[p] = [] + else: + data[p] = [] + e = AccountUpdateEvent() + self.core.pullManager.addEvent(e) + return data + + def sendChange(self): + e = AccountUpdateEvent() + self.core.pullManager.addEvent(e) diff --git a/pyload/manager/CaptchaManager.py b/pyload/manager/CaptchaManager.py new file mode 100644 index 000000000..0ba876ae8 --- /dev/null +++ b/pyload/manager/CaptchaManager.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay, RaNaN +""" + +from time import time +from traceback import print_exc +from threading import Lock + +class CaptchaManager: + def __init__(self, core): + self.lock = Lock() + self.core = core + self.tasks = [] #task store, for outgoing tasks only + + self.ids = 0 #only for internal purpose + + def newTask(self, img, format, file, result_type): + task = CaptchaTask(self.ids, img, format, file, result_type) + self.ids += 1 + return task + + def removeTask(self, task): + self.lock.acquire() + if task in self.tasks: + self.tasks.remove(task) + self.lock.release() + + def getTask(self): + self.lock.acquire() + for task in self.tasks: + if task.status in ("waiting", "shared-user"): + self.lock.release() + return task + self.lock.release() + return None + + def getTaskByID(self, tid): + self.lock.acquire() + for task in self.tasks: + if task.id == str(tid): #task ids are strings + self.lock.release() + return task + self.lock.release() + return None + + def handleCaptcha(self, task): + cli = self.core.isClientConnected() + + if cli: #client connected -> should solve the captcha + task.setWaiting(50) #wait 50 sec for response + + for plugin in self.core.hookManager.activePlugins(): + try: + plugin.newCaptchaTask(task) + except: + if self.core.debug: + print_exc() + + if task.handler or cli: #the captcha was handled + self.tasks.append(task) + return True + + task.error = _("No Client connected for captcha decrypting") + + return False + + +class CaptchaTask: + def __init__(self, id, img, format, file, result_type='textual'): + self.id = str(id) + self.captchaImg = img + self.captchaFormat = format + self.captchaFile = file + self.captchaResultType = result_type + self.handler = [] #the hook plugins that will take care of the solution + self.result = None + self.waitUntil = None + self.error = None #error message + + self.status = "init" + self.data = {} #handler can store data here + + def getCaptcha(self): + return self.captchaImg, self.captchaFormat, self.captchaResultType + + def setResult(self, text): + if self.isTextual(): + self.result = text + if self.isPositional(): + try: + parts = text.split(',') + self.result = (int(parts[0]), int(parts[1])) + except: + self.result = None + + def getResult(self): + try: + res = self.result.encode("utf8", "replace") + except: + res = self.result + + return res + + def getStatus(self): + return self.status + + def setWaiting(self, sec): + """ let the captcha wait secs for the solution """ + self.waitUntil = max(time() + sec, self.waitUntil) + self.status = "waiting" + + def isWaiting(self): + if self.result or self.error or time() > self.waitUntil: + return False + + return True + + def isTextual(self): + """ returns if text is written on the captcha """ + return self.captchaResultType == 'textual' + + def isPositional(self): + """ returns if user have to click a specific region on the captcha """ + return self.captchaResultType == 'positional' + + def setWatingForUser(self, exclusive): + if exclusive: + self.status = "user" + else: + self.status = "shared-user" + + def timedOut(self): + return time() > self.waitUntil + + def invalid(self): + """ indicates the captcha was not correct """ + [x.captchaInvalid(self) for x in self.handler] + + def correct(self): + [x.captchaCorrect(self) for x in self.handler] + + def __str__(self): + return "" % self.id diff --git a/pyload/manager/HookManager.py b/pyload/manager/HookManager.py new file mode 100644 index 000000000..6f5477aeb --- /dev/null +++ b/pyload/manager/HookManager.py @@ -0,0 +1,305 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN, mkaay + @interface-version: 0.1 +""" +import __builtin__ + +import traceback +from thread import start_new_thread +from threading import RLock + +from types import MethodType + +from pyload.manager.thread.PluginThread import HookThread +from pyload.manager.PluginManager import literal_eval +from utils import lock + +class HookManager: + """Manages hooks, delegates and handles Events. + + Every plugin can define events, \ + but some very usefull events are called by the Core. + Contrary to overwriting hook methods you can use event listener, + which provides additional entry point in the control flow. + Only do very short tasks or use threads. + + **Known Events:** + Most hook methods exists as events. These are the additional known events. + + ===================== ============== ================================== + Name Arguments Description + ===================== ============== ================================== + downloadPreparing fid A download was just queued and will be prepared now. + downloadStarts fid A plugin will immediately starts the download afterwards. + linksAdded links, pid Someone just added links, you are able to modify the links. + allDownloadsProcessed Every link was handled, pyload would idle afterwards. + allDownloadsFinished Every download in queue is finished. + unrarFinished folder, fname An Unrar job finished + configChanged The config was changed via the api. + pluginConfigChanged The plugin config changed, due to api or internal process. + ===================== ============== ================================== + + | Notes: + | allDownloadsProcessed is *always* called before allDownloadsFinished. + | configChanged is *always* called before pluginConfigChanged. + + + """ + + def __init__(self, core): + self.core = core + self.config = self.core.config + + __builtin__.hookManager = self #needed to let hooks register themself + + self.log = self.core.log + self.plugins = [] + self.pluginMap = {} + self.methods = {} #dict of names and list of methods usable by rpc + + self.events = {} # contains events + + #registering callback for config event + self.config.pluginCB = MethodType(self.dispatchEvent, "pluginConfigChanged", basestring) + + self.addEvent("pluginConfigChanged", self.manageHooks) + + self.lock = RLock() + self.createIndex() + + def try_catch(func): + def new(*args): + try: + return func(*args) + except Exception, e: + args[0].log.error(_("Error executing hooks: %s") % str(e)) + if args[0].core.debug: + traceback.print_exc() + + return new + + + def addRPC(self, plugin, func, doc): + plugin = plugin.rpartition(".")[2] + doc = doc.strip() if doc else "" + + if plugin in self.methods: + self.methods[plugin][func] = doc + else: + self.methods[plugin] = {func: doc} + + def callRPC(self, plugin, func, args, parse): + if not args: args = tuple() + if parse: + args = tuple([literal_eval(x) for x in args]) + + plugin = self.pluginMap[plugin] + f = getattr(plugin, func) + return f(*args) + + + def createIndex(self): + plugins = [] + + active = [] + deactive = [] + + for pluginname in self.core.pluginManager.hookPlugins: + try: + #hookClass = getattr(plugin, plugin.__name__) + + if self.config.getPlugin(pluginname, "activated"): + pluginClass = self.core.pluginManager.loadClass("hooks", pluginname) + if not pluginClass: continue + + plugin = pluginClass(self.core, self) + plugins.append(plugin) + self.pluginMap[pluginClass.__name__] = plugin + if plugin.isActivated(): + active.append(pluginClass.__name__) + else: + deactive.append(pluginname) + + + except: + self.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) + if self.core.debug: + traceback.print_exc() + + self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active))) + self.log.info(_("Deactivate plugins: %s") % ", ".join(sorted(deactive))) + + self.plugins = plugins + + def manageHooks(self, plugin, name, value): + if name == "activated" and value: + self.activateHook(plugin) + elif name == "activated" and not value: + self.deactivateHook(plugin) + + def activateHook(self, plugin): + + #check if already loaded + for inst in self.plugins: + if inst.__name__ == plugin: + return + + pluginClass = self.core.pluginManager.loadClass("hooks", plugin) + + if not pluginClass: return + + self.log.debug("Plugin loaded: %s" % plugin) + + plugin = pluginClass(self.core, self) + self.plugins.append(plugin) + self.pluginMap[pluginClass.__name__] = plugin + + # call core Ready + start_new_thread(plugin.coreReady, tuple()) + + def deactivateHook(self, plugin): + + hook = None + for inst in self.plugins: + if inst.__name__ == plugin: + hook = inst + + if not hook: return + + self.log.debug("Plugin unloaded: %s" % plugin) + + hook.unload() + + #remove periodic call + self.log.debug("Removed callback %s" % self.core.scheduler.removeJob(hook.cb)) + self.plugins.remove(hook) + del self.pluginMap[hook.__name__] + + + @try_catch + def coreReady(self): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.coreReady() + + self.dispatchEvent("coreReady") + + @try_catch + def coreExiting(self): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.coreExiting() + + self.dispatchEvent("coreExiting") + + @lock + def downloadPreparing(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadPreparing(pyfile) + + self.dispatchEvent("downloadPreparing", pyfile) + + @lock + def downloadFinished(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadFinished(pyfile) + + self.dispatchEvent("downloadFinished", pyfile) + + @lock + @try_catch + def downloadFailed(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadFailed(pyfile) + + self.dispatchEvent("downloadFailed", pyfile) + + @lock + def packageFinished(self, package): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.packageFinished(package) + + self.dispatchEvent("packageFinished", package) + + @lock + def beforeReconnecting(self, ip): + for plugin in self.plugins: + plugin.beforeReconnecting(ip) + + self.dispatchEvent("beforeReconnecting", ip) + + @lock + def afterReconnecting(self, ip): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.afterReconnecting(ip) + + self.dispatchEvent("afterReconnecting", ip) + + def startThread(self, function, *args, **kwargs): + t = HookThread(self.core.threadManager, function, args, kwargs) + + def activePlugins(self): + """ returns all active plugins """ + return [x for x in self.plugins if x.isActivated()] + + def getAllInfo(self): + """returns info stored by hook plugins""" + info = {} + for name, plugin in self.pluginMap.iteritems(): + if plugin.info: + #copy and convert so str + info[name] = dict([(x, str(y) if not isinstance(y, basestring) else y) for x, y in plugin.info.iteritems()]) + return info + + + def getInfo(self, plugin): + info = {} + if plugin in self.pluginMap and self.pluginMap[plugin].info: + info = dict([(x, str(y) if not isinstance(y, basestring) else y) + for x, y in self.pluginMap[plugin].info.iteritems()]) + + return info + + def addEvent(self, event, func): + """Adds an event listener for event name""" + if event in self.events: + self.events[event].append(func) + else: + self.events[event] = [func] + + def removeEvent(self, event, func): + """removes previously added event listener""" + if event in self.events: + self.events[event].remove(func) + + def dispatchEvent(self, event, *args): + """dispatches event with args""" + if event in self.events: + for f in self.events[event]: + try: + f(*args) + except Exception, e: + self.log.warning("Error calling event handler %s: %s, %s, %s" + % (event, f, args, str(e))) + if self.core.debug: + traceback.print_exc() diff --git a/pyload/manager/PluginManager.py b/pyload/manager/PluginManager.py new file mode 100644 index 000000000..56e59237c --- /dev/null +++ b/pyload/manager/PluginManager.py @@ -0,0 +1,356 @@ +# -*- coding: utf-8 -*- + +import re +import sys + +from itertools import chain +from os import listdir, makedirs +from os.path import isfile, join, exists, abspath +from sys import version_info +from traceback import print_exc + +from SafeEval import const_eval as literal_eval + +from pyload.config.Parser import IGNORE + + +class PluginManager: + ROOT = "pyload.plugins." + USERROOT = "userplugins." + TYPES = ("accounts", "container", "crypter", "hooks", "hoster", "internal", "ocr") + + PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') + VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)') + CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) + DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') + + + def __init__(self, core): + self.core = core + + self.config = core.config + self.log = core.log + + self.plugins = {} + self.createIndex() + + #register for import hook + sys.meta_path.append(self) + + + def createIndex(self): + """create information for all plugins available""" + + sys.path.append(abspath("")) + + if not exists("userplugins"): + makedirs("userplugins") + if not exists(join("userplugins", "__init__.py")): + f = open(join("userplugins", "__init__.py"), "wb") + f.close() + + self.plugins['crypter'] = self.crypterPlugins = self.parse("crypter", pattern=True) + self.plugins['container'] = self.containerPlugins = self.parse("container", pattern=True) + self.plugins['hoster'] = self.hosterPlugins = self.parse("hoster", pattern=True) + + self.plugins['ocr'] = self.captchaPlugins = self.parse("ocr") + self.plugins['accounts'] = self.accountPlugins = self.parse("accounts") + self.plugins['hooks'] = self.hookPlugins = self.parse("hooks") + self.plugins['internal'] = self.internalPlugins = self.parse("internal") + + self.log.debug("created index of plugins") + + def parse(self, folder, pattern=False, home={}): + """ + returns dict with information + home contains parsed plugins from pyload. + + { + name : {path, version, config, (pattern, re), (plugin, class)} + } + + """ + plugins = {} + if home: + pfolder = join("userplugins", folder) + if not exists(pfolder): + makedirs(pfolder) + if not exists(join(pfolder, "__init__.py")): + f = open(join(pfolder, "__init__.py"), "wb") + f.close() + + else: + pfolder = join(pypath, "pyload", "plugins", folder) + + for f in listdir(pfolder): + if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( + "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): + data = open(join(pfolder, f)) + content = data.read() + data.close() + + if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): + continue + elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): + continue + elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): + continue + + name = f[:-3] + if name[-1] == ".": name = name[:-4] + + version = self.VERSION.findall(content) + if version: + version = float(version[0][1]) + else: + version = 0 + + # home contains plugins from pyload root + if home and name in home: + if home[name]['v'] >= version: + continue + + if name in IGNORE or (folder, name) in IGNORE: + continue + + plugins[name] = {} + plugins[name]['v'] = version + + module = f.replace(".pyc", "").replace(".py", "") + + # the plugin is loaded from user directory + plugins[name]['user'] = True if home else False + plugins[name]['name'] = module + + if pattern: + pattern = self.PATTERN.findall(content) + + if pattern: + pattern = pattern[0][1] + else: + pattern = "^unmachtable$" + + plugins[name]['pattern'] = pattern + + try: + plugins[name]['re'] = re.compile(pattern) + except: + self.log.error(_("%s has a invalid pattern.") % name) + + + # internals have no config + if folder == "internal": + self.config.deleteConfig(name) + continue + + config = self.CONFIG.findall(content) + if config: + config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) + desc = self.DESC.findall(content) + desc = desc[0][1] if desc else "" + + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] + + if folder == "hooks": + append = True + for item in config: + if item[0] == "activated": append = False + + # activated flag missing + if append: config.append(["activated", "bool", "Activated", False]) + + try: + self.config.addPluginConfig(name, config, desc) + except: + self.log.error("Invalid config in %s: %s" % (name, config)) + + elif folder == "hooks": #force config creation + desc = self.DESC.findall(content) + desc = desc[0][1] if desc else "" + config = (["activated", "bool", "Activated", False],) + + try: + self.config.addPluginConfig(name, config, desc) + except: + self.log.error("Invalid config in %s: %s" % (name, config)) + + if not home: + temp = self.parse(folder, pattern, plugins) + plugins.update(temp) + + return plugins + + + def parseUrls(self, urls): + """parse plugins for given list of urls""" + + last = None + res = [] # tupels of (url, plugin) + + for url in urls: + if type(url) not in (str, unicode, buffer): continue + found = False + + if last and last[1]['re'].match(url): + res.append((url, last[0])) + continue + + for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), + self.containerPlugins.iteritems()): + if value['re'].match(url): + res.append((url, name)) + last = (name, value) + found = True + break + + if not found: + res.append((url, "BasePlugin")) + + return res + + def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): + for ptype in pluginlist: + if name in self.plugins[ptype]: + return self.plugins[ptype][name], ptype + return None, None + + def getPlugin(self, name, original=False): + """return plugin module from hoster|decrypter|container""" + plugin, type = self.findPlugin(name) + + if not plugin: + self.log.warning("Plugin %s not found." % name) + plugin = self.hosterPlugins['BasePlugin'] + + if "new_module" in plugin and not original: + return plugin['new_module'] + + return self.loadModule(type, name) + + def getPluginName(self, name): + """ used to obtain new name if other plugin was injected""" + plugin, type = self.findPlugin(name) + + if "new_name" in plugin: + return plugin['new_name'] + + return name + + def loadModule(self, type, name): + """ Returns loaded module for plugin + + :param type: plugin type, subfolder of pyload.plugins + :param name: + """ + plugins = self.plugins[type] + if name in plugins: + if "module" in plugins[name]: return plugins[name]['module'] + try: + module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]['name']), globals(), locals(), + plugins[name]['name']) + plugins[name]['module'] = module #cache import, maybe unneeded + return module + except Exception, e: + self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) + if self.core.debug: + print_exc() + + def loadClass(self, type, name): + """Returns the class of a plugin with the same name""" + module = self.loadModule(type, name) + if module: return getattr(module, name) + + def getAccountPlugins(self): + """return list of account plugin names""" + return self.accountPlugins.keys() + + def find_module(self, fullname, path=None): + #redirecting imports if necesarry + if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins + if fullname.startswith(self.USERROOT): user = 1 + else: user = 0 #used as bool and int + + split = fullname.split(".") + if len(split) != 4 - user: return + type, name = split[2 - user:4 - user] + + if type in self.plugins and name in self.plugins[type]: + #userplugin is a newer version + if not user and self.plugins[type][name]['user']: + return self + #imported from userdir, but pyloads is newer + if user and not self.plugins[type][name]['user']: + return self + + + def load_module(self, name, replace=True): + if name not in sys.modules: #could be already in modules + if replace: + if self.ROOT in name: + newname = name.replace(self.ROOT, self.USERROOT) + else: + newname = name.replace(self.USERROOT, self.ROOT) + else: newname = name + + base, plugin = newname.rsplit(".", 1) + + self.log.debug("Redirected import %s -> %s" % (name, newname)) + + module = __import__(newname, globals(), locals(), [plugin]) + #inject under new an old name + sys.modules[name] = module + sys.modules[newname] = module + + return sys.modules[name] + + + def reloadPlugins(self, type_plugins): + """ reload and reindex plugins """ + if not type_plugins: + return None + + self.log.debug("Request reload of plugins: %s" % type_plugins) + + reloaded = [] + + as_dict = {} + for t,n in type_plugins: + if t in ("hooks", "internal"): #: do not reload hooks or internals, because would cause to much side effects + continue + elif t in as_dict: + as_dict[t].append(n) + else: + as_dict[t] = [n] + + for type in as_dict.iterkeys(): + for plugin in as_dict[type]: + if plugin in self.plugins[type] and "module" in self.plugins[type][plugin]: + self.log.debug("Reloading %s" % plugin) + id = (type, plugin) + try: + reload(self.plugins[type][plugin]['module']) + except Exception, e: + self.log.error("Error when reloading %s" % id, str(e)) + continue + else: + reloaded.append(id) + + #index creation + self.plugins['crypter'] = self.crypterPlugins = self.parse("crypter", pattern=True) + self.plugins['container'] = self.containerPlugins = self.parse("container", pattern=True) + self.plugins['hoster'] = self.hosterPlugins = self.parse("hoster", pattern=True) + self.plugins['ocr'] = self.captchaPlugins = self.parse("ocr") + self.plugins['accounts'] = self.accountPlugins = self.parse("accounts") + + if "accounts" in as_dict: #: accounts needs to be reloaded + self.core.accountManager.initPlugins() + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + + return reloaded #: return a list of the plugins successfully reloaded + + def reloadPlugin(self, type_plugin): + """ reload and reindex ONE plugin """ + return True if self.reloadPlugins(type_plugin) else False diff --git a/pyload/manager/RemoteManager.py b/pyload/manager/RemoteManager.py new file mode 100644 index 000000000..e53e317e3 --- /dev/null +++ b/pyload/manager/RemoteManager.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from threading import Thread +from traceback import print_exc + +class BackendBase(Thread): + def __init__(self, manager): + Thread.__init__(self) + self.m = manager + self.core = manager.core + self.enabled = True + self.running = False + + def run(self): + self.running = True + try: + self.serve() + except Exception, e: + self.core.log.error(_("Remote backend error: %s") % e) + if self.core.debug: + print_exc() + finally: + self.running = False + + def setup(self, host, port): + pass + + def checkDeps(self): + return True + + def serve(self): + pass + + def shutdown(self): + pass + + def stop(self): + self.enabled = False# set flag and call shutdowm message, so thread can react + self.shutdown() + + +class RemoteManager: + available = [] + + def __init__(self, core): + self.core = core + self.backends = [] + + if self.core.remote: + self.available.append("ThriftBackend") +# else: +# self.available.append("SocketBackend") + + + def startBackends(self): + host = self.core.config["remote"]["listenaddr"] + port = self.core.config["remote"]["port"] + + for b in self.available: + klass = getattr(__import__("pyload.remote.%s" % b, globals(), locals(), [b], -1), b) + backend = klass(self) + if not backend.checkDeps(): + continue + try: + backend.setup(host, port) + self.core.log.info(_("Starting %(name)s: %(addr)s:%(port)s") % {"name": b, "addr": host, "port": port}) + except Exception, e: + self.core.log.error(_("Failed loading backend %(name)s | %(error)s") % {"name": b, "error": str(e)}) + if self.core.debug: + print_exc() + else: + backend.start() + self.backends.append(backend) + + port += 1 diff --git a/pyload/manager/ThreadManager.py b/pyload/manager/ThreadManager.py new file mode 100644 index 000000000..1073f8040 --- /dev/null +++ b/pyload/manager/ThreadManager.py @@ -0,0 +1,317 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from os.path import exists, join +import re +from subprocess import Popen +from threading import Event, Lock +from time import sleep, time +from traceback import print_exc +from random import choice + +import pycurl + +from pyload.manager.thread import PluginThread +from pyload.datatypes.PyFile import PyFile +from pyload.network.RequestFactory import getURL +from pyload.utils import freeSpace, lock + + +class ThreadManager: + """manages the download threads, assign jobs, reconnect etc""" + + + def __init__(self, core): + """Constructor""" + self.core = core + self.log = core.log + + self.threads = [] # thread list + self.localThreads = [] #hook+decrypter threads + + self.pause = True + + self.reconnecting = Event() + self.reconnecting.clear() + self.downloaded = 0 #number of files downloaded since last cleanup + + self.lock = Lock() + + # some operations require to fetch url info from hoster, so we caching them so it wont be done twice + # contains a timestamp and will be purged after timeout + self.infoCache = {} + + # pool of ids for online check + self.resultIDs = 0 + + # threads which are fetching hoster results + self.infoResults = {} + #timeout for cache purge + self.timestamp = 0 + + pycurl.global_init(pycurl.GLOBAL_DEFAULT) + + for i in range(0, self.core.config.get("download", "max_downloads")): + self.createThread() + + + def createThread(self): + """create a download thread""" + + thread = PluginThread.DownloadThread(self) + self.threads.append(thread) + + def createInfoThread(self, data, pid): + """ + start a thread whichs fetches online status and other infos + data = [ .. () .. ] + """ + self.timestamp = time() + 5 * 60 + + PluginThread.InfoThread(self, data, pid) + + @lock + def createResultThread(self, data, add=False): + """ creates a thread to fetch online status, returns result id """ + self.timestamp = time() + 5 * 60 + + rid = self.resultIDs + self.resultIDs += 1 + + PluginThread.InfoThread(self, data, rid=rid, add=add) + + return rid + + + @lock + def getInfoResult(self, rid): + """returns result and clears it""" + self.timestamp = time() + 5 * 60 + + if rid in self.infoResults: + data = self.infoResults[rid] + self.infoResults[rid] = {} + return data + else: + return {} + + @lock + def setInfoResults(self, rid, result): + self.infoResults[rid].update(result) + + def getActiveFiles(self): + active = [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)] + + for t in self.localThreads: + active.extend(t.getActiveFiles()) + + return active + + def processingIds(self): + """get a id list of all pyfiles processed""" + return [x.id for x in self.getActiveFiles()] + + + def work(self): + """run all task which have to be done (this is for repetivive call by core)""" + try: + self.tryReconnect() + except Exception, e: + self.log.error(_("Reconnect Failed: %s") % str(e) ) + self.reconnecting.clear() + if self.core.debug: + print_exc() + self.checkThreadCount() + + try: + self.assignJob() + except Exception, e: + self.log.warning("Assign job error", e) + if self.core.debug: + print_exc() + + sleep(0.5) + self.assignJob() + #it may be failed non critical so we try it again + + if (self.infoCache or self.infoResults) and self.timestamp < time(): + self.infoCache.clear() + self.infoResults.clear() + self.log.debug("Cleared Result cache") + + #-------------------------------------------------------------------------- + def tryReconnect(self): + """checks if reconnect needed""" + + if not (self.core.config["reconnect"]["activated"] and self.core.api.isTimeReconnect()): + return False + + active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active] + + if not (0 < active.count(True) == len(active)): + return False + + if not exists(self.core.config['reconnect']['method']): + if exists(join(pypath, self.core.config['reconnect']['method'])): + self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method']) + else: + self.core.config["reconnect"]["activated"] = False + self.log.warning(_("Reconnect script not found!")) + return + + self.reconnecting.set() + + #Do reconnect + self.log.info(_("Starting reconnect")) + + while [x.active.plugin.waiting for x in self.threads if x.active].count(True) != 0: + sleep(0.25) + + ip = self.getIP() + + self.core.hookManager.beforeReconnecting(ip) + + self.log.debug("Old IP: %s" % ip) + + try: + reconn = Popen(self.core.config['reconnect']['method'], bufsize=-1, shell=True)#, stdout=subprocess.PIPE) + except: + self.log.warning(_("Failed executing reconnect script!")) + self.core.config["reconnect"]["activated"] = False + self.reconnecting.clear() + if self.core.debug: + print_exc() + return + + reconn.wait() + sleep(1) + ip = self.getIP() + self.core.hookManager.afterReconnecting(ip) + + self.log.info(_("Reconnected, new IP: %s") % ip) + + self.reconnecting.clear() + + def getIP(self): + """retrieve current ip""" + services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"), + ("http://checkip.dyndns.org/",".*Current IP Address: (\S+).*")] + + ip = "" + for i in range(10): + try: + sv = choice(services) + ip = getURL(sv[0]) + ip = re.match(sv[1], ip).group(1) + break + except: + ip = "" + sleep(1) + + return ip + + #-------------------------------------------------------------------------- + def checkThreadCount(self): + """checks if there are need for increasing or reducing thread count""" + + if len(self.threads) == self.core.config.get("download", "max_downloads"): + return True + elif len(self.threads) < self.core.config.get("download", "max_downloads"): + self.createThread() + else: + free = [x for x in self.threads if not x.active] + if free: + free[0].put("quit") + + + def cleanPycurl(self): + """ make a global curl cleanup (currently ununused) """ + if self.processingIds(): + return False + pycurl.global_cleanup() + pycurl.global_init(pycurl.GLOBAL_DEFAULT) + self.downloaded = 0 + self.log.debug("Cleaned up pycurl") + return True + + #-------------------------------------------------------------------------- + def assignJob(self): + """assing a job to a thread if possible""" + + if self.pause or not self.core.api.isTimeDownload(): return + + #if self.downloaded > 20: + # if not self.cleanPyCurl(): return + + free = [x for x in self.threads if not x.active] + + inuse = set([(x.active.pluginname, self.getLimit(x)) for x in self.threads if x.active and x.active.hasPlugin() and x.active.plugin.account]) + inuse = map(lambda x: (x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) ,inuse) + onlimit = [x[0] for x in inuse if x[1] > 0 and x[2] >= x[1]] + + occ = [x.active.pluginname for x in self.threads if x.active and x.active.hasPlugin() and not x.active.plugin.multiDL] + onlimit + + occ.sort() + occ = tuple(set(occ)) + job = self.core.files.getJob(occ) + if job: + try: + job.initPlugin() + except Exception, e: + self.log.critical(str(e)) + print_exc() + job.setStatus("failed") + job.error = str(e) + job.release() + return + + if job.plugin.__type__ == "hoster": + spaceLeft = freeSpace(self.core.config["general"]["download_folder"]) / 1024 / 1024 + if spaceLeft < self.core.config["general"]["min_free_space"]: + self.log.warning(_("Not enough space left on device")) + self.pause = True + + if free and not self.pause: + thread = free[0] + #self.downloaded += 1 + + thread.put(job) + else: + #put job back + if occ not in self.core.files.jobCache: + self.core.files.jobCache[occ] = [] + self.core.files.jobCache[occ].append(job.id) + + #check for decrypt jobs + job = self.core.files.getDecryptJob() + if job: + job.initPlugin() + thread = PluginThread.DecrypterThread(self, job) + + + else: + thread = PluginThread.DecrypterThread(self, job) + + def getLimit(self, thread): + limit = thread.active.plugin.account.getAccountData(thread.active.plugin.user)["options"].get("limitDL", ["0"])[0] + return int(limit) + + def cleanup(self): + """do global cleanup, should be called when finished with pycurl""" + pycurl.global_cleanup() diff --git a/pyload/manager/__init__.py b/pyload/manager/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/manager/event/PullEvents.py b/pyload/manager/event/PullEvents.py new file mode 100644 index 000000000..0739b4ec8 --- /dev/null +++ b/pyload/manager/event/PullEvents.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from time import time +from pyload.utils import uniqify + +class PullManager: + def __init__(self, core): + self.core = core + self.clients = [] + + def newClient(self, uuid): + self.clients.append(Client(uuid)) + + def clean(self): + for n, client in enumerate(self.clients): + if client.lastActive + 30 < time(): + del self.clients[n] + + def getEvents(self, uuid): + events = [] + validUuid = False + for client in self.clients: + if client.uuid == uuid: + client.lastActive = time() + validUuid = True + while client.newEvents(): + events.append(client.popEvent().toList()) + break + if not validUuid: + self.newClient(uuid) + events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] + return uniqify(events) + + def addEvent(self, event): + for client in self.clients: + client.addEvent(event) + +class Client: + def __init__(self, uuid): + self.uuid = uuid + self.lastActive = time() + self.events = [] + + def newEvents(self): + return len(self.events) > 0 + + def popEvent(self): + if not len(self.events): + return None + return self.events.pop(0) + + def addEvent(self, event): + self.events.append(event) + +class UpdateEvent: + def __init__(self, itype, iid, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.destination = destination + + def toList(self): + return ["update", self.destination, self.type, self.id] + +class RemoveEvent: + def __init__(self, itype, iid, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.destination = destination + + def toList(self): + return ["remove", self.destination, self.type, self.id] + +class InsertEvent: + def __init__(self, itype, iid, after, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.after = after + self.destination = destination + + def toList(self): + return ["insert", self.destination, self.type, self.id, self.after] + +class ReloadAllEvent: + def __init__(self, destination): + assert destination == "queue" or destination == "collector" + self.destination = destination + + def toList(self): + return ["reload", self.destination] + +class AccountUpdateEvent: + def toList(self): + return ["account"] + +class ConfigUpdateEvent: + def toList(self): + return ["config"] diff --git a/pyload/manager/event/Scheduler.py b/pyload/manager/event/Scheduler.py new file mode 100644 index 000000000..71b5f96af --- /dev/null +++ b/pyload/manager/event/Scheduler.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from time import time +from heapq import heappop, heappush +from thread import start_new_thread +from threading import Lock + +class AlreadyCalled(Exception): + pass + + +class Deferred: + def __init__(self): + self.call = [] + self.result = () + + def addCallback(self, f, *cargs, **ckwargs): + self.call.append((f, cargs, ckwargs)) + + def callback(self, *args, **kwargs): + if self.result: + raise AlreadyCalled + self.result = (args, kwargs) + for f, cargs, ckwargs in self.call: + args += tuple(cargs) + kwargs.update(ckwargs) + f(*args ** kwargs) + + +class Scheduler: + def __init__(self, core): + self.core = core + + self.queue = PriorityQueue() + + def addJob(self, t, call, args=[], kwargs={}, threaded=True): + d = Deferred() + t += time() + j = Job(t, call, args, kwargs, d, threaded) + self.queue.put((t, j)) + return d + + + def removeJob(self, d): + """ + :param d: defered object + :return: if job was deleted + """ + index = -1 + + for i, j in enumerate(self.queue): + if j[1].deferred == d: + index = i + + if index >= 0: + del self.queue[index] + return True + + return False + + def work(self): + while True: + t, j = self.queue.get() + if not j: + break + else: + if t <= time(): + j.start() + else: + self.queue.put((t, j)) + break + + +class Job: + def __init__(self, time, call, args=[], kwargs={}, deferred=None, threaded=True): + self.time = float(time) + self.call = call + self.args = args + self.kwargs = kwargs + self.deferred = deferred + self.threaded = threaded + + def run(self): + ret = self.call(*self.args, **self.kwargs) + if self.deferred is None: + return + else: + self.deferred.callback(ret) + + def start(self): + if self.threaded: + start_new_thread(self.run, ()) + else: + self.run() + + +class PriorityQueue: + """ a non blocking priority queue """ + + def __init__(self): + self.queue = [] + self.lock = Lock() + + def __iter__(self): + return iter(self.queue) + + def __delitem__(self, key): + del self.queue[key] + + def put(self, element): + self.lock.acquire() + heappush(self.queue, element) + self.lock.release() + + def get(self): + """ return element or None """ + self.lock.acquire() + try: + el = heappop(self.queue) + return el + except IndexError: + return None, None + finally: + self.lock.release() diff --git a/pyload/manager/event/__init__.py b/pyload/manager/event/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/manager/thread/PluginThread.py b/pyload/manager/thread/PluginThread.py new file mode 100644 index 000000000..5c274fa46 --- /dev/null +++ b/pyload/manager/thread/PluginThread.py @@ -0,0 +1,675 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.datatypes.PyFile import PyFile +from pyload.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload +from pyload.utils.packagetools import parseNames +from pyload.utils import safe_join +from pyload.api import OnlineStatus + +class PluginThread(Thread): + """abstract base class for thread types""" + + #-------------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + Thread.__init__(self) + self.setDaemon(True) + self.m = manager #thread manager + + + def writeDebugReport(self, pyfile): + """ writes a + :return: + """ + + dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) + dump = self.getDebugDump(pyfile) + + try: + import zipfile + + zip = zipfile.ZipFile(dump_name, "w") + + for f in listdir(join("tmp", pyfile.pluginname)): + try: + # avoid encoding errors + zip.write(join("tmp", pyfile.pluginname, f), safe_join(pyfile.pluginname, f)) + except: + pass + + info = zipfile.ZipInfo(safe_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) + info.external_attr = 0644 << 16L # change permissions + + zip.writestr(info, dump) + zip.close() + + if not stat(dump_name).st_size: + raise Exception("Empty Zipfile") + + except Exception, e: + self.m.log.debug("Error creating zip file: %s" % e) + + dump_name = dump_name.replace(".zip", ".txt") + f = open(dump_name, "wb") + f.write(dump) + f.close() + + self.m.core.log.info("Debug Report written to %s" % dump_name) + + def getDebugDump(self, pyfile): + dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( + self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version__, format_exc()) + + tb = exc_info()[2] + stack = [] + while tb: + stack.append(tb.tb_frame) + tb = tb.tb_next + + for frame in stack[1:]: + dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno) + + for key, value in frame.f_locals.items(): + dump += "\t%20s = " % key + try: + dump += pformat(value) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + del frame + + del stack #delete it just to be sure... + + dump += "\n\nPLUGIN OBJECT DUMP: \n\n" + + for name in dir(pyfile.plugin): + attr = getattr(pyfile.plugin, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + dump += "\nPYFILE OBJECT DUMP: \n\n" + + for name in dir(pyfile): + attr = getattr(pyfile, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + if pyfile.pluginname in self.m.core.config.plugin: + dump += "\n\nCONFIG: \n\n" + dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" + + return dump + + def clean(self, pyfile): + """ set thread unactive and release pyfile """ + self.active = False + pyfile.release() + + +class DownloadThread(PluginThread): + """thread for downloading files from 'real' hoster plugins""" + + #-------------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + PluginThread.__init__(self, manager) + + self.queue = Queue() # job queue + self.active = False + + self.start() + + #-------------------------------------------------------------------------- + def run(self): + """run method""" + pyfile = None + + while True: + del pyfile + self.active = self.queue.get() + pyfile = self.active + + if self.active == "quit": + self.active = False + self.m.threads.remove(self) + return True + + try: + if not pyfile.hasPlugin(): continue + #this pyfile was deleted while queueing + + pyfile.plugin.checkForSameFiles(starting=True) + self.m.log.info(_("Download starts: %s" % pyfile.name)) + + # start download + self.m.core.hookManager.downloadPreparing(pyfile) + pyfile.plugin.preprocessing(self) + + self.m.log.info(_("Download finished: %s") % pyfile.name) + self.m.core.hookManager.downloadFinished(pyfile) + self.m.core.files.checkPackageFinished(pyfile) + + except NotImplementedError: + self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + pyfile.setStatus("failed") + pyfile.error = "Plugin does not work" + self.clean(pyfile) + continue + + except Abort: + try: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + except: + pass + + pyfile.setStatus("aborted") + + self.clean(pyfile) + continue + + except Reconnect: + self.queue.put(pyfile) + #pyfile.req.clearCookies() + + while self.m.reconnecting.isSet(): + sleep(0.5) + + continue + + except Retry, e: + reason = e.args[0] + self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) + self.queue.put(pyfile) + continue + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % pyfile.name) + elif msg == "temp. offline": + pyfile.setStatus("temp. offline") + self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + self.m.core.hookManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + except error, e: + if len(e.args) == 2: + code, msg = e.args + else: + code = 0 + msg = e.args + + self.m.log.debug("pycurl exception %s: %s" % (code, msg)) + + if code in (7, 18, 28, 52, 56): + self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) + wait = time() + 60 + + pyfile.waitUntil = wait + pyfile.setStatus("waiting") + while time() < wait: + sleep(1) + if pyfile.abort: + break + + if pyfile.abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + self.clean(pyfile) + else: + self.queue.put(pyfile) + + continue + + else: + pyfile.setStatus("failed") + self.m.log.error("pycurl error %s: %s" % (code, msg)) + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.hookManager.downloadFailed(pyfile) + + self.clean(pyfile) + continue + + except SkipDownload, e: + pyfile.setStatus("skipped") + + self.m.log.info( + _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) + + self.clean(pyfile) + + self.m.core.files.checkPackageFinished(pyfile) + + self.active = False + self.m.core.files.save() + + continue + + + except Exception, e: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.hookManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + finally: + self.m.core.files.save() + pyfile.checkIfProcessed() + exc_clear() + + #pyfile.plugin.req.clean() + + self.active = False + pyfile.finishIfDone() + self.m.core.files.save() + + + def put(self, job): + """assing job to thread""" + self.queue.put(job) + + + def stop(self): + """stops the thread""" + self.put("quit") + + +class DecrypterThread(PluginThread): + """thread for decrypting""" + + def __init__(self, manager, pyfile): + """constructor""" + PluginThread.__init__(self, manager) + + self.active = pyfile + manager.localThreads.append(self) + + pyfile.setStatus("decrypting") + + self.start() + + def getActiveFiles(self): + return [self.active] + + def run(self): + """run method""" + + pyfile = self.active + retry = False + + try: + self.m.log.info(_("Decrypting starts: %s") % self.active.name) + self.active.plugin.preprocessing(self) + + except NotImplementedError: + self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) + return + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + self.active.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % self.active.name) + else: + self.active.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": msg}) + self.active.error = msg + + return + + except Abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + return + + except Retry: + self.m.log.info(_("Retrying %s") % self.active.name) + retry = True + return self.run() + + except Exception, e: + self.active.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": str(e)}) + self.active.error = str(e) + + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + return + + + finally: + if not retry: + self.active.release() + self.active = False + self.m.core.files.save() + self.m.localThreads.remove(self) + exc_clear() + + + #self.m.core.hookManager.downloadFinished(pyfile) + + + #self.m.localThreads.remove(self) + #self.active.finishIfDone() + if not retry: + pyfile.delete() + + +class HookThread(PluginThread): + """thread for hooks""" + + #-------------------------------------------------------------------------- + def __init__(self, m, function, args, kwargs): + """Constructor""" + PluginThread.__init__(self, m) + + self.f = function + self.args = args + self.kwargs = kwargs + + self.active = [] + + m.localThreads.append(self) + + self.start() + + def getActiveFiles(self): + return self.active + + def addActive(self, pyfile): + """ Adds a pyfile to active list and thus will be displayed on overview""" + if pyfile not in self.active: + self.active.append(pyfile) + + def finishFile(self, pyfile): + if pyfile in self.active: + self.active.remove(pyfile) + + pyfile.finishIfDone() + + def run(self): + try: + try: + self.kwargs["thread"] = self + self.f(*self.args, **self.kwargs) + except TypeError, e: + #dirty method to filter out exceptions + if "unexpected keyword argument 'thread'" not in e.args[0]: + raise + + del self.kwargs["thread"] + self.f(*self.args, **self.kwargs) + finally: + local = copy(self.active) + for x in local: + self.finishFile(x) + + self.m.localThreads.remove(self) + + +class InfoThread(PluginThread): + def __init__(self, manager, data, pid=-1, rid=-1, add=False): + """Constructor""" + PluginThread.__init__(self, manager) + + self.data = data + self.pid = pid # package id + # [ .. (name, plugin) .. ] + + self.rid = rid #result id + self.add = add #add packages instead of return result + + self.cache = [] #accumulated data + + self.start() + + def run(self): + """run method""" + + plugins = {} + container = [] + + for url, plugin in self.data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + + # filter out container plugins + for name in self.m.core.pluginManager.containerPlugins: + if name in plugins: + container.extend([(name, url) for url in plugins[name]]) + + del plugins[name] + + #directly write to database + if self.pid > -1: + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) + self.m.core.files.save() + + elif self.add: + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateCache(pluginname, result) + + packs = parseNames([(name, url) for name, x, y, url in self.cache]) + + self.m.log.debug("Fetched and generated %d packages" % len(packs)) + + for k, v in packs: + self.m.core.api.addPackage(k, v) + + #empty cache + del self.cache[:] + + else: #post the results + + + for name, url in container: + #attach container content + try: + data = self.decryptContainer(name, url) + except: + print_exc() + self.m.log.error("Could not decrypt container.") + data = [] + + for url, plugin in data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + self.m.infoResults[self.rid] = {} + + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + + #force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateResult(pluginname, result, True) + + self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} + + self.m.timestamp = time() + 5 * 60 + + + def updateDB(self, plugin, result): + self.m.core.files.updateFileInfo(result, self.pid) + + def updateResult(self, plugin, result, force=False): + #parse package name and generate result + #accumulate results + + self.cache.extend(result) + + if len(self.cache) >= 20 or force: + #used for package generating + tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) + for name, size, status, url in self.cache] + + data = parseNames(tmp) + result = {} + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + self.m.setInfoResults(self.rid, result) + + self.cache = [] + + def updateCache(self, plugin, result): + self.cache.extend(result) + + def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): + try: + result = [] #result loaded from cache + process = [] #urls to process + for url in urls: + if url in self.m.infoCache: + result.append(self.m.infoCache[url]) + else: + process.append(url) + + if result: + self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) + cb(pluginname, result) + + if process: + self.m.log.debug("Run Info Fetching for %s" % pluginname) + for result in plugin.getInfo(process): + #result = [ .. (name, size, status, url) .. ] + if not type(result) == list: result = [result] + + for res in result: + self.m.infoCache[res[3]] = res + + cb(pluginname, result) + + self.m.log.debug("Finished Info Fetching for %s" % pluginname) + except Exception, e: + self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % + {"name": pluginname, "err": str(e)}) + if self.m.core.debug: + print_exc() + + # generate default results + if err: + result = [(url, 0, 3, url) for url in urls] + cb(pluginname, result) + + + def decryptContainer(self, plugin, url): + data = [] + # only works on container plugins + + self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) + + # dummy pyfile + pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) + + pyfile.initPlugin() + + # little plugin lifecycle + try: + pyfile.plugin.setup() + pyfile.plugin.loadToDisk() + pyfile.plugin.decrypt(pyfile) + pyfile.plugin.deleteTmp() + + for pack in pyfile.plugin.packages: + pyfile.plugin.urls.extend(pack[1]) + + data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) + + self.m.log.debug("Got %d links." % len(data)) + + except Exception, e: + self.m.log.debug("Pre decrypting error: %s" % str(e)) + finally: + pyfile.release() + + return data diff --git a/pyload/manager/thread/ServerThread.py b/pyload/manager/thread/ServerThread.py new file mode 100644 index 000000000..7de3b1ca1 --- /dev/null +++ b/pyload/manager/thread/ServerThread.py @@ -0,0 +1,108 @@ +from __future__ import with_statement +from os.path import exists + +import os +import threading +import logging + +core = None +setup = None +log = logging.getLogger("log") + +class WebServer(threading.Thread): + def __init__(self, pycore): + global core + threading.Thread.__init__(self) + self.core = pycore + core = pycore + self.running = True + self.server = pycore.config['webinterface']['server'] + self.https = pycore.config['webinterface']['https'] + self.cert = pycore.config["ssl"]["cert"] + self.key = pycore.config["ssl"]["key"] + self.host = pycore.config['webinterface']['host'] + self.port = pycore.config['webinterface']['port'] + + self.setDaemon(True) + + def run(self): + import pyload.webui as webinterface + global webinterface + + reset = False + + if self.https and (not exists(self.cert) or not exists(self.key)): + log.warning(_("SSL certificates not found.")) + self.https = False + + if self.server in ("lighttpd", "nginx"): + log.warning(_("Sorry, we dropped support for starting %s directly within pyLoad") % self.server) + log.warning(_("You can use the threaded server which offers good performance and ssl,")) + log.warning(_("of course you can still use your existing %s with pyLoads fastcgi server") % self.server) + log.warning(_("sample configs are located in the pyload/web/servers directory")) + reset = True + elif self.server == "fastcgi": + try: + import flup + except: + log.warning(_("Can't use %(server)s, python-flup is not installed!") % { + "server": self.server}) + reset = True + + if reset or self.server == "lightweight": + if os.name != "nt": + try: + import bjoern + except Exception, e: + log.error(_("Error importing lightweight server: %s") % e) + log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) + log.warning(_("Copy the boern.so to pyload/lib folder or use setup.py install")) + log.warning(_("Of course you need to be familiar with linux and know how to compile software")) + self.server = "builtin" + else: + self.core.log.info(_("Server set to threaded, due to known performance problems on windows.")) + self.core.config['webinterface']['server'] = "threaded" + self.server = "threaded" + + if self.server == "threaded": + self.start_threaded() + elif self.server == "fastcgi": + self.start_fcgi() + elif self.server == "lightweight": + self.start_lightweight() + else: + self.start_builtin() + + def start_builtin(self): + + if self.https: + log.warning(_("This server offers no SSL, please consider using threaded instead")) + + self.core.log.info(_("Starting builtin webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_simple(host=self.host, port=self.port) + + def start_threaded(self): + if self.https: + self.core.log.info(_("Starting threaded SSL webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + else: + self.cert = "" + self.key = "" + self.core.log.info(_("Starting threaded webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + + webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) + + def start_fcgi(self): + + self.core.log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_fcgi(host=self.host, port=self.port) + + + def start_lightweight(self): + if self.https: + log.warning(_("This server offers no SSL, please consider using threaded instead")) + + self.core.log.info(_("Starting lightweight webserver (bjoern): %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_lightweight(host=self.host, port=self.port) + + def quit(self): + self.running = False diff --git a/pyload/manager/thread/__init__.py b/pyload/manager/thread/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/plugins/AccountManager.py b/pyload/plugins/AccountManager.py deleted file mode 100644 index 67909072f..000000000 --- a/pyload/plugins/AccountManager.py +++ /dev/null @@ -1,173 +0,0 @@ -# -*- coding: utf-8 -*- - -from os.path import exists -from shutil import copy - -from threading import Lock - -from pyload.PullEvents import AccountUpdateEvent -from pyload.utils import chmod, lock - -ACC_VERSION = 1 - - -class AccountManager: - """manages all accounts""" - - #-------------------------------------------------------------------------- - def __init__(self, core): - """Constructor""" - - self.core = core - self.lock = Lock() - - self.initPlugins() - self.saveAccounts() # save to add categories to conf - - def initPlugins(self): - self.accounts = {} # key = ( plugin ) - self.plugins = {} - - self.initAccountPlugins() - self.loadAccounts() - - def getAccountPlugin(self, plugin): - """get account instance for plugin or None if anonymous""" - if plugin in self.accounts: - if plugin not in self.plugins: - try: - self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin]) - except TypeError: # The account class no longer exists (blacklisted plugin). Skipping the account to avoid crash - return None - - return self.plugins[plugin] - else: - return None - - def getAccountPlugins(self): - """ get all account instances""" - - plugins = [] - for plugin in self.accounts.keys(): - plugins.append(self.getAccountPlugin(plugin)) - - return plugins - - #-------------------------------------------------------------------------- - def loadAccounts(self): - """loads all accounts available""" - - if not exists("accounts.conf"): - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - - f = open("accounts.conf", "rb") - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - f.close() - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - self.core.log.warning(_("Account settings deleted, due to new config format.")) - return - - plugin = "" - name = "" - - for line in content[1:]: - line = line.strip() - - if not line: continue - if line.startswith("#"): continue - if line.startswith("version"): continue - - if line.endswith(":") and line.count(":") == 1: - plugin = line[:-1] - self.accounts[plugin] = {} - - elif line.startswith("@"): - try: - option = line[1:].split() - self.accounts[plugin][name]['options'][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) - except: - pass - - elif ":" in line: - name, sep, pw = line.partition(":") - self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - - #-------------------------------------------------------------------------- - def saveAccounts(self): - """save all account information""" - - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION) + "\n") - - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin+":\n") - - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data['password']) ) - if data['options']: - for option, values in data['options'].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) - - f.close() - chmod(f.name, 0600) - - #-------------------------------------------------------------------------- - def initAccountPlugins(self): - """init names""" - for name in self.core.pluginManager.getAccountPlugins(): - self.accounts[name] = {} - - @lock - def updateAccount(self, plugin , user, password=None, options={}): - """add or update account""" - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - updated = p.updateAccounts(user, password, options) - #since accounts is a ref in plugin self.accounts doesnt need to be updated here - - self.saveAccounts() - if updated: p.scheduleRefresh(user, force=False) - - @lock - def removeAccount(self, plugin, user): - """remove account""" - - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - p.removeAccount(user) - - self.saveAccounts() - - @lock - def getAccountInfos(self, force=True, refresh=False): - data = {} - - if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - force = False - - for p in self.accounts.keys(): - if self.accounts[p]: - p = self.getAccountPlugin(p) - if p: - data[p.__name__] = p.getAllAccounts(force) - else: # When an account has been skipped, p is None - data[p] = [] - else: - data[p] = [] - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) - return data - - def sendChange(self): - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) diff --git a/pyload/plugins/PluginManager.py b/pyload/plugins/PluginManager.py deleted file mode 100644 index be467b41a..000000000 --- a/pyload/plugins/PluginManager.py +++ /dev/null @@ -1,356 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -import sys - -from itertools import chain -from os import listdir, makedirs -from os.path import isfile, join, exists, abspath -from sys import version_info -from traceback import print_exc - -from SafeEval import const_eval as literal_eval - -from pyload.ConfigParser import IGNORE - - -class PluginManager: - ROOT = "pyload.plugins." - USERROOT = "userplugins." - TYPES = ("accounts", "container", "crypter", "hooks", "hoster", "internal", "ocr") - - PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') - VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)') - CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) - DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') - - - def __init__(self, core): - self.core = core - - self.config = core.config - self.log = core.log - - self.plugins = {} - self.createIndex() - - #register for import hook - sys.meta_path.append(self) - - - def createIndex(self): - """create information for all plugins available""" - - sys.path.append(abspath("")) - - if not exists("userplugins"): - makedirs("userplugins") - if not exists(join("userplugins", "__init__.py")): - f = open(join("userplugins", "__init__.py"), "wb") - f.close() - - self.plugins['crypter'] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins['container'] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins['hoster'] = self.hosterPlugins = self.parse("hoster", pattern=True) - - self.plugins['ocr'] = self.captchaPlugins = self.parse("ocr") - self.plugins['accounts'] = self.accountPlugins = self.parse("accounts") - self.plugins['hooks'] = self.hookPlugins = self.parse("hooks") - self.plugins['internal'] = self.internalPlugins = self.parse("internal") - - self.log.debug("created index of plugins") - - def parse(self, folder, pattern=False, home={}): - """ - returns dict with information - home contains parsed plugins from pyload. - - { - name : {path, version, config, (pattern, re), (plugin, class)} - } - - """ - plugins = {} - if home: - pfolder = join("userplugins", folder) - if not exists(pfolder): - makedirs(pfolder) - if not exists(join(pfolder, "__init__.py")): - f = open(join(pfolder, "__init__.py"), "wb") - f.close() - - else: - pfolder = join(pypath, "pyload", "plugins", folder) - - for f in listdir(pfolder): - if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( - "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - data = open(join(pfolder, f)) - content = data.read() - data.close() - - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): - continue - elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): - continue - elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): - continue - - name = f[:-3] - if name[-1] == ".": name = name[:-4] - - version = self.VERSION.findall(content) - if version: - version = float(version[0][1]) - else: - version = 0 - - # home contains plugins from pyload root - if home and name in home: - if home[name]['v'] >= version: - continue - - if name in IGNORE or (folder, name) in IGNORE: - continue - - plugins[name] = {} - plugins[name]['v'] = version - - module = f.replace(".pyc", "").replace(".py", "") - - # the plugin is loaded from user directory - plugins[name]['user'] = True if home else False - plugins[name]['name'] = module - - if pattern: - pattern = self.PATTERN.findall(content) - - if pattern: - pattern = pattern[0][1] - else: - pattern = "^unmachtable$" - - plugins[name]['pattern'] = pattern - - try: - plugins[name]['re'] = re.compile(pattern) - except: - self.log.error(_("%s has a invalid pattern.") % name) - - - # internals have no config - if folder == "internal": - self.config.deleteConfig(name) - continue - - config = self.CONFIG.findall(content) - if config: - config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] - - if folder == "hooks": - append = True - for item in config: - if item[0] == "activated": append = False - - # activated flag missing - if append: config.append(["activated", "bool", "Activated", False]) - - try: - self.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) - - elif folder == "hooks": #force config creation - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - config = (["activated", "bool", "Activated", False],) - - try: - self.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) - - if not home: - temp = self.parse(folder, pattern, plugins) - plugins.update(temp) - - return plugins - - - def parseUrls(self, urls): - """parse plugins for given list of urls""" - - last = None - res = [] # tupels of (url, plugin) - - for url in urls: - if type(url) not in (str, unicode, buffer): continue - found = False - - if last and last[1]['re'].match(url): - res.append((url, last[0])) - continue - - for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), - self.containerPlugins.iteritems()): - if value['re'].match(url): - res.append((url, name)) - last = (name, value) - found = True - break - - if not found: - res.append((url, "BasePlugin")) - - return res - - def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): - for ptype in pluginlist: - if name in self.plugins[ptype]: - return self.plugins[ptype][name], ptype - return None, None - - def getPlugin(self, name, original=False): - """return plugin module from hoster|decrypter|container""" - plugin, type = self.findPlugin(name) - - if not plugin: - self.log.warning("Plugin %s not found." % name) - plugin = self.hosterPlugins['BasePlugin'] - - if "new_module" in plugin and not original: - return plugin['new_module'] - - return self.loadModule(type, name) - - def getPluginName(self, name): - """ used to obtain new name if other plugin was injected""" - plugin, type = self.findPlugin(name) - - if "new_name" in plugin: - return plugin['new_name'] - - return name - - def loadModule(self, type, name): - """ Returns loaded module for plugin - - :param type: plugin type, subfolder of pyload.plugins - :param name: - """ - plugins = self.plugins[type] - if name in plugins: - if "module" in plugins[name]: return plugins[name]['module'] - try: - module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]['name']), globals(), locals(), - plugins[name]['name']) - plugins[name]['module'] = module #cache import, maybe unneeded - return module - except Exception, e: - self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) - if self.core.debug: - print_exc() - - def loadClass(self, type, name): - """Returns the class of a plugin with the same name""" - module = self.loadModule(type, name) - if module: return getattr(module, name) - - def getAccountPlugins(self): - """return list of account plugin names""" - return self.accountPlugins.keys() - - def find_module(self, fullname, path=None): - #redirecting imports if necesarry - if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins - if fullname.startswith(self.USERROOT): user = 1 - else: user = 0 #used as bool and int - - split = fullname.split(".") - if len(split) != 4 - user: return - type, name = split[2 - user:4 - user] - - if type in self.plugins and name in self.plugins[type]: - #userplugin is a newer version - if not user and self.plugins[type][name]['user']: - return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name]['user']: - return self - - - def load_module(self, name, replace=True): - if name not in sys.modules: #could be already in modules - if replace: - if self.ROOT in name: - newname = name.replace(self.ROOT, self.USERROOT) - else: - newname = name.replace(self.USERROOT, self.ROOT) - else: newname = name - - base, plugin = newname.rsplit(".", 1) - - self.log.debug("Redirected import %s -> %s" % (name, newname)) - - module = __import__(newname, globals(), locals(), [plugin]) - #inject under new an old name - sys.modules[name] = module - sys.modules[newname] = module - - return sys.modules[name] - - - def reloadPlugins(self, type_plugins): - """ reload and reindex plugins """ - if not type_plugins: - return None - - self.log.debug("Request reload of plugins: %s" % type_plugins) - - reloaded = [] - - as_dict = {} - for t,n in type_plugins: - if t in ("hooks", "internal"): #: do not reload hooks or internals, because would cause to much side effects - continue - elif t in as_dict: - as_dict[t].append(n) - else: - as_dict[t] = [n] - - for type in as_dict.iterkeys(): - for plugin in as_dict[type]: - if plugin in self.plugins[type] and "module" in self.plugins[type][plugin]: - self.log.debug("Reloading %s" % plugin) - id = (type, plugin) - try: - reload(self.plugins[type][plugin]['module']) - except Exception, e: - self.log.error("Error when reloading %s" % id, str(e)) - continue - else: - reloaded.append(id) - - #index creation - self.plugins['crypter'] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins['container'] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins['hoster'] = self.hosterPlugins = self.parse("hoster", pattern=True) - self.plugins['ocr'] = self.captchaPlugins = self.parse("ocr") - self.plugins['accounts'] = self.accountPlugins = self.parse("accounts") - - if "accounts" in as_dict: #: accounts needs to be reloaded - self.core.accountManager.initPlugins() - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - - return reloaded #: return a list of the plugins successfully reloaded - - def reloadPlugin(self, type_plugin): - """ reload and reindex ONE plugin """ - return True if self.reloadPlugins(type_plugin) else False diff --git a/pyload/plugins/accounts/BayfilesCom.py b/pyload/plugins/accounts/BayfilesCom.py index a42ac6457..38537da0e 100644 --- a/pyload/plugins/accounts/BayfilesCom.py +++ b/pyload/plugins/accounts/BayfilesCom.py @@ -3,7 +3,7 @@ from time import time from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class BayfilesCom(Account): diff --git a/pyload/plugins/accounts/FastixRu.py b/pyload/plugins/accounts/FastixRu.py index a7b939a2c..69840fa2c 100644 --- a/pyload/plugins/accounts/FastixRu.py +++ b/pyload/plugins/accounts/FastixRu.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class FastixRu(Account): diff --git a/pyload/plugins/accounts/FilecloudIo.py b/pyload/plugins/accounts/FilecloudIo.py index 5e6b001e8..dc764d218 100644 --- a/pyload/plugins/accounts/FilecloudIo.py +++ b/pyload/plugins/accounts/FilecloudIo.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class FilecloudIo(Account): diff --git a/pyload/plugins/accounts/FileserveCom.py b/pyload/plugins/accounts/FileserveCom.py index 211023991..99f4430c2 100644 --- a/pyload/plugins/accounts/FileserveCom.py +++ b/pyload/plugins/accounts/FileserveCom.py @@ -3,7 +3,7 @@ from time import mktime, strptime from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class FileserveCom(Account): diff --git a/pyload/plugins/accounts/FourSharedCom.py b/pyload/plugins/accounts/FourSharedCom.py index a62749c43..217bd2874 100644 --- a/pyload/plugins/accounts/FourSharedCom.py +++ b/pyload/plugins/accounts/FourSharedCom.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class FourSharedCom(Account): diff --git a/pyload/plugins/accounts/FreeWayMe.py b/pyload/plugins/accounts/FreeWayMe.py index baca53cd4..5106067a9 100644 --- a/pyload/plugins/accounts/FreeWayMe.py +++ b/pyload/plugins/accounts/FreeWayMe.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class FreeWayMe(Account): diff --git a/pyload/plugins/accounts/LetitbitNet.py b/pyload/plugins/accounts/LetitbitNet.py index 88e679f8e..bce97d378 100644 --- a/pyload/plugins/accounts/LetitbitNet.py +++ b/pyload/plugins/accounts/LetitbitNet.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -# from pyload.common.json_layer import json_loads, json_dumps +# from pyload.utils import json_loads, json_dumps class LetitbitNet(Account): diff --git a/pyload/plugins/accounts/LinksnappyCom.py b/pyload/plugins/accounts/LinksnappyCom.py index 4ac046988..35fc881b0 100644 --- a/pyload/plugins/accounts/LinksnappyCom.py +++ b/pyload/plugins/accounts/LinksnappyCom.py @@ -3,7 +3,7 @@ from hashlib import md5 from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class LinksnappyCom(Account): diff --git a/pyload/plugins/accounts/MegaDebridEu.py b/pyload/plugins/accounts/MegaDebridEu.py index b449d7246..5ba8d3577 100644 --- a/pyload/plugins/accounts/MegaDebridEu.py +++ b/pyload/plugins/accounts/MegaDebridEu.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class MegaDebridEu(Account): diff --git a/pyload/plugins/accounts/MultiDebridCom.py b/pyload/plugins/accounts/MultiDebridCom.py index fa10c92cc..cca6a9849 100644 --- a/pyload/plugins/accounts/MultiDebridCom.py +++ b/pyload/plugins/accounts/MultiDebridCom.py @@ -3,7 +3,7 @@ from time import time from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class MultiDebridCom(Account): diff --git a/pyload/plugins/accounts/OboomCom.py b/pyload/plugins/accounts/OboomCom.py index 3ef6f9d58..b281a289a 100644 --- a/pyload/plugins/accounts/OboomCom.py +++ b/pyload/plugins/accounts/OboomCom.py @@ -4,7 +4,7 @@ import time from beaker.crypto.pbkdf2 import PBKDF2 -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Account import Account diff --git a/pyload/plugins/accounts/OverLoadMe.py b/pyload/plugins/accounts/OverLoadMe.py index ba5a58158..18a3db645 100644 --- a/pyload/plugins/accounts/OverLoadMe.py +++ b/pyload/plugins/accounts/OverLoadMe.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class OverLoadMe(Account): diff --git a/pyload/plugins/accounts/PremiumizeMe.py b/pyload/plugins/accounts/PremiumizeMe.py index 822ca8db9..ecb03afda 100644 --- a/pyload/plugins/accounts/PremiumizeMe.py +++ b/pyload/plugins/accounts/PremiumizeMe.py @@ -2,7 +2,7 @@ from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class PremiumizeMe(Account): diff --git a/pyload/plugins/accounts/RPNetBiz.py b/pyload/plugins/accounts/RPNetBiz.py index 0e600b4e3..c2b7cad21 100644 --- a/pyload/plugins/accounts/RPNetBiz.py +++ b/pyload/plugins/accounts/RPNetBiz.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class RPNetBiz(Account): diff --git a/pyload/plugins/accounts/RapidgatorNet.py b/pyload/plugins/accounts/RapidgatorNet.py index 391d7ffcb..c8f129c5b 100644 --- a/pyload/plugins/accounts/RapidgatorNet.py +++ b/pyload/plugins/accounts/RapidgatorNet.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class RapidgatorNet(Account): diff --git a/pyload/plugins/accounts/SimplyPremiumCom.py b/pyload/plugins/accounts/SimplyPremiumCom.py index 5958c1f34..3a385c477 100644 --- a/pyload/plugins/accounts/SimplyPremiumCom.py +++ b/pyload/plugins/accounts/SimplyPremiumCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Account import Account diff --git a/pyload/plugins/accounts/UnrestrictLi.py b/pyload/plugins/accounts/UnrestrictLi.py index 39a75f959..6e878b556 100644 --- a/pyload/plugins/accounts/UnrestrictLi.py +++ b/pyload/plugins/accounts/UnrestrictLi.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class UnrestrictLi(Account): diff --git a/pyload/plugins/crypter/DailymotionBatch.py b/pyload/plugins/crypter/DailymotionBatch.py index d44350c6b..2c818d8bc 100644 --- a/pyload/plugins/crypter/DailymotionBatch.py +++ b/pyload/plugins/crypter/DailymotionBatch.py @@ -4,7 +4,7 @@ import re from urlparse import urljoin -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Crypter import Crypter from pyload.utils import safe_join diff --git a/pyload/plugins/crypter/GooGl.py b/pyload/plugins/crypter/GooGl.py index ae48c61b5..93f3456cc 100644 --- a/pyload/plugins/crypter/GooGl.py +++ b/pyload/plugins/crypter/GooGl.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Crypter import Crypter -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class GooGl(Crypter): diff --git a/pyload/plugins/crypter/LinkSaveIn.py b/pyload/plugins/crypter/LinkSaveIn.py index 84dd8172e..4a56606c8 100644 --- a/pyload/plugins/crypter/LinkSaveIn.py +++ b/pyload/plugins/crypter/LinkSaveIn.py @@ -9,7 +9,7 @@ import re from Crypto.Cipher import AES from pyload.plugins.Crypter import Crypter -from pyload.unescape import unescape +from pyload.utils import html_unescape class LinkSaveIn(Crypter): @@ -153,7 +153,7 @@ class LinkSaveIn(Crypter): dlLink = re.search(r'http://linksave\.in/dl-\w+', jseval).group(0) self.logDebug("JsEngine returns value [%s] for redirection link" % dlLink) response = self.load(dlLink) - link = unescape(re.search(r'", self.req.load("http://1kh.de/l/" + id)).group(1)) self.urls.append(new_link) diff --git a/pyload/plugins/crypter/SafelinkingNet.py b/pyload/plugins/crypter/SafelinkingNet.py index 543fb32fb..9c68ba915 100644 --- a/pyload/plugins/crypter/SafelinkingNet.py +++ b/pyload/plugins/crypter/SafelinkingNet.py @@ -6,7 +6,7 @@ from pycurl import FOLLOWLOCATION from BeautifulSoup import BeautifulSoup -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Crypter import Crypter from pyload.plugins.internal.CaptchaService import SolveMedia diff --git a/pyload/plugins/crypter/SerienjunkiesOrg.py b/pyload/plugins/crypter/SerienjunkiesOrg.py index a1745c6c9..713086cb9 100644 --- a/pyload/plugins/crypter/SerienjunkiesOrg.py +++ b/pyload/plugins/crypter/SerienjunkiesOrg.py @@ -8,7 +8,7 @@ from time import sleep from BeautifulSoup import BeautifulSoup from pyload.plugins.Crypter import Crypter -from pyload.unescape import unescape +from pyload.utils import html_unescape class SerienjunkiesOrg(Crypter): @@ -48,7 +48,7 @@ class SerienjunkiesOrg(Crypter): soup = BeautifulSoup(src) packageName = self.pyfile.package().name if self.getConfig("changeNameSJ") == "Show": - found = unescape(soup.find("h2").find("a").string.split(' –')[0]) + found = html_unescape(soup.find("h2").find("a").string.split(' –')[0]) if found: packageName = found @@ -71,7 +71,7 @@ class SerienjunkiesOrg(Crypter): post = soup.find("div", attrs={"class": "post-content"}) ps = post.findAll("p") - seasonName = unescape(soup.find("a", attrs={"rel": "bookmark"}).string).replace("–", "-") + seasonName = html_unescape(soup.find("a", attrs={"rel": "bookmark"}).string).replace("–", "-") groups = {} gid = -1 for p in ps: @@ -79,7 +79,7 @@ class SerienjunkiesOrg(Crypter): var = p.findAll("strong") opts = {"Sprache": "", "Format": ""} for v in var: - n = unescape(v.string).strip() + n = html_unescape(v.string).strip() n = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', n) if n.strip() not in opts: continue @@ -198,7 +198,7 @@ class SerienjunkiesOrg(Crypter): soup = BeautifulSoup(src) post = soup.find("div", attrs={"id": "page_post"}) ps = post.findAll("p") - found = unescape(soup.find("h2").find("a").string.split(' –')[0]) + found = html_unescape(soup.find("h2").find("a").string.split(' –')[0]) if found: seasonName = found @@ -209,7 +209,7 @@ class SerienjunkiesOrg(Crypter): var = p.findAll("strong") opts = {"Sprache": "", "Format": ""} for v in var: - n = unescape(v.string).strip() + n = html_unescape(v.string).strip() n = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', n) if n.strip() not in opts: continue diff --git a/pyload/plugins/crypter/TurbobitNetFolder.py b/pyload/plugins/crypter/TurbobitNetFolder.py index 48b28c28a..c7786b7be 100644 --- a/pyload/plugins/crypter/TurbobitNetFolder.py +++ b/pyload/plugins/crypter/TurbobitNetFolder.py @@ -3,7 +3,7 @@ import re from pyload.plugins.internal.SimpleCrypter import SimpleCrypter -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class TurbobitNetFolder(SimpleCrypter): diff --git a/pyload/plugins/crypter/YoutubeBatch.py b/pyload/plugins/crypter/YoutubeBatch.py index 5b7cb6530..bc72e04ea 100644 --- a/pyload/plugins/crypter/YoutubeBatch.py +++ b/pyload/plugins/crypter/YoutubeBatch.py @@ -4,7 +4,7 @@ import re from urlparse import urljoin -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Crypter import Crypter from pyload.utils import safe_join diff --git a/pyload/plugins/hooks/DeathByCaptcha.py b/pyload/plugins/hooks/DeathByCaptcha.py index 57bf9031f..6db91b8c1 100644 --- a/pyload/plugins/hooks/DeathByCaptcha.py +++ b/pyload/plugins/hooks/DeathByCaptcha.py @@ -9,7 +9,7 @@ from pycurl import FORM_FILE, HTTPHEADER from thread import start_new_thread from time import sleep -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.HTTPRequest import BadHeader from pyload.network.RequestFactory import getRequest from pyload.plugins.Hook import Hook diff --git a/pyload/plugins/hooks/FastixRu.py b/pyload/plugins/hooks/FastixRu.py index 966bc6bd3..9e2abd92a 100644 --- a/pyload/plugins/hooks/FastixRu.py +++ b/pyload/plugins/hooks/FastixRu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/IRCInterface.py b/pyload/plugins/hooks/IRCInterface.py index af8d8fa69..ef1fa2a09 100644 --- a/pyload/plugins/hooks/IRCInterface.py +++ b/pyload/plugins/hooks/IRCInterface.py @@ -10,7 +10,7 @@ from threading import Thread from time import sleep from traceback import print_exc -from pyload.Api import PackageDoesNotExists, FileDoesNotExists +from pyload.api import PackageDoesNotExists, FileDoesNotExists from pyload.network.RequestFactory import getURL from pyload.plugins.Hook import Hook from pyload.utils import formatSize diff --git a/pyload/plugins/hooks/LinksnappyCom.py b/pyload/plugins/hooks/LinksnappyCom.py index f662ae4e9..9086e3c2a 100644 --- a/pyload/plugins/hooks/LinksnappyCom.py +++ b/pyload/plugins/hooks/LinksnappyCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/MegaDebridEu.py b/pyload/plugins/hooks/MegaDebridEu.py index da151f9aa..8c8894c9b 100644 --- a/pyload/plugins/hooks/MegaDebridEu.py +++ b/pyload/plugins/hooks/MegaDebridEu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/MultiDebridCom.py b/pyload/plugins/hooks/MultiDebridCom.py index 7d9b6526a..6aa5ba3eb 100644 --- a/pyload/plugins/hooks/MultiDebridCom.py +++ b/pyload/plugins/hooks/MultiDebridCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/PremiumizeMe.py b/pyload/plugins/hooks/PremiumizeMe.py index 70bc4a0f2..b6d89adb6 100644 --- a/pyload/plugins/hooks/PremiumizeMe.py +++ b/pyload/plugins/hooks/PremiumizeMe.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/RPNetBiz.py b/pyload/plugins/hooks/RPNetBiz.py index e119e6451..b46e326e6 100644 --- a/pyload/plugins/hooks/RPNetBiz.py +++ b/pyload/plugins/hooks/RPNetBiz.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/SimplyPremiumCom.py b/pyload/plugins/hooks/SimplyPremiumCom.py index 8e9bc5e1e..f0df36b22 100644 --- a/pyload/plugins/hooks/SimplyPremiumCom.py +++ b/pyload/plugins/hooks/SimplyPremiumCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/UnSkipOnFail.py b/pyload/plugins/hooks/UnSkipOnFail.py index fd3b35a0a..941ce4fc7 100644 --- a/pyload/plugins/hooks/UnSkipOnFail.py +++ b/pyload/plugins/hooks/UnSkipOnFail.py @@ -2,7 +2,7 @@ from os.path import basename -from pyload.PyFile import PyFile +from pyload.datatypes.PyFile import PyFile from pyload.plugins.Hook import Hook from pyload.utils import fs_encode diff --git a/pyload/plugins/hooks/UnrestrictLi.py b/pyload/plugins/hooks/UnrestrictLi.py index 1562bdf24..2ca5ad907 100644 --- a/pyload/plugins/hooks/UnrestrictLi.py +++ b/pyload/plugins/hooks/UnrestrictLi.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hoster/AlldebridCom.py b/pyload/plugins/hoster/AlldebridCom.py index 1b115f19e..bdb5b1599 100644 --- a/pyload/plugins/hoster/AlldebridCom.py +++ b/pyload/plugins/hoster/AlldebridCom.py @@ -5,7 +5,7 @@ import re from random import randrange from urllib import unquote -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster from pyload.utils import parseFileSize diff --git a/pyload/plugins/hoster/BayfilesCom.py b/pyload/plugins/hoster/BayfilesCom.py index ea4bd3ca5..5906ade75 100644 --- a/pyload/plugins/hoster/BayfilesCom.py +++ b/pyload/plugins/hoster/BayfilesCom.py @@ -4,7 +4,7 @@ import re from time import time -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/plugins/hoster/DailymotionCom.py b/pyload/plugins/hoster/DailymotionCom.py index 0ae4c697b..5692fa652 100644 --- a/pyload/plugins/hoster/DailymotionCom.py +++ b/pyload/plugins/hoster/DailymotionCom.py @@ -2,8 +2,8 @@ import re -from pyload.PyFile import statusMap -from pyload.common.json_layer import json_loads +from pyload.datatypes.PyFile import statusMap +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/DlFreeFr.py b/pyload/plugins/hoster/DlFreeFr.py index 387e11efc..0365754bc 100644 --- a/pyload/plugins/hoster/DlFreeFr.py +++ b/pyload/plugins/hoster/DlFreeFr.py @@ -3,7 +3,7 @@ import pycurl import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.Browser import Browser from pyload.network.CookieJar import CookieJar from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns diff --git a/pyload/plugins/hoster/ExtabitCom.py b/pyload/plugins/hoster/ExtabitCom.py index 38479410e..78172ef02 100644 --- a/pyload/plugins/hoster/ExtabitCom.py +++ b/pyload/plugins/hoster/ExtabitCom.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.hoster.UnrestrictLi import secondsToMidnight from pyload.plugins.internal.CaptchaService import ReCaptcha diff --git a/pyload/plugins/hoster/FastixRu.py b/pyload/plugins/hoster/FastixRu.py index e031e3e55..aa1794047 100644 --- a/pyload/plugins/hoster/FastixRu.py +++ b/pyload/plugins/hoster/FastixRu.py @@ -5,7 +5,7 @@ import re from random import randrange from urllib import unquote -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/FilecloudIo.py b/pyload/plugins/hoster/FilecloudIo.py index 05753a67e..9cf9306d1 100644 --- a/pyload/plugins/hoster/FilecloudIo.py +++ b/pyload/plugins/hoster/FilecloudIo.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.CaptchaService import ReCaptcha from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/plugins/hoster/FilepostCom.py b/pyload/plugins/hoster/FilepostCom.py index ac2ae4845..382971c61 100644 --- a/pyload/plugins/hoster/FilepostCom.py +++ b/pyload/plugins/hoster/FilepostCom.py @@ -4,7 +4,7 @@ import re from time import time -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.CaptchaService import ReCaptcha from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/plugins/hoster/FileserveCom.py b/pyload/plugins/hoster/FileserveCom.py index 15830b759..5892cd96a 100644 --- a/pyload/plugins/hoster/FileserveCom.py +++ b/pyload/plugins/hoster/FileserveCom.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.Hoster import Hoster from pyload.plugins.Plugin import chunks diff --git a/pyload/plugins/hoster/IfileIt.py b/pyload/plugins/hoster/IfileIt.py index 2707edd5a..fc26e2f23 100644 --- a/pyload/plugins/hoster/IfileIt.py +++ b/pyload/plugins/hoster/IfileIt.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.CaptchaService import ReCaptcha from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/plugins/hoster/LetitbitNet.py b/pyload/plugins/hoster/LetitbitNet.py index 3a8e28a90..a28f06571 100644 --- a/pyload/plugins/hoster/LetitbitNet.py +++ b/pyload/plugins/hoster/LetitbitNet.py @@ -10,7 +10,7 @@ import re from urllib import urlencode, urlopen -from pyload.common.json_layer import json_loads, json_dumps +from pyload.utils import json_loads, json_dumps from pyload.plugins.hoster.UnrestrictLi import secondsToMidnight from pyload.plugins.internal.CaptchaService import ReCaptcha from pyload.plugins.internal.SimpleHoster import SimpleHoster diff --git a/pyload/plugins/hoster/LinksnappyCom.py b/pyload/plugins/hoster/LinksnappyCom.py index aed74d09b..e7cc61391 100644 --- a/pyload/plugins/hoster/LinksnappyCom.py +++ b/pyload/plugins/hoster/LinksnappyCom.py @@ -4,7 +4,7 @@ import re from urlparse import urlsplit -from pyload.common.json_layer import json_loads, json_dumps +from pyload.utils import json_loads, json_dumps from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/MegaDebridEu.py b/pyload/plugins/hoster/MegaDebridEu.py index 6c980009e..ed3747aed 100644 --- a/pyload/plugins/hoster/MegaDebridEu.py +++ b/pyload/plugins/hoster/MegaDebridEu.py @@ -4,7 +4,7 @@ import re from urllib import unquote_plus -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/MegaNz.py b/pyload/plugins/hoster/MegaNz.py index 801df9c9d..2e70b5dc6 100644 --- a/pyload/plugins/hoster/MegaNz.py +++ b/pyload/plugins/hoster/MegaNz.py @@ -9,7 +9,7 @@ from array import array from base64 import standard_b64decode from os import remove -from pyload.common.json_layer import json_loads, json_dumps +from pyload.utils import json_loads, json_dumps from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/MegacrypterCom.py b/pyload/plugins/hoster/MegacrypterCom.py index 0cc17bfe3..cdc019fdf 100644 --- a/pyload/plugins/hoster/MegacrypterCom.py +++ b/pyload/plugins/hoster/MegacrypterCom.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads, json_dumps +from pyload.utils import json_loads, json_dumps from pyload.plugins.hoster.MegaNz import MegaNz diff --git a/pyload/plugins/hoster/MultiDebridCom.py b/pyload/plugins/hoster/MultiDebridCom.py index 765022eef..bd7d9460c 100644 --- a/pyload/plugins/hoster/MultiDebridCom.py +++ b/pyload/plugins/hoster/MultiDebridCom.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/MyvideoDe.py b/pyload/plugins/hoster/MyvideoDe.py index 4ce75b4a2..06cfb9c63 100644 --- a/pyload/plugins/hoster/MyvideoDe.py +++ b/pyload/plugins/hoster/MyvideoDe.py @@ -3,7 +3,7 @@ import re from pyload.plugins.Hoster import Hoster -from pyload.unescape import unescape +from pyload.utils import html_unescape class MyvideoDe(Hoster): @@ -35,7 +35,7 @@ class MyvideoDe(Hoster): def get_file_name(self): file_name_pattern = r"

(.*)

" - return unescape(re.search(file_name_pattern, self.html).group(1).replace("/", "") + '.flv') + return html_unescape(re.search(file_name_pattern, self.html).group(1).replace("/", "") + '.flv') def file_exists(self): self.download_html() diff --git a/pyload/plugins/hoster/OboomCom.py b/pyload/plugins/hoster/OboomCom.py index 04efa31b7..8c0d9d760 100644 --- a/pyload/plugins/hoster/OboomCom.py +++ b/pyload/plugins/hoster/OboomCom.py @@ -5,7 +5,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster from pyload.plugins.internal.CaptchaService import ReCaptcha diff --git a/pyload/plugins/hoster/OverLoadMe.py b/pyload/plugins/hoster/OverLoadMe.py index 8061b2e1d..e27972fc2 100644 --- a/pyload/plugins/hoster/OverLoadMe.py +++ b/pyload/plugins/hoster/OverLoadMe.py @@ -5,7 +5,7 @@ import re from random import randrange from urllib import unquote -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster from pyload.utils import parseFileSize diff --git a/pyload/plugins/hoster/PremiumizeMe.py b/pyload/plugins/hoster/PremiumizeMe.py index 16649f492..45c011084 100644 --- a/pyload/plugins/hoster/PremiumizeMe.py +++ b/pyload/plugins/hoster/PremiumizeMe.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/RPNetBiz.py b/pyload/plugins/hoster/RPNetBiz.py index e305c35ce..8a7bec097 100644 --- a/pyload/plugins/hoster/RPNetBiz.py +++ b/pyload/plugins/hoster/RPNetBiz.py @@ -3,7 +3,7 @@ import re from pyload.plugins.Hoster import Hoster -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class RPNetBiz(Hoster): diff --git a/pyload/plugins/hoster/RapidgatorNet.py b/pyload/plugins/hoster/RapidgatorNet.py index 46fe285b7..cc13f7097 100644 --- a/pyload/plugins/hoster/RapidgatorNet.py +++ b/pyload/plugins/hoster/RapidgatorNet.py @@ -4,7 +4,7 @@ import re from pycurl import HTTPHEADER -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.HTTPRequest import BadHeader from pyload.plugins.hoster.UnrestrictLi import secondsToMidnight from pyload.plugins.internal.CaptchaService import AdsCaptcha, ReCaptcha, SolveMedia diff --git a/pyload/plugins/hoster/RealdebridCom.py b/pyload/plugins/hoster/RealdebridCom.py index a458cc5d0..08f7b9329 100644 --- a/pyload/plugins/hoster/RealdebridCom.py +++ b/pyload/plugins/hoster/RealdebridCom.py @@ -6,7 +6,7 @@ from random import randrange from urllib import quote, unquote from time import time -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster from pyload.utils import parseFileSize diff --git a/pyload/plugins/hoster/RedtubeCom.py b/pyload/plugins/hoster/RedtubeCom.py index 42c24628e..9f5d2d0d9 100644 --- a/pyload/plugins/hoster/RedtubeCom.py +++ b/pyload/plugins/hoster/RedtubeCom.py @@ -3,7 +3,7 @@ import re from pyload.plugins.Hoster import Hoster -from pyload.unescape import unescape +from pyload.utils import html_unescape class RedtubeCom(Hoster): @@ -36,7 +36,7 @@ class RedtubeCom(Hoster): if not self.html: self.download_html() - file_url = unescape(re.search(r'hashlink=(http.*?)"', self.html).group(1)) + file_url = html_unescape(re.search(r'hashlink=(http.*?)"', self.html).group(1)) return file_url diff --git a/pyload/plugins/hoster/UlozTo.py b/pyload/plugins/hoster/UlozTo.py index 2f1fdc595..c3957aaa0 100644 --- a/pyload/plugins/hoster/UlozTo.py +++ b/pyload/plugins/hoster/UlozTo.py @@ -3,7 +3,7 @@ import re import time -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/plugins/hoster/UnrestrictLi.py b/pyload/plugins/hoster/UnrestrictLi.py index 2cad6616f..c0c1e3965 100644 --- a/pyload/plugins/hoster/UnrestrictLi.py +++ b/pyload/plugins/hoster/UnrestrictLi.py @@ -4,7 +4,7 @@ import re from datetime import datetime, timedelta -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/UploadingCom.py b/pyload/plugins/hoster/UploadingCom.py index a7c328eec..882cb863f 100644 --- a/pyload/plugins/hoster/UploadingCom.py +++ b/pyload/plugins/hoster/UploadingCom.py @@ -4,7 +4,7 @@ import re from pycurl import HTTPHEADER -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp diff --git a/pyload/plugins/hoster/XHamsterCom.py b/pyload/plugins/hoster/XHamsterCom.py index 0afb94b74..aa406cc45 100644 --- a/pyload/plugins/hoster/XHamsterCom.py +++ b/pyload/plugins/hoster/XHamsterCom.py @@ -4,7 +4,7 @@ import re from urllib import unquote -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/YibaishiwuCom.py b/pyload/plugins/hoster/YibaishiwuCom.py index b6d06d234..24cd0e9fc 100644 --- a/pyload/plugins/hoster/YibaishiwuCom.py +++ b/pyload/plugins/hoster/YibaishiwuCom.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/remote/ClickAndLoadBackend.py b/pyload/remote/ClickAndLoadBackend.py index 82167f6e3..365364a3b 100644 --- a/pyload/remote/ClickAndLoadBackend.py +++ b/pyload/remote/ClickAndLoadBackend.py @@ -27,7 +27,7 @@ try: except: pass -from RemoteManager import BackendBase +from pyload.manager.RemoteManager import BackendBase core = None js = None diff --git a/pyload/remote/RemoteManager.py b/pyload/remote/RemoteManager.py deleted file mode 100644 index e53e317e3..000000000 --- a/pyload/remote/RemoteManager.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from threading import Thread -from traceback import print_exc - -class BackendBase(Thread): - def __init__(self, manager): - Thread.__init__(self) - self.m = manager - self.core = manager.core - self.enabled = True - self.running = False - - def run(self): - self.running = True - try: - self.serve() - except Exception, e: - self.core.log.error(_("Remote backend error: %s") % e) - if self.core.debug: - print_exc() - finally: - self.running = False - - def setup(self, host, port): - pass - - def checkDeps(self): - return True - - def serve(self): - pass - - def shutdown(self): - pass - - def stop(self): - self.enabled = False# set flag and call shutdowm message, so thread can react - self.shutdown() - - -class RemoteManager: - available = [] - - def __init__(self, core): - self.core = core - self.backends = [] - - if self.core.remote: - self.available.append("ThriftBackend") -# else: -# self.available.append("SocketBackend") - - - def startBackends(self): - host = self.core.config["remote"]["listenaddr"] - port = self.core.config["remote"]["port"] - - for b in self.available: - klass = getattr(__import__("pyload.remote.%s" % b, globals(), locals(), [b], -1), b) - backend = klass(self) - if not backend.checkDeps(): - continue - try: - backend.setup(host, port) - self.core.log.info(_("Starting %(name)s: %(addr)s:%(port)s") % {"name": b, "addr": host, "port": port}) - except Exception, e: - self.core.log.error(_("Failed loading backend %(name)s | %(error)s") % {"name": b, "error": str(e)}) - if self.core.debug: - print_exc() - else: - backend.start() - self.backends.append(backend) - - port += 1 diff --git a/pyload/remote/SocketBackend.py b/pyload/remote/SocketBackend.py index 1a157cf1d..c85e59f42 100644 --- a/pyload/remote/SocketBackend.py +++ b/pyload/remote/SocketBackend.py @@ -2,7 +2,7 @@ import SocketServer -from RemoteManager import BackendBase +from pyload.manager.RemoteManager import BackendBase class RequestHandler(SocketServer.BaseRequestHandler): diff --git a/pyload/remote/ThriftBackend.py b/pyload/remote/ThriftBackend.py index 2c65578be..609a3c158 100644 --- a/pyload/remote/ThriftBackend.py +++ b/pyload/remote/ThriftBackend.py @@ -17,7 +17,7 @@ """ from os.path import exists -from pyload.remote.RemoteManager import BackendBase +from pyload.manager.RemoteManager import BackendBase from thriftbackend.Processor import Processor from thriftbackend.Protocol import ProtocolFactory diff --git a/pyload/setup.py b/pyload/setup.py deleted file mode 100644 index 6a98cc6f3..000000000 --- a/pyload/setup.py +++ /dev/null @@ -1,538 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import sys - -import pyload.common.pylgettext as gettext - -from getpass import getpass -from os import makedirs -from os.path import abspath, dirname, exists, join -from subprocess import PIPE, call - -from pyload.utils import get_console_encoding, versiontuple - - -class Setup: - """ pyLoads initial setup configuration assistant """ - - def __init__(self, path, config): - self.path = path - self.config = config - self.stdin_encoding = get_console_encoding(sys.stdin.encoding) - - - def start(self): - langs = self.config.getMetaData("general", "language")["type"].split(";") - lang = self.ask(u"Choose setup language", "en", langs) - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), languages=[lang, "en"], fallback=True) - translation.install(True) - - #Input shorthand for yes - self.yes = _("y") - #Input shorthand for no - self.no = _("n") - - # print - # print _("Would you like to configure pyLoad via Webinterface?") - # print _("You need a Browser and a connection to this PC for it.") - # viaweb = self.ask(_("Start initial webinterface for configuration?"), "y", bool=True) - # if viaweb: - # try: - # from pyload.threads import ServerThread - # ServerThread.setup = self - # import pyload.webui as webinterface - # webinterface.run_simple() - # return False - # except Exception, e: - # print "Setup failed with this error: ", e - # print "Falling back to commandline setup." - - print - print - print _("Welcome to the pyLoad Configuration Assistant.") - print _("It will check your system and make a basic setup in order to run pyLoad.") - print - print _("The value in brackets [] always is the default value,") - print _("in case you don't want to change it or you are unsure what to choose, just hit enter.") - print _( - "Don't forget: You can always rerun this assistant with --setup or -s parameter, when you start pyload.py .") - print _("If you have any problems with this assistant hit STRG-C,") - print _("to abort and don't let him start with pyload.py automatically anymore.") - print - print - raw_input(_("When you are ready for system check, hit enter.")) - print - print - - basic, ssl, captcha, web, js = self.system_check() - print - print - - if not basic: - print _("You need pycurl, sqlite and python 2.5, 2.6 or 2.7 to run pyLoad.") - print _("Please correct this and re-run pyLoad.") - print - print _("Setup will now close.") - print - print - raw_input(_("Press Enter to exit.")) - return False - - raw_input(_("System check finished, hit enter to see your status report.")) - print - print - print _("## Status ##") - print - - avail = [] - if self.check_module("Crypto"): - avail.append(_("container decrypting")) - if ssl: - avail.append(_("ssl connection")) - if captcha: - avail.append(_("automatic captcha decryption")) - if web: - avail.append(_("webinterface")) - if js: - avail.append(_("extended Click'N'Load")) - - string = "" - - for av in avail: - string += ", " + av - - print _("AVAILABLE FEATURES:") + string[1:] - print - - if len(avail) < 5: - print _("MISSING FEATURES: ") - - if not self.check_module("Crypto"): - print _("- no py-crypto available") - print _("You need this if you want to decrypt container files.") - print - - if not ssl: - print _("- no SSL available") - print _("This is needed if you want to establish a secure connection to core or webinterface.") - print _("If you only want to access locally to pyLoad ssl is not usefull.") - print - - if not captcha: - print _("- no Captcha Recognition available") - print _("Only needed for some hosters and as freeuser.") - print - - if not js: - print _("- no JavaScript engine found") - print _("You will need this for some Click'N'Load links. Install Spidermonkey, ossp-js, pyv8 or rhino") - print - - print - print _("You can abort the setup now and fix some dependicies if you want.") - - print - con = self.ask(_("Continue with setup?"), self.yes, bool=True) - - if not con: - return False - - print - print - print _("CURRENT CONFIG PATH: %s") % abspath("") - print - print _("NOTE: If you use pyLoad on a server or the home partition lives on an iternal flash it may be a good idea to change it.") - path = self.ask(_("Do you want to change the config path?"), self.no, bool=True) - if path: - print - self.conf_path() - #calls exit when changed - - print - print _("Do you want to configure login data and basic settings?") - print _("This is recommend for first run.") - con = self.ask(_("Make basic setup?"), self.yes, bool=True) - - if con: - print - print - self.conf_basic() - - if ssl: - print - print _("Do you want to configure ssl?") - ssl = self.ask(_("Configure ssl?"), self.no, bool=True) - if ssl: - print - print - self.conf_ssl() - - if web: - print - print _("Do you want to configure webinterface?") - web = self.ask(_("Configure webinterface?"), self.yes, bool=True) - if web: - print - print - self.conf_web() - - print - print - print _("Setup finished successfully!") - print - print - raw_input(_("Hit enter to exit and restart pyLoad.")) - return True - - - def system_check(self): - """ make a systemcheck and return the results""" - - print _("## System Information ##") - print - print _("Platform: %s") % sys.platform - print _("Operating System: %s") % os.name - print _("Python: %s") % sys.version.replace("\n", "") - print - print - - print _("## System Check ##") - print - - if sys.version_info[:2] > (2, 7): - print _("Your python version is to new, Please use Python 2.6/2.7") - python = False - elif sys.version_info[:2] < (2, 5): - print _("Your python version is to old, Please use at least Python 2.5") - python = False - else: - print _("Python Version: OK") - python = True - - curl = self.check_module("pycurl") - self.print_dep("pycurl", curl) - - sqlite = self.check_module("sqlite3") - self.print_dep("sqlite3", sqlite) - - basic = python and curl and sqlite - - print - - crypto = self.check_module("Crypto") - self.print_dep("pycrypto", crypto) - - ssl = self.check_module("OpenSSL") - self.print_dep("py-OpenSSL", ssl) - - print - - pil = self.check_module("PIL.Image") - self.print_dep("PIL/Pillow", pil) - - if os.name == "nt": - tesser = self.check_prog([join(pypath, "tesseract", "tesseract.exe"), "-v"]) - else: - tesser = self.check_prog(["tesseract", "-v"]) - - self.print_dep("tesseract", tesser) - - captcha = pil and tesser - - print - - try: - import jinja2 - - v = jinja2.__version__ - if v and versiontuple(v) < (2, 5, 0): - jinja = False - else: - jinja = True - except: - jinja = False - - jinja = self.print_dep("jinja2", jinja) - - beaker = self.check_module("beaker") - self.print_dep("beaker", beaker) - - bjoern = self.check_module("bjoern") - self.print_dep("bjoern", bjoern) - - web = sqlite and beaker - - from pyload.common import JsEngine - js = True if JsEngine.ENGINE else False - self.print_dep(_("JS engine"), js) - - if not jinja: - print - print - print _("WARNING: Your installed jinja2 version %s seems too old.") % jinja2.__version__ - print _("You can safely continue but if the webinterface is not working,") - print _("please upgrade or uninstall it, because pyLoad self-includes jinja2 libary.") - - return basic, ssl, captcha, web, js - - - def conf_basic(self): - print _("## Basic Setup ##") - - print - print _("The following logindata is valid for CLI and webinterface.") - - from pyload.database import DatabaseBackend - - db = DatabaseBackend(None) - db.setup() - print _("NOTE: Consider a password of 10 or more symbols if you expect to access from outside your local network (ex. internet).") - print - username = self.ask(_("Username"), "User") - password = self.ask("", "", password=True) - db.addUser(username, password) - db.shutdown() - - print - print _("External clients (GUI, CLI or other) need remote access to work over the network.") - print _("However, if you only want to use the webinterface you may disable it to save ram.") - self.config["remote"]["activated"] = self.ask(_("Enable remote access"), self.no, bool=True) - - print - langs = self.config.getMetaData("general", "language") - self.config["general"]["language"] = self.ask(_("Choose pyLoad language"), "en", langs["type"].split(";")) - - print - self.config["general"]["download_folder"] = self.ask(_("Download folder"), "Downloads") - print - self.config["download"]["max_downloads"] = self.ask(_("Max parallel downloads"), "3") - print - reconnect = self.ask(_("Use Reconnect?"), self.no, bool=True) - self.config["reconnect"]["activated"] = reconnect - if reconnect: - self.config["reconnect"]["method"] = self.ask(_("Reconnect script location"), "./reconnect.sh") - - - def conf_web(self): - print _("## Webinterface Setup ##") - - print - self.config["webinterface"]["activated"] = self.ask(_("Activate webinterface?"), self.yes, bool=True) - print - print _("Listen address, if you use 127.0.0.1 or localhost, the webinterface will only accessible locally.") - self.config["webinterface"]["host"] = self.ask(_("Address"), "0.0.0.0") - self.config["webinterface"]["port"] = self.ask(_("Port"), "8000") - print - print _("pyLoad offers several server backends, now following a short explanation.") - print "- builtin:", _("Default server; best choice if you plan to use pyLoad just for you.") - print "- threaded:", _("Support SSL connection and can serve simultaneously more client flawlessly.") - print "- fastcgi:", _( - "Can be used by apache, lighttpd, etc.; needs to be properly configured before.") - if os.name != "nt": - print "- lightweight:", _("Very fast alternative to builtin; requires libev and bjoern packages.") - - print - print _("NOTE: In some rare cases the builtin server is not working, if you notice problems with the webinterface") - print _("come back here and change the builtin server to the threaded one here.") - - if os.name == "nt": - servers = ["builtin", "threaded", "fastcgi"] - default = "threaded" - else: - servers = ["builtin", "threaded", "fastcgi", "lightweight"] - default = "lightweight" if self.check_module("bjoern") else "builtin" - - self.config["webinterface"]["server"] = self.ask(_("Server"), default, servers) - - - def conf_ssl(self): - print _("## SSL Setup ##") - print - print _("Execute these commands from pyLoad config folder to make ssl certificates:") - print - print "openssl genrsa -out ssl.key 1024" - print "openssl req -new -key ssl.key -out ssl.csr" - print "openssl req -days 36500 -x509 -key ssl.key -in ssl.csr > ssl.crt " - print - print _("If you're done and everything went fine, you can activate ssl now.") - - self.config["ssl"]["activated"] = self.ask(_("Activate SSL?"), self.yes, bool=True) - - - def set_user(self): - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), - languages=[self.config["general"]["language"], "en"], fallback=True) - translation.install(True) - - from pyload.database import DatabaseBackend - - db = DatabaseBackend(None) - db.setup() - - noaction = True - try: - while True: - print _("Select action") - print _("1 - Create/Edit user") - print _("2 - List users") - print _("3 - Remove user") - print _("4 - Quit") - action = raw_input("[1]/2/3/4: ") - if not action in ("1", "2", "3", "4"): - continue - elif action == "1": - print - username = self.ask(_("Username"), "User") - password = self.ask("", "", password=True) - db.addUser(username, password) - noaction = False - elif action == "2": - print - print _("Users") - print "-----" - users = db.listUsers() - noaction = False - for user in users: - print user - print "-----" - print - elif action == "3": - print - username = self.ask(_("Username"), "") - if username: - db.removeUser(username) - noaction = False - elif action == "4": - break - finally: - if not noaction: - db.shutdown() - - - def conf_path(self, trans=False): - if trans: - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), - languages=[self.config["general"]["language"], "en"], fallback=True) - translation.install(True) - - print _("Setting new config path, current configuration will not be transfered!") - path = self.ask(_("CONFIG PATH"), abspath("")) - try: - path = join(pypath, path) - if not exists(path): - makedirs(path) - f = open(join(pypath, "pyload", "config", "configdir"), "wb") - f.write(path) - f.close() - print - print - print _("pyLoad config path changed, setup will now close!") - print - print - raw_input(_("Press Enter to exit.")) - sys.exit() - except Exception, e: - print _("Setting config path failed: %s") % str(e) - - - def print_dep(self, name, value): - """Print Status of dependency""" - if value: - print _("%s: OK") % name - else: - print _("%s: MISSING") % name - - - def check_module(self, module): - try: - __import__(module) - return True - except: - return False - - - def check_prog(self, command): - pipe = PIPE - try: - call(command, stdout=pipe, stderr=pipe) - return True - except: - return False - - - def ask(self, qst, default, answers=[], bool=False, password=False): - """produce one line to asking for input""" - if answers: - info = "(" - - for i, answer in enumerate(answers): - info += (", " if i != 0 else "") + str((answer == default and "[%s]" % answer) or answer) - - info += ")" - elif bool: - if default == self.yes: - info = "([%s]/%s)" % (self.yes, self.no) - else: - info = "(%s/[%s])" % (self.yes, self.no) - else: - info = "[%s]" % default - - if password: - p1 = True - p2 = False - pwlen = 8 - while p1 != p2: - # getpass(_("Password: ")) will crash on systems with broken locales (Win, NAS) - sys.stdout.write(_("Password: ")) - p1 = getpass("") - - if len(p1) < pwlen: - print _("Password too short! Use at least %s symbols." % pwlen) - continue - elif not p1.isalnum(): - print _("Password must be alphanumeric.") - continue - - sys.stdout.write(_("Password (again): ")) - p2 = getpass("") - - if p1 == p2: - return p1 - else: - print _("Passwords did not match.") - - while True: - try: - input = raw_input(qst + " %s: " % info) - except KeyboardInterrupt: - print "\nSetup interrupted" - sys.exit() - - input = input.decode(self.stdin_encoding) - - if input.strip() == "": - input = default - - if bool: - # yes, true, t are inputs for booleans with value true - if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]: - return True - # no, false, f are inputs for booleans with value false - elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]: - return False - else: - print _("Invalid Input") - continue - - if not answers: - return input - - else: - if input in answers: - return input - else: - print _("Invalid Input") diff --git a/pyload/threads/PluginThread.py b/pyload/threads/PluginThread.py deleted file mode 100644 index ca1d396dc..000000000 --- a/pyload/threads/PluginThread.py +++ /dev/null @@ -1,675 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from Queue import Queue -from threading import Thread -from os import listdir, stat -from os.path import join -from time import sleep, time, strftime, gmtime -from traceback import print_exc, format_exc -from pprint import pformat -from sys import exc_info, exc_clear -from copy import copy -from types import MethodType - -from pycurl import error - -from pyload.PyFile import PyFile -from pyload.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload -from pyload.common.packagetools import parseNames -from pyload.utils import safe_join -from pyload.Api import OnlineStatus - -class PluginThread(Thread): - """abstract base class for thread types""" - - #-------------------------------------------------------------------------- - def __init__(self, manager): - """Constructor""" - Thread.__init__(self) - self.setDaemon(True) - self.m = manager #thread manager - - - def writeDebugReport(self, pyfile): - """ writes a - :return: - """ - - dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) - dump = self.getDebugDump(pyfile) - - try: - import zipfile - - zip = zipfile.ZipFile(dump_name, "w") - - for f in listdir(join("tmp", pyfile.pluginname)): - try: - # avoid encoding errors - zip.write(join("tmp", pyfile.pluginname, f), safe_join(pyfile.pluginname, f)) - except: - pass - - info = zipfile.ZipInfo(safe_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) - info.external_attr = 0644 << 16L # change permissions - - zip.writestr(info, dump) - zip.close() - - if not stat(dump_name).st_size: - raise Exception("Empty Zipfile") - - except Exception, e: - self.m.log.debug("Error creating zip file: %s" % e) - - dump_name = dump_name.replace(".zip", ".txt") - f = open(dump_name, "wb") - f.write(dump) - f.close() - - self.m.core.log.info("Debug Report written to %s" % dump_name) - - def getDebugDump(self, pyfile): - dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( - self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version__, format_exc()) - - tb = exc_info()[2] - stack = [] - while tb: - stack.append(tb.tb_frame) - tb = tb.tb_next - - for frame in stack[1:]: - dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, - frame.f_code.co_filename, - frame.f_lineno) - - for key, value in frame.f_locals.items(): - dump += "\t%20s = " % key - try: - dump += pformat(value) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - del frame - - del stack #delete it just to be sure... - - dump += "\n\nPLUGIN OBJECT DUMP: \n\n" - - for name in dir(pyfile.plugin): - attr = getattr(pyfile.plugin, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - dump += "\nPYFILE OBJECT DUMP: \n\n" - - for name in dir(pyfile): - attr = getattr(pyfile, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - if pyfile.pluginname in self.m.core.config.plugin: - dump += "\n\nCONFIG: \n\n" - dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" - - return dump - - def clean(self, pyfile): - """ set thread unactive and release pyfile """ - self.active = False - pyfile.release() - - -class DownloadThread(PluginThread): - """thread for downloading files from 'real' hoster plugins""" - - #-------------------------------------------------------------------------- - def __init__(self, manager): - """Constructor""" - PluginThread.__init__(self, manager) - - self.queue = Queue() # job queue - self.active = False - - self.start() - - #-------------------------------------------------------------------------- - def run(self): - """run method""" - pyfile = None - - while True: - del pyfile - self.active = self.queue.get() - pyfile = self.active - - if self.active == "quit": - self.active = False - self.m.threads.remove(self) - return True - - try: - if not pyfile.hasPlugin(): continue - #this pyfile was deleted while queueing - - pyfile.plugin.checkForSameFiles(starting=True) - self.m.log.info(_("Download starts: %s" % pyfile.name)) - - # start download - self.m.core.hookManager.downloadPreparing(pyfile) - pyfile.plugin.preprocessing(self) - - self.m.log.info(_("Download finished: %s") % pyfile.name) - self.m.core.hookManager.downloadFinished(pyfile) - self.m.core.files.checkPackageFinished(pyfile) - - except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) - pyfile.setStatus("failed") - pyfile.error = "Plugin does not work" - self.clean(pyfile) - continue - - except Abort: - try: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - except: - pass - - pyfile.setStatus("aborted") - - self.clean(pyfile) - continue - - except Reconnect: - self.queue.put(pyfile) - #pyfile.req.clearCookies() - - while self.m.reconnecting.isSet(): - sleep(0.5) - - continue - - except Retry, e: - reason = e.args[0] - self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) - self.queue.put(pyfile) - continue - - except Fail, e: - msg = e.args[0] - - if msg == "offline": - pyfile.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % pyfile.name) - elif msg == "temp. offline": - pyfile.setStatus("temp. offline") - self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) - else: - pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) - pyfile.error = msg - - self.m.core.hookManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - except error, e: - if len(e.args) == 2: - code, msg = e.args - else: - code = 0 - msg = e.args - - self.m.log.debug("pycurl exception %s: %s" % (code, msg)) - - if code in (7, 18, 28, 52, 56): - self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) - wait = time() + 60 - - pyfile.waitUntil = wait - pyfile.setStatus("waiting") - while time() < wait: - sleep(1) - if pyfile.abort: - break - - if pyfile.abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") - - self.clean(pyfile) - else: - self.queue.put(pyfile) - - continue - - else: - pyfile.setStatus("failed") - self.m.log.error("pycurl error %s: %s" % (code, msg)) - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - self.m.core.hookManager.downloadFailed(pyfile) - - self.clean(pyfile) - continue - - except SkipDownload, e: - pyfile.setStatus("skipped") - - self.m.log.info( - _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) - - self.clean(pyfile) - - self.m.core.files.checkPackageFinished(pyfile) - - self.active = False - self.m.core.files.save() - - continue - - - except Exception, e: - pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) - pyfile.error = str(e) - - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - self.m.core.hookManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - finally: - self.m.core.files.save() - pyfile.checkIfProcessed() - exc_clear() - - #pyfile.plugin.req.clean() - - self.active = False - pyfile.finishIfDone() - self.m.core.files.save() - - - def put(self, job): - """assing job to thread""" - self.queue.put(job) - - - def stop(self): - """stops the thread""" - self.put("quit") - - -class DecrypterThread(PluginThread): - """thread for decrypting""" - - def __init__(self, manager, pyfile): - """constructor""" - PluginThread.__init__(self, manager) - - self.active = pyfile - manager.localThreads.append(self) - - pyfile.setStatus("decrypting") - - self.start() - - def getActiveFiles(self): - return [self.active] - - def run(self): - """run method""" - - pyfile = self.active - retry = False - - try: - self.m.log.info(_("Decrypting starts: %s") % self.active.name) - self.active.plugin.preprocessing(self) - - except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) - return - - except Fail, e: - msg = e.args[0] - - if msg == "offline": - self.active.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % self.active.name) - else: - self.active.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": msg}) - self.active.error = msg - - return - - except Abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") - - return - - except Retry: - self.m.log.info(_("Retrying %s") % self.active.name) - retry = True - return self.run() - - except Exception, e: - self.active.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": str(e)}) - self.active.error = str(e) - - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - return - - - finally: - if not retry: - self.active.release() - self.active = False - self.m.core.files.save() - self.m.localThreads.remove(self) - exc_clear() - - - #self.m.core.hookManager.downloadFinished(pyfile) - - - #self.m.localThreads.remove(self) - #self.active.finishIfDone() - if not retry: - pyfile.delete() - - -class HookThread(PluginThread): - """thread for hooks""" - - #-------------------------------------------------------------------------- - def __init__(self, m, function, args, kwargs): - """Constructor""" - PluginThread.__init__(self, m) - - self.f = function - self.args = args - self.kwargs = kwargs - - self.active = [] - - m.localThreads.append(self) - - self.start() - - def getActiveFiles(self): - return self.active - - def addActive(self, pyfile): - """ Adds a pyfile to active list and thus will be displayed on overview""" - if pyfile not in self.active: - self.active.append(pyfile) - - def finishFile(self, pyfile): - if pyfile in self.active: - self.active.remove(pyfile) - - pyfile.finishIfDone() - - def run(self): - try: - try: - self.kwargs["thread"] = self - self.f(*self.args, **self.kwargs) - except TypeError, e: - #dirty method to filter out exceptions - if "unexpected keyword argument 'thread'" not in e.args[0]: - raise - - del self.kwargs["thread"] - self.f(*self.args, **self.kwargs) - finally: - local = copy(self.active) - for x in local: - self.finishFile(x) - - self.m.localThreads.remove(self) - - -class InfoThread(PluginThread): - def __init__(self, manager, data, pid=-1, rid=-1, add=False): - """Constructor""" - PluginThread.__init__(self, manager) - - self.data = data - self.pid = pid # package id - # [ .. (name, plugin) .. ] - - self.rid = rid #result id - self.add = add #add packages instead of return result - - self.cache = [] #accumulated data - - self.start() - - def run(self): - """run method""" - - plugins = {} - container = [] - - for url, plugin in self.data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - - # filter out container plugins - for name in self.m.core.pluginManager.containerPlugins: - if name in plugins: - container.extend([(name, url) for url in plugins[name]]) - - del plugins[name] - - #directly write to database - if self.pid > -1: - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) - self.m.core.files.save() - - elif self.add: - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) - - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateCache(pluginname, result) - - packs = parseNames([(name, url) for name, x, y, url in self.cache]) - - self.m.log.debug("Fetched and generated %d packages" % len(packs)) - - for k, v in packs: - self.m.core.api.addPackage(k, v) - - #empty cache - del self.cache[:] - - else: #post the results - - - for name, url in container: - #attach container content - try: - data = self.decryptContainer(name, url) - except: - print_exc() - self.m.log.error("Could not decrypt container.") - data = [] - - for url, plugin in data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - self.m.infoResults[self.rid] = {} - - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - - #force to process cache - if self.cache: - self.updateResult(pluginname, [], True) - - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateResult(pluginname, result, True) - - self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} - - self.m.timestamp = time() + 5 * 60 - - - def updateDB(self, plugin, result): - self.m.core.files.updateFileInfo(result, self.pid) - - def updateResult(self, plugin, result, force=False): - #parse package name and generate result - #accumulate results - - self.cache.extend(result) - - if len(self.cache) >= 20 or force: - #used for package generating - tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) - for name, size, status, url in self.cache] - - data = parseNames(tmp) - result = {} - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - self.m.setInfoResults(self.rid, result) - - self.cache = [] - - def updateCache(self, plugin, result): - self.cache.extend(result) - - def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): - try: - result = [] #result loaded from cache - process = [] #urls to process - for url in urls: - if url in self.m.infoCache: - result.append(self.m.infoCache[url]) - else: - process.append(url) - - if result: - self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) - cb(pluginname, result) - - if process: - self.m.log.debug("Run Info Fetching for %s" % pluginname) - for result in plugin.getInfo(process): - #result = [ .. (name, size, status, url) .. ] - if not type(result) == list: result = [result] - - for res in result: - self.m.infoCache[res[3]] = res - - cb(pluginname, result) - - self.m.log.debug("Finished Info Fetching for %s" % pluginname) - except Exception, e: - self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % - {"name": pluginname, "err": str(e)}) - if self.m.core.debug: - print_exc() - - # generate default results - if err: - result = [(url, 0, 3, url) for url in urls] - cb(pluginname, result) - - - def decryptContainer(self, plugin, url): - data = [] - # only works on container plugins - - self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) - - # dummy pyfile - pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) - - pyfile.initPlugin() - - # little plugin lifecycle - try: - pyfile.plugin.setup() - pyfile.plugin.loadToDisk() - pyfile.plugin.decrypt(pyfile) - pyfile.plugin.deleteTmp() - - for pack in pyfile.plugin.packages: - pyfile.plugin.urls.extend(pack[1]) - - data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) - - self.m.log.debug("Got %d links." % len(data)) - - except Exception, e: - self.m.log.debug("Pre decrypting error: %s" % str(e)) - finally: - pyfile.release() - - return data diff --git a/pyload/threads/ServerThread.py b/pyload/threads/ServerThread.py deleted file mode 100644 index 7de3b1ca1..000000000 --- a/pyload/threads/ServerThread.py +++ /dev/null @@ -1,108 +0,0 @@ -from __future__ import with_statement -from os.path import exists - -import os -import threading -import logging - -core = None -setup = None -log = logging.getLogger("log") - -class WebServer(threading.Thread): - def __init__(self, pycore): - global core - threading.Thread.__init__(self) - self.core = pycore - core = pycore - self.running = True - self.server = pycore.config['webinterface']['server'] - self.https = pycore.config['webinterface']['https'] - self.cert = pycore.config["ssl"]["cert"] - self.key = pycore.config["ssl"]["key"] - self.host = pycore.config['webinterface']['host'] - self.port = pycore.config['webinterface']['port'] - - self.setDaemon(True) - - def run(self): - import pyload.webui as webinterface - global webinterface - - reset = False - - if self.https and (not exists(self.cert) or not exists(self.key)): - log.warning(_("SSL certificates not found.")) - self.https = False - - if self.server in ("lighttpd", "nginx"): - log.warning(_("Sorry, we dropped support for starting %s directly within pyLoad") % self.server) - log.warning(_("You can use the threaded server which offers good performance and ssl,")) - log.warning(_("of course you can still use your existing %s with pyLoads fastcgi server") % self.server) - log.warning(_("sample configs are located in the pyload/web/servers directory")) - reset = True - elif self.server == "fastcgi": - try: - import flup - except: - log.warning(_("Can't use %(server)s, python-flup is not installed!") % { - "server": self.server}) - reset = True - - if reset or self.server == "lightweight": - if os.name != "nt": - try: - import bjoern - except Exception, e: - log.error(_("Error importing lightweight server: %s") % e) - log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) - log.warning(_("Copy the boern.so to pyload/lib folder or use setup.py install")) - log.warning(_("Of course you need to be familiar with linux and know how to compile software")) - self.server = "builtin" - else: - self.core.log.info(_("Server set to threaded, due to known performance problems on windows.")) - self.core.config['webinterface']['server'] = "threaded" - self.server = "threaded" - - if self.server == "threaded": - self.start_threaded() - elif self.server == "fastcgi": - self.start_fcgi() - elif self.server == "lightweight": - self.start_lightweight() - else: - self.start_builtin() - - def start_builtin(self): - - if self.https: - log.warning(_("This server offers no SSL, please consider using threaded instead")) - - self.core.log.info(_("Starting builtin webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_simple(host=self.host, port=self.port) - - def start_threaded(self): - if self.https: - self.core.log.info(_("Starting threaded SSL webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - else: - self.cert = "" - self.key = "" - self.core.log.info(_("Starting threaded webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - - webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) - - def start_fcgi(self): - - self.core.log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_fcgi(host=self.host, port=self.port) - - - def start_lightweight(self): - if self.https: - log.warning(_("This server offers no SSL, please consider using threaded instead")) - - self.core.log.info(_("Starting lightweight webserver (bjoern): %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_lightweight(host=self.host, port=self.port) - - def quit(self): - self.running = False diff --git a/pyload/threads/__init__.py b/pyload/threads/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pyload/unescape.py b/pyload/unescape.py deleted file mode 100644 index 7bcc1aa72..000000000 --- a/pyload/unescape.py +++ /dev/null @@ -1,3 +0,0 @@ -from pyload.utils import html_unescape -#deprecated -unescape = html_unescape diff --git a/pyload/utils.py b/pyload/utils.py deleted file mode 100644 index b9eb8c88e..000000000 --- a/pyload/utils.py +++ /dev/null @@ -1,227 +0,0 @@ -# -*- coding: utf-8 -*- - -""" Store all useful functions here """ - -import os -import sys -import time -import re -from os.path import join -from string import maketrans -from htmlentitydefs import name2codepoint - - -def chmod(*args): - try: - os.chmod(*args) - except: - pass - - -def decode(string): - """ decode string with utf if possible """ - try: - return string.decode("utf8", "replace") - except: - return string - - -def remove_chars(string, repl): - """ removes all chars in repl from string""" - if type(repl) == unicode: - for badc in list(repl): - string = string.replace(badc, "") - return string - else: - if type(string) == str: - return string.translate(maketrans("", ""), repl) - elif type(string) == unicode: - return string.translate(dict([(ord(s), None) for s in repl])) - - -def safe_filename(name): - """ remove bad chars """ - name = name.encode('ascii', 'replace') # Non-ASCII chars usually breaks file saving. Replacing. - if os.name == 'nt': - return remove_chars(name, u'\00\01\02\03\04\05\06\07\10\11\12\13\14\15\16\17\20\21\22\23\24\25\26\27\30\31\32' - u'\33\34\35\36\37/\\?%*:|"<>') - else: - return remove_chars(name, u'\0/\\"') - -#: Deprecated method -def save_path(name): - return safe_filename(name) - - -def safe_join(*args): - """ joins a path, encoding aware """ - return fs_encode(join(*[x if type(x) == unicode else decode(x) for x in args])) - -#: Deprecated method -def save_join(*args): - return safe_join(*args) - - -# File System Encoding functions: -# Use fs_encode before accesing files on disk, it will encode the string properly - -if sys.getfilesystemencoding().startswith('ANSI'): - def fs_encode(string): - try: - string = string.encode('utf-8') - finally: - return string - - fs_decode = decode #decode utf8 - -else: - fs_encode = fs_decode = lambda x: x # do nothing - - -def get_console_encoding(enc): - if os.name == "nt": - if enc == "cp65001": # aka UTF-8 - print "WARNING: Windows codepage 65001 is not supported." - enc = "cp850" - else: - enc = "utf8" - - return enc - - -def compare_time(start, end): - start = map(int, start) - end = map(int, end) - - if start == end: return True - - now = list(time.localtime()[3:5]) - if start < now < end: return True - elif start > end and (now > start or now < end): return True - elif start < now > end < start: return True - else: return False - - -def formatSize(size): - """formats size of bytes""" - size = int(size) - steps = 0 - sizes = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB") - while size > 1000: - size /= 1024.0 - steps += 1 - return "%.2f %s" % (size, sizes[steps]) - - -def formatSpeed(speed): - return formatSize(speed) + "/s" - - -def freeSpace(folder): - if os.name == "nt": - import ctypes - - free_bytes = ctypes.c_ulonglong(0) - ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(folder), None, None, ctypes.pointer(free_bytes)) - return free_bytes.value - else: - s = os.statvfs(folder) - return s.f_bsize * s.f_bavail - - -def fs_bsize(path): - """ get optimal file system buffer size (in bytes) for I/O calls """ - path = fs_encode(path) - - if os.name == "nt": - import ctypes - - drive = "%s\\" % os.path.splitdrive(path)[0] - cluster_sectors, sector_size = ctypes.c_longlong(0) - ctypes.windll.kernel32.GetDiskFreeSpaceW(ctypes.c_wchar_p(drive), ctypes.pointer(cluster_sectors), ctypes.pointer(sector_size), None, None) - return cluster_sectors * sector_size - else: - return os.statvfs(path).f_bsize - - -def uniqify(seq): #: Originally by Dave Kirby - """ removes duplicates from list, preserve order """ - seen = set() - seen_add = seen.add - return [x for x in seq if x not in seen and not seen_add(x)] - - -def parseFileSize(string, unit=None): #returns bytes - if not unit: - m = re.match(r"([\d.,]+) *([a-zA-Z]*)", string.strip().lower()) - if m: - traffic = float(m.group(1).replace(",", ".")) - unit = m.group(2) - else: - return 0 - else: - if isinstance(string, basestring): - traffic = float(string.replace(",", ".")) - else: - traffic = string - - #ignore case - unit = unit.lower().strip() - - if unit in ("eb", "ebyte", "exabyte", "eib", "e"): - traffic *= 1 << 60 - elif unit in ("pb", "pbyte", "petabyte", "pib", "p"): - traffic *= 1 << 50 - elif unit in ("tb", "tbyte", "terabyte", "tib", "t"): - traffic *= 1 << 40 - elif unit in ("gb", "gbyte", "gigabyte", "gib", "g", "gig"): - traffic *= 1 << 30 - elif unit in ("mb", "mbyte", "megabyte", "mib", "m"): - traffic *= 1 << 20 - elif unit in ("kb", "kbyte", "kilobyte", "kib", "k"): - traffic *= 1 << 10 - - return traffic - - -def lock(func): - def new(*args): - #print "Handler: %s args: %s" % (func, args[1:]) - args[0].lock.acquire() - try: - return func(*args) - finally: - args[0].lock.release() - - return new - - -def fixup(m): - text = m.group(0) - if text[:2] == "&#": - # character reference - try: - if text[:3] == "&#x": - return unichr(int(text[3:-1], 16)) - else: - return unichr(int(text[2:-1])) - except ValueError: - pass - else: - # named entity - try: - name = text[1:-1] - text = unichr(name2codepoint[name]) - except KeyError: - pass - - return text # leave as is - - -def html_unescape(text): - """Removes HTML or XML character references and entities from a text string""" - return re.sub("&#?\w+;", fixup, text) - - -def versiontuple(v): #: By kindall (http://stackoverflow.com/a/11887825) - return tuple(map(int, (v.split(".")))) diff --git a/pyload/utils/JsEngine.py b/pyload/utils/JsEngine.py new file mode 100644 index 000000000..46789f64d --- /dev/null +++ b/pyload/utils/JsEngine.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from imp import find_module +from os.path import join, exists +from urllib import quote + + +ENGINE = "" + +DEBUG = False +JS = False +PYV8 = False +RHINO = False + + +if not ENGINE: + try: + import subprocess + + subprocess.Popen(["js", "-v"], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + p = subprocess.Popen(["js", "-e", "print(23+19)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + #integrity check + if out.strip() == "42": + ENGINE = "js" + JS = True + except: + pass + +if not ENGINE or DEBUG: + try: + find_module("PyV8") + ENGINE = "pyv8" + PYV8 = True + except: + pass + +if not ENGINE or DEBUG: + try: + path = "" #path where to find rhino + + if exists("/usr/share/java/js.jar"): + path = "/usr/share/java/js.jar" + elif exists("js.jar"): + path = "js.jar" + elif exists(join(pypath, "js.jar")): #may raises an exception, but js.jar wasnt found anyway + path = join(pypath, "js.jar") + + if not path: + raise Exception + + import subprocess + + p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", "print(23+19)"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + #integrity check + if out.strip() == "42": + ENGINE = "rhino" + RHINO = True + except: + pass + +class JsEngine: + def __init__(self): + self.engine = ENGINE + self.init = False + + def __nonzero__(self): + return False if not ENGINE else True + + def eval(self, script): + if not self.init: + if ENGINE == "pyv8" or (DEBUG and PYV8): + import PyV8 + global PyV8 + + self.init = True + + if type(script) == unicode: + script = script.encode("utf8") + + if not ENGINE: + raise Exception("No JS Engine") + + if not DEBUG: + if ENGINE == "pyv8": + return self.eval_pyv8(script) + elif ENGINE == "js": + return self.eval_js(script) + elif ENGINE == "rhino": + return self.eval_rhino(script) + else: + results = [] + if PYV8: + res = self.eval_pyv8(script) + print "PyV8:", res + results.append(res) + if JS: + res = self.eval_js(script) + print "JS:", res + results.append(res) + if RHINO: + res = self.eval_rhino(script) + print "Rhino:", res + results.append(res) + + warning = False + for x in results: + for y in results: + if x != y: + warning = True + + if warning: print "### WARNING ###: Different results" + + return results[0] + + def eval_pyv8(self, script): + rt = PyV8.JSContext() + rt.enter() + return rt.eval(script) + + def eval_js(self, script): + script = "print(eval(unescape('%s')))" % quote(script) + p = subprocess.Popen(["js", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) + out, err = p.communicate() + res = out.strip() + return res + + def eval_rhino(self, script): + script = "print(eval(unescape('%s')))" % quote(script) + p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", script], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) + out, err = p.communicate() + res = out.strip() + return res.decode("utf8").encode("ISO-8859-1") + + def error(self): + return _("No js engine detected, please install either Spidermonkey, ossp-js, pyv8 or rhino") diff --git a/pyload/utils/__init__.py b/pyload/utils/__init__.py new file mode 100644 index 000000000..b9eb8c88e --- /dev/null +++ b/pyload/utils/__init__.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- + +""" Store all useful functions here """ + +import os +import sys +import time +import re +from os.path import join +from string import maketrans +from htmlentitydefs import name2codepoint + + +def chmod(*args): + try: + os.chmod(*args) + except: + pass + + +def decode(string): + """ decode string with utf if possible """ + try: + return string.decode("utf8", "replace") + except: + return string + + +def remove_chars(string, repl): + """ removes all chars in repl from string""" + if type(repl) == unicode: + for badc in list(repl): + string = string.replace(badc, "") + return string + else: + if type(string) == str: + return string.translate(maketrans("", ""), repl) + elif type(string) == unicode: + return string.translate(dict([(ord(s), None) for s in repl])) + + +def safe_filename(name): + """ remove bad chars """ + name = name.encode('ascii', 'replace') # Non-ASCII chars usually breaks file saving. Replacing. + if os.name == 'nt': + return remove_chars(name, u'\00\01\02\03\04\05\06\07\10\11\12\13\14\15\16\17\20\21\22\23\24\25\26\27\30\31\32' + u'\33\34\35\36\37/\\?%*:|"<>') + else: + return remove_chars(name, u'\0/\\"') + +#: Deprecated method +def save_path(name): + return safe_filename(name) + + +def safe_join(*args): + """ joins a path, encoding aware """ + return fs_encode(join(*[x if type(x) == unicode else decode(x) for x in args])) + +#: Deprecated method +def save_join(*args): + return safe_join(*args) + + +# File System Encoding functions: +# Use fs_encode before accesing files on disk, it will encode the string properly + +if sys.getfilesystemencoding().startswith('ANSI'): + def fs_encode(string): + try: + string = string.encode('utf-8') + finally: + return string + + fs_decode = decode #decode utf8 + +else: + fs_encode = fs_decode = lambda x: x # do nothing + + +def get_console_encoding(enc): + if os.name == "nt": + if enc == "cp65001": # aka UTF-8 + print "WARNING: Windows codepage 65001 is not supported." + enc = "cp850" + else: + enc = "utf8" + + return enc + + +def compare_time(start, end): + start = map(int, start) + end = map(int, end) + + if start == end: return True + + now = list(time.localtime()[3:5]) + if start < now < end: return True + elif start > end and (now > start or now < end): return True + elif start < now > end < start: return True + else: return False + + +def formatSize(size): + """formats size of bytes""" + size = int(size) + steps = 0 + sizes = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB") + while size > 1000: + size /= 1024.0 + steps += 1 + return "%.2f %s" % (size, sizes[steps]) + + +def formatSpeed(speed): + return formatSize(speed) + "/s" + + +def freeSpace(folder): + if os.name == "nt": + import ctypes + + free_bytes = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(folder), None, None, ctypes.pointer(free_bytes)) + return free_bytes.value + else: + s = os.statvfs(folder) + return s.f_bsize * s.f_bavail + + +def fs_bsize(path): + """ get optimal file system buffer size (in bytes) for I/O calls """ + path = fs_encode(path) + + if os.name == "nt": + import ctypes + + drive = "%s\\" % os.path.splitdrive(path)[0] + cluster_sectors, sector_size = ctypes.c_longlong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceW(ctypes.c_wchar_p(drive), ctypes.pointer(cluster_sectors), ctypes.pointer(sector_size), None, None) + return cluster_sectors * sector_size + else: + return os.statvfs(path).f_bsize + + +def uniqify(seq): #: Originally by Dave Kirby + """ removes duplicates from list, preserve order """ + seen = set() + seen_add = seen.add + return [x for x in seq if x not in seen and not seen_add(x)] + + +def parseFileSize(string, unit=None): #returns bytes + if not unit: + m = re.match(r"([\d.,]+) *([a-zA-Z]*)", string.strip().lower()) + if m: + traffic = float(m.group(1).replace(",", ".")) + unit = m.group(2) + else: + return 0 + else: + if isinstance(string, basestring): + traffic = float(string.replace(",", ".")) + else: + traffic = string + + #ignore case + unit = unit.lower().strip() + + if unit in ("eb", "ebyte", "exabyte", "eib", "e"): + traffic *= 1 << 60 + elif unit in ("pb", "pbyte", "petabyte", "pib", "p"): + traffic *= 1 << 50 + elif unit in ("tb", "tbyte", "terabyte", "tib", "t"): + traffic *= 1 << 40 + elif unit in ("gb", "gbyte", "gigabyte", "gib", "g", "gig"): + traffic *= 1 << 30 + elif unit in ("mb", "mbyte", "megabyte", "mib", "m"): + traffic *= 1 << 20 + elif unit in ("kb", "kbyte", "kilobyte", "kib", "k"): + traffic *= 1 << 10 + + return traffic + + +def lock(func): + def new(*args): + #print "Handler: %s args: %s" % (func, args[1:]) + args[0].lock.acquire() + try: + return func(*args) + finally: + args[0].lock.release() + + return new + + +def fixup(m): + text = m.group(0) + if text[:2] == "&#": + # character reference + try: + if text[:3] == "&#x": + return unichr(int(text[3:-1], 16)) + else: + return unichr(int(text[2:-1])) + except ValueError: + pass + else: + # named entity + try: + name = text[1:-1] + text = unichr(name2codepoint[name]) + except KeyError: + pass + + return text # leave as is + + +def html_unescape(text): + """Removes HTML or XML character references and entities from a text string""" + return re.sub("&#?\w+;", fixup, text) + + +def versiontuple(v): #: By kindall (http://stackoverflow.com/a/11887825) + return tuple(map(int, (v.split(".")))) diff --git a/pyload/utils/packagetools.py b/pyload/utils/packagetools.py new file mode 100644 index 000000000..d5ab4d182 --- /dev/null +++ b/pyload/utils/packagetools.py @@ -0,0 +1,136 @@ +# JDownloader/src/jd/controlling/LinkGrabberPackager.java + +import re +from urlparse import urlparse + +def matchFirst(string, *args): + """ matches against list of regexp and returns first match""" + for patternlist in args: + for pattern in patternlist: + r = pattern.search(string) + if r is not None: + name = r.group(1) + return name + + return string + + +def parseNames(files): + """ Generates packages names from name, data lists + + :param files: list of (name, data) + :return: packagenames mapt to data lists (eg. urls) + """ + packs = {} + + endings = "\\.(3gp|7zip|7z|abr|ac3|aiff|aifc|aif|ai|au|avi|bin|bz2|cbr|cbz|ccf|cue|cvd|chm|dta|deb|divx|djvu|dlc|dmg|doc|docx|dot|eps|exe|ff|flv|f4v|gsd|gif|gz|iwd|iso|ipsw|java|jar|jpg|jpeg|jdeatme|load|mws|mw|m4v|m4a|mkv|mp2|mp3|mp4|mov|movie|mpeg|mpe|mpg|msi|msu|msp|nfo|npk|oga|ogg|ogv|otrkey|pkg|png|pdf|pptx|ppt|pps|ppz|pot|psd|qt|rmvb|rm|rar|ram|ra|rev|rnd|r\\d+|rpm|run|rsdf|rtf|sh(!?tml)|srt|snd|sfv|swf|tar|tif|tiff|ts|txt|viv|vivo|vob|wav|wmv|xla|xls|xpi|zeno|zip|z\\d+|_[_a-z]{2}|\\d+$)" + + rarPats = [re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I), + re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I), + re.compile("(.*)\\.rar$", re.I), + re.compile("(.*)\\.r\\d+$", re.I), + re.compile("(.*)(\\.|_|-)\\d+$", re.I)] + + zipPats = [re.compile("(.*)\\.zip$", re.I), + re.compile("(.*)\\.z\\d+$", re.I), + re.compile("(?is).*\\.7z\\.[\\d]+$", re.I), + re.compile("(.*)\\.a.$", re.I)] + + ffsjPats = [re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)"), + re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I)] + + iszPats = [re.compile("(.*)\\.isz$", re.I), + re.compile("(.*)\\.i\\d{2}$", re.I)] + + pat1 = re.compile("(\\.?CD\\d+)", re.I) + pat2 = re.compile("(\\.?part\\d+)", re.I) + + pat3 = re.compile("(.+)[\\.\\-_]+$") + pat4 = re.compile("(.+)\\.\\d+\\.xtm$") + + for file, url in files: + patternMatch = False + + if file is None: + continue + + # remove trailing / + name = file.rstrip('/') + + # extract last path part .. if there is a path + split = name.rsplit("/", 1) + if len(split) > 1: + name = split.pop(1) + + #check if a already existing package may be ok for this file + # found = False + # for pack in packs: + # if pack in file: + # packs[pack].append(url) + # found = True + # break + # + # if found: continue + + # unrar pattern, 7zip/zip and hjmerge pattern, isz pattern, FFSJ pattern + before = name + name = matchFirst(name, rarPats, zipPats, iszPats, ffsjPats) + if before != name: + patternMatch = True + + # xtremsplit pattern + r = pat4.search(name) + if r is not None: + name = r.group(1) + + # remove part and cd pattern + r = pat1.search(name) + if r is not None: + name = name.replace(r.group(0), "") + patternMatch = True + + r = pat2.search(name) + if r is not None: + name = name.replace(r.group(0), "") + patternMatch = True + + # additional checks if extension pattern matched + if patternMatch: + # remove extension + index = name.rfind(".") + if index <= 0: + index = name.rfind("_") + if index > 0: + length = len(name) - index + if length <= 4: + name = name[:-length] + + # remove endings like . _ - + r = pat3.search(name) + if r is not None: + name = r.group(1) + + # replace . and _ with space + name = name.replace(".", " ") + name = name.replace("_", " ") + + name = name.strip() + else: + name = "" + + # fallback: package by hoster + if not name: + name = urlparse(file).hostname + if name: name = name.replace("www.", "") + + # fallback : default name + if not name: + name = "unknown" + + # build mapping + if name in packs: + packs[name].append(url) + else: + packs[name] = [url] + + return packs diff --git a/pyload/utils/pylgettext.py b/pyload/utils/pylgettext.py new file mode 100644 index 000000000..cab631cf4 --- /dev/null +++ b/pyload/utils/pylgettext.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +from gettext import * + +_searchdirs = None + +origfind = find + +def setpaths(pathlist): + global _searchdirs + if isinstance(pathlist, list): + _searchdirs = pathlist + else: + _searchdirs = list(pathlist) + + +def addpath(path): + global _searchdirs + if _searchdirs is None: + _searchdirs = list(path) + else: + if path not in _searchdirs: + _searchdirs.append(path) + + +def delpath(path): + global _searchdirs + if _searchdirs is not None: + if path in _searchdirs: + _searchdirs.remove(path) + + +def clearpath(): + global _searchdirs + if _searchdirs is not None: + _searchdirs = None + + +def find(domain, localedir=None, languages=None, all=False): + if _searchdirs is None: + return origfind(domain, localedir, languages, all) + searches = [localedir] + _searchdirs + results = list() + for dir in searches: + res = origfind(domain, dir, languages, all) + if all is False: + results.append(res) + else: + results.extend(res) + if all is False: + results = filter(lambda x: x is not None, results) + if len(results) == 0: + return None + else: + return results[0] + else: + return results + +#Is there a smarter/cleaner pythonic way for this? +translation.func_globals['find'] = find diff --git a/pyload/webui/__init__.py b/pyload/webui/__init__.py index c421b632e..6173c57df 100644 --- a/pyload/webui/__init__.py +++ b/pyload/webui/__init__.py @@ -17,7 +17,7 @@ """ import sys -import pyload.common.pylgettext as gettext +import pyload.utils.pylgettext as gettext import os from os.path import join, abspath, dirname, exists @@ -40,7 +40,7 @@ from middlewares import StripPathMiddleware, GZipMiddleWare, PrefixMiddleware SETUP = None PYLOAD = None -from pyload.threads import ServerThread +from pyload.manager.thread import ServerThread if not ServerThread.core: if ServerThread.setup: @@ -52,7 +52,7 @@ else: PYLOAD = ServerThread.core.api config = ServerThread.core.config -from pyload.common.JsEngine import JsEngine +from pyload.utils.JsEngine import JsEngine JS = JsEngine() diff --git a/pyload/webui/app/api.py b/pyload/webui/app/api.py index 7050b78dc..286061c2a 100644 --- a/pyload/webui/app/api.py +++ b/pyload/webui/app/api.py @@ -9,9 +9,9 @@ from bottle import route, request, response, HTTPError from utils import toDict, set_session from pyload.webui import PYLOAD -from pyload.common.json_layer import json +from pyload.utils import json from SafeEval import const_eval as literal_eval -from pyload.Api import BaseObject +from pyload.api import BaseObject # json encoder that accepts TBase objects class TBaseEncoder(json.JSONEncoder): diff --git a/pyload/webui/app/utils.py b/pyload/webui/app/utils.py index 895696c19..d5fa66a35 100644 --- a/pyload/webui/app/utils.py +++ b/pyload/webui/app/utils.py @@ -22,7 +22,7 @@ from bottle import request, HTTPError, redirect, ServerAdapter from pyload.webui import env, THEME -from pyload.Api import has_permission, PERMS, ROLE +from pyload.api import has_permission, PERMS, ROLE def render_to_response(file, args={}, proc=[]): for p in proc: -- cgit v1.2.3