From 91115fd577f20704ef7f2e74c4527ffbb0730a09 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 8 Sep 2014 01:08:03 +0200 Subject: Restructure pyload file tree (1) --- docs/access_api.rst | 8 +- docs/module_overview.rst | 2 +- docs/write_hooks.rst | 8 +- pavement.py | 412 +++++++++++ pyload/Api.py | 1030 -------------------------- pyload/CaptchaManager.py | 158 ---- pyload/ConfigParser.py | 373 ---------- pyload/Core.py | 50 +- pyload/HookManager.py | 305 -------- pyload/PullEvents.py | 120 --- pyload/PyFile.py | 284 ------- pyload/PyPackage.py | 79 -- pyload/Scheduler.py | 141 ---- pyload/ThreadManager.py | 317 -------- pyload/api/__init__.py | 1030 ++++++++++++++++++++++++++ pyload/cli/Cli.py | 6 +- pyload/cli/ManageFiles.py | 2 +- pyload/common/APIExerciser.py | 157 ---- pyload/common/ImportDebugger.py | 19 - pyload/common/JsEngine.py | 155 ---- pyload/common/__init__.py | 0 pyload/common/json_layer.py | 12 - pyload/common/packagetools.py | 136 ---- pyload/common/pavement.py | 412 ----------- pyload/common/pylgettext.py | 60 -- pyload/common/test_api.py | 20 - pyload/common/test_json.py | 48 -- pyload/config/Parser.py | 373 ++++++++++ pyload/config/Setup.py | 538 ++++++++++++++ pyload/database/FileDatabase.py | 6 +- pyload/datatypes/PyFile.py | 284 +++++++ pyload/datatypes/PyPackage.py | 79 ++ pyload/datatypes/__init__.py | 0 pyload/debug.py | 94 --- pyload/forwarder.py | 61 -- pyload/manager/AccountManager.py | 173 +++++ pyload/manager/CaptchaManager.py | 158 ++++ pyload/manager/HookManager.py | 305 ++++++++ pyload/manager/PluginManager.py | 356 +++++++++ pyload/manager/RemoteManager.py | 91 +++ pyload/manager/ThreadManager.py | 317 ++++++++ pyload/manager/__init__.py | 0 pyload/manager/event/PullEvents.py | 120 +++ pyload/manager/event/Scheduler.py | 141 ++++ pyload/manager/event/__init__.py | 0 pyload/manager/thread/PluginThread.py | 675 +++++++++++++++++ pyload/manager/thread/ServerThread.py | 108 +++ pyload/manager/thread/__init__.py | 0 pyload/plugins/AccountManager.py | 173 ----- pyload/plugins/PluginManager.py | 356 --------- pyload/plugins/accounts/BayfilesCom.py | 2 +- pyload/plugins/accounts/FastixRu.py | 2 +- pyload/plugins/accounts/FilecloudIo.py | 2 +- pyload/plugins/accounts/FileserveCom.py | 2 +- pyload/plugins/accounts/FourSharedCom.py | 2 +- pyload/plugins/accounts/FreeWayMe.py | 2 +- pyload/plugins/accounts/LetitbitNet.py | 2 +- pyload/plugins/accounts/LinksnappyCom.py | 2 +- pyload/plugins/accounts/MegaDebridEu.py | 2 +- pyload/plugins/accounts/MultiDebridCom.py | 2 +- pyload/plugins/accounts/OboomCom.py | 2 +- pyload/plugins/accounts/OverLoadMe.py | 2 +- pyload/plugins/accounts/PremiumizeMe.py | 2 +- pyload/plugins/accounts/RPNetBiz.py | 2 +- pyload/plugins/accounts/RapidgatorNet.py | 2 +- pyload/plugins/accounts/SimplyPremiumCom.py | 2 +- pyload/plugins/accounts/UnrestrictLi.py | 2 +- pyload/plugins/crypter/DailymotionBatch.py | 2 +- pyload/plugins/crypter/GooGl.py | 2 +- pyload/plugins/crypter/LinkSaveIn.py | 6 +- pyload/plugins/crypter/MediafireComFolder.py | 2 +- pyload/plugins/crypter/MultiuploadCom.py | 2 +- pyload/plugins/crypter/OneKhDe.py | 4 +- pyload/plugins/crypter/SafelinkingNet.py | 2 +- pyload/plugins/crypter/SerienjunkiesOrg.py | 12 +- pyload/plugins/crypter/TurbobitNetFolder.py | 2 +- pyload/plugins/crypter/YoutubeBatch.py | 2 +- pyload/plugins/hooks/DeathByCaptcha.py | 2 +- pyload/plugins/hooks/FastixRu.py | 2 +- pyload/plugins/hooks/IRCInterface.py | 2 +- pyload/plugins/hooks/LinksnappyCom.py | 2 +- pyload/plugins/hooks/MegaDebridEu.py | 2 +- pyload/plugins/hooks/MultiDebridCom.py | 2 +- pyload/plugins/hooks/PremiumizeMe.py | 2 +- pyload/plugins/hooks/RPNetBiz.py | 2 +- pyload/plugins/hooks/SimplyPremiumCom.py | 2 +- pyload/plugins/hooks/UnSkipOnFail.py | 2 +- pyload/plugins/hooks/UnrestrictLi.py | 2 +- pyload/plugins/hoster/AlldebridCom.py | 2 +- pyload/plugins/hoster/BayfilesCom.py | 2 +- pyload/plugins/hoster/DailymotionCom.py | 4 +- pyload/plugins/hoster/DlFreeFr.py | 2 +- pyload/plugins/hoster/ExtabitCom.py | 2 +- pyload/plugins/hoster/FastixRu.py | 2 +- pyload/plugins/hoster/FilecloudIo.py | 2 +- pyload/plugins/hoster/FilepostCom.py | 2 +- pyload/plugins/hoster/FileserveCom.py | 2 +- pyload/plugins/hoster/IfileIt.py | 2 +- pyload/plugins/hoster/LetitbitNet.py | 2 +- pyload/plugins/hoster/LinksnappyCom.py | 2 +- pyload/plugins/hoster/MegaDebridEu.py | 2 +- pyload/plugins/hoster/MegaNz.py | 2 +- pyload/plugins/hoster/MegacrypterCom.py | 2 +- pyload/plugins/hoster/MultiDebridCom.py | 2 +- pyload/plugins/hoster/MyvideoDe.py | 4 +- pyload/plugins/hoster/OboomCom.py | 2 +- pyload/plugins/hoster/OverLoadMe.py | 2 +- pyload/plugins/hoster/PremiumizeMe.py | 2 +- pyload/plugins/hoster/RPNetBiz.py | 2 +- pyload/plugins/hoster/RapidgatorNet.py | 2 +- pyload/plugins/hoster/RealdebridCom.py | 2 +- pyload/plugins/hoster/RedtubeCom.py | 4 +- pyload/plugins/hoster/UlozTo.py | 2 +- pyload/plugins/hoster/UnrestrictLi.py | 2 +- pyload/plugins/hoster/UploadingCom.py | 2 +- pyload/plugins/hoster/XHamsterCom.py | 2 +- pyload/plugins/hoster/YibaishiwuCom.py | 2 +- pyload/remote/ClickAndLoadBackend.py | 2 +- pyload/remote/RemoteManager.py | 91 --- pyload/remote/SocketBackend.py | 2 +- pyload/remote/ThriftBackend.py | 2 +- pyload/setup.py | 538 -------------- pyload/threads/PluginThread.py | 675 ----------------- pyload/threads/ServerThread.py | 108 --- pyload/threads/__init__.py | 0 pyload/unescape.py | 3 - pyload/utils.py | 227 ------ pyload/utils/JsEngine.py | 155 ++++ pyload/utils/__init__.py | 227 ++++++ pyload/utils/packagetools.py | 136 ++++ pyload/utils/pylgettext.py | 60 ++ pyload/webui/__init__.py | 6 +- pyload/webui/app/api.py | 4 +- pyload/webui/app/utils.py | 2 +- scripts/clonedigger.sh | 4 - scripts/code_analysis.sh | 25 - tests/APIExerciser.py | 157 ++++ tests/clonedigger.sh | 4 + tests/code_analysis.sh | 25 + tests/test_api.py | 21 + tests/test_json.py | 48 ++ 141 files changed, 6115 insertions(+), 6315 deletions(-) create mode 100644 pavement.py delete mode 100644 pyload/Api.py delete mode 100644 pyload/CaptchaManager.py delete mode 100644 pyload/ConfigParser.py delete mode 100644 pyload/HookManager.py delete mode 100644 pyload/PullEvents.py delete mode 100644 pyload/PyFile.py delete mode 100644 pyload/PyPackage.py delete mode 100644 pyload/Scheduler.py delete mode 100644 pyload/ThreadManager.py create mode 100644 pyload/api/__init__.py delete mode 100644 pyload/common/APIExerciser.py delete mode 100644 pyload/common/ImportDebugger.py delete mode 100644 pyload/common/JsEngine.py delete mode 100644 pyload/common/__init__.py delete mode 100644 pyload/common/json_layer.py delete mode 100644 pyload/common/packagetools.py delete mode 100644 pyload/common/pavement.py delete mode 100644 pyload/common/pylgettext.py delete mode 100644 pyload/common/test_api.py delete mode 100644 pyload/common/test_json.py create mode 100644 pyload/config/Parser.py create mode 100644 pyload/config/Setup.py create mode 100644 pyload/datatypes/PyFile.py create mode 100644 pyload/datatypes/PyPackage.py create mode 100644 pyload/datatypes/__init__.py delete mode 100644 pyload/debug.py delete mode 100644 pyload/forwarder.py create mode 100644 pyload/manager/AccountManager.py create mode 100644 pyload/manager/CaptchaManager.py create mode 100644 pyload/manager/HookManager.py create mode 100644 pyload/manager/PluginManager.py create mode 100644 pyload/manager/RemoteManager.py create mode 100644 pyload/manager/ThreadManager.py create mode 100644 pyload/manager/__init__.py create mode 100644 pyload/manager/event/PullEvents.py create mode 100644 pyload/manager/event/Scheduler.py create mode 100644 pyload/manager/event/__init__.py create mode 100644 pyload/manager/thread/PluginThread.py create mode 100644 pyload/manager/thread/ServerThread.py create mode 100644 pyload/manager/thread/__init__.py delete mode 100644 pyload/plugins/AccountManager.py delete mode 100644 pyload/plugins/PluginManager.py delete mode 100644 pyload/remote/RemoteManager.py delete mode 100644 pyload/setup.py delete mode 100644 pyload/threads/PluginThread.py delete mode 100644 pyload/threads/ServerThread.py delete mode 100644 pyload/threads/__init__.py delete mode 100644 pyload/unescape.py delete mode 100644 pyload/utils.py create mode 100644 pyload/utils/JsEngine.py create mode 100644 pyload/utils/__init__.py create mode 100644 pyload/utils/packagetools.py create mode 100644 pyload/utils/pylgettext.py delete mode 100644 scripts/clonedigger.sh delete mode 100644 scripts/code_analysis.sh create mode 100644 tests/APIExerciser.py create mode 100644 tests/clonedigger.sh create mode 100644 tests/code_analysis.sh create mode 100644 tests/test_api.py create mode 100644 tests/test_json.py diff --git a/docs/access_api.rst b/docs/access_api.rst index d83698aa3..58fe56ca4 100644 --- a/docs/access_api.rst +++ b/docs/access_api.rst @@ -13,12 +13,12 @@ First of all, you need to know what you can do with our API. It lets you do all retrieving download status, manage queue, manage accounts, modify config and so on. This document is not intended to explain every function in detail, for a complete listing -see :class:`Api `. +see :class:`Api `. Of course its possible to access the ``core.api`` attribute in plugins and hooks, but much more interesting is the possibillity to call function from different programs written in many different languages. -pyLoad uses thrift as backend and provides its :class:`Api ` as service. +pyLoad uses thrift as backend and provides its :class:`Api ` as service. More information about thrift can be found here http://wiki.apache.org/thrift/. @@ -92,7 +92,7 @@ so pyLoad can authenticate you. Calling Methods =============== -In general you can use any method listed at the :class:`Api ` documentation, which is also available to +In general you can use any method listed at the :class:`Api ` documentation, which is also available to the thriftbackend. Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address @@ -107,7 +107,7 @@ Passing parameters To pass arguments you have two choices. Either use positional arguments, eg ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword arguments -supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names in the :class:`Api ` +supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names in the :class:`Api ` documentation. It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because diff --git a/docs/module_overview.rst b/docs/module_overview.rst index 8cea3874b..d76386ed7 100644 --- a/docs/module_overview.rst +++ b/docs/module_overview.rst @@ -6,7 +6,7 @@ You can find an overview of some important classes here: .. autosummary:: :toctree: pyload - pyload.Api.Api + pyload.api.Api pyload.plugins.Plugin.Base pyload.plugins.Plugin.Plugin pyload.plugins.Crypter.Crypter diff --git a/docs/write_hooks.rst b/docs/write_hooks.rst index 5508f5a0c..ebf2ed368 100644 --- a/docs/write_hooks.rst +++ b/docs/write_hooks.rst @@ -6,7 +6,7 @@ Hooks A Hook is a python file which is located at :file:`pyload/plugins/hooks`. The :class:`HookManager ` will load it automatically on startup. Only one instance exists over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager `, -do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. +do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. The UpdateManager, CaptchaTrader, UnRar and many more are realised as hooks. Hook header @@ -102,7 +102,7 @@ available as event and not accessible through overwriting hook methods. However Providing RPC services ---------------------- -You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. +You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. So probably clients want to be able to interact with your hook to request it's state or invoke some action. Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: @@ -118,7 +118,7 @@ Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: def invoke(self, arg): print "Invoked with", arg -Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. +Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. Here is a basic example: :: #Assuming client is a ThriftClient or Api object @@ -128,7 +128,7 @@ Here is a basic example: :: Providing status information ---------------------------- -Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. +Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. Just store everything in ``self.info``. :: diff --git a/pavement.py b/pavement.py new file mode 100644 index 000000000..9b2dc98b3 --- /dev/null +++ b/pavement.py @@ -0,0 +1,412 @@ +# -*- coding: utf-8 -*- + +from paver.easy import * +from paver.setuputils import setup +from paver.doctools import cog + +import os +import sys +import shutil +import re +from glob import glob +from tempfile import mkdtemp +from urllib import urlretrieve +from subprocess import call, Popen, PIPE +from zipfile import ZipFile + +PROJECT_DIR = path(__file__).dirname() +sys.path.append(PROJECT_DIR) + +options = environment.options +path("pyload").mkdir() + +extradeps = [] +if sys.version_info <= (2, 5): + extradeps += 'simplejson' + +setup( + name="pyload", + version="0.4.10", + description='Fast, lightweight and full featured download manager.', + long_description=open(PROJECT_DIR / "README.md").read(), + keywords = ("pyload", "download-manager", "one-click-hoster", "download"), + url="http://pyload.org", + download_url='http://pyload.org/download', + license='GPL v3', + author="pyLoad Team", + author_email="support@pyload.org", + platforms = ('Any',), + #package_dir={'pyload': "src"}, + packages=["pyload"], + #package_data=find_package_data(), + #data_files=[], + include_package_data=True, + exclude_package_data={'pyload': ["docs*", "scripts*", "tests*"]}, #exluced from build but not from sdist + # 'bottle >= 0.10.0' not in list, because its small and contain little modifications + install_requires=['thrift >= 0.8.0', 'jinja2', 'pycurl', 'Beaker', 'BeautifulSoup >= 3.2, < 3.3'] + extradeps, + extras_require={ + 'SSL': ["pyOpenSSL"], + 'DLC': ['pycrypto'], + 'lightweight webserver': ['bjoern'], + 'RSS plugins': ['feedparser'], + }, + #setup_requires=["setuptools_hg"], + entry_points={ + 'console_scripts': [ + 'pyLoadCore = pyLoadCore:main', + 'pyLoadCli = pyLoadCli:main' + ]}, + zip_safe=False, + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Topic :: Internet :: WWW/HTTP", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: GNU General Public License (GPL)", + "Operating System :: OS Independent", + "Programming Language :: Python :: 2" + ] +) + +options( + sphinx=Bunch( + builddir="_build", + sourcedir="" + ), + get_source=Bunch( + src="https://bitbucket.org/spoob/pyload/get/tip.zip", + rev=None, + clean=False + ), + thrift=Bunch( + path="../thrift/trunk/compiler/cpp/thrift", + gen="" + ), + virtualenv=Bunch( + dir="env", + python="python2", + virtual="virtualenv2", + ), + cog=Bunch( + pattern="*.py", + ) +) + +# xgettext args +xargs = ["--language=Python", "--add-comments=L10N", + "--from-code=utf-8", "--copyright-holder=pyLoad Team", "--package-name=pyLoad", + "--package-version=%s" % options.version, "--msgid-bugs-address='bugs@pyload.org'"] + +@task +@needs('cog') +def html(): + """Build html documentation""" + module = path("docs") / "pyload" + pyload.rmtree() + call_task('paver.doctools.html') + + +@task +@cmdopts([ + ('src=', 's', 'Url to source'), + ('rev=', 'r', "HG revision"), + ("clean", 'c', 'Delete old source folder') +]) +def get_source(options): + """ Downloads pyload source from bitbucket tip or given rev""" + if options.rev: options.url = "https://bitbucket.org/spoob/pyload/get/%s.zip" % options.rev + + pyload = path("pyload") + + if len(pyload.listdir()) and not options.clean: + return + elif pyload.exists(): + pyload.rmtree() + + urlretrieve(options.src, "pyload_src.zip") + zip = ZipFile("pyload_src.zip") + zip.extractall() + path("pyload_src.zip").remove() + + folder = [x for x in path(".").dirs() if x.name.startswith("spoob-pyload-")][0] + folder.move(pyload) + + change_mode(pyload, 0644) + change_mode(pyload, 0755, folder=True) + + for file in pyload.files(): + if file.name.endswith(".py"): + file.chmod(0755) + + (pyload / ".hgtags").remove() + (pyload / ".gitignore").remove() + #(pyload / "docs").rmtree() + + f = open(pyload / "__init__.py", "wb") + f.close() + + #options.setup.packages = find_packages() + #options.setup.package_data = find_package_data() + + +@task +@needs('clean', 'generate_setup', 'minilib', 'get_source', 'setuptools.command.sdist') +def sdist(): + """ Build source code package with distutils """ + + +@task +@cmdopts([ + ('path=', 'p', 'Thrift path'), + ('gen=', 'g', "Extra --gen option") +]) +def thrift(options): + """ Generate Thrift stubs """ + + print "add import for TApplicationException manually as long it is not fixed" + + outdir = path("pyload") / "remote" / "thriftbackend" + (outdir / "gen-py").rmtree() + + cmd = [options.thrift.path, "-strict", "-o", outdir, "--gen", "py:slots, dynamic", outdir / "pyload.thrift"] + + if options.gen: + cmd.insert(len(cmd) - 1, "--gen") + cmd.insert(len(cmd) - 1, options.gen) + + print "running", cmd + + p = Popen(cmd) + p.communicate() + + (outdir / "thriftgen").rmtree() + (outdir / "gen-py").move(outdir / "thriftgen") + + #create light ttypes + from pyload.remote.socketbackend.create_ttypes import main + main() + +@task +def compile_js(): + """ Compile .coffee files to javascript""" + + root = path("pyload") / "web" / "media" / "js" + for f in root.glob("*.coffee"): + print "generate", f + coffee = Popen(["coffee", "-cbs"], stdin=open(f, "rb"), stdout=PIPE) + yui = Popen(["yuicompressor", "--type", "js"], stdin=coffee.stdout, stdout=PIPE) + coffee.stdout.close() + content = yui.communicate()[0] + with open(root / f.name.replace(".coffee", ".js"), "wb") as js: + js.write("{% autoescape true %}\n") + js.write(content) + js.write("\n{% endautoescape %}") + + +@task +def generate_locale(): + """ Generates localization files """ + + EXCLUDE = ["BeautifulSoup.py", "pyload/cli", "web/locale", "web/ajax", "web/cnl", "web/pyload", + "setup.py"] + makepot("core", path("pyload"), EXCLUDE, "./pyload.py\n") + + makepot("cli", path("pyload") / "cli", [], includes="./pyload-cli.py\n") + makepot("setup", "", [], includes="./pyload/setup.py\n") + + EXCLUDE = ["ServerThread.py", "web/media/default"] + + # strings from js files + strings = set() + + for fi in path("pyload/web").walkfiles(): + if not fi.name.endswith(".js") and not fi.endswith(".coffee"): continue + with open(fi, "rb") as c: + content = c.read() + + strings.update(re.findall(r"_\s*\(\s*\"([^\"]+)", content)) + strings.update(re.findall(r"_\s*\(\s*\'([^\']+)", content)) + + trans = path("pyload") / "web" / "translations.js" + + with open(trans, "wb") as js: + for s in strings: + js.write('_("%s")\n' % s) + + makepot("django", path("pyload/web"), EXCLUDE, "./%s\n" % trans.relpath(), [".py", ".html"], ["--language=Python"]) + + trans.remove() + + path("includes.txt").remove() + + print "Locale generated" + + +@task +@cmdopts([ + ('key=', 'k', 'api key') +]) +def upload_translations(options): + """ Uploads the locale files to translation server """ + tmp = path(mkdtemp()) + + shutil.copy('locale/crowdin.yaml', tmp) + os.mkdir(tmp / 'pyLoad') + for f in glob('locale/*.pot'): + if os.path.isfile(f): + shutil.copy(f, tmp / 'pyLoad') + + config = tmp / 'crowdin.yaml' + content = open(config, 'rb').read() + content = content.format(key=options.key, tmp=tmp) + f = open(config, 'wb') + f.write(content) + f.close() + + call(['crowdin-cli', '-c', config, 'upload', 'source']) + + shutil.rmtree(tmp) + + print "Translations uploaded" + + +@task +@cmdopts([ + ('key=', 'k', 'api key') +]) +def download_translations(options): + """ Downloads the translated files from translation server """ + tmp = path(mkdtemp()) + + shutil.copy('locale/crowdin.yaml', tmp) + os.mkdir(tmp / 'pyLoad') + for f in glob('locale/*.pot'): + if os.path.isfile(f): + shutil.copy(f, tmp / 'pyLoad') + + config = tmp / 'crowdin.yaml' + content = open(config, 'rb').read() + content = content.format(key=options.key, tmp=tmp) + f = open(config, 'wb') + f.write(content) + f.close() + + call(['crowdin-cli', '-c', config, 'download']) + + for language in (tmp / 'pyLoad').listdir(): + if not language.isdir(): + continue + + target = path('locale') / language.basename() + print "Copy language %s" % target + if target.exists(): + shutil.rmtree(target) + + shutil.copytree(language, target) + + shutil.rmtree(tmp) + + +@task +def compile_translations(): + """ Compile PO files to MO """ + for language in path('locale').listdir(): + if not language.isdir(): + continue + + for f in glob(language / 'LC_MESSAGES' / '*.po'): + print "Compiling %s" % f + call(['msgfmt', '-o', f.replace('.po', '.mo'), f]) + + +@task +def tests(): + call(["nosetests2"]) + +@task +def virtualenv(options): + """Setup virtual environment""" + if path(options.dir).exists(): + return + + call([options.virtual, "--no-site-packages", "--python", options.python, options.dir]) + print "$ source %s/bin/activate" % options.dir + + +@task +def clean_env(): + """Deletes the virtual environment""" + env = path(options.virtualenv.dir) + if env.exists(): + env.rmtree() + + +@task +@needs('generate_setup', 'minilib', 'get_source', 'virtualenv') +def env_install(): + """Install pyLoad into the virtualenv""" + venv = options.virtualenv + call([path(venv.dir) / "bin" / "easy_install", "."]) + + +@task +def clean(): + """Cleans build directories""" + path("build").rmtree() + path("dist").rmtree() + + +#helper functions + +def walk_trans(path, EXCLUDE, endings=[".py"]): + result = "" + + for f in path.walkfiles(): + if [True for x in EXCLUDE if x in f.dirname().relpath()]: continue + if f.name in EXCLUDE: continue + + for e in endings: + if f.name.endswith(e): + result += "./%s\n" % f.relpath() + break + + return result + + +def makepot(domain, p, excludes=[], includes="", endings=[".py"], xxargs=[]): + print "Generate %s.pot" % domain + + f = open("includes.txt", "wb") + if includes: + f.write(includes) + + if p: + f.write(walk_trans(path(p), excludes, endings)) + + f.close() + + call(["xgettext", "--files-from=includes.txt", "--default-domain=%s" % domain] + xargs + xxargs) + + # replace charset und move file + with open("%s.po" % domain, "rb") as f: + content = f.read() + + path("%s.po" % domain).remove() + content = content.replace("charset=CHARSET", "charset=UTF-8") + + with open("locale/%s.pot" % domain, "wb") as f: + f.write(content) + + +def change_owner(dir, uid, gid): + for p in dir.walk(): + p.chown(uid, gid) + + +def change_mode(dir, mode, folder=False): + for p in dir.walk(): + if folder and p.isdir(): + p.chmod(mode) + elif p.isfile() and not folder: + p.chmod(mode) diff --git a/pyload/Api.py b/pyload/Api.py deleted file mode 100644 index 066d490ec..000000000 --- a/pyload/Api.py +++ /dev/null @@ -1,1030 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from base64 import standard_b64encode -from os.path import join -from time import time -import re - -from PyFile import PyFile -from utils import freeSpace, compare_time -from common.packagetools import parseNames -from network.RequestFactory import getURL -from remote import activated - -if activated: - try: - from remote.thriftbackend.thriftgen.pyload.ttypes import * - from remote.thriftbackend.thriftgen.pyload.Pyload import Iface - BaseObject = TBase - except ImportError: - print "Thrift not imported" - from remote.socketbackend.ttypes import * -else: - from remote.socketbackend.ttypes import * - -# contains function names mapped to their permissions -# unlisted functions are for admins only -permMap = {} - -# decorator only called on init, never initialized, so has no effect on runtime -def permission(bits): - class _Dec(object): - def __new__(cls, func, *args, **kwargs): - permMap[func.__name__] = bits - return func - - return _Dec - - -urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&]*)", re.IGNORECASE) - -class PERMS: - ALL = 0 # requires no permission, but login - ADD = 1 # can add packages - DELETE = 2 # can delete packages - STATUS = 4 # see and change server status - LIST = 16 # see queue and collector - MODIFY = 32 # moddify some attribute of downloads - DOWNLOAD = 64 # can download from webinterface - SETTINGS = 128 # can access settings - ACCOUNTS = 256 # can access accounts - LOGS = 512 # can see server logs - -class ROLE: - ADMIN = 0 #admin has all permissions implicit - USER = 1 - -def has_permission(userperms, perms): - # bytewise or perms before if needed - return perms == (userperms & perms) - - -class Api(Iface): - """ - **pyLoads API** - - This is accessible either internal via core.api or via thrift backend. - - see Thrift specification file remote/thriftbackend/pyload.thrift\ - for information about data structures and what methods are usuable with rpc. - - Most methods requires specific permissions, please look at the source code if you need to know.\ - These can be configured via webinterface. - Admin user have all permissions, and are the only ones who can access the methods with no specific permission. - """ - - EXTERNAL = Iface # let the json api know which methods are external - - def __init__(self, core): - self.core = core - - def _convertPyFile(self, p): - f = FileData(p["id"], p["url"], p["name"], p["plugin"], p["size"], - p["format_size"], p["status"], p["statusmsg"], - p["package"], p["error"], p["order"]) - return f - - def _convertConfigFormat(self, c): - sections = {} - for sectionName, sub in c.iteritems(): - section = ConfigSection(sectionName, sub["desc"]) - items = [] - for key, data in sub.iteritems(): - if key in ("desc", "outline"): - continue - item = ConfigItem() - item.name = key - item.description = data["desc"] - item.value = str(data["value"]) if not isinstance(data["value"], basestring) else data["value"] - item.type = data["type"] - items.append(item) - section.items = items - sections[sectionName] = section - if "outline" in sub: - section.outline = sub["outline"] - return sections - - @permission(PERMS.SETTINGS) - def getConfigValue(self, category, option, section="core"): - """Retrieve config value. - - :param category: name of category, or plugin - :param option: config option - :param section: 'plugin' or 'core' - :return: config value as string - """ - if section == "core": - value = self.core.config[category][option] - else: - value = self.core.config.getPlugin(category, option) - - return str(value) if not isinstance(value, basestring) else value - - @permission(PERMS.SETTINGS) - def setConfigValue(self, category, option, value, section="core"): - """Set new config value. - - :param category: - :param option: - :param value: new config value - :param section: 'plugin' or 'core - """ - self.core.hookManager.dispatchEvent("configChanged", category, option, value, section) - - if section == "core": - self.core.config[category][option] = value - - if option in ("limit_speed", "max_speed"): #not so nice to update the limit - self.core.requestFactory.updateBucket() - - elif section == "plugin": - self.core.config.setPlugin(category, option, value) - - @permission(PERMS.SETTINGS) - def getConfig(self): - """Retrieves complete config of core. - - :return: list of `ConfigSection` - """ - return self._convertConfigFormat(self.core.config.config) - - def getConfigDict(self): - """Retrieves complete config in dict format, not for RPC. - - :return: dict - """ - return self.core.config.config - - @permission(PERMS.SETTINGS) - def getPluginConfig(self): - """Retrieves complete config for all plugins. - - :return: list of `ConfigSection` - """ - return self._convertConfigFormat(self.core.config.plugin) - - def getPluginConfigDict(self): - """Plugin config as dict, not for RPC. - - :return: dict - """ - return self.core.config.plugin - - - @permission(PERMS.STATUS) - def pauseServer(self): - """Pause server: Tt wont start any new downloads, but nothing gets aborted.""" - self.core.threadManager.pause = True - - @permission(PERMS.STATUS) - def unpauseServer(self): - """Unpause server: New Downloads will be started.""" - self.core.threadManager.pause = False - - @permission(PERMS.STATUS) - def togglePause(self): - """Toggle pause state. - - :return: new pause state - """ - self.core.threadManager.pause ^= True - return self.core.threadManager.pause - - @permission(PERMS.STATUS) - def toggleReconnect(self): - """Toggle reconnect activation. - - :return: new reconnect state - """ - self.core.config["reconnect"]["activated"] ^= True - return self.core.config["reconnect"]["activated"] - - @permission(PERMS.LIST) - def statusServer(self): - """Some general information about the current status of pyLoad. - - :return: `ServerStatus` - """ - serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), - self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, - not self.core.threadManager.pause and self.isTimeDownload(), - self.core.config['reconnect']['activated'] and self.isTimeReconnect()) - - for pyfile in [x.active for x in self.core.threadManager.threads if x.active and isinstance(x.active, PyFile)]: - serverStatus.speed += pyfile.getSpeed() #bytes/s - - return serverStatus - - @permission(PERMS.STATUS) - def freeSpace(self): - """Available free space at download directory in bytes""" - return freeSpace(self.core.config["general"]["download_folder"]) - - @permission(PERMS.ALL) - def getServerVersion(self): - """pyLoad Core version """ - return self.core.version - - def kill(self): - """Clean way to quit pyLoad""" - self.core.do_kill = True - - def restart(self): - """Restart pyload core""" - self.core.do_restart = True - - @permission(PERMS.LOGS) - def getLog(self, offset=0): - """Returns most recent log entries. - - :param offset: line offset - :return: List of log entries - """ - filename = join(self.core.config['log']['log_folder'], 'log.txt') - try: - fh = open(filename, "r") - lines = fh.readlines() - fh.close() - if offset >= len(lines): - return [] - return lines[offset:] - except: - return ['No log available'] - - @permission(PERMS.STATUS) - def isTimeDownload(self): - """Checks if pyload will start new downloads according to time in config. - - :return: bool - """ - start = self.core.config['downloadTime']['start'].split(":") - end = self.core.config['downloadTime']['end'].split(":") - return compare_time(start, end) - - @permission(PERMS.STATUS) - def isTimeReconnect(self): - """Checks if pyload will try to make a reconnect - - :return: bool - """ - start = self.core.config['reconnect']['startTime'].split(":") - end = self.core.config['reconnect']['endTime'].split(":") - return compare_time(start, end) and self.core.config["reconnect"]["activated"] - - @permission(PERMS.LIST) - def statusDownloads(self): - """ Status off all currently running downloads. - - :return: list of `DownloadStatus` - """ - data = [] - for pyfile in self.core.threadManager.getActiveFiles(): - if not isinstance(pyfile, PyFile): - continue - - data.append(DownloadInfo( - pyfile.id, pyfile.name, pyfile.getSpeed(), pyfile.getETA(), pyfile.formatETA(), - pyfile.getBytesLeft(), pyfile.getSize(), pyfile.formatSize(), pyfile.getPercent(), - pyfile.status, pyfile.getStatusName(), pyfile.formatWait(), - pyfile.waitUntil, pyfile.packageid, pyfile.package().name, pyfile.pluginname)) - - return data - - @permission(PERMS.ADD) - def addPackage(self, name, links, dest=Destination.Queue): - """Adds a package, with links to desired destination. - - :param name: name of the new package - :param links: list of urls - :param dest: `Destination` - :return: package id of the new package - """ - if self.core.config['general']['folder_per_package']: - folder = name - else: - folder = "" - - folder = folder.replace("http://", "").replace(":", "").replace("/", "_").replace("\\", "_") - - pid = self.core.files.addPackage(name, folder, dest) - - self.core.files.addLinks(links, pid) - - self.core.log.info(_("Added package %(name)s containing %(count)d links") % {"name": name, "count": len(links)}) - - self.core.files.save() - - return pid - - @permission(PERMS.ADD) - def parseURLs(self, html=None, url=None): - """Parses html content or any arbitaty text for links and returns result of `checkURLs` - - :param html: html source - :return: - """ - urls = [] - - if html: - urls += [x[0] for x in urlmatcher.findall(html)] - - if url: - page = getURL(url) - urls += [x[0] for x in urlmatcher.findall(page)] - - # remove duplicates - return self.checkURLs(set(urls)) - - - @permission(PERMS.ADD) - def checkURLs(self, urls): - """ Gets urls and returns pluginname mapped to list of matches urls. - - :param urls: - :return: {plugin: urls} - """ - data = self.core.pluginManager.parseUrls(urls) - plugins = {} - - for url, plugin in data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - return plugins - - @permission(PERMS.ADD) - def checkOnlineStatus(self, urls): - """ initiates online status check - - :param urls: - :return: initial set of data as `OnlineCheck` instance containing the result id - """ - data = self.core.pluginManager.parseUrls(urls) - - rid = self.core.threadManager.createResultThread(data, False) - - tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] - data = parseNames(tmp) - result = {} - - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - return OnlineCheck(rid, result) - - @permission(PERMS.ADD) - def checkOnlineStatusContainer(self, urls, container, data): - """ checks online status of urls and a submited container file - - :param urls: list of urls - :param container: container file name - :param data: file content - :return: online check - """ - th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb") - th.write(str(data)) - th.close() - - return self.checkOnlineStatus(urls + [th.name]) - - @permission(PERMS.ADD) - def pollResults(self, rid): - """ Polls the result available for ResultID - - :param rid: `ResultID` - :return: `OnlineCheck`, if rid is -1 then no more data available - """ - result = self.core.threadManager.getInfoResult(rid) - - if "ALL_INFO_FETCHED" in result: - del result["ALL_INFO_FETCHED"] - return OnlineCheck(-1, result) - else: - return OnlineCheck(rid, result) - - - @permission(PERMS.ADD) - def generatePackages(self, links): - """ Parses links, generates packages names from urls - - :param links: list of urls - :return: package names mapped to urls - """ - result = parseNames((x, x) for x in links) - return result - - @permission(PERMS.ADD) - def generateAndAddPackages(self, links, dest=Destination.Queue): - """Generates and add packages - - :param links: list of urls - :param dest: `Destination` - :return: list of package ids - """ - return [self.addPackage(name, urls, dest) for name, urls - in self.generatePackages(links).iteritems()] - - @permission(PERMS.ADD) - def checkAndAddPackages(self, links, dest=Destination.Queue): - """Checks online status, retrieves names, and will add packages.\ - Because of this packages are not added immediatly, only for internal use. - - :param links: list of urls - :param dest: `Destination` - :return: None - """ - data = self.core.pluginManager.parseUrls(links) - self.core.threadManager.createResultThread(data, True) - - - @permission(PERMS.LIST) - def getPackageData(self, pid): - """Returns complete information about package, and included files. - - :param pid: package id - :return: `PackageData` with .links attribute - """ - data = self.core.files.getPackageData(int(pid)) - - if not data: - raise PackageDoesNotExists(pid) - - pdata = PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], - data["queue"], data["order"], - links=[self._convertPyFile(x) for x in data["links"].itervalues()]) - - return pdata - - @permission(PERMS.LIST) - def getPackageInfo(self, pid): - """Returns information about package, without detailed information about containing files - - :param pid: package id - :return: `PackageData` with .fid attribute - """ - data = self.core.files.getPackageData(int(pid)) - - if not data: - raise PackageDoesNotExists(pid) - - pdata = PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], - data["queue"], data["order"], - fids=[int(x) for x in data["links"]]) - - return pdata - - @permission(PERMS.LIST) - def getFileData(self, fid): - """Get complete information about a specific file. - - :param fid: file id - :return: `FileData` - """ - info = self.core.files.getFileData(int(fid)) - if not info: - raise FileDoesNotExists(fid) - - fdata = self._convertPyFile(info.values()[0]) - return fdata - - @permission(PERMS.DELETE) - def deleteFiles(self, fids): - """Deletes several file entries from pyload. - - :param fids: list of file ids - """ - for id in fids: - self.core.files.deleteLink(int(id)) - - self.core.files.save() - - @permission(PERMS.DELETE) - def deletePackages(self, pids): - """Deletes packages and containing links. - - :param pids: list of package ids - """ - for id in pids: - self.core.files.deletePackage(int(id)) - - self.core.files.save() - - @permission(PERMS.LIST) - def getQueue(self): - """Returns info about queue and packages, **not** about files, see `getQueueData` \ - or `getPackageData` instead. - - :return: list of `PackageInfo` - """ - return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - pack["linkstotal"]) - for pack in self.core.files.getInfoData(Destination.Queue).itervalues()] - - @permission(PERMS.LIST) - def getQueueData(self): - """Return complete data about everything in queue, this is very expensive use it sparely.\ - See `getQueue` for alternative. - - :return: list of `PackageData` - """ - return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) - for pack in self.core.files.getCompleteData(Destination.Queue).itervalues()] - - @permission(PERMS.LIST) - def getCollector(self): - """same as `getQueue` for collector. - - :return: list of `PackageInfo` - """ - return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - pack["linkstotal"]) - for pack in self.core.files.getInfoData(Destination.Collector).itervalues()] - - @permission(PERMS.LIST) - def getCollectorData(self): - """same as `getQueueData` for collector. - - :return: list of `PackageInfo` - """ - return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) - for pack in self.core.files.getCompleteData(Destination.Collector).itervalues()] - - - @permission(PERMS.ADD) - def addFiles(self, pid, links): - """Adds files to specific package. - - :param pid: package id - :param links: list of urls - """ - self.core.files.addLinks(links, int(pid)) - - self.core.log.info(_("Added %(count)d links to package #%(package)d ") % {"count": len(links), "package": pid}) - self.core.files.save() - - @permission(PERMS.MODIFY) - def pushToQueue(self, pid): - """Moves package from Collector to Queue. - - :param pid: package id - """ - self.core.files.setPackageLocation(pid, Destination.Queue) - - @permission(PERMS.MODIFY) - def pullFromQueue(self, pid): - """Moves package from Queue to Collector. - - :param pid: package id - """ - self.core.files.setPackageLocation(pid, Destination.Collector) - - @permission(PERMS.MODIFY) - def restartPackage(self, pid): - """Restarts a package, resets every containing files. - - :param pid: package id - """ - self.core.files.restartPackage(int(pid)) - - @permission(PERMS.MODIFY) - def restartFile(self, fid): - """Resets file status, so it will be downloaded again. - - :param fid: file id - """ - self.core.files.restartFile(int(fid)) - - @permission(PERMS.MODIFY) - def recheckPackage(self, pid): - """Proofes online status of all files in a package, also a default action when package is added. - - :param pid: - :return: - """ - self.core.files.reCheckPackage(int(pid)) - - @permission(PERMS.MODIFY) - def stopAllDownloads(self): - """Aborts all running downloads.""" - - pyfiles = self.core.files.cache.values() - for pyfile in pyfiles: - pyfile.abortDownload() - - @permission(PERMS.MODIFY) - def stopDownloads(self, fids): - """Aborts specific downloads. - - :param fids: list of file ids - :return: - """ - pyfiles = self.core.files.cache.values() - - for pyfile in pyfiles: - if pyfile.id in fids: - pyfile.abortDownload() - - @permission(PERMS.MODIFY) - def setPackageName(self, pid, name): - """Renames a package. - - :param pid: package id - :param name: new package name - """ - pack = self.core.files.getPackage(pid) - pack.name = name - pack.sync() - - @permission(PERMS.MODIFY) - def movePackage(self, destination, pid): - """Set a new package location. - - :param destination: `Destination` - :param pid: package id - """ - if destination not in (0, 1): return - self.core.files.setPackageLocation(pid, destination) - - @permission(PERMS.MODIFY) - def moveFiles(self, fids, pid): - """Move multiple files to another package - - :param fids: list of file ids - :param pid: destination package - :return: - """ - #TODO: implement - pass - - - @permission(PERMS.ADD) - def uploadContainer(self, filename, data): - """Uploads and adds a container file to pyLoad. - - :param filename: filename, extension is important so it can correctly decrypted - :param data: file content - """ - th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb") - th.write(str(data)) - th.close() - - self.addPackage(th.name, [th.name], Destination.Queue) - - @permission(PERMS.MODIFY) - def orderPackage(self, pid, position): - """Gives a package a new position. - - :param pid: package id - :param position: - """ - self.core.files.reorderPackage(pid, position) - - @permission(PERMS.MODIFY) - def orderFile(self, fid, position): - """Gives a new position to a file within its package. - - :param fid: file id - :param position: - """ - self.core.files.reorderFile(fid, position) - - @permission(PERMS.MODIFY) - def setPackageData(self, pid, data): - """Allows to modify several package attributes. - - :param pid: package id - :param data: dict that maps attribute to desired value - """ - p = self.core.files.getPackage(pid) - if not p: raise PackageDoesNotExists(pid) - - for key, value in data.iteritems(): - if key == "id": continue - setattr(p, key, value) - - p.sync() - self.core.files.save() - - @permission(PERMS.DELETE) - def deleteFinished(self): - """Deletes all finished files and completly finished packages. - - :return: list of deleted package ids - """ - return self.core.files.deleteFinishedLinks() - - @permission(PERMS.MODIFY) - def restartFailed(self): - """Restarts all failed failes.""" - self.core.files.restartFailed() - - @permission(PERMS.LIST) - def getPackageOrder(self, destination): - """Returns information about package order. - - :param destination: `Destination` - :return: dict mapping order to package id - """ - - packs = self.core.files.getInfoData(destination) - order = {} - - for pid in packs: - pack = self.core.files.getPackageData(int(pid)) - while pack["order"] in order.keys(): #just in case - pack["order"] += 1 - order[pack["order"]] = pack["id"] - return order - - @permission(PERMS.LIST) - def getFileOrder(self, pid): - """Information about file order within package. - - :param pid: - :return: dict mapping order to file id - """ - rawData = self.core.files.getPackageData(int(pid)) - order = {} - for id, pyfile in rawData["links"].iteritems(): - while pyfile["order"] in order.keys(): #just in case - pyfile["order"] += 1 - order[pyfile["order"]] = pyfile["id"] - return order - - - @permission(PERMS.STATUS) - def isCaptchaWaiting(self): - """Indicates wether a captcha task is available - - :return: bool - """ - self.core.lastClientConnected = time() - task = self.core.captchaManager.getTask() - return not task is None - - @permission(PERMS.STATUS) - def getCaptchaTask(self, exclusive=False): - """Returns a captcha task - - :param exclusive: unused - :return: `CaptchaTask` - """ - self.core.lastClientConnected = time() - task = self.core.captchaManager.getTask() - if task: - task.setWatingForUser(exclusive=exclusive) - data, type, result = task.getCaptcha() - t = CaptchaTask(int(task.id), standard_b64encode(data), type, result) - return t - else: - return CaptchaTask(-1) - - @permission(PERMS.STATUS) - def getCaptchaTaskStatus(self, tid): - """Get information about captcha task - - :param tid: task id - :return: string - """ - self.core.lastClientConnected = time() - t = self.core.captchaManager.getTaskByID(tid) - return t.getStatus() if t else "" - - @permission(PERMS.STATUS) - def setCaptchaResult(self, tid, result): - """Set result for a captcha task - - :param tid: task id - :param result: captcha result - """ - self.core.lastClientConnected = time() - task = self.core.captchaManager.getTaskByID(tid) - if task: - task.setResult(result) - self.core.captchaManager.removeTask(task) - - - @permission(PERMS.STATUS) - def getEvents(self, uuid): - """Lists occured events, may be affected to changes in future. - - :param uuid: - :return: list of `Events` - """ - events = self.core.pullManager.getEvents(uuid) - newEvents = [] - - def convDest(d): - return Destination.Queue if d == "queue" else Destination.Collector - - for e in events: - event = EventInfo() - event.eventname = e[0] - if e[0] in ("update", "remove", "insert"): - event.id = e[3] - event.type = ElementType.Package if e[2] == "pack" else ElementType.File - event.destination = convDest(e[1]) - elif e[0] == "order": - if e[1]: - event.id = e[1] - event.type = ElementType.Package if e[2] == "pack" else ElementType.File - event.destination = convDest(e[3]) - elif e[0] == "reload": - event.destination = convDest(e[1]) - newEvents.append(event) - return newEvents - - @permission(PERMS.ACCOUNTS) - def getAccounts(self, refresh): - """Get information about all entered accounts. - - :param refresh: reload account info - :return: list of `AccountInfo` - """ - accs = self.core.accountManager.getAccountInfos(False, refresh) - accounts = [] - for group in accs.values(): - accounts.extend([AccountInfo(acc["validuntil"], acc["login"], acc["options"], acc["valid"], - acc["trafficleft"], acc["maxtraffic"], acc["premium"], acc["type"]) - for acc in group]) - return accounts - - @permission(PERMS.ALL) - def getAccountTypes(self): - """All available account types. - - :return: list - """ - return self.core.accountManager.accounts.keys() - - @permission(PERMS.ACCOUNTS) - def updateAccount(self, plugin, account, password=None, options={}): - """Changes pw/options for specific account.""" - self.core.accountManager.updateAccount(plugin, account, password, options) - - @permission(PERMS.ACCOUNTS) - def removeAccount(self, plugin, account): - """Remove account from pyload. - - :param plugin: pluginname - :param account: accountname - """ - self.core.accountManager.removeAccount(plugin, account) - - @permission(PERMS.ALL) - def login(self, username, password, remoteip=None): - """Login into pyLoad, this **must** be called when using rpc before any methods can be used. - - :param username: - :param password: - :param remoteip: Omit this argument, its only used internal - :return: bool indicating login was successful - """ - return True if self.checkAuth(username, password, remoteip) else False - - def checkAuth(self, username, password, remoteip=None): - """Check authentication and returns details - - :param username: - :param password: - :param remoteip: - :return: dict with info, empty when login is incorrect - """ - if self.core.config["remote"]["nolocalauth"] and remoteip == "127.0.0.1": - return "local" - else: - return self.core.db.checkAuth(username, password) - - def isAuthorized(self, func, userdata): - """checks if the user is authorized for specific method - - :param func: function name - :param userdata: dictionary of user data - :return: boolean - """ - if userdata == "local" or userdata["role"] == ROLE.ADMIN: - return True - elif func in permMap and has_permission(userdata["permission"], permMap[func]): - return True - else: - return False - - - @permission(PERMS.ALL) - def getUserData(self, username, password): - """similar to `checkAuth` but returns UserData thrift type """ - user = self.checkAuth(username, password) - if user: - return UserData(user["name"], user["email"], user["role"], user["permission"], user["template"]) - else: - return UserData() - - - def getAllUserData(self): - """returns all known user and info""" - res = {} - for user, data in self.core.db.getAllUserData().iteritems(): - res[user] = UserData(user, data["email"], data["role"], data["permission"], data["template"]) - - return res - - @permission(PERMS.STATUS) - def getServices(self): - """ A dict of available services, these can be defined by hook plugins. - - :return: dict with this style: {"plugin": {"method": "description"}} - """ - data = {} - for plugin, funcs in self.core.hookManager.methods.iteritems(): - data[plugin] = funcs - - return data - - @permission(PERMS.STATUS) - def hasService(self, plugin, func): - """Checks wether a service is available. - - :param plugin: - :param func: - :return: bool - """ - cont = self.core.hookManager.methods - return plugin in cont and func in cont[plugin] - - @permission(PERMS.STATUS) - def call(self, info): - """Calls a service (a method in hook plugin). - - :param info: `ServiceCall` - :return: result - :raises: ServiceDoesNotExists, when its not available - :raises: ServiceException, when a exception was raised - """ - plugin = info.plugin - func = info.func - args = info.arguments - parse = info.parseArguments - - if not self.hasService(plugin, func): - raise ServiceDoesNotExists(plugin, func) - - try: - ret = self.core.hookManager.callRPC(plugin, func, args, parse) - return str(ret) - except Exception, e: - raise ServiceException(e.message) - - @permission(PERMS.STATUS) - def getAllInfo(self): - """Returns all information stored by hook plugins. Values are always strings - - :return: {"plugin": {"name": value}} - """ - return self.core.hookManager.getAllInfo() - - @permission(PERMS.STATUS) - def getInfoByPlugin(self, plugin): - """Returns information stored by a specific plugin. - - :param plugin: pluginname - :return: dict of attr names mapped to value {"name": value} - """ - return self.core.hookManager.getInfo(plugin) - - def changePassword(self, user, oldpw, newpw): - """ changes password for specific user """ - return self.core.db.changePassword(user, oldpw, newpw) - - def setUserPermission(self, user, permission, role): - self.core.db.setPermission(user, permission) - self.core.db.setRole(user, role) diff --git a/pyload/CaptchaManager.py b/pyload/CaptchaManager.py deleted file mode 100644 index 0ba876ae8..000000000 --- a/pyload/CaptchaManager.py +++ /dev/null @@ -1,158 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay, RaNaN -""" - -from time import time -from traceback import print_exc -from threading import Lock - -class CaptchaManager: - def __init__(self, core): - self.lock = Lock() - self.core = core - self.tasks = [] #task store, for outgoing tasks only - - self.ids = 0 #only for internal purpose - - def newTask(self, img, format, file, result_type): - task = CaptchaTask(self.ids, img, format, file, result_type) - self.ids += 1 - return task - - def removeTask(self, task): - self.lock.acquire() - if task in self.tasks: - self.tasks.remove(task) - self.lock.release() - - def getTask(self): - self.lock.acquire() - for task in self.tasks: - if task.status in ("waiting", "shared-user"): - self.lock.release() - return task - self.lock.release() - return None - - def getTaskByID(self, tid): - self.lock.acquire() - for task in self.tasks: - if task.id == str(tid): #task ids are strings - self.lock.release() - return task - self.lock.release() - return None - - def handleCaptcha(self, task): - cli = self.core.isClientConnected() - - if cli: #client connected -> should solve the captcha - task.setWaiting(50) #wait 50 sec for response - - for plugin in self.core.hookManager.activePlugins(): - try: - plugin.newCaptchaTask(task) - except: - if self.core.debug: - print_exc() - - if task.handler or cli: #the captcha was handled - self.tasks.append(task) - return True - - task.error = _("No Client connected for captcha decrypting") - - return False - - -class CaptchaTask: - def __init__(self, id, img, format, file, result_type='textual'): - self.id = str(id) - self.captchaImg = img - self.captchaFormat = format - self.captchaFile = file - self.captchaResultType = result_type - self.handler = [] #the hook plugins that will take care of the solution - self.result = None - self.waitUntil = None - self.error = None #error message - - self.status = "init" - self.data = {} #handler can store data here - - def getCaptcha(self): - return self.captchaImg, self.captchaFormat, self.captchaResultType - - def setResult(self, text): - if self.isTextual(): - self.result = text - if self.isPositional(): - try: - parts = text.split(',') - self.result = (int(parts[0]), int(parts[1])) - except: - self.result = None - - def getResult(self): - try: - res = self.result.encode("utf8", "replace") - except: - res = self.result - - return res - - def getStatus(self): - return self.status - - def setWaiting(self, sec): - """ let the captcha wait secs for the solution """ - self.waitUntil = max(time() + sec, self.waitUntil) - self.status = "waiting" - - def isWaiting(self): - if self.result or self.error or time() > self.waitUntil: - return False - - return True - - def isTextual(self): - """ returns if text is written on the captcha """ - return self.captchaResultType == 'textual' - - def isPositional(self): - """ returns if user have to click a specific region on the captcha """ - return self.captchaResultType == 'positional' - - def setWatingForUser(self, exclusive): - if exclusive: - self.status = "user" - else: - self.status = "shared-user" - - def timedOut(self): - return time() > self.waitUntil - - def invalid(self): - """ indicates the captcha was not correct """ - [x.captchaInvalid(self) for x in self.handler] - - def correct(self): - [x.captchaCorrect(self) for x in self.handler] - - def __str__(self): - return "" % self.id diff --git a/pyload/ConfigParser.py b/pyload/ConfigParser.py deleted file mode 100644 index 64ce6b10e..000000000 --- a/pyload/ConfigParser.py +++ /dev/null @@ -1,373 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import with_statement -from time import sleep -from os.path import exists, join -from shutil import copy - -from traceback import print_exc -from utils import chmod - -# ignore these plugin configs, mainly because plugins were wiped out -IGNORE = ( - "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('hooks', 'UnRar'), - 'EasyShareCom', 'FlyshareCz' - ) - -CONF_VERSION = 1 - -class ConfigParser: - """ - holds and manage the configuration - - current dict layout: - - { - - section: { - option: { - value: - type: - desc: - } - desc: - - } - - """ - - - def __init__(self): - """Constructor""" - self.config = {} # the config values - self.plugin = {} # the config for plugins - self.oldRemoteData = {} - - self.pluginCB = None # callback when plugin config value is changed - - self.checkVersion() - - self.readConfig() - - self.deleteOldPlugins() - - - def checkVersion(self, n=0): - """determines if config need to be copied""" - try: - if not exists("pyload.conf"): - copy(join(pypath, "pyload", "config", "default.conf"), "pyload.conf") - - if not exists("plugin.conf"): - f = open("plugin.conf", "wb") - f.write("version: " + str(CONF_VERSION)) - f.close() - - f = open("pyload.conf", "rb") - v = f.readline() - f.close() - v = v[v.find(":") + 1:].strip() - - if not v or int(v) < CONF_VERSION: - copy(join(pypath, "pyload", "config", "default.conf"), "pyload.conf") - print "Old version of config was replaced" - - f = open("plugin.conf", "rb") - v = f.readline() - f.close() - v = v[v.find(":") + 1:].strip() - - if not v or int(v) < CONF_VERSION: - f = open("plugin.conf", "wb") - f.write("version: " + str(CONF_VERSION)) - f.close() - print "Old version of plugin-config replaced" - except: - if n < 3: - sleep(0.3) - self.checkVersion(n + 1) - else: - raise - - def readConfig(self): - """reads the config file""" - - self.config = self.parseConfig(join(pypath, "pyload", "config", "default.conf")) - self.plugin = self.parseConfig("plugin.conf") - - try: - homeconf = self.parseConfig("pyload.conf") - if "username" in homeconf["remote"]: - if "password" in homeconf["remote"]: - self.oldRemoteData = {"username": homeconf["remote"]["username"]["value"], - "password": homeconf["remote"]["username"]["value"]} - del homeconf["remote"]["password"] - del homeconf["remote"]["username"] - self.updateValues(homeconf, self.config) - - except Exception, e: - print "Config Warning" - print_exc() - - - def parseConfig(self, config): - """parses a given configfile""" - - f = open(config) - - config = f.read() - - config = config.splitlines()[1:] - - conf = {} - - section, option, value, typ, desc = "", "", "", "", "" - - listmode = False - - for line in config: - comment = line.rfind("#") - if line.find(":", comment) < 0 > line.find("=", comment) and comment > 0 and line[comment - 1].isspace(): - line = line.rpartition("#") # removes comments - if line[1]: - line = line[0] - else: - line = line[2] - - line = line.strip() - - try: - if line == "": - continue - elif line.endswith(":"): - section, none, desc = line[:-1].partition('-') - section = section.strip() - desc = desc.replace('"', "").strip() - conf[section] = {"desc": desc} - else: - if listmode: - if line.endswith("]"): - listmode = False - line = line.replace("]", "") - - value += [self.cast(typ, x.strip()) for x in line.split(",") if x] - - if not listmode: - conf[section][option] = {"desc": desc, - "type": typ, - "value": value} - - - else: - content, none, value = line.partition("=") - - content, none, desc = content.partition(":") - - desc = desc.replace('"', "").strip() - - typ, none, option = content.strip().rpartition(" ") - - value = value.strip() - - if value.startswith("["): - if value.endswith("]"): - listmode = False - value = value[:-1] - else: - listmode = True - - value = [self.cast(typ, x.strip()) for x in value[1:].split(",") if x] - else: - value = self.cast(typ, value) - - if not listmode: - conf[section][option] = {"desc": desc, - "type": typ, - "value": value} - - except Exception, e: - print "Config Warning" - print_exc() - - f.close() - return conf - - - def updateValues(self, config, dest): - """sets the config values from a parsed config file to values in destination""" - - for section in config.iterkeys(): - if section in dest: - for option in config[section].iterkeys(): - if option in ("desc", "outline"): continue - - if option in dest[section]: - dest[section][option]["value"] = config[section][option]["value"] - - #else: - # dest[section][option] = config[section][option] - - - #else: - # dest[section] = config[section] - - def saveConfig(self, config, filename): - """saves config to filename""" - with open(filename, "wb") as f: - chmod(filename, 0600) - f.write("version: %i \n" % CONF_VERSION) - for section in config.iterkeys(): - f.write('\n%s - "%s":\n' % (section, config[section]["desc"])) - - for option, data in config[section].iteritems(): - if option in ("desc", "outline"): continue - - if isinstance(data["value"], list): - value = "[ \n" - for x in data["value"]: - value += "\t\t" + str(x) + ",\n" - value += "\t\t]\n" - else: - if type(data["value"]) in (str, unicode): - value = data["value"] + "\n" - else: - value = str(data["value"]) + "\n" - try: - f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value)) - except UnicodeEncodeError: - f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value.encode("utf8"))) - - def cast(self, typ, value): - """cast value to given format""" - if type(value) not in (str, unicode): - return value - - elif typ == "int": - return int(value) - elif typ == "bool": - return True if value.lower() in ("1", "true", "on", "an", "yes") else False - elif typ == "time": - if not value: value = "0:00" - if not ":" in value: value += ":00" - return value - elif typ in ("str", "file", "folder"): - try: - return value.encode("utf8") - except: - return value - else: - return value - - - def save(self): - """saves the configs to disk""" - - self.saveConfig(self.config, "pyload.conf") - self.saveConfig(self.plugin, "plugin.conf") - - - def __getitem__(self, section): - """provides dictonary like access: c['section']['option']""" - return Section(self, section) - - - def get(self, section, option): - """get value""" - val = self.config[section][option]["value"] - try: - if type(val) in (str, unicode): - return val.decode("utf8") - else: - return val - except: - return val - - def set(self, section, option, value): - """set value""" - - value = self.cast(self.config[section][option]["type"], value) - - self.config[section][option]["value"] = value - self.save() - - def getPlugin(self, plugin, option): - """gets a value for a plugin""" - val = self.plugin[plugin][option]["value"] - try: - if type(val) in (str, unicode): - return val.decode("utf8") - else: - return val - except: - return val - - def setPlugin(self, plugin, option, value): - """sets a value for a plugin""" - - value = self.cast(self.plugin[plugin][option]["type"], value) - - if self.pluginCB: self.pluginCB(plugin, option, value) - - self.plugin[plugin][option]["value"] = value - self.save() - - def getMetaData(self, section, option): - """ get all config data for an option """ - return self.config[section][option] - - def addPluginConfig(self, name, config, outline=""): - """adds config options with tuples (name, type, desc, default)""" - if name not in self.plugin: - conf = {"desc": name, - "outline": outline} - self.plugin[name] = conf - else: - conf = self.plugin[name] - conf["outline"] = outline - - for item in config: - if item[0] in conf: - conf[item[0]]["type"] = item[1] - conf[item[0]]["desc"] = item[2] - else: - conf[item[0]] = { - "desc": item[2], - "type": item[1], - "value": self.cast(item[1], item[3]) - } - - values = [x[0] for x in config] + ["desc", "outline"] - #delete old values - for item in conf.keys(): - if item not in values: - del conf[item] - - def deleteConfig(self, name): - """Removes a plugin config""" - if name in self.plugin: - del self.plugin[name] - - - def deleteOldPlugins(self): - """ remove old plugins from config """ - - for name in IGNORE: - if name in self.plugin: - del self.plugin[name] - - -class Section: - """provides dictionary like access for configparser""" - - def __init__(self, parser, section): - """Constructor""" - self.parser = parser - self.section = section - - def __getitem__(self, item): - """getitem""" - return self.parser.get(self.section, item) - - def __setitem__(self, item, value): - """setitem""" - self.parser.set(self.section, item, value) diff --git a/pyload/Core.py b/pyload/Core.py index 5f506b980..3e78a5bea 100644 --- a/pyload/Core.py +++ b/pyload/Core.py @@ -25,7 +25,7 @@ CURRENT_VERSION = '0.4.10' import __builtin__ from getopt import getopt, GetoptError -import pyload.common.pylgettext as gettext +import pyload.utils.pylgettext as gettext from imp import find_module import logging import logging.handlers @@ -40,17 +40,17 @@ from time import time, sleep from traceback import print_exc from pyload import InitHomeDir -from pyload.plugins.AccountManager import AccountManager -from pyload.CaptchaManager import CaptchaManager -from pyload.ConfigParser import ConfigParser -from pyload.plugins.PluginManager import PluginManager -from pyload.PullEvents import PullManager +from pyload.manager.AccountManager import AccountManager +from pyload.manager.CaptchaManager import CaptchaManager +from pyload.config.Parser import ConfigParser +from pyload.manager.PluginManager import PluginManager +from pyload.manager.event.PullEvents import PullManager from pyload.network.RequestFactory import RequestFactory -from pyload.threads.ServerThread import WebServer -from pyload.Scheduler import Scheduler -from pyload.common.JsEngine import JsEngine +from pyload.manager.thread.ServerThread import WebServer +from pyload.manager.event.Scheduler import Scheduler +from pyload.utils.JsEngine import JsEngine from pyload import remote -from pyload.remote.RemoteManager import RemoteManager +from pyload.manager.RemoteManager import RemoteManager from pyload.database import DatabaseBackend, FileHandler from pyload.utils import freeSpace, formatSize, get_console_encoding @@ -99,21 +99,21 @@ class Core(object): elif option in ("-d", "--debug"): self.doDebug = True elif option in ("-u", "--user"): - from pyload.setup import Setup + from pyload.config.Setup import SetupAssistant as Setup self.config = ConfigParser() s = Setup(pypath, self.config) s.set_user() exit() elif option in ("-s", "--setup"): - from pyload.setup import Setup + from pyload.config.Setup import SetupAssistant as Setup self.config = ConfigParser() s = Setup(pypath, self.config) s.start() exit() elif option == "--changedir": - from pyload.setup import Setup + from pyload.config.Setup import SetupAssistant as Setup self.config = ConfigParser() s = Setup(pypath, self.config) @@ -263,7 +263,7 @@ class Core(object): self.version = CURRENT_VERSION if not exists("pyload.conf"): - from pyload.setup import Setup + from pyload.config.Setup import SetupAssistant as Setup print "This is your first start, running configuration assistent now." self.config = ConfigParser() @@ -376,14 +376,14 @@ class Core(object): self.lastClientConnected = 0 # later imported because they would trigger api import, and remote value not set correctly - from pyload import Api - from pyload.HookManager import HookManager - from pyload.ThreadManager import ThreadManager + from pyload import api + from pyload.manager.HookManager import HookManager + from pyload.manager.ThreadManager import ThreadManager - if Api.activated != self.remote: + if api.activated != self.remote: self.log.warning("Import error: API remote status not correct.") - self.api = Api.Api(self) + self.api = api.Api(self) self.scheduler = Scheduler(self) @@ -439,18 +439,6 @@ class Core(object): self.log.info(_("pyLoad is up and running")) - #test api -# from pyload.common.APIExerciser import startApiExerciser -# startApiExerciser(self, 3) - - #some memory stats -# from guppy import hpy -# hp=hpy() -# import objgraph -# objgraph.show_most_common_types(limit=20) -# import memdebug -# memdebug.start(8002) - locals().clear() while True: diff --git a/pyload/HookManager.py b/pyload/HookManager.py deleted file mode 100644 index 7545b4d60..000000000 --- a/pyload/HookManager.py +++ /dev/null @@ -1,305 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN, mkaay - @interface-version: 0.1 -""" -import __builtin__ - -import traceback -from thread import start_new_thread -from threading import RLock - -from types import MethodType - -from pyload.threads.PluginThread import HookThread -from pyload.plugins.PluginManager import literal_eval -from utils import lock - -class HookManager: - """Manages hooks, delegates and handles Events. - - Every plugin can define events, \ - but some very usefull events are called by the Core. - Contrary to overwriting hook methods you can use event listener, - which provides additional entry point in the control flow. - Only do very short tasks or use threads. - - **Known Events:** - Most hook methods exists as events. These are the additional known events. - - ===================== ============== ================================== - Name Arguments Description - ===================== ============== ================================== - downloadPreparing fid A download was just queued and will be prepared now. - downloadStarts fid A plugin will immediately starts the download afterwards. - linksAdded links, pid Someone just added links, you are able to modify the links. - allDownloadsProcessed Every link was handled, pyload would idle afterwards. - allDownloadsFinished Every download in queue is finished. - unrarFinished folder, fname An Unrar job finished - configChanged The config was changed via the api. - pluginConfigChanged The plugin config changed, due to api or internal process. - ===================== ============== ================================== - - | Notes: - | allDownloadsProcessed is *always* called before allDownloadsFinished. - | configChanged is *always* called before pluginConfigChanged. - - - """ - - def __init__(self, core): - self.core = core - self.config = self.core.config - - __builtin__.hookManager = self #needed to let hooks register themself - - self.log = self.core.log - self.plugins = [] - self.pluginMap = {} - self.methods = {} #dict of names and list of methods usable by rpc - - self.events = {} # contains events - - #registering callback for config event - self.config.pluginCB = MethodType(self.dispatchEvent, "pluginConfigChanged", basestring) - - self.addEvent("pluginConfigChanged", self.manageHooks) - - self.lock = RLock() - self.createIndex() - - def try_catch(func): - def new(*args): - try: - return func(*args) - except Exception, e: - args[0].log.error(_("Error executing hooks: %s") % str(e)) - if args[0].core.debug: - traceback.print_exc() - - return new - - - def addRPC(self, plugin, func, doc): - plugin = plugin.rpartition(".")[2] - doc = doc.strip() if doc else "" - - if plugin in self.methods: - self.methods[plugin][func] = doc - else: - self.methods[plugin] = {func: doc} - - def callRPC(self, plugin, func, args, parse): - if not args: args = tuple() - if parse: - args = tuple([literal_eval(x) for x in args]) - - plugin = self.pluginMap[plugin] - f = getattr(plugin, func) - return f(*args) - - - def createIndex(self): - plugins = [] - - active = [] - deactive = [] - - for pluginname in self.core.pluginManager.hookPlugins: - try: - #hookClass = getattr(plugin, plugin.__name__) - - if self.config.getPlugin(pluginname, "activated"): - pluginClass = self.core.pluginManager.loadClass("hooks", pluginname) - if not pluginClass: continue - - plugin = pluginClass(self.core, self) - plugins.append(plugin) - self.pluginMap[pluginClass.__name__] = plugin - if plugin.isActivated(): - active.append(pluginClass.__name__) - else: - deactive.append(pluginname) - - - except: - self.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) - if self.core.debug: - traceback.print_exc() - - self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active))) - self.log.info(_("Deactivate plugins: %s") % ", ".join(sorted(deactive))) - - self.plugins = plugins - - def manageHooks(self, plugin, name, value): - if name == "activated" and value: - self.activateHook(plugin) - elif name == "activated" and not value: - self.deactivateHook(plugin) - - def activateHook(self, plugin): - - #check if already loaded - for inst in self.plugins: - if inst.__name__ == plugin: - return - - pluginClass = self.core.pluginManager.loadClass("hooks", plugin) - - if not pluginClass: return - - self.log.debug("Plugin loaded: %s" % plugin) - - plugin = pluginClass(self.core, self) - self.plugins.append(plugin) - self.pluginMap[pluginClass.__name__] = plugin - - # call core Ready - start_new_thread(plugin.coreReady, tuple()) - - def deactivateHook(self, plugin): - - hook = None - for inst in self.plugins: - if inst.__name__ == plugin: - hook = inst - - if not hook: return - - self.log.debug("Plugin unloaded: %s" % plugin) - - hook.unload() - - #remove periodic call - self.log.debug("Removed callback %s" % self.core.scheduler.removeJob(hook.cb)) - self.plugins.remove(hook) - del self.pluginMap[hook.__name__] - - - @try_catch - def coreReady(self): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.coreReady() - - self.dispatchEvent("coreReady") - - @try_catch - def coreExiting(self): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.coreExiting() - - self.dispatchEvent("coreExiting") - - @lock - def downloadPreparing(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.downloadPreparing(pyfile) - - self.dispatchEvent("downloadPreparing", pyfile) - - @lock - def downloadFinished(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.downloadFinished(pyfile) - - self.dispatchEvent("downloadFinished", pyfile) - - @lock - @try_catch - def downloadFailed(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.downloadFailed(pyfile) - - self.dispatchEvent("downloadFailed", pyfile) - - @lock - def packageFinished(self, package): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.packageFinished(package) - - self.dispatchEvent("packageFinished", package) - - @lock - def beforeReconnecting(self, ip): - for plugin in self.plugins: - plugin.beforeReconnecting(ip) - - self.dispatchEvent("beforeReconnecting", ip) - - @lock - def afterReconnecting(self, ip): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.afterReconnecting(ip) - - self.dispatchEvent("afterReconnecting", ip) - - def startThread(self, function, *args, **kwargs): - t = HookThread(self.core.threadManager, function, args, kwargs) - - def activePlugins(self): - """ returns all active plugins """ - return [x for x in self.plugins if x.isActivated()] - - def getAllInfo(self): - """returns info stored by hook plugins""" - info = {} - for name, plugin in self.pluginMap.iteritems(): - if plugin.info: - #copy and convert so str - info[name] = dict([(x, str(y) if not isinstance(y, basestring) else y) for x, y in plugin.info.iteritems()]) - return info - - - def getInfo(self, plugin): - info = {} - if plugin in self.pluginMap and self.pluginMap[plugin].info: - info = dict([(x, str(y) if not isinstance(y, basestring) else y) - for x, y in self.pluginMap[plugin].info.iteritems()]) - - return info - - def addEvent(self, event, func): - """Adds an event listener for event name""" - if event in self.events: - self.events[event].append(func) - else: - self.events[event] = [func] - - def removeEvent(self, event, func): - """removes previously added event listener""" - if event in self.events: - self.events[event].remove(func) - - def dispatchEvent(self, event, *args): - """dispatches event with args""" - if event in self.events: - for f in self.events[event]: - try: - f(*args) - except Exception, e: - self.log.warning("Error calling event handler %s: %s, %s, %s" - % (event, f, args, str(e))) - if self.core.debug: - traceback.print_exc() diff --git a/pyload/PullEvents.py b/pyload/PullEvents.py deleted file mode 100644 index 0739b4ec8..000000000 --- a/pyload/PullEvents.py +++ /dev/null @@ -1,120 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from time import time -from pyload.utils import uniqify - -class PullManager: - def __init__(self, core): - self.core = core - self.clients = [] - - def newClient(self, uuid): - self.clients.append(Client(uuid)) - - def clean(self): - for n, client in enumerate(self.clients): - if client.lastActive + 30 < time(): - del self.clients[n] - - def getEvents(self, uuid): - events = [] - validUuid = False - for client in self.clients: - if client.uuid == uuid: - client.lastActive = time() - validUuid = True - while client.newEvents(): - events.append(client.popEvent().toList()) - break - if not validUuid: - self.newClient(uuid) - events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] - return uniqify(events) - - def addEvent(self, event): - for client in self.clients: - client.addEvent(event) - -class Client: - def __init__(self, uuid): - self.uuid = uuid - self.lastActive = time() - self.events = [] - - def newEvents(self): - return len(self.events) > 0 - - def popEvent(self): - if not len(self.events): - return None - return self.events.pop(0) - - def addEvent(self, event): - self.events.append(event) - -class UpdateEvent: - def __init__(self, itype, iid, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.destination = destination - - def toList(self): - return ["update", self.destination, self.type, self.id] - -class RemoveEvent: - def __init__(self, itype, iid, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.destination = destination - - def toList(self): - return ["remove", self.destination, self.type, self.id] - -class InsertEvent: - def __init__(self, itype, iid, after, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.after = after - self.destination = destination - - def toList(self): - return ["insert", self.destination, self.type, self.id, self.after] - -class ReloadAllEvent: - def __init__(self, destination): - assert destination == "queue" or destination == "collector" - self.destination = destination - - def toList(self): - return ["reload", self.destination] - -class AccountUpdateEvent: - def toList(self): - return ["account"] - -class ConfigUpdateEvent: - def toList(self): - return ["config"] diff --git a/pyload/PyFile.py b/pyload/PyFile.py deleted file mode 100644 index c4ce71570..000000000 --- a/pyload/PyFile.py +++ /dev/null @@ -1,284 +0,0 @@ -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN - @author: mkaay -""" - -from pyload.PullEvents import UpdateEvent -from pyload.utils import formatSize, lock - -from time import sleep, time - -from threading import RLock - -statusMap = { - "finished": 0, - "offline": 1, - "online": 2, - "queued": 3, - "skipped": 4, - "waiting": 5, - "temp. offline": 6, - "starting": 7, - "failed": 8, - "aborted": 9, - "decrypting": 10, - "custom": 11, - "downloading": 12, - "processing": 13, - "unknown": 14, -} - - -def setSize(self, value): - self._size = int(value) - -class PyFile(object): - """ - Represents a file object at runtime - """ - __slots__ = ("m", "id", "url", "name", "size", "_size", "status", "pluginname", "packageid", - "error", "order", "lock", "plugin", "waitUntil", "active", "abort", "statusname", - "reconnected", "progress", "maxprogress", "pluginmodule", "pluginclass") - - def __init__(self, manager, id, url, name, size, status, error, pluginname, package, order): - self.m = manager - - self.id = int(id) - self.url = url - self.name = name - self.size = size - self.status = status - self.pluginname = pluginname - self.packageid = package #should not be used, use package() instead - self.error = error - self.order = order - # database information ends here - - self.lock = RLock() - - self.plugin = None - #self.download = None - - self.waitUntil = 0 # time() + time to wait - - # status attributes - self.active = False #obsolete? - self.abort = False - self.reconnected = False - - self.statusname = None - - self.progress = 0 - self.maxprogress = 100 - - self.m.cache[int(id)] = self - - - # will convert all sizes to ints - size = property(lambda self: self._size, setSize) - - def __repr__(self): - return "PyFile %s: %s@%s" % (self.id, self.name, self.pluginname) - - @lock - def initPlugin(self): - """ inits plugin instance """ - if not self.plugin: - self.pluginmodule = self.m.core.pluginManager.getPlugin(self.pluginname) - self.pluginclass = getattr(self.pluginmodule, self.m.core.pluginManager.getPluginName(self.pluginname)) - self.plugin = self.pluginclass(self) - - @lock - def hasPlugin(self): - """Thread safe way to determine this file has initialized plugin attribute - - :return: - """ - return hasattr(self, "plugin") and self.plugin - - def package(self): - """ return package instance""" - return self.m.getPackage(self.packageid) - - def setStatus(self, status): - self.status = statusMap[status] - self.sync() #@TODO needed aslong no better job approving exists - - def setCustomStatus(self, msg, status="processing"): - self.statusname = msg - self.setStatus(status) - - def getStatusName(self): - if self.status not in (13, 14) or not self.statusname: - return self.m.statusMsg[self.status] - else: - return self.statusname - - def hasStatus(self, status): - return statusMap[status] == self.status - - def sync(self): - """sync PyFile instance with database""" - self.m.updateLink(self) - - @lock - def release(self): - """sync and remove from cache""" - # file has valid package - if self.packageid > 0: - self.sync() - - if hasattr(self, "plugin") and self.plugin: - self.plugin.clean() - del self.plugin - - self.m.releaseLink(self.id) - - def delete(self): - """delete pyfile from database""" - self.m.deleteLink(self.id) - - def toDict(self): - """return dict with all information for interface""" - return self.toDbDict() - - def toDbDict(self): - """return data as dict for databse - - format: - - { - id: {'url': url, 'name': name ... } - } - - """ - return { - self.id: { - 'id': self.id, - 'url': self.url, - 'name': self.name, - 'plugin': self.pluginname, - 'size': self.getSize(), - 'format_size': self.formatSize(), - 'status': self.status, - 'statusmsg': self.getStatusName(), - 'package': self.packageid, - 'error': self.error, - 'order': self.order - } - } - - def abortDownload(self): - """abort pyfile if possible""" - while self.id in self.m.core.threadManager.processingIds(): - self.abort = True - if self.plugin and self.plugin.req: - self.plugin.req.abortDownloads() - sleep(0.1) - - self.abort = False - if self.hasPlugin() and self.plugin.req: - self.plugin.req.abortDownloads() - - self.release() - - def finishIfDone(self): - """set status to finish and release file if every thread is finished with it""" - - if self.id in self.m.core.threadManager.processingIds(): - return False - - self.setStatus("finished") - self.release() - self.m.checkAllLinksFinished() - return True - - def checkIfProcessed(self): - self.m.checkAllLinksProcessed(self.id) - - def formatWait(self): - """ formats and return wait time in humanreadable format """ - seconds = self.waitUntil - time() - - if seconds < 0: return "00:00:00" - - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - - def formatSize(self): - """ formats size to readable format """ - return formatSize(self.getSize()) - - def formatETA(self): - """ formats eta to readable format """ - seconds = self.getETA() - - if seconds < 0: return "00:00:00" - - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - - def getSpeed(self): - """ calculates speed """ - try: - return self.plugin.req.speed - except: - return 0 - - def getETA(self): - """ gets established time of arrival""" - try: - return self.getBytesLeft() / self.getSpeed() - except: - return 0 - - def getBytesLeft(self): - """ gets bytes left """ - try: - return self.getSize() - self.plugin.req.arrived - except: - return 0 - - def getPercent(self): - """ get % of download """ - if self.status == 12: - try: - return self.plugin.req.percent - except: - return 0 - else: - return self.progress - - def getSize(self): - """ get size of download """ - try: - if self.plugin.req.size: - return self.plugin.req.size - else: - return self.size - except: - return self.size - - def notifyChange(self): - e = UpdateEvent("file", self.id, "collector" if not self.package().queue else "queue") - self.m.core.pullManager.addEvent(e) - - def setProgress(self, value): - if not value == self.progress: - self.progress = value - self.notifyChange() diff --git a/pyload/PyPackage.py b/pyload/PyPackage.py deleted file mode 100644 index 3961d8a70..000000000 --- a/pyload/PyPackage.py +++ /dev/null @@ -1,79 +0,0 @@ -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN - @author: mkaay -""" - -from pyload.PullEvents import UpdateEvent -from pyload.utils import safe_filename - -class PyPackage: - """ - Represents a package object at runtime - """ - def __init__(self, manager, id, name, folder, site, password, queue, order): - self.m = manager - self.m.packageCache[int(id)] = self - - self.id = int(id) - self.name = name - self._folder = folder - self.site = site - self.password = password - self.queue = queue - self.order = order - self.setFinished = False - - @property - def folder(self): - return safe_filename(self._folder) - - def toDict(self): - """ Returns a dictionary representation of the data. - - :return: dict: {id: { attr: value }} - """ - return { - self.id: { - 'id': self.id, - 'name': self.name, - 'folder': self.folder, - 'site': self.site, - 'password': self.password, - 'queue': self.queue, - 'order': self.order, - 'links': {} - } - } - - def getChildren(self): - """get information about contained links""" - return self.m.getPackageData(self.id)["links"] - - def sync(self): - """sync with db""" - self.m.updatePackage(self) - - def release(self): - """sync and delete from cache""" - self.sync() - self.m.releasePackage(self.id) - - def delete(self): - self.m.deletePackage(self.id) - - def notifyChange(self): - e = UpdateEvent("pack", self.id, "collector" if not self.queue else "queue") - self.m.core.pullManager.addEvent(e) diff --git a/pyload/Scheduler.py b/pyload/Scheduler.py deleted file mode 100644 index 71b5f96af..000000000 --- a/pyload/Scheduler.py +++ /dev/null @@ -1,141 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from time import time -from heapq import heappop, heappush -from thread import start_new_thread -from threading import Lock - -class AlreadyCalled(Exception): - pass - - -class Deferred: - def __init__(self): - self.call = [] - self.result = () - - def addCallback(self, f, *cargs, **ckwargs): - self.call.append((f, cargs, ckwargs)) - - def callback(self, *args, **kwargs): - if self.result: - raise AlreadyCalled - self.result = (args, kwargs) - for f, cargs, ckwargs in self.call: - args += tuple(cargs) - kwargs.update(ckwargs) - f(*args ** kwargs) - - -class Scheduler: - def __init__(self, core): - self.core = core - - self.queue = PriorityQueue() - - def addJob(self, t, call, args=[], kwargs={}, threaded=True): - d = Deferred() - t += time() - j = Job(t, call, args, kwargs, d, threaded) - self.queue.put((t, j)) - return d - - - def removeJob(self, d): - """ - :param d: defered object - :return: if job was deleted - """ - index = -1 - - for i, j in enumerate(self.queue): - if j[1].deferred == d: - index = i - - if index >= 0: - del self.queue[index] - return True - - return False - - def work(self): - while True: - t, j = self.queue.get() - if not j: - break - else: - if t <= time(): - j.start() - else: - self.queue.put((t, j)) - break - - -class Job: - def __init__(self, time, call, args=[], kwargs={}, deferred=None, threaded=True): - self.time = float(time) - self.call = call - self.args = args - self.kwargs = kwargs - self.deferred = deferred - self.threaded = threaded - - def run(self): - ret = self.call(*self.args, **self.kwargs) - if self.deferred is None: - return - else: - self.deferred.callback(ret) - - def start(self): - if self.threaded: - start_new_thread(self.run, ()) - else: - self.run() - - -class PriorityQueue: - """ a non blocking priority queue """ - - def __init__(self): - self.queue = [] - self.lock = Lock() - - def __iter__(self): - return iter(self.queue) - - def __delitem__(self, key): - del self.queue[key] - - def put(self, element): - self.lock.acquire() - heappush(self.queue, element) - self.lock.release() - - def get(self): - """ return element or None """ - self.lock.acquire() - try: - el = heappop(self.queue) - return el - except IndexError: - return None, None - finally: - self.lock.release() diff --git a/pyload/ThreadManager.py b/pyload/ThreadManager.py deleted file mode 100644 index d9a6e1b8c..000000000 --- a/pyload/ThreadManager.py +++ /dev/null @@ -1,317 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from os.path import exists, join -import re -from subprocess import Popen -from threading import Event, Lock -from time import sleep, time -from traceback import print_exc -from random import choice - -import pycurl - -from pyload.threads import PluginThread -from pyload.PyFile import PyFile -from pyload.network.RequestFactory import getURL -from pyload.utils import freeSpace, lock - - -class ThreadManager: - """manages the download threads, assign jobs, reconnect etc""" - - - def __init__(self, core): - """Constructor""" - self.core = core - self.log = core.log - - self.threads = [] # thread list - self.localThreads = [] #hook+decrypter threads - - self.pause = True - - self.reconnecting = Event() - self.reconnecting.clear() - self.downloaded = 0 #number of files downloaded since last cleanup - - self.lock = Lock() - - # some operations require to fetch url info from hoster, so we caching them so it wont be done twice - # contains a timestamp and will be purged after timeout - self.infoCache = {} - - # pool of ids for online check - self.resultIDs = 0 - - # threads which are fetching hoster results - self.infoResults = {} - #timeout for cache purge - self.timestamp = 0 - - pycurl.global_init(pycurl.GLOBAL_DEFAULT) - - for i in range(0, self.core.config.get("download", "max_downloads")): - self.createThread() - - - def createThread(self): - """create a download thread""" - - thread = PluginThread.DownloadThread(self) - self.threads.append(thread) - - def createInfoThread(self, data, pid): - """ - start a thread whichs fetches online status and other infos - data = [ .. () .. ] - """ - self.timestamp = time() + 5 * 60 - - PluginThread.InfoThread(self, data, pid) - - @lock - def createResultThread(self, data, add=False): - """ creates a thread to fetch online status, returns result id """ - self.timestamp = time() + 5 * 60 - - rid = self.resultIDs - self.resultIDs += 1 - - PluginThread.InfoThread(self, data, rid=rid, add=add) - - return rid - - - @lock - def getInfoResult(self, rid): - """returns result and clears it""" - self.timestamp = time() + 5 * 60 - - if rid in self.infoResults: - data = self.infoResults[rid] - self.infoResults[rid] = {} - return data - else: - return {} - - @lock - def setInfoResults(self, rid, result): - self.infoResults[rid].update(result) - - def getActiveFiles(self): - active = [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)] - - for t in self.localThreads: - active.extend(t.getActiveFiles()) - - return active - - def processingIds(self): - """get a id list of all pyfiles processed""" - return [x.id for x in self.getActiveFiles()] - - - def work(self): - """run all task which have to be done (this is for repetivive call by core)""" - try: - self.tryReconnect() - except Exception, e: - self.log.error(_("Reconnect Failed: %s") % str(e) ) - self.reconnecting.clear() - if self.core.debug: - print_exc() - self.checkThreadCount() - - try: - self.assignJob() - except Exception, e: - self.log.warning("Assign job error", e) - if self.core.debug: - print_exc() - - sleep(0.5) - self.assignJob() - #it may be failed non critical so we try it again - - if (self.infoCache or self.infoResults) and self.timestamp < time(): - self.infoCache.clear() - self.infoResults.clear() - self.log.debug("Cleared Result cache") - - #-------------------------------------------------------------------------- - def tryReconnect(self): - """checks if reconnect needed""" - - if not (self.core.config["reconnect"]["activated"] and self.core.api.isTimeReconnect()): - return False - - active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active] - - if not (0 < active.count(True) == len(active)): - return False - - if not exists(self.core.config['reconnect']['method']): - if exists(join(pypath, self.core.config['reconnect']['method'])): - self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method']) - else: - self.core.config["reconnect"]["activated"] = False - self.log.warning(_("Reconnect script not found!")) - return - - self.reconnecting.set() - - #Do reconnect - self.log.info(_("Starting reconnect")) - - while [x.active.plugin.waiting for x in self.threads if x.active].count(True) != 0: - sleep(0.25) - - ip = self.getIP() - - self.core.hookManager.beforeReconnecting(ip) - - self.log.debug("Old IP: %s" % ip) - - try: - reconn = Popen(self.core.config['reconnect']['method'], bufsize=-1, shell=True)#, stdout=subprocess.PIPE) - except: - self.log.warning(_("Failed executing reconnect script!")) - self.core.config["reconnect"]["activated"] = False - self.reconnecting.clear() - if self.core.debug: - print_exc() - return - - reconn.wait() - sleep(1) - ip = self.getIP() - self.core.hookManager.afterReconnecting(ip) - - self.log.info(_("Reconnected, new IP: %s") % ip) - - self.reconnecting.clear() - - def getIP(self): - """retrieve current ip""" - services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"), - ("http://checkip.dyndns.org/",".*Current IP Address: (\S+).*")] - - ip = "" - for i in range(10): - try: - sv = choice(services) - ip = getURL(sv[0]) - ip = re.match(sv[1], ip).group(1) - break - except: - ip = "" - sleep(1) - - return ip - - #-------------------------------------------------------------------------- - def checkThreadCount(self): - """checks if there are need for increasing or reducing thread count""" - - if len(self.threads) == self.core.config.get("download", "max_downloads"): - return True - elif len(self.threads) < self.core.config.get("download", "max_downloads"): - self.createThread() - else: - free = [x for x in self.threads if not x.active] - if free: - free[0].put("quit") - - - def cleanPycurl(self): - """ make a global curl cleanup (currently ununused) """ - if self.processingIds(): - return False - pycurl.global_cleanup() - pycurl.global_init(pycurl.GLOBAL_DEFAULT) - self.downloaded = 0 - self.log.debug("Cleaned up pycurl") - return True - - #-------------------------------------------------------------------------- - def assignJob(self): - """assing a job to a thread if possible""" - - if self.pause or not self.core.api.isTimeDownload(): return - - #if self.downloaded > 20: - # if not self.cleanPyCurl(): return - - free = [x for x in self.threads if not x.active] - - inuse = set([(x.active.pluginname, self.getLimit(x)) for x in self.threads if x.active and x.active.hasPlugin() and x.active.plugin.account]) - inuse = map(lambda x: (x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) ,inuse) - onlimit = [x[0] for x in inuse if x[1] > 0 and x[2] >= x[1]] - - occ = [x.active.pluginname for x in self.threads if x.active and x.active.hasPlugin() and not x.active.plugin.multiDL] + onlimit - - occ.sort() - occ = tuple(set(occ)) - job = self.core.files.getJob(occ) - if job: - try: - job.initPlugin() - except Exception, e: - self.log.critical(str(e)) - print_exc() - job.setStatus("failed") - job.error = str(e) - job.release() - return - - if job.plugin.__type__ == "hoster": - spaceLeft = freeSpace(self.core.config["general"]["download_folder"]) / 1024 / 1024 - if spaceLeft < self.core.config["general"]["min_free_space"]: - self.log.warning(_("Not enough space left on device")) - self.pause = True - - if free and not self.pause: - thread = free[0] - #self.downloaded += 1 - - thread.put(job) - else: - #put job back - if occ not in self.core.files.jobCache: - self.core.files.jobCache[occ] = [] - self.core.files.jobCache[occ].append(job.id) - - #check for decrypt jobs - job = self.core.files.getDecryptJob() - if job: - job.initPlugin() - thread = PluginThread.DecrypterThread(self, job) - - - else: - thread = PluginThread.DecrypterThread(self, job) - - def getLimit(self, thread): - limit = thread.active.plugin.account.getAccountData(thread.active.plugin.user)["options"].get("limitDL", ["0"])[0] - return int(limit) - - def cleanup(self): - """do global cleanup, should be called when finished with pycurl""" - pycurl.global_cleanup() diff --git a/pyload/api/__init__.py b/pyload/api/__init__.py new file mode 100644 index 000000000..3da1e13e1 --- /dev/null +++ b/pyload/api/__init__.py @@ -0,0 +1,1030 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from base64 import standard_b64encode +from os.path import join +from time import time +import re + +from pyload.datatypes.PyFile import PyFile +from utils import freeSpace, compare_time +from pyload.utils.packagetools import parseNames +from network.RequestFactory import getURL +from remote import activated + +if activated: + try: + from remote.thriftbackend.thriftgen.pyload.ttypes import * + from remote.thriftbackend.thriftgen.pyload.Pyload import Iface + BaseObject = TBase + except ImportError: + print "Thrift not imported" + from remote.socketbackend.ttypes import * +else: + from remote.socketbackend.ttypes import * + +# contains function names mapped to their permissions +# unlisted functions are for admins only +permMap = {} + +# decorator only called on init, never initialized, so has no effect on runtime +def permission(bits): + class _Dec(object): + def __new__(cls, func, *args, **kwargs): + permMap[func.__name__] = bits + return func + + return _Dec + + +urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-=\\\.&]*)", re.IGNORECASE) + +class PERMS: + ALL = 0 # requires no permission, but login + ADD = 1 # can add packages + DELETE = 2 # can delete packages + STATUS = 4 # see and change server status + LIST = 16 # see queue and collector + MODIFY = 32 # moddify some attribute of downloads + DOWNLOAD = 64 # can download from webinterface + SETTINGS = 128 # can access settings + ACCOUNTS = 256 # can access accounts + LOGS = 512 # can see server logs + +class ROLE: + ADMIN = 0 #admin has all permissions implicit + USER = 1 + +def has_permission(userperms, perms): + # bytewise or perms before if needed + return perms == (userperms & perms) + + +class Api(Iface): + """ + **pyLoads API** + + This is accessible either internal via core.api or via thrift backend. + + see Thrift specification file remote/thriftbackend/pyload.thrift\ + for information about data structures and what methods are usuable with rpc. + + Most methods requires specific permissions, please look at the source code if you need to know.\ + These can be configured via webinterface. + Admin user have all permissions, and are the only ones who can access the methods with no specific permission. + """ + + EXTERNAL = Iface # let the json api know which methods are external + + def __init__(self, core): + self.core = core + + def _convertPyFile(self, p): + f = FileData(p["id"], p["url"], p["name"], p["plugin"], p["size"], + p["format_size"], p["status"], p["statusmsg"], + p["package"], p["error"], p["order"]) + return f + + def _convertConfigFormat(self, c): + sections = {} + for sectionName, sub in c.iteritems(): + section = ConfigSection(sectionName, sub["desc"]) + items = [] + for key, data in sub.iteritems(): + if key in ("desc", "outline"): + continue + item = ConfigItem() + item.name = key + item.description = data["desc"] + item.value = str(data["value"]) if not isinstance(data["value"], basestring) else data["value"] + item.type = data["type"] + items.append(item) + section.items = items + sections[sectionName] = section + if "outline" in sub: + section.outline = sub["outline"] + return sections + + @permission(PERMS.SETTINGS) + def getConfigValue(self, category, option, section="core"): + """Retrieve config value. + + :param category: name of category, or plugin + :param option: config option + :param section: 'plugin' or 'core' + :return: config value as string + """ + if section == "core": + value = self.core.config[category][option] + else: + value = self.core.config.getPlugin(category, option) + + return str(value) if not isinstance(value, basestring) else value + + @permission(PERMS.SETTINGS) + def setConfigValue(self, category, option, value, section="core"): + """Set new config value. + + :param category: + :param option: + :param value: new config value + :param section: 'plugin' or 'core + """ + self.core.hookManager.dispatchEvent("configChanged", category, option, value, section) + + if section == "core": + self.core.config[category][option] = value + + if option in ("limit_speed", "max_speed"): #not so nice to update the limit + self.core.requestFactory.updateBucket() + + elif section == "plugin": + self.core.config.setPlugin(category, option, value) + + @permission(PERMS.SETTINGS) + def getConfig(self): + """Retrieves complete config of core. + + :return: list of `ConfigSection` + """ + return self._convertConfigFormat(self.core.config.config) + + def getConfigDict(self): + """Retrieves complete config in dict format, not for RPC. + + :return: dict + """ + return self.core.config.config + + @permission(PERMS.SETTINGS) + def getPluginConfig(self): + """Retrieves complete config for all plugins. + + :return: list of `ConfigSection` + """ + return self._convertConfigFormat(self.core.config.plugin) + + def getPluginConfigDict(self): + """Plugin config as dict, not for RPC. + + :return: dict + """ + return self.core.config.plugin + + + @permission(PERMS.STATUS) + def pauseServer(self): + """Pause server: Tt wont start any new downloads, but nothing gets aborted.""" + self.core.threadManager.pause = True + + @permission(PERMS.STATUS) + def unpauseServer(self): + """Unpause server: New Downloads will be started.""" + self.core.threadManager.pause = False + + @permission(PERMS.STATUS) + def togglePause(self): + """Toggle pause state. + + :return: new pause state + """ + self.core.threadManager.pause ^= True + return self.core.threadManager.pause + + @permission(PERMS.STATUS) + def toggleReconnect(self): + """Toggle reconnect activation. + + :return: new reconnect state + """ + self.core.config["reconnect"]["activated"] ^= True + return self.core.config["reconnect"]["activated"] + + @permission(PERMS.LIST) + def statusServer(self): + """Some general information about the current status of pyLoad. + + :return: `ServerStatus` + """ + serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), + self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, + not self.core.threadManager.pause and self.isTimeDownload(), + self.core.config['reconnect']['activated'] and self.isTimeReconnect()) + + for pyfile in [x.active for x in self.core.threadManager.threads if x.active and isinstance(x.active, PyFile)]: + serverStatus.speed += pyfile.getSpeed() #bytes/s + + return serverStatus + + @permission(PERMS.STATUS) + def freeSpace(self): + """Available free space at download directory in bytes""" + return freeSpace(self.core.config["general"]["download_folder"]) + + @permission(PERMS.ALL) + def getServerVersion(self): + """pyLoad Core version """ + return self.core.version + + def kill(self): + """Clean way to quit pyLoad""" + self.core.do_kill = True + + def restart(self): + """Restart pyload core""" + self.core.do_restart = True + + @permission(PERMS.LOGS) + def getLog(self, offset=0): + """Returns most recent log entries. + + :param offset: line offset + :return: List of log entries + """ + filename = join(self.core.config['log']['log_folder'], 'log.txt') + try: + fh = open(filename, "r") + lines = fh.readlines() + fh.close() + if offset >= len(lines): + return [] + return lines[offset:] + except: + return ['No log available'] + + @permission(PERMS.STATUS) + def isTimeDownload(self): + """Checks if pyload will start new downloads according to time in config. + + :return: bool + """ + start = self.core.config['downloadTime']['start'].split(":") + end = self.core.config['downloadTime']['end'].split(":") + return compare_time(start, end) + + @permission(PERMS.STATUS) + def isTimeReconnect(self): + """Checks if pyload will try to make a reconnect + + :return: bool + """ + start = self.core.config['reconnect']['startTime'].split(":") + end = self.core.config['reconnect']['endTime'].split(":") + return compare_time(start, end) and self.core.config["reconnect"]["activated"] + + @permission(PERMS.LIST) + def statusDownloads(self): + """ Status off all currently running downloads. + + :return: list of `DownloadStatus` + """ + data = [] + for pyfile in self.core.threadManager.getActiveFiles(): + if not isinstance(pyfile, PyFile): + continue + + data.append(DownloadInfo( + pyfile.id, pyfile.name, pyfile.getSpeed(), pyfile.getETA(), pyfile.formatETA(), + pyfile.getBytesLeft(), pyfile.getSize(), pyfile.formatSize(), pyfile.getPercent(), + pyfile.status, pyfile.getStatusName(), pyfile.formatWait(), + pyfile.waitUntil, pyfile.packageid, pyfile.package().name, pyfile.pluginname)) + + return data + + @permission(PERMS.ADD) + def addPackage(self, name, links, dest=Destination.Queue): + """Adds a package, with links to desired destination. + + :param name: name of the new package + :param links: list of urls + :param dest: `Destination` + :return: package id of the new package + """ + if self.core.config['general']['folder_per_package']: + folder = name + else: + folder = "" + + folder = folder.replace("http://", "").replace(":", "").replace("/", "_").replace("\\", "_") + + pid = self.core.files.addPackage(name, folder, dest) + + self.core.files.addLinks(links, pid) + + self.core.log.info(_("Added package %(name)s containing %(count)d links") % {"name": name, "count": len(links)}) + + self.core.files.save() + + return pid + + @permission(PERMS.ADD) + def parseURLs(self, html=None, url=None): + """Parses html content or any arbitaty text for links and returns result of `checkURLs` + + :param html: html source + :return: + """ + urls = [] + + if html: + urls += [x[0] for x in urlmatcher.findall(html)] + + if url: + page = getURL(url) + urls += [x[0] for x in urlmatcher.findall(page)] + + # remove duplicates + return self.checkURLs(set(urls)) + + + @permission(PERMS.ADD) + def checkURLs(self, urls): + """ Gets urls and returns pluginname mapped to list of matches urls. + + :param urls: + :return: {plugin: urls} + """ + data = self.core.pluginManager.parseUrls(urls) + plugins = {} + + for url, plugin in data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + return plugins + + @permission(PERMS.ADD) + def checkOnlineStatus(self, urls): + """ initiates online status check + + :param urls: + :return: initial set of data as `OnlineCheck` instance containing the result id + """ + data = self.core.pluginManager.parseUrls(urls) + + rid = self.core.threadManager.createResultThread(data, False) + + tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] + data = parseNames(tmp) + result = {} + + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + return OnlineCheck(rid, result) + + @permission(PERMS.ADD) + def checkOnlineStatusContainer(self, urls, container, data): + """ checks online status of urls and a submited container file + + :param urls: list of urls + :param container: container file name + :param data: file content + :return: online check + """ + th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb") + th.write(str(data)) + th.close() + + return self.checkOnlineStatus(urls + [th.name]) + + @permission(PERMS.ADD) + def pollResults(self, rid): + """ Polls the result available for ResultID + + :param rid: `ResultID` + :return: `OnlineCheck`, if rid is -1 then no more data available + """ + result = self.core.threadManager.getInfoResult(rid) + + if "ALL_INFO_FETCHED" in result: + del result["ALL_INFO_FETCHED"] + return OnlineCheck(-1, result) + else: + return OnlineCheck(rid, result) + + + @permission(PERMS.ADD) + def generatePackages(self, links): + """ Parses links, generates packages names from urls + + :param links: list of urls + :return: package names mapped to urls + """ + result = parseNames((x, x) for x in links) + return result + + @permission(PERMS.ADD) + def generateAndAddPackages(self, links, dest=Destination.Queue): + """Generates and add packages + + :param links: list of urls + :param dest: `Destination` + :return: list of package ids + """ + return [self.addPackage(name, urls, dest) for name, urls + in self.generatePackages(links).iteritems()] + + @permission(PERMS.ADD) + def checkAndAddPackages(self, links, dest=Destination.Queue): + """Checks online status, retrieves names, and will add packages.\ + Because of this packages are not added immediatly, only for internal use. + + :param links: list of urls + :param dest: `Destination` + :return: None + """ + data = self.core.pluginManager.parseUrls(links) + self.core.threadManager.createResultThread(data, True) + + + @permission(PERMS.LIST) + def getPackageData(self, pid): + """Returns complete information about package, and included files. + + :param pid: package id + :return: `PackageData` with .links attribute + """ + data = self.core.files.getPackageData(int(pid)) + + if not data: + raise PackageDoesNotExists(pid) + + pdata = PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], + data["queue"], data["order"], + links=[self._convertPyFile(x) for x in data["links"].itervalues()]) + + return pdata + + @permission(PERMS.LIST) + def getPackageInfo(self, pid): + """Returns information about package, without detailed information about containing files + + :param pid: package id + :return: `PackageData` with .fid attribute + """ + data = self.core.files.getPackageData(int(pid)) + + if not data: + raise PackageDoesNotExists(pid) + + pdata = PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], + data["queue"], data["order"], + fids=[int(x) for x in data["links"]]) + + return pdata + + @permission(PERMS.LIST) + def getFileData(self, fid): + """Get complete information about a specific file. + + :param fid: file id + :return: `FileData` + """ + info = self.core.files.getFileData(int(fid)) + if not info: + raise FileDoesNotExists(fid) + + fdata = self._convertPyFile(info.values()[0]) + return fdata + + @permission(PERMS.DELETE) + def deleteFiles(self, fids): + """Deletes several file entries from pyload. + + :param fids: list of file ids + """ + for id in fids: + self.core.files.deleteLink(int(id)) + + self.core.files.save() + + @permission(PERMS.DELETE) + def deletePackages(self, pids): + """Deletes packages and containing links. + + :param pids: list of package ids + """ + for id in pids: + self.core.files.deletePackage(int(id)) + + self.core.files.save() + + @permission(PERMS.LIST) + def getQueue(self): + """Returns info about queue and packages, **not** about files, see `getQueueData` \ + or `getPackageData` instead. + + :return: list of `PackageInfo` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + pack["linkstotal"]) + for pack in self.core.files.getInfoData(Destination.Queue).itervalues()] + + @permission(PERMS.LIST) + def getQueueData(self): + """Return complete data about everything in queue, this is very expensive use it sparely.\ + See `getQueue` for alternative. + + :return: list of `PackageData` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) + for pack in self.core.files.getCompleteData(Destination.Queue).itervalues()] + + @permission(PERMS.LIST) + def getCollector(self): + """same as `getQueue` for collector. + + :return: list of `PackageInfo` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + pack["linkstotal"]) + for pack in self.core.files.getInfoData(Destination.Collector).itervalues()] + + @permission(PERMS.LIST) + def getCollectorData(self): + """same as `getQueueData` for collector. + + :return: list of `PackageInfo` + """ + return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) + for pack in self.core.files.getCompleteData(Destination.Collector).itervalues()] + + + @permission(PERMS.ADD) + def addFiles(self, pid, links): + """Adds files to specific package. + + :param pid: package id + :param links: list of urls + """ + self.core.files.addLinks(links, int(pid)) + + self.core.log.info(_("Added %(count)d links to package #%(package)d ") % {"count": len(links), "package": pid}) + self.core.files.save() + + @permission(PERMS.MODIFY) + def pushToQueue(self, pid): + """Moves package from Collector to Queue. + + :param pid: package id + """ + self.core.files.setPackageLocation(pid, Destination.Queue) + + @permission(PERMS.MODIFY) + def pullFromQueue(self, pid): + """Moves package from Queue to Collector. + + :param pid: package id + """ + self.core.files.setPackageLocation(pid, Destination.Collector) + + @permission(PERMS.MODIFY) + def restartPackage(self, pid): + """Restarts a package, resets every containing files. + + :param pid: package id + """ + self.core.files.restartPackage(int(pid)) + + @permission(PERMS.MODIFY) + def restartFile(self, fid): + """Resets file status, so it will be downloaded again. + + :param fid: file id + """ + self.core.files.restartFile(int(fid)) + + @permission(PERMS.MODIFY) + def recheckPackage(self, pid): + """Proofes online status of all files in a package, also a default action when package is added. + + :param pid: + :return: + """ + self.core.files.reCheckPackage(int(pid)) + + @permission(PERMS.MODIFY) + def stopAllDownloads(self): + """Aborts all running downloads.""" + + pyfiles = self.core.files.cache.values() + for pyfile in pyfiles: + pyfile.abortDownload() + + @permission(PERMS.MODIFY) + def stopDownloads(self, fids): + """Aborts specific downloads. + + :param fids: list of file ids + :return: + """ + pyfiles = self.core.files.cache.values() + + for pyfile in pyfiles: + if pyfile.id in fids: + pyfile.abortDownload() + + @permission(PERMS.MODIFY) + def setPackageName(self, pid, name): + """Renames a package. + + :param pid: package id + :param name: new package name + """ + pack = self.core.files.getPackage(pid) + pack.name = name + pack.sync() + + @permission(PERMS.MODIFY) + def movePackage(self, destination, pid): + """Set a new package location. + + :param destination: `Destination` + :param pid: package id + """ + if destination not in (0, 1): return + self.core.files.setPackageLocation(pid, destination) + + @permission(PERMS.MODIFY) + def moveFiles(self, fids, pid): + """Move multiple files to another package + + :param fids: list of file ids + :param pid: destination package + :return: + """ + #TODO: implement + pass + + + @permission(PERMS.ADD) + def uploadContainer(self, filename, data): + """Uploads and adds a container file to pyLoad. + + :param filename: filename, extension is important so it can correctly decrypted + :param data: file content + """ + th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb") + th.write(str(data)) + th.close() + + self.addPackage(th.name, [th.name], Destination.Queue) + + @permission(PERMS.MODIFY) + def orderPackage(self, pid, position): + """Gives a package a new position. + + :param pid: package id + :param position: + """ + self.core.files.reorderPackage(pid, position) + + @permission(PERMS.MODIFY) + def orderFile(self, fid, position): + """Gives a new position to a file within its package. + + :param fid: file id + :param position: + """ + self.core.files.reorderFile(fid, position) + + @permission(PERMS.MODIFY) + def setPackageData(self, pid, data): + """Allows to modify several package attributes. + + :param pid: package id + :param data: dict that maps attribute to desired value + """ + p = self.core.files.getPackage(pid) + if not p: raise PackageDoesNotExists(pid) + + for key, value in data.iteritems(): + if key == "id": continue + setattr(p, key, value) + + p.sync() + self.core.files.save() + + @permission(PERMS.DELETE) + def deleteFinished(self): + """Deletes all finished files and completly finished packages. + + :return: list of deleted package ids + """ + return self.core.files.deleteFinishedLinks() + + @permission(PERMS.MODIFY) + def restartFailed(self): + """Restarts all failed failes.""" + self.core.files.restartFailed() + + @permission(PERMS.LIST) + def getPackageOrder(self, destination): + """Returns information about package order. + + :param destination: `Destination` + :return: dict mapping order to package id + """ + + packs = self.core.files.getInfoData(destination) + order = {} + + for pid in packs: + pack = self.core.files.getPackageData(int(pid)) + while pack["order"] in order.keys(): #just in case + pack["order"] += 1 + order[pack["order"]] = pack["id"] + return order + + @permission(PERMS.LIST) + def getFileOrder(self, pid): + """Information about file order within package. + + :param pid: + :return: dict mapping order to file id + """ + rawData = self.core.files.getPackageData(int(pid)) + order = {} + for id, pyfile in rawData["links"].iteritems(): + while pyfile["order"] in order.keys(): #just in case + pyfile["order"] += 1 + order[pyfile["order"]] = pyfile["id"] + return order + + + @permission(PERMS.STATUS) + def isCaptchaWaiting(self): + """Indicates wether a captcha task is available + + :return: bool + """ + self.core.lastClientConnected = time() + task = self.core.captchaManager.getTask() + return not task is None + + @permission(PERMS.STATUS) + def getCaptchaTask(self, exclusive=False): + """Returns a captcha task + + :param exclusive: unused + :return: `CaptchaTask` + """ + self.core.lastClientConnected = time() + task = self.core.captchaManager.getTask() + if task: + task.setWatingForUser(exclusive=exclusive) + data, type, result = task.getCaptcha() + t = CaptchaTask(int(task.id), standard_b64encode(data), type, result) + return t + else: + return CaptchaTask(-1) + + @permission(PERMS.STATUS) + def getCaptchaTaskStatus(self, tid): + """Get information about captcha task + + :param tid: task id + :return: string + """ + self.core.lastClientConnected = time() + t = self.core.captchaManager.getTaskByID(tid) + return t.getStatus() if t else "" + + @permission(PERMS.STATUS) + def setCaptchaResult(self, tid, result): + """Set result for a captcha task + + :param tid: task id + :param result: captcha result + """ + self.core.lastClientConnected = time() + task = self.core.captchaManager.getTaskByID(tid) + if task: + task.setResult(result) + self.core.captchaManager.removeTask(task) + + + @permission(PERMS.STATUS) + def getEvents(self, uuid): + """Lists occured events, may be affected to changes in future. + + :param uuid: + :return: list of `Events` + """ + events = self.core.pullManager.getEvents(uuid) + newEvents = [] + + def convDest(d): + return Destination.Queue if d == "queue" else Destination.Collector + + for e in events: + event = EventInfo() + event.eventname = e[0] + if e[0] in ("update", "remove", "insert"): + event.id = e[3] + event.type = ElementType.Package if e[2] == "pack" else ElementType.File + event.destination = convDest(e[1]) + elif e[0] == "order": + if e[1]: + event.id = e[1] + event.type = ElementType.Package if e[2] == "pack" else ElementType.File + event.destination = convDest(e[3]) + elif e[0] == "reload": + event.destination = convDest(e[1]) + newEvents.append(event) + return newEvents + + @permission(PERMS.ACCOUNTS) + def getAccounts(self, refresh): + """Get information about all entered accounts. + + :param refresh: reload account info + :return: list of `AccountInfo` + """ + accs = self.core.accountManager.getAccountInfos(False, refresh) + accounts = [] + for group in accs.values(): + accounts.extend([AccountInfo(acc["validuntil"], acc["login"], acc["options"], acc["valid"], + acc["trafficleft"], acc["maxtraffic"], acc["premium"], acc["type"]) + for acc in group]) + return accounts + + @permission(PERMS.ALL) + def getAccountTypes(self): + """All available account types. + + :return: list + """ + return self.core.accountManager.accounts.keys() + + @permission(PERMS.ACCOUNTS) + def updateAccount(self, plugin, account, password=None, options={}): + """Changes pw/options for specific account.""" + self.core.accountManager.updateAccount(plugin, account, password, options) + + @permission(PERMS.ACCOUNTS) + def removeAccount(self, plugin, account): + """Remove account from pyload. + + :param plugin: pluginname + :param account: accountname + """ + self.core.accountManager.removeAccount(plugin, account) + + @permission(PERMS.ALL) + def login(self, username, password, remoteip=None): + """Login into pyLoad, this **must** be called when using rpc before any methods can be used. + + :param username: + :param password: + :param remoteip: Omit this argument, its only used internal + :return: bool indicating login was successful + """ + return True if self.checkAuth(username, password, remoteip) else False + + def checkAuth(self, username, password, remoteip=None): + """Check authentication and returns details + + :param username: + :param password: + :param remoteip: + :return: dict with info, empty when login is incorrect + """ + if self.core.config["remote"]["nolocalauth"] and remoteip == "127.0.0.1": + return "local" + else: + return self.core.db.checkAuth(username, password) + + def isAuthorized(self, func, userdata): + """checks if the user is authorized for specific method + + :param func: function name + :param userdata: dictionary of user data + :return: boolean + """ + if userdata == "local" or userdata["role"] == ROLE.ADMIN: + return True + elif func in permMap and has_permission(userdata["permission"], permMap[func]): + return True + else: + return False + + + @permission(PERMS.ALL) + def getUserData(self, username, password): + """similar to `checkAuth` but returns UserData thrift type """ + user = self.checkAuth(username, password) + if user: + return UserData(user["name"], user["email"], user["role"], user["permission"], user["template"]) + else: + return UserData() + + + def getAllUserData(self): + """returns all known user and info""" + res = {} + for user, data in self.core.db.getAllUserData().iteritems(): + res[user] = UserData(user, data["email"], data["role"], data["permission"], data["template"]) + + return res + + @permission(PERMS.STATUS) + def getServices(self): + """ A dict of available services, these can be defined by hook plugins. + + :return: dict with this style: {"plugin": {"method": "description"}} + """ + data = {} + for plugin, funcs in self.core.hookManager.methods.iteritems(): + data[plugin] = funcs + + return data + + @permission(PERMS.STATUS) + def hasService(self, plugin, func): + """Checks wether a service is available. + + :param plugin: + :param func: + :return: bool + """ + cont = self.core.hookManager.methods + return plugin in cont and func in cont[plugin] + + @permission(PERMS.STATUS) + def call(self, info): + """Calls a service (a method in hook plugin). + + :param info: `ServiceCall` + :return: result + :raises: ServiceDoesNotExists, when its not available + :raises: ServiceException, when a exception was raised + """ + plugin = info.plugin + func = info.func + args = info.arguments + parse = info.parseArguments + + if not self.hasService(plugin, func): + raise ServiceDoesNotExists(plugin, func) + + try: + ret = self.core.hookManager.callRPC(plugin, func, args, parse) + return str(ret) + except Exception, e: + raise ServiceException(e.message) + + @permission(PERMS.STATUS) + def getAllInfo(self): + """Returns all information stored by hook plugins. Values are always strings + + :return: {"plugin": {"name": value}} + """ + return self.core.hookManager.getAllInfo() + + @permission(PERMS.STATUS) + def getInfoByPlugin(self, plugin): + """Returns information stored by a specific plugin. + + :param plugin: pluginname + :return: dict of attr names mapped to value {"name": value} + """ + return self.core.hookManager.getInfo(plugin) + + def changePassword(self, user, oldpw, newpw): + """ changes password for specific user """ + return self.core.db.changePassword(user, oldpw, newpw) + + def setUserPermission(self, user, permission, role): + self.core.db.setPermission(user, permission) + self.core.db.setRole(user, role) diff --git a/pyload/cli/Cli.py b/pyload/cli/Cli.py index 447829ef7..f05e98b1a 100644 --- a/pyload/cli/Cli.py +++ b/pyload/cli/Cli.py @@ -19,7 +19,7 @@ from __future__ import with_statement from getopt import GetoptError, getopt -import pyload.common.pylgettext as gettext +import pyload.utils.pylgettext as gettext import os from os import _exit from os.path import join, exists, abspath, basename @@ -29,7 +29,7 @@ from threading import Thread, Lock from time import sleep from traceback import print_exc -import ConfigParser +from pyload.config.Parser import ConfigParser from codecs import getwriter @@ -44,7 +44,7 @@ from pyload import InitHomeDir from pyload.cli.printer import * from pyload.cli import AddPackage, ManageFiles -from pyload.Api import Destination +from pyload.api import Destination from pyload.utils import formatSize, decode from pyload.remote.thriftbackend.ThriftClient import ThriftClient, NoConnection, NoSSL, WrongLogin, ConnectionClosed from Getch import Getch diff --git a/pyload/cli/ManageFiles.py b/pyload/cli/ManageFiles.py index fba96b990..6901e3731 100644 --- a/pyload/cli/ManageFiles.py +++ b/pyload/cli/ManageFiles.py @@ -23,7 +23,7 @@ from time import time from Handler import Handler from printer import * -from pyload.Api import Destination, PackageData +from pyload.api import Destination, PackageData class ManageFiles(Handler): """ possibility to manage queue/collector """ diff --git a/pyload/common/APIExerciser.py b/pyload/common/APIExerciser.py deleted file mode 100644 index 886c72a4a..000000000 --- a/pyload/common/APIExerciser.py +++ /dev/null @@ -1,157 +0,0 @@ -# -*- coding: utf-8 -*- - -import string -from threading import Thread -from random import choice, random, sample, randint -from time import time, sleep -from math import floor -import gc - -from traceback import print_exc, format_exc - -from pyload.remote.thriftbackend.ThriftClient import ThriftClient, Destination - -def createURLs(): - """ create some urls, some may fail """ - urls = [] - for x in range(0, randint(20, 100)): - name = "DEBUG_API" - if randint(0, 5) == 5: - name = "" #this link will fail - - urls.append(name + "".join(sample(string.ascii_letters, randint(10, 20)))) - - return urls - -AVOID = (0, 3, 8) - -idPool = 0 -sumCalled = 0 - - -def startApiExerciser(core, n): - for i in range(n): - APIExerciser(core).start() - -class APIExerciser(Thread): - - - def __init__(self, core, thrift=False, user=None, pw=None): - global idPool - - Thread.__init__(self) - self.setDaemon(True) - self.core = core - self.count = 0 #number of methods - self.time = time() - - if thrift: - self.api = ThriftClient(user=user, password=pw) - else: - self.api = core.api - - - self.id = idPool - - idPool += 1 - - #self.start() - - def run(self): - - self.core.log.info("API Excerciser started %d" % self.id) - - out = open("error.log", "ab") - #core errors are not logged of course - out.write("\n" + "Starting\n") - out.flush() - - while True: - try: - self.testAPI() - except Exception: - self.core.log.error("Excerciser %d throw an execption" % self.id) - print_exc() - out.write(format_exc() + 2 * "\n") - out.flush() - - if not self.count % 100: - self.core.log.info("Exerciser %d tested %d api calls" % (self.id, self.count)) - if not self.count % 1000: - out.flush() - - if not sumCalled % 1000: #not thread safe - self.core.log.info("Exercisers tested %d api calls" % sumCalled) - persec = sumCalled / (time() - self.time) - self.core.log.info("Approx. %.2f calls per second." % persec) - self.core.log.info("Approx. %.2f ms per call." % (1000 / persec)) - self.core.log.info("Collected garbage: %d" % gc.collect()) - - - #sleep(random() / 500) - - def testAPI(self): - global sumCalled - - m = ["statusDownloads", "statusServer", "addPackage", "getPackageData", "getFileData", "deleteFiles", - "deletePackages", "getQueue", "getCollector", "getQueueData", "getCollectorData", "isCaptchaWaiting", - "getCaptchaTask", "stopAllDownloads", "getAllInfo", "getServices" , "getAccounts", "getAllUserData"] - - method = choice(m) - #print "Testing:", method - - if hasattr(self, method): - res = getattr(self, method)() - else: - res = getattr(self.api, method)() - - self.count += 1 - sumCalled += 1 - - #print res - - def addPackage(self): - name = "".join(sample(string.ascii_letters, 10)) - urls = createURLs() - - self.api.addPackage(name, urls, choice([Destination.Queue, Destination.Collector])) - - - def deleteFiles(self): - info = self.api.getQueueData() - if not info: return - - pack = choice(info) - fids = pack.links - - if len(fids): - fids = [f.fid for f in sample(fids, randint(1, max(len(fids) / 2, 1)))] - self.api.deleteFiles(fids) - - - def deletePackages(self): - info = choice([self.api.getQueue(), self.api.getCollector()]) - if not info: return - - pids = [p.pid for p in info] - if len(pids): - pids = sample(pids, randint(1, max(floor(len(pids) / 2.5), 1))) - self.api.deletePackages(pids) - - def getFileData(self): - info = self.api.getQueueData() - if info: - p = choice(info) - if p.links: - self.api.getFileData(choice(p.links).fid) - - def getPackageData(self): - info = self.api.getQueue() - if info: - self.api.getPackageData(choice(info).pid) - - def getAccounts(self): - self.api.getAccounts(False) - - def getCaptchaTask(self): - self.api.getCaptchaTask(False) diff --git a/pyload/common/ImportDebugger.py b/pyload/common/ImportDebugger.py deleted file mode 100644 index ae3aef629..000000000 --- a/pyload/common/ImportDebugger.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys - -class ImportDebugger(object): - - def __init__(self): - self.imported = {} - - def find_module(self, name, path=None): - - if name not in self.imported: - self.imported[name] = 0 - - self.imported[name] += 1 - - print name, path - -sys.meta_path.append(ImportDebugger()) diff --git a/pyload/common/JsEngine.py b/pyload/common/JsEngine.py deleted file mode 100644 index 46789f64d..000000000 --- a/pyload/common/JsEngine.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from imp import find_module -from os.path import join, exists -from urllib import quote - - -ENGINE = "" - -DEBUG = False -JS = False -PYV8 = False -RHINO = False - - -if not ENGINE: - try: - import subprocess - - subprocess.Popen(["js", "-v"], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - p = subprocess.Popen(["js", "-e", "print(23+19)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - #integrity check - if out.strip() == "42": - ENGINE = "js" - JS = True - except: - pass - -if not ENGINE or DEBUG: - try: - find_module("PyV8") - ENGINE = "pyv8" - PYV8 = True - except: - pass - -if not ENGINE or DEBUG: - try: - path = "" #path where to find rhino - - if exists("/usr/share/java/js.jar"): - path = "/usr/share/java/js.jar" - elif exists("js.jar"): - path = "js.jar" - elif exists(join(pypath, "js.jar")): #may raises an exception, but js.jar wasnt found anyway - path = join(pypath, "js.jar") - - if not path: - raise Exception - - import subprocess - - p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", "print(23+19)"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - #integrity check - if out.strip() == "42": - ENGINE = "rhino" - RHINO = True - except: - pass - -class JsEngine: - def __init__(self): - self.engine = ENGINE - self.init = False - - def __nonzero__(self): - return False if not ENGINE else True - - def eval(self, script): - if not self.init: - if ENGINE == "pyv8" or (DEBUG and PYV8): - import PyV8 - global PyV8 - - self.init = True - - if type(script) == unicode: - script = script.encode("utf8") - - if not ENGINE: - raise Exception("No JS Engine") - - if not DEBUG: - if ENGINE == "pyv8": - return self.eval_pyv8(script) - elif ENGINE == "js": - return self.eval_js(script) - elif ENGINE == "rhino": - return self.eval_rhino(script) - else: - results = [] - if PYV8: - res = self.eval_pyv8(script) - print "PyV8:", res - results.append(res) - if JS: - res = self.eval_js(script) - print "JS:", res - results.append(res) - if RHINO: - res = self.eval_rhino(script) - print "Rhino:", res - results.append(res) - - warning = False - for x in results: - for y in results: - if x != y: - warning = True - - if warning: print "### WARNING ###: Different results" - - return results[0] - - def eval_pyv8(self, script): - rt = PyV8.JSContext() - rt.enter() - return rt.eval(script) - - def eval_js(self, script): - script = "print(eval(unescape('%s')))" % quote(script) - p = subprocess.Popen(["js", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) - out, err = p.communicate() - res = out.strip() - return res - - def eval_rhino(self, script): - script = "print(eval(unescape('%s')))" % quote(script) - p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", script], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) - out, err = p.communicate() - res = out.strip() - return res.decode("utf8").encode("ISO-8859-1") - - def error(self): - return _("No js engine detected, please install either Spidermonkey, ossp-js, pyv8 or rhino") diff --git a/pyload/common/__init__.py b/pyload/common/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pyload/common/json_layer.py b/pyload/common/json_layer.py deleted file mode 100644 index bb3937cdc..000000000 --- a/pyload/common/json_layer.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- - -# abstraction layer for json operations - -try: # since python 2.6 - import json - from json import loads as json_loads - from json import dumps as json_dumps -except ImportError: #use system simplejson if available - import simplejson as json - from simplejson import loads as json_loads - from simplejson import dumps as json_dumps diff --git a/pyload/common/packagetools.py b/pyload/common/packagetools.py deleted file mode 100644 index d5ab4d182..000000000 --- a/pyload/common/packagetools.py +++ /dev/null @@ -1,136 +0,0 @@ -# JDownloader/src/jd/controlling/LinkGrabberPackager.java - -import re -from urlparse import urlparse - -def matchFirst(string, *args): - """ matches against list of regexp and returns first match""" - for patternlist in args: - for pattern in patternlist: - r = pattern.search(string) - if r is not None: - name = r.group(1) - return name - - return string - - -def parseNames(files): - """ Generates packages names from name, data lists - - :param files: list of (name, data) - :return: packagenames mapt to data lists (eg. urls) - """ - packs = {} - - endings = "\\.(3gp|7zip|7z|abr|ac3|aiff|aifc|aif|ai|au|avi|bin|bz2|cbr|cbz|ccf|cue|cvd|chm|dta|deb|divx|djvu|dlc|dmg|doc|docx|dot|eps|exe|ff|flv|f4v|gsd|gif|gz|iwd|iso|ipsw|java|jar|jpg|jpeg|jdeatme|load|mws|mw|m4v|m4a|mkv|mp2|mp3|mp4|mov|movie|mpeg|mpe|mpg|msi|msu|msp|nfo|npk|oga|ogg|ogv|otrkey|pkg|png|pdf|pptx|ppt|pps|ppz|pot|psd|qt|rmvb|rm|rar|ram|ra|rev|rnd|r\\d+|rpm|run|rsdf|rtf|sh(!?tml)|srt|snd|sfv|swf|tar|tif|tiff|ts|txt|viv|vivo|vob|wav|wmv|xla|xls|xpi|zeno|zip|z\\d+|_[_a-z]{2}|\\d+$)" - - rarPats = [re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I), - re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I), - re.compile("(.*)\\.rar$", re.I), - re.compile("(.*)\\.r\\d+$", re.I), - re.compile("(.*)(\\.|_|-)\\d+$", re.I)] - - zipPats = [re.compile("(.*)\\.zip$", re.I), - re.compile("(.*)\\.z\\d+$", re.I), - re.compile("(?is).*\\.7z\\.[\\d]+$", re.I), - re.compile("(.*)\\.a.$", re.I)] - - ffsjPats = [re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)"), - re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I)] - - iszPats = [re.compile("(.*)\\.isz$", re.I), - re.compile("(.*)\\.i\\d{2}$", re.I)] - - pat1 = re.compile("(\\.?CD\\d+)", re.I) - pat2 = re.compile("(\\.?part\\d+)", re.I) - - pat3 = re.compile("(.+)[\\.\\-_]+$") - pat4 = re.compile("(.+)\\.\\d+\\.xtm$") - - for file, url in files: - patternMatch = False - - if file is None: - continue - - # remove trailing / - name = file.rstrip('/') - - # extract last path part .. if there is a path - split = name.rsplit("/", 1) - if len(split) > 1: - name = split.pop(1) - - #check if a already existing package may be ok for this file - # found = False - # for pack in packs: - # if pack in file: - # packs[pack].append(url) - # found = True - # break - # - # if found: continue - - # unrar pattern, 7zip/zip and hjmerge pattern, isz pattern, FFSJ pattern - before = name - name = matchFirst(name, rarPats, zipPats, iszPats, ffsjPats) - if before != name: - patternMatch = True - - # xtremsplit pattern - r = pat4.search(name) - if r is not None: - name = r.group(1) - - # remove part and cd pattern - r = pat1.search(name) - if r is not None: - name = name.replace(r.group(0), "") - patternMatch = True - - r = pat2.search(name) - if r is not None: - name = name.replace(r.group(0), "") - patternMatch = True - - # additional checks if extension pattern matched - if patternMatch: - # remove extension - index = name.rfind(".") - if index <= 0: - index = name.rfind("_") - if index > 0: - length = len(name) - index - if length <= 4: - name = name[:-length] - - # remove endings like . _ - - r = pat3.search(name) - if r is not None: - name = r.group(1) - - # replace . and _ with space - name = name.replace(".", " ") - name = name.replace("_", " ") - - name = name.strip() - else: - name = "" - - # fallback: package by hoster - if not name: - name = urlparse(file).hostname - if name: name = name.replace("www.", "") - - # fallback : default name - if not name: - name = "unknown" - - # build mapping - if name in packs: - packs[name].append(url) - else: - packs[name] = [url] - - return packs diff --git a/pyload/common/pavement.py b/pyload/common/pavement.py deleted file mode 100644 index 9b2dc98b3..000000000 --- a/pyload/common/pavement.py +++ /dev/null @@ -1,412 +0,0 @@ -# -*- coding: utf-8 -*- - -from paver.easy import * -from paver.setuputils import setup -from paver.doctools import cog - -import os -import sys -import shutil -import re -from glob import glob -from tempfile import mkdtemp -from urllib import urlretrieve -from subprocess import call, Popen, PIPE -from zipfile import ZipFile - -PROJECT_DIR = path(__file__).dirname() -sys.path.append(PROJECT_DIR) - -options = environment.options -path("pyload").mkdir() - -extradeps = [] -if sys.version_info <= (2, 5): - extradeps += 'simplejson' - -setup( - name="pyload", - version="0.4.10", - description='Fast, lightweight and full featured download manager.', - long_description=open(PROJECT_DIR / "README.md").read(), - keywords = ("pyload", "download-manager", "one-click-hoster", "download"), - url="http://pyload.org", - download_url='http://pyload.org/download', - license='GPL v3', - author="pyLoad Team", - author_email="support@pyload.org", - platforms = ('Any',), - #package_dir={'pyload': "src"}, - packages=["pyload"], - #package_data=find_package_data(), - #data_files=[], - include_package_data=True, - exclude_package_data={'pyload': ["docs*", "scripts*", "tests*"]}, #exluced from build but not from sdist - # 'bottle >= 0.10.0' not in list, because its small and contain little modifications - install_requires=['thrift >= 0.8.0', 'jinja2', 'pycurl', 'Beaker', 'BeautifulSoup >= 3.2, < 3.3'] + extradeps, - extras_require={ - 'SSL': ["pyOpenSSL"], - 'DLC': ['pycrypto'], - 'lightweight webserver': ['bjoern'], - 'RSS plugins': ['feedparser'], - }, - #setup_requires=["setuptools_hg"], - entry_points={ - 'console_scripts': [ - 'pyLoadCore = pyLoadCore:main', - 'pyLoadCli = pyLoadCli:main' - ]}, - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Topic :: Internet :: WWW/HTTP", - "Environment :: Console", - "Environment :: Web Environment", - "Intended Audience :: End Users/Desktop", - "License :: OSI Approved :: GNU General Public License (GPL)", - "Operating System :: OS Independent", - "Programming Language :: Python :: 2" - ] -) - -options( - sphinx=Bunch( - builddir="_build", - sourcedir="" - ), - get_source=Bunch( - src="https://bitbucket.org/spoob/pyload/get/tip.zip", - rev=None, - clean=False - ), - thrift=Bunch( - path="../thrift/trunk/compiler/cpp/thrift", - gen="" - ), - virtualenv=Bunch( - dir="env", - python="python2", - virtual="virtualenv2", - ), - cog=Bunch( - pattern="*.py", - ) -) - -# xgettext args -xargs = ["--language=Python", "--add-comments=L10N", - "--from-code=utf-8", "--copyright-holder=pyLoad Team", "--package-name=pyLoad", - "--package-version=%s" % options.version, "--msgid-bugs-address='bugs@pyload.org'"] - -@task -@needs('cog') -def html(): - """Build html documentation""" - module = path("docs") / "pyload" - pyload.rmtree() - call_task('paver.doctools.html') - - -@task -@cmdopts([ - ('src=', 's', 'Url to source'), - ('rev=', 'r', "HG revision"), - ("clean", 'c', 'Delete old source folder') -]) -def get_source(options): - """ Downloads pyload source from bitbucket tip or given rev""" - if options.rev: options.url = "https://bitbucket.org/spoob/pyload/get/%s.zip" % options.rev - - pyload = path("pyload") - - if len(pyload.listdir()) and not options.clean: - return - elif pyload.exists(): - pyload.rmtree() - - urlretrieve(options.src, "pyload_src.zip") - zip = ZipFile("pyload_src.zip") - zip.extractall() - path("pyload_src.zip").remove() - - folder = [x for x in path(".").dirs() if x.name.startswith("spoob-pyload-")][0] - folder.move(pyload) - - change_mode(pyload, 0644) - change_mode(pyload, 0755, folder=True) - - for file in pyload.files(): - if file.name.endswith(".py"): - file.chmod(0755) - - (pyload / ".hgtags").remove() - (pyload / ".gitignore").remove() - #(pyload / "docs").rmtree() - - f = open(pyload / "__init__.py", "wb") - f.close() - - #options.setup.packages = find_packages() - #options.setup.package_data = find_package_data() - - -@task -@needs('clean', 'generate_setup', 'minilib', 'get_source', 'setuptools.command.sdist') -def sdist(): - """ Build source code package with distutils """ - - -@task -@cmdopts([ - ('path=', 'p', 'Thrift path'), - ('gen=', 'g', "Extra --gen option") -]) -def thrift(options): - """ Generate Thrift stubs """ - - print "add import for TApplicationException manually as long it is not fixed" - - outdir = path("pyload") / "remote" / "thriftbackend" - (outdir / "gen-py").rmtree() - - cmd = [options.thrift.path, "-strict", "-o", outdir, "--gen", "py:slots, dynamic", outdir / "pyload.thrift"] - - if options.gen: - cmd.insert(len(cmd) - 1, "--gen") - cmd.insert(len(cmd) - 1, options.gen) - - print "running", cmd - - p = Popen(cmd) - p.communicate() - - (outdir / "thriftgen").rmtree() - (outdir / "gen-py").move(outdir / "thriftgen") - - #create light ttypes - from pyload.remote.socketbackend.create_ttypes import main - main() - -@task -def compile_js(): - """ Compile .coffee files to javascript""" - - root = path("pyload") / "web" / "media" / "js" - for f in root.glob("*.coffee"): - print "generate", f - coffee = Popen(["coffee", "-cbs"], stdin=open(f, "rb"), stdout=PIPE) - yui = Popen(["yuicompressor", "--type", "js"], stdin=coffee.stdout, stdout=PIPE) - coffee.stdout.close() - content = yui.communicate()[0] - with open(root / f.name.replace(".coffee", ".js"), "wb") as js: - js.write("{% autoescape true %}\n") - js.write(content) - js.write("\n{% endautoescape %}") - - -@task -def generate_locale(): - """ Generates localization files """ - - EXCLUDE = ["BeautifulSoup.py", "pyload/cli", "web/locale", "web/ajax", "web/cnl", "web/pyload", - "setup.py"] - makepot("core", path("pyload"), EXCLUDE, "./pyload.py\n") - - makepot("cli", path("pyload") / "cli", [], includes="./pyload-cli.py\n") - makepot("setup", "", [], includes="./pyload/setup.py\n") - - EXCLUDE = ["ServerThread.py", "web/media/default"] - - # strings from js files - strings = set() - - for fi in path("pyload/web").walkfiles(): - if not fi.name.endswith(".js") and not fi.endswith(".coffee"): continue - with open(fi, "rb") as c: - content = c.read() - - strings.update(re.findall(r"_\s*\(\s*\"([^\"]+)", content)) - strings.update(re.findall(r"_\s*\(\s*\'([^\']+)", content)) - - trans = path("pyload") / "web" / "translations.js" - - with open(trans, "wb") as js: - for s in strings: - js.write('_("%s")\n' % s) - - makepot("django", path("pyload/web"), EXCLUDE, "./%s\n" % trans.relpath(), [".py", ".html"], ["--language=Python"]) - - trans.remove() - - path("includes.txt").remove() - - print "Locale generated" - - -@task -@cmdopts([ - ('key=', 'k', 'api key') -]) -def upload_translations(options): - """ Uploads the locale files to translation server """ - tmp = path(mkdtemp()) - - shutil.copy('locale/crowdin.yaml', tmp) - os.mkdir(tmp / 'pyLoad') - for f in glob('locale/*.pot'): - if os.path.isfile(f): - shutil.copy(f, tmp / 'pyLoad') - - config = tmp / 'crowdin.yaml' - content = open(config, 'rb').read() - content = content.format(key=options.key, tmp=tmp) - f = open(config, 'wb') - f.write(content) - f.close() - - call(['crowdin-cli', '-c', config, 'upload', 'source']) - - shutil.rmtree(tmp) - - print "Translations uploaded" - - -@task -@cmdopts([ - ('key=', 'k', 'api key') -]) -def download_translations(options): - """ Downloads the translated files from translation server """ - tmp = path(mkdtemp()) - - shutil.copy('locale/crowdin.yaml', tmp) - os.mkdir(tmp / 'pyLoad') - for f in glob('locale/*.pot'): - if os.path.isfile(f): - shutil.copy(f, tmp / 'pyLoad') - - config = tmp / 'crowdin.yaml' - content = open(config, 'rb').read() - content = content.format(key=options.key, tmp=tmp) - f = open(config, 'wb') - f.write(content) - f.close() - - call(['crowdin-cli', '-c', config, 'download']) - - for language in (tmp / 'pyLoad').listdir(): - if not language.isdir(): - continue - - target = path('locale') / language.basename() - print "Copy language %s" % target - if target.exists(): - shutil.rmtree(target) - - shutil.copytree(language, target) - - shutil.rmtree(tmp) - - -@task -def compile_translations(): - """ Compile PO files to MO """ - for language in path('locale').listdir(): - if not language.isdir(): - continue - - for f in glob(language / 'LC_MESSAGES' / '*.po'): - print "Compiling %s" % f - call(['msgfmt', '-o', f.replace('.po', '.mo'), f]) - - -@task -def tests(): - call(["nosetests2"]) - -@task -def virtualenv(options): - """Setup virtual environment""" - if path(options.dir).exists(): - return - - call([options.virtual, "--no-site-packages", "--python", options.python, options.dir]) - print "$ source %s/bin/activate" % options.dir - - -@task -def clean_env(): - """Deletes the virtual environment""" - env = path(options.virtualenv.dir) - if env.exists(): - env.rmtree() - - -@task -@needs('generate_setup', 'minilib', 'get_source', 'virtualenv') -def env_install(): - """Install pyLoad into the virtualenv""" - venv = options.virtualenv - call([path(venv.dir) / "bin" / "easy_install", "."]) - - -@task -def clean(): - """Cleans build directories""" - path("build").rmtree() - path("dist").rmtree() - - -#helper functions - -def walk_trans(path, EXCLUDE, endings=[".py"]): - result = "" - - for f in path.walkfiles(): - if [True for x in EXCLUDE if x in f.dirname().relpath()]: continue - if f.name in EXCLUDE: continue - - for e in endings: - if f.name.endswith(e): - result += "./%s\n" % f.relpath() - break - - return result - - -def makepot(domain, p, excludes=[], includes="", endings=[".py"], xxargs=[]): - print "Generate %s.pot" % domain - - f = open("includes.txt", "wb") - if includes: - f.write(includes) - - if p: - f.write(walk_trans(path(p), excludes, endings)) - - f.close() - - call(["xgettext", "--files-from=includes.txt", "--default-domain=%s" % domain] + xargs + xxargs) - - # replace charset und move file - with open("%s.po" % domain, "rb") as f: - content = f.read() - - path("%s.po" % domain).remove() - content = content.replace("charset=CHARSET", "charset=UTF-8") - - with open("locale/%s.pot" % domain, "wb") as f: - f.write(content) - - -def change_owner(dir, uid, gid): - for p in dir.walk(): - p.chown(uid, gid) - - -def change_mode(dir, mode, folder=False): - for p in dir.walk(): - if folder and p.isdir(): - p.chmod(mode) - elif p.isfile() and not folder: - p.chmod(mode) diff --git a/pyload/common/pylgettext.py b/pyload/common/pylgettext.py deleted file mode 100644 index cab631cf4..000000000 --- a/pyload/common/pylgettext.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- - -from gettext import * - -_searchdirs = None - -origfind = find - -def setpaths(pathlist): - global _searchdirs - if isinstance(pathlist, list): - _searchdirs = pathlist - else: - _searchdirs = list(pathlist) - - -def addpath(path): - global _searchdirs - if _searchdirs is None: - _searchdirs = list(path) - else: - if path not in _searchdirs: - _searchdirs.append(path) - - -def delpath(path): - global _searchdirs - if _searchdirs is not None: - if path in _searchdirs: - _searchdirs.remove(path) - - -def clearpath(): - global _searchdirs - if _searchdirs is not None: - _searchdirs = None - - -def find(domain, localedir=None, languages=None, all=False): - if _searchdirs is None: - return origfind(domain, localedir, languages, all) - searches = [localedir] + _searchdirs - results = list() - for dir in searches: - res = origfind(domain, dir, languages, all) - if all is False: - results.append(res) - else: - results.extend(res) - if all is False: - results = filter(lambda x: x is not None, results) - if len(results) == 0: - return None - else: - return results[0] - else: - return results - -#Is there a smarter/cleaner pythonic way for this? -translation.func_globals['find'] = find diff --git a/pyload/common/test_api.py b/pyload/common/test_api.py deleted file mode 100644 index 4efaa35d6..000000000 --- a/pyload/common/test_api.py +++ /dev/null @@ -1,20 +0,0 @@ -# -*- coding: utf-8 -*- - -from pyload.common import APIExerciser -from nose.tools import nottest - - -class TestApi: - - def __init__(self): - self.api = APIExerciser.APIExerciser(None, True, "TestUser", "pwhere") - - def test_login(self): - assert self.api.api.login("crapp", "wrong pw") is False - - #takes really long, only test when needed - @nottest - def test_random(self): - - for i in range(0, 100): - self.api.testAPI() diff --git a/pyload/common/test_json.py b/pyload/common/test_json.py deleted file mode 100644 index 320a42d4f..000000000 --- a/pyload/common/test_json.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- - -from urllib import urlencode -from urllib2 import urlopen, HTTPError -from json import loads - -from logging import log - -url = "http://localhost:8001/api/%s" - -class TestJson: - - def call(self, name, post=None): - if not post: post = {} - post["session"] = self.key - u = urlopen(url % name, data=urlencode(post)) - return loads(u.read()) - - def setUp(self): - u = urlopen(url % "login", data=urlencode({"username": "TestUser", "password": "pwhere"})) - self.key = loads(u.read()) - assert self.key is not False - - def test_wronglogin(self): - u = urlopen(url % "login", data=urlencode({"username": "crap", "password": "wrongpw"})) - assert loads(u.read()) is False - - def test_access(self): - try: - urlopen(url % "getServerVersion") - except HTTPError, e: - assert e.code == 403 - else: - assert False - - def test_status(self): - ret = self.call("statusServer") - log(1, str(ret)) - assert "pause" in ret - assert "queue" in ret - - def test_unknown_method(self): - try: - self.call("notExisting") - except HTTPError, e: - assert e.code == 404 - else: - assert False diff --git a/pyload/config/Parser.py b/pyload/config/Parser.py new file mode 100644 index 000000000..64ce6b10e --- /dev/null +++ b/pyload/config/Parser.py @@ -0,0 +1,373 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement +from time import sleep +from os.path import exists, join +from shutil import copy + +from traceback import print_exc +from utils import chmod + +# ignore these plugin configs, mainly because plugins were wiped out +IGNORE = ( + "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('hooks', 'UnRar'), + 'EasyShareCom', 'FlyshareCz' + ) + +CONF_VERSION = 1 + +class ConfigParser: + """ + holds and manage the configuration + + current dict layout: + + { + + section: { + option: { + value: + type: + desc: + } + desc: + + } + + """ + + + def __init__(self): + """Constructor""" + self.config = {} # the config values + self.plugin = {} # the config for plugins + self.oldRemoteData = {} + + self.pluginCB = None # callback when plugin config value is changed + + self.checkVersion() + + self.readConfig() + + self.deleteOldPlugins() + + + def checkVersion(self, n=0): + """determines if config need to be copied""" + try: + if not exists("pyload.conf"): + copy(join(pypath, "pyload", "config", "default.conf"), "pyload.conf") + + if not exists("plugin.conf"): + f = open("plugin.conf", "wb") + f.write("version: " + str(CONF_VERSION)) + f.close() + + f = open("pyload.conf", "rb") + v = f.readline() + f.close() + v = v[v.find(":") + 1:].strip() + + if not v or int(v) < CONF_VERSION: + copy(join(pypath, "pyload", "config", "default.conf"), "pyload.conf") + print "Old version of config was replaced" + + f = open("plugin.conf", "rb") + v = f.readline() + f.close() + v = v[v.find(":") + 1:].strip() + + if not v or int(v) < CONF_VERSION: + f = open("plugin.conf", "wb") + f.write("version: " + str(CONF_VERSION)) + f.close() + print "Old version of plugin-config replaced" + except: + if n < 3: + sleep(0.3) + self.checkVersion(n + 1) + else: + raise + + def readConfig(self): + """reads the config file""" + + self.config = self.parseConfig(join(pypath, "pyload", "config", "default.conf")) + self.plugin = self.parseConfig("plugin.conf") + + try: + homeconf = self.parseConfig("pyload.conf") + if "username" in homeconf["remote"]: + if "password" in homeconf["remote"]: + self.oldRemoteData = {"username": homeconf["remote"]["username"]["value"], + "password": homeconf["remote"]["username"]["value"]} + del homeconf["remote"]["password"] + del homeconf["remote"]["username"] + self.updateValues(homeconf, self.config) + + except Exception, e: + print "Config Warning" + print_exc() + + + def parseConfig(self, config): + """parses a given configfile""" + + f = open(config) + + config = f.read() + + config = config.splitlines()[1:] + + conf = {} + + section, option, value, typ, desc = "", "", "", "", "" + + listmode = False + + for line in config: + comment = line.rfind("#") + if line.find(":", comment) < 0 > line.find("=", comment) and comment > 0 and line[comment - 1].isspace(): + line = line.rpartition("#") # removes comments + if line[1]: + line = line[0] + else: + line = line[2] + + line = line.strip() + + try: + if line == "": + continue + elif line.endswith(":"): + section, none, desc = line[:-1].partition('-') + section = section.strip() + desc = desc.replace('"', "").strip() + conf[section] = {"desc": desc} + else: + if listmode: + if line.endswith("]"): + listmode = False + line = line.replace("]", "") + + value += [self.cast(typ, x.strip()) for x in line.split(",") if x] + + if not listmode: + conf[section][option] = {"desc": desc, + "type": typ, + "value": value} + + + else: + content, none, value = line.partition("=") + + content, none, desc = content.partition(":") + + desc = desc.replace('"', "").strip() + + typ, none, option = content.strip().rpartition(" ") + + value = value.strip() + + if value.startswith("["): + if value.endswith("]"): + listmode = False + value = value[:-1] + else: + listmode = True + + value = [self.cast(typ, x.strip()) for x in value[1:].split(",") if x] + else: + value = self.cast(typ, value) + + if not listmode: + conf[section][option] = {"desc": desc, + "type": typ, + "value": value} + + except Exception, e: + print "Config Warning" + print_exc() + + f.close() + return conf + + + def updateValues(self, config, dest): + """sets the config values from a parsed config file to values in destination""" + + for section in config.iterkeys(): + if section in dest: + for option in config[section].iterkeys(): + if option in ("desc", "outline"): continue + + if option in dest[section]: + dest[section][option]["value"] = config[section][option]["value"] + + #else: + # dest[section][option] = config[section][option] + + + #else: + # dest[section] = config[section] + + def saveConfig(self, config, filename): + """saves config to filename""" + with open(filename, "wb") as f: + chmod(filename, 0600) + f.write("version: %i \n" % CONF_VERSION) + for section in config.iterkeys(): + f.write('\n%s - "%s":\n' % (section, config[section]["desc"])) + + for option, data in config[section].iteritems(): + if option in ("desc", "outline"): continue + + if isinstance(data["value"], list): + value = "[ \n" + for x in data["value"]: + value += "\t\t" + str(x) + ",\n" + value += "\t\t]\n" + else: + if type(data["value"]) in (str, unicode): + value = data["value"] + "\n" + else: + value = str(data["value"]) + "\n" + try: + f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value)) + except UnicodeEncodeError: + f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value.encode("utf8"))) + + def cast(self, typ, value): + """cast value to given format""" + if type(value) not in (str, unicode): + return value + + elif typ == "int": + return int(value) + elif typ == "bool": + return True if value.lower() in ("1", "true", "on", "an", "yes") else False + elif typ == "time": + if not value: value = "0:00" + if not ":" in value: value += ":00" + return value + elif typ in ("str", "file", "folder"): + try: + return value.encode("utf8") + except: + return value + else: + return value + + + def save(self): + """saves the configs to disk""" + + self.saveConfig(self.config, "pyload.conf") + self.saveConfig(self.plugin, "plugin.conf") + + + def __getitem__(self, section): + """provides dictonary like access: c['section']['option']""" + return Section(self, section) + + + def get(self, section, option): + """get value""" + val = self.config[section][option]["value"] + try: + if type(val) in (str, unicode): + return val.decode("utf8") + else: + return val + except: + return val + + def set(self, section, option, value): + """set value""" + + value = self.cast(self.config[section][option]["type"], value) + + self.config[section][option]["value"] = value + self.save() + + def getPlugin(self, plugin, option): + """gets a value for a plugin""" + val = self.plugin[plugin][option]["value"] + try: + if type(val) in (str, unicode): + return val.decode("utf8") + else: + return val + except: + return val + + def setPlugin(self, plugin, option, value): + """sets a value for a plugin""" + + value = self.cast(self.plugin[plugin][option]["type"], value) + + if self.pluginCB: self.pluginCB(plugin, option, value) + + self.plugin[plugin][option]["value"] = value + self.save() + + def getMetaData(self, section, option): + """ get all config data for an option """ + return self.config[section][option] + + def addPluginConfig(self, name, config, outline=""): + """adds config options with tuples (name, type, desc, default)""" + if name not in self.plugin: + conf = {"desc": name, + "outline": outline} + self.plugin[name] = conf + else: + conf = self.plugin[name] + conf["outline"] = outline + + for item in config: + if item[0] in conf: + conf[item[0]]["type"] = item[1] + conf[item[0]]["desc"] = item[2] + else: + conf[item[0]] = { + "desc": item[2], + "type": item[1], + "value": self.cast(item[1], item[3]) + } + + values = [x[0] for x in config] + ["desc", "outline"] + #delete old values + for item in conf.keys(): + if item not in values: + del conf[item] + + def deleteConfig(self, name): + """Removes a plugin config""" + if name in self.plugin: + del self.plugin[name] + + + def deleteOldPlugins(self): + """ remove old plugins from config """ + + for name in IGNORE: + if name in self.plugin: + del self.plugin[name] + + +class Section: + """provides dictionary like access for configparser""" + + def __init__(self, parser, section): + """Constructor""" + self.parser = parser + self.section = section + + def __getitem__(self, item): + """getitem""" + return self.parser.get(self.section, item) + + def __setitem__(self, item, value): + """setitem""" + self.parser.set(self.section, item, value) diff --git a/pyload/config/Setup.py b/pyload/config/Setup.py new file mode 100644 index 000000000..63c6188c4 --- /dev/null +++ b/pyload/config/Setup.py @@ -0,0 +1,538 @@ +# -*- coding: utf-8 -*- + +import os +import sys + +import pyload.utils.pylgettext as gettext + +from getpass import getpass +from os import makedirs +from os.path import abspath, dirname, exists, join +from subprocess import PIPE, call + +from pyload.utils import get_console_encoding, versiontuple + + +class SetupAssistant: + """ pyLoads initial setup configuration assistant """ + + def __init__(self, path, config): + self.path = path + self.config = config + self.stdin_encoding = get_console_encoding(sys.stdin.encoding) + + + def start(self): + langs = self.config.getMetaData("general", "language")["type"].split(";") + lang = self.ask(u"Choose setup language", "en", langs) + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("setup", join(self.path, "locale"), languages=[lang, "en"], fallback=True) + translation.install(True) + + #Input shorthand for yes + self.yes = _("y") + #Input shorthand for no + self.no = _("n") + + # print + # print _("Would you like to configure pyLoad via Webinterface?") + # print _("You need a Browser and a connection to this PC for it.") + # viaweb = self.ask(_("Start initial webinterface for configuration?"), "y", bool=True) + # if viaweb: + # try: + # from pyload.manager.thread import ServerThread + # ServerThread.setup = self + # import pyload.webui as webinterface + # webinterface.run_simple() + # return False + # except Exception, e: + # print "Setup failed with this error: ", e + # print "Falling back to commandline setup." + + print + print + print _("Welcome to the pyLoad Configuration Assistant.") + print _("It will check your system and make a basic setup in order to run pyLoad.") + print + print _("The value in brackets [] always is the default value,") + print _("in case you don't want to change it or you are unsure what to choose, just hit enter.") + print _( + "Don't forget: You can always rerun this assistant with --setup or -s parameter, when you start pyload.py .") + print _("If you have any problems with this assistant hit STRG-C,") + print _("to abort and don't let him start with pyload.py automatically anymore.") + print + print + raw_input(_("When you are ready for system check, hit enter.")) + print + print + + basic, ssl, captcha, web, js = self.system_check() + print + print + + if not basic: + print _("You need pycurl, sqlite and python 2.5, 2.6 or 2.7 to run pyLoad.") + print _("Please correct this and re-run pyLoad.") + print + print _("Setup will now close.") + print + print + raw_input(_("Press Enter to exit.")) + return False + + raw_input(_("System check finished, hit enter to see your status report.")) + print + print + print _("## Status ##") + print + + avail = [] + if self.check_module("Crypto"): + avail.append(_("container decrypting")) + if ssl: + avail.append(_("ssl connection")) + if captcha: + avail.append(_("automatic captcha decryption")) + if web: + avail.append(_("webinterface")) + if js: + avail.append(_("extended Click'N'Load")) + + string = "" + + for av in avail: + string += ", " + av + + print _("AVAILABLE FEATURES:") + string[1:] + print + + if len(avail) < 5: + print _("MISSING FEATURES: ") + + if not self.check_module("Crypto"): + print _("- no py-crypto available") + print _("You need this if you want to decrypt container files.") + print + + if not ssl: + print _("- no SSL available") + print _("This is needed if you want to establish a secure connection to core or webinterface.") + print _("If you only want to access locally to pyLoad ssl is not usefull.") + print + + if not captcha: + print _("- no Captcha Recognition available") + print _("Only needed for some hosters and as freeuser.") + print + + if not js: + print _("- no JavaScript engine found") + print _("You will need this for some Click'N'Load links. Install Spidermonkey, ossp-js, pyv8 or rhino") + print + + print + print _("You can abort the setup now and fix some dependicies if you want.") + + print + con = self.ask(_("Continue with setup?"), self.yes, bool=True) + + if not con: + return False + + print + print + print _("CURRENT CONFIG PATH: %s") % abspath("") + print + print _("NOTE: If you use pyLoad on a server or the home partition lives on an iternal flash it may be a good idea to change it.") + path = self.ask(_("Do you want to change the config path?"), self.no, bool=True) + if path: + print + self.conf_path() + #calls exit when changed + + print + print _("Do you want to configure login data and basic settings?") + print _("This is recommend for first run.") + con = self.ask(_("Make basic setup?"), self.yes, bool=True) + + if con: + print + print + self.conf_basic() + + if ssl: + print + print _("Do you want to configure ssl?") + ssl = self.ask(_("Configure ssl?"), self.no, bool=True) + if ssl: + print + print + self.conf_ssl() + + if web: + print + print _("Do you want to configure webinterface?") + web = self.ask(_("Configure webinterface?"), self.yes, bool=True) + if web: + print + print + self.conf_web() + + print + print + print _("Setup finished successfully!") + print + print + raw_input(_("Hit enter to exit and restart pyLoad.")) + return True + + + def system_check(self): + """ make a systemcheck and return the results""" + + print _("## System Information ##") + print + print _("Platform: %s") % sys.platform + print _("Operating System: %s") % os.name + print _("Python: %s") % sys.version.replace("\n", "") + print + print + + print _("## System Check ##") + print + + if sys.version_info[:2] > (2, 7): + print _("Your python version is to new, Please use Python 2.6/2.7") + python = False + elif sys.version_info[:2] < (2, 5): + print _("Your python version is to old, Please use at least Python 2.5") + python = False + else: + print _("Python Version: OK") + python = True + + curl = self.check_module("pycurl") + self.print_dep("pycurl", curl) + + sqlite = self.check_module("sqlite3") + self.print_dep("sqlite3", sqlite) + + basic = python and curl and sqlite + + print + + crypto = self.check_module("Crypto") + self.print_dep("pycrypto", crypto) + + ssl = self.check_module("OpenSSL") + self.print_dep("py-OpenSSL", ssl) + + print + + pil = self.check_module("PIL.Image") + self.print_dep("PIL/Pillow", pil) + + if os.name == "nt": + tesser = self.check_prog([join(pypath, "tesseract", "tesseract.exe"), "-v"]) + else: + tesser = self.check_prog(["tesseract", "-v"]) + + self.print_dep("tesseract", tesser) + + captcha = pil and tesser + + print + + try: + import jinja2 + + v = jinja2.__version__ + if v and versiontuple(v) < (2, 5, 0): + jinja = False + else: + jinja = True + except: + jinja = False + + jinja = self.print_dep("jinja2", jinja) + + beaker = self.check_module("beaker") + self.print_dep("beaker", beaker) + + bjoern = self.check_module("bjoern") + self.print_dep("bjoern", bjoern) + + web = sqlite and beaker + + from pyload.utils import JsEngine + js = True if JsEngine.ENGINE else False + self.print_dep(_("JS engine"), js) + + if not jinja: + print + print + print _("WARNING: Your installed jinja2 version %s seems too old.") % jinja2.__version__ + print _("You can safely continue but if the webinterface is not working,") + print _("please upgrade or uninstall it, because pyLoad self-includes jinja2 libary.") + + return basic, ssl, captcha, web, js + + + def conf_basic(self): + print _("## Basic Setup ##") + + print + print _("The following logindata is valid for CLI and webinterface.") + + from pyload.database import DatabaseBackend + + db = DatabaseBackend(None) + db.setup() + print _("NOTE: Consider a password of 10 or more symbols if you expect to access from outside your local network (ex. internet).") + print + username = self.ask(_("Username"), "User") + password = self.ask("", "", password=True) + db.addUser(username, password) + db.shutdown() + + print + print _("External clients (GUI, CLI or other) need remote access to work over the network.") + print _("However, if you only want to use the webinterface you may disable it to save ram.") + self.config["remote"]["activated"] = self.ask(_("Enable remote access"), self.no, bool=True) + + print + langs = self.config.getMetaData("general", "language") + self.config["general"]["language"] = self.ask(_("Choose pyLoad language"), "en", langs["type"].split(";")) + + print + self.config["general"]["download_folder"] = self.ask(_("Download folder"), "Downloads") + print + self.config["download"]["max_downloads"] = self.ask(_("Max parallel downloads"), "3") + print + reconnect = self.ask(_("Use Reconnect?"), self.no, bool=True) + self.config["reconnect"]["activated"] = reconnect + if reconnect: + self.config["reconnect"]["method"] = self.ask(_("Reconnect script location"), "./reconnect.sh") + + + def conf_web(self): + print _("## Webinterface Setup ##") + + print + self.config["webinterface"]["activated"] = self.ask(_("Activate webinterface?"), self.yes, bool=True) + print + print _("Listen address, if you use 127.0.0.1 or localhost, the webinterface will only accessible locally.") + self.config["webinterface"]["host"] = self.ask(_("Address"), "0.0.0.0") + self.config["webinterface"]["port"] = self.ask(_("Port"), "8000") + print + print _("pyLoad offers several server backends, now following a short explanation.") + print "- builtin:", _("Default server; best choice if you plan to use pyLoad just for you.") + print "- threaded:", _("Support SSL connection and can serve simultaneously more client flawlessly.") + print "- fastcgi:", _( + "Can be used by apache, lighttpd, etc.; needs to be properly configured before.") + if os.name != "nt": + print "- lightweight:", _("Very fast alternative to builtin; requires libev and bjoern packages.") + + print + print _("NOTE: In some rare cases the builtin server is not working, if you notice problems with the webinterface") + print _("come back here and change the builtin server to the threaded one here.") + + if os.name == "nt": + servers = ["builtin", "threaded", "fastcgi"] + default = "threaded" + else: + servers = ["builtin", "threaded", "fastcgi", "lightweight"] + default = "lightweight" if self.check_module("bjoern") else "builtin" + + self.config["webinterface"]["server"] = self.ask(_("Server"), default, servers) + + + def conf_ssl(self): + print _("## SSL Setup ##") + print + print _("Execute these commands from pyLoad config folder to make ssl certificates:") + print + print "openssl genrsa -out ssl.key 1024" + print "openssl req -new -key ssl.key -out ssl.csr" + print "openssl req -days 36500 -x509 -key ssl.key -in ssl.csr > ssl.crt " + print + print _("If you're done and everything went fine, you can activate ssl now.") + + self.config["ssl"]["activated"] = self.ask(_("Activate SSL?"), self.yes, bool=True) + + + def set_user(self): + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("setup", join(self.path, "locale"), + languages=[self.config["general"]["language"], "en"], fallback=True) + translation.install(True) + + from pyload.database import DatabaseBackend + + db = DatabaseBackend(None) + db.setup() + + noaction = True + try: + while True: + print _("Select action") + print _("1 - Create/Edit user") + print _("2 - List users") + print _("3 - Remove user") + print _("4 - Quit") + action = raw_input("[1]/2/3/4: ") + if not action in ("1", "2", "3", "4"): + continue + elif action == "1": + print + username = self.ask(_("Username"), "User") + password = self.ask("", "", password=True) + db.addUser(username, password) + noaction = False + elif action == "2": + print + print _("Users") + print "-----" + users = db.listUsers() + noaction = False + for user in users: + print user + print "-----" + print + elif action == "3": + print + username = self.ask(_("Username"), "") + if username: + db.removeUser(username) + noaction = False + elif action == "4": + break + finally: + if not noaction: + db.shutdown() + + + def conf_path(self, trans=False): + if trans: + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("setup", join(self.path, "locale"), + languages=[self.config["general"]["language"], "en"], fallback=True) + translation.install(True) + + print _("Setting new config path, current configuration will not be transfered!") + path = self.ask(_("CONFIG PATH"), abspath("")) + try: + path = join(pypath, path) + if not exists(path): + makedirs(path) + f = open(join(pypath, "pyload", "config", "configdir"), "wb") + f.write(path) + f.close() + print + print + print _("pyLoad config path changed, setup will now close!") + print + print + raw_input(_("Press Enter to exit.")) + sys.exit() + except Exception, e: + print _("Setting config path failed: %s") % str(e) + + + def print_dep(self, name, value): + """Print Status of dependency""" + if value: + print _("%s: OK") % name + else: + print _("%s: MISSING") % name + + + def check_module(self, module): + try: + __import__(module) + return True + except: + return False + + + def check_prog(self, command): + pipe = PIPE + try: + call(command, stdout=pipe, stderr=pipe) + return True + except: + return False + + + def ask(self, qst, default, answers=[], bool=False, password=False): + """produce one line to asking for input""" + if answers: + info = "(" + + for i, answer in enumerate(answers): + info += (", " if i != 0 else "") + str((answer == default and "[%s]" % answer) or answer) + + info += ")" + elif bool: + if default == self.yes: + info = "([%s]/%s)" % (self.yes, self.no) + else: + info = "(%s/[%s])" % (self.yes, self.no) + else: + info = "[%s]" % default + + if password: + p1 = True + p2 = False + pwlen = 8 + while p1 != p2: + # getpass(_("Password: ")) will crash on systems with broken locales (Win, NAS) + sys.stdout.write(_("Password: ")) + p1 = getpass("") + + if len(p1) < pwlen: + print _("Password too short! Use at least %s symbols." % pwlen) + continue + elif not p1.isalnum(): + print _("Password must be alphanumeric.") + continue + + sys.stdout.write(_("Password (again): ")) + p2 = getpass("") + + if p1 == p2: + return p1 + else: + print _("Passwords did not match.") + + while True: + try: + input = raw_input(qst + " %s: " % info) + except KeyboardInterrupt: + print "\nSetup interrupted" + sys.exit() + + input = input.decode(self.stdin_encoding) + + if input.strip() == "": + input = default + + if bool: + # yes, true, t are inputs for booleans with value true + if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]: + return True + # no, false, f are inputs for booleans with value false + elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]: + return False + else: + print _("Invalid Input") + continue + + if not answers: + return input + + else: + if input in answers: + return input + else: + print _("Invalid Input") diff --git a/pyload/database/FileDatabase.py b/pyload/database/FileDatabase.py index fb30735bb..54e5450e8 100644 --- a/pyload/database/FileDatabase.py +++ b/pyload/database/FileDatabase.py @@ -21,9 +21,9 @@ from threading import RLock from time import time from pyload.utils import formatSize, lock -from pyload.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent -from pyload.PyPackage import PyPackage -from pyload.PyFile import PyFile +from pyload.manager.event.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent +from pyload.datatypes.PyPackage import PyPackage +from pyload.datatypes.PyFile import PyFile from pyload.database import style, DatabaseBackend try: diff --git a/pyload/datatypes/PyFile.py b/pyload/datatypes/PyFile.py new file mode 100644 index 000000000..be3129681 --- /dev/null +++ b/pyload/datatypes/PyFile.py @@ -0,0 +1,284 @@ +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN + @author: mkaay +""" + +from pyload.manager.event.PullEvents import UpdateEvent +from pyload.utils import formatSize, lock + +from time import sleep, time + +from threading import RLock + +statusMap = { + "finished": 0, + "offline": 1, + "online": 2, + "queued": 3, + "skipped": 4, + "waiting": 5, + "temp. offline": 6, + "starting": 7, + "failed": 8, + "aborted": 9, + "decrypting": 10, + "custom": 11, + "downloading": 12, + "processing": 13, + "unknown": 14, +} + + +def setSize(self, value): + self._size = int(value) + +class PyFile(object): + """ + Represents a file object at runtime + """ + __slots__ = ("m", "id", "url", "name", "size", "_size", "status", "pluginname", "packageid", + "error", "order", "lock", "plugin", "waitUntil", "active", "abort", "statusname", + "reconnected", "progress", "maxprogress", "pluginmodule", "pluginclass") + + def __init__(self, manager, id, url, name, size, status, error, pluginname, package, order): + self.m = manager + + self.id = int(id) + self.url = url + self.name = name + self.size = size + self.status = status + self.pluginname = pluginname + self.packageid = package #should not be used, use package() instead + self.error = error + self.order = order + # database information ends here + + self.lock = RLock() + + self.plugin = None + #self.download = None + + self.waitUntil = 0 # time() + time to wait + + # status attributes + self.active = False #obsolete? + self.abort = False + self.reconnected = False + + self.statusname = None + + self.progress = 0 + self.maxprogress = 100 + + self.m.cache[int(id)] = self + + + # will convert all sizes to ints + size = property(lambda self: self._size, setSize) + + def __repr__(self): + return "PyFile %s: %s@%s" % (self.id, self.name, self.pluginname) + + @lock + def initPlugin(self): + """ inits plugin instance """ + if not self.plugin: + self.pluginmodule = self.m.core.pluginManager.getPlugin(self.pluginname) + self.pluginclass = getattr(self.pluginmodule, self.m.core.pluginManager.getPluginName(self.pluginname)) + self.plugin = self.pluginclass(self) + + @lock + def hasPlugin(self): + """Thread safe way to determine this file has initialized plugin attribute + + :return: + """ + return hasattr(self, "plugin") and self.plugin + + def package(self): + """ return package instance""" + return self.m.getPackage(self.packageid) + + def setStatus(self, status): + self.status = statusMap[status] + self.sync() #@TODO needed aslong no better job approving exists + + def setCustomStatus(self, msg, status="processing"): + self.statusname = msg + self.setStatus(status) + + def getStatusName(self): + if self.status not in (13, 14) or not self.statusname: + return self.m.statusMsg[self.status] + else: + return self.statusname + + def hasStatus(self, status): + return statusMap[status] == self.status + + def sync(self): + """sync PyFile instance with database""" + self.m.updateLink(self) + + @lock + def release(self): + """sync and remove from cache""" + # file has valid package + if self.packageid > 0: + self.sync() + + if hasattr(self, "plugin") and self.plugin: + self.plugin.clean() + del self.plugin + + self.m.releaseLink(self.id) + + def delete(self): + """delete pyfile from database""" + self.m.deleteLink(self.id) + + def toDict(self): + """return dict with all information for interface""" + return self.toDbDict() + + def toDbDict(self): + """return data as dict for databse + + format: + + { + id: {'url': url, 'name': name ... } + } + + """ + return { + self.id: { + 'id': self.id, + 'url': self.url, + 'name': self.name, + 'plugin': self.pluginname, + 'size': self.getSize(), + 'format_size': self.formatSize(), + 'status': self.status, + 'statusmsg': self.getStatusName(), + 'package': self.packageid, + 'error': self.error, + 'order': self.order + } + } + + def abortDownload(self): + """abort pyfile if possible""" + while self.id in self.m.core.threadManager.processingIds(): + self.abort = True + if self.plugin and self.plugin.req: + self.plugin.req.abortDownloads() + sleep(0.1) + + self.abort = False + if self.hasPlugin() and self.plugin.req: + self.plugin.req.abortDownloads() + + self.release() + + def finishIfDone(self): + """set status to finish and release file if every thread is finished with it""" + + if self.id in self.m.core.threadManager.processingIds(): + return False + + self.setStatus("finished") + self.release() + self.m.checkAllLinksFinished() + return True + + def checkIfProcessed(self): + self.m.checkAllLinksProcessed(self.id) + + def formatWait(self): + """ formats and return wait time in humanreadable format """ + seconds = self.waitUntil - time() + + if seconds < 0: return "00:00:00" + + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + + def formatSize(self): + """ formats size to readable format """ + return formatSize(self.getSize()) + + def formatETA(self): + """ formats eta to readable format """ + seconds = self.getETA() + + if seconds < 0: return "00:00:00" + + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + + def getSpeed(self): + """ calculates speed """ + try: + return self.plugin.req.speed + except: + return 0 + + def getETA(self): + """ gets established time of arrival""" + try: + return self.getBytesLeft() / self.getSpeed() + except: + return 0 + + def getBytesLeft(self): + """ gets bytes left """ + try: + return self.getSize() - self.plugin.req.arrived + except: + return 0 + + def getPercent(self): + """ get % of download """ + if self.status == 12: + try: + return self.plugin.req.percent + except: + return 0 + else: + return self.progress + + def getSize(self): + """ get size of download """ + try: + if self.plugin.req.size: + return self.plugin.req.size + else: + return self.size + except: + return self.size + + def notifyChange(self): + e = UpdateEvent("file", self.id, "collector" if not self.package().queue else "queue") + self.m.core.pullManager.addEvent(e) + + def setProgress(self, value): + if not value == self.progress: + self.progress = value + self.notifyChange() diff --git a/pyload/datatypes/PyPackage.py b/pyload/datatypes/PyPackage.py new file mode 100644 index 000000000..c8d3e6096 --- /dev/null +++ b/pyload/datatypes/PyPackage.py @@ -0,0 +1,79 @@ +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN + @author: mkaay +""" + +from pyload.manager.event.PullEvents import UpdateEvent +from pyload.utils import safe_filename + +class PyPackage: + """ + Represents a package object at runtime + """ + def __init__(self, manager, id, name, folder, site, password, queue, order): + self.m = manager + self.m.packageCache[int(id)] = self + + self.id = int(id) + self.name = name + self._folder = folder + self.site = site + self.password = password + self.queue = queue + self.order = order + self.setFinished = False + + @property + def folder(self): + return safe_filename(self._folder) + + def toDict(self): + """ Returns a dictionary representation of the data. + + :return: dict: {id: { attr: value }} + """ + return { + self.id: { + 'id': self.id, + 'name': self.name, + 'folder': self.folder, + 'site': self.site, + 'password': self.password, + 'queue': self.queue, + 'order': self.order, + 'links': {} + } + } + + def getChildren(self): + """get information about contained links""" + return self.m.getPackageData(self.id)["links"] + + def sync(self): + """sync with db""" + self.m.updatePackage(self) + + def release(self): + """sync and delete from cache""" + self.sync() + self.m.releasePackage(self.id) + + def delete(self): + self.m.deletePackage(self.id) + + def notifyChange(self): + e = UpdateEvent("pack", self.id, "collector" if not self.queue else "queue") + self.m.core.pullManager.addEvent(e) diff --git a/pyload/datatypes/__init__.py b/pyload/datatypes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/debug.py b/pyload/debug.py deleted file mode 100644 index 54a159e78..000000000 --- a/pyload/debug.py +++ /dev/null @@ -1,94 +0,0 @@ -#coding:utf-8 - -import re -import InitHomeDir -from os import listdir - -class Wrapper(object): - pass - -def filter_info(line): - if "object at 0x" in line: - return False - elif " at line " in line: - return False - elif " . - - @author: RaNaN -""" - -from sys import argv -from sys import exit - -import socket -import thread - -from traceback import print_exc - -class Forwarder: - - def __init__(self, extip, extport=9666): - print "Start portforwarding to %s:%s" % (extip, extport) - proxy(extip, extport, 9666) - - -def proxy(*settings): - while True: - server(*settings) - -def server(*settings): - try: - dock_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - dock_socket.bind(("127.0.0.1", settings[2])) - dock_socket.listen(5) - while True: - client_socket = dock_socket.accept()[0] - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.connect((settings[0], settings[1])) - thread.start_new_thread(forward, (client_socket, server_socket)) - thread.start_new_thread(forward, (server_socket, client_socket)) - except Exception: - print_exc() - -def forward(source, destination): - string = ' ' - while string: - string = source.recv(1024) - if string: - destination.sendall(string) - else: - #source.shutdown(socket.SHUT_RD) - destination.shutdown(socket.SHUT_WR) diff --git a/pyload/manager/AccountManager.py b/pyload/manager/AccountManager.py new file mode 100644 index 000000000..d1958f4b6 --- /dev/null +++ b/pyload/manager/AccountManager.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- + +from os.path import exists +from shutil import copy + +from threading import Lock + +from pyload.manager.event.PullEvents import AccountUpdateEvent +from pyload.utils import chmod, lock + +ACC_VERSION = 1 + + +class AccountManager: + """manages all accounts""" + + #-------------------------------------------------------------------------- + def __init__(self, core): + """Constructor""" + + self.core = core + self.lock = Lock() + + self.initPlugins() + self.saveAccounts() # save to add categories to conf + + def initPlugins(self): + self.accounts = {} # key = ( plugin ) + self.plugins = {} + + self.initAccountPlugins() + self.loadAccounts() + + def getAccountPlugin(self, plugin): + """get account instance for plugin or None if anonymous""" + if plugin in self.accounts: + if plugin not in self.plugins: + try: + self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin]) + except TypeError: # The account class no longer exists (blacklisted plugin). Skipping the account to avoid crash + return None + + return self.plugins[plugin] + else: + return None + + def getAccountPlugins(self): + """ get all account instances""" + + plugins = [] + for plugin in self.accounts.keys(): + plugins.append(self.getAccountPlugin(plugin)) + + return plugins + + #-------------------------------------------------------------------------- + def loadAccounts(self): + """loads all accounts available""" + + if not exists("accounts.conf"): + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION)) + f.close() + + f = open("accounts.conf", "rb") + content = f.readlines() + version = content[0].split(":")[1].strip() if content else "" + f.close() + + if not version or int(version) < ACC_VERSION: + copy("accounts.conf", "accounts.backup") + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION)) + f.close() + self.core.log.warning(_("Account settings deleted, due to new config format.")) + return + + plugin = "" + name = "" + + for line in content[1:]: + line = line.strip() + + if not line: continue + if line.startswith("#"): continue + if line.startswith("version"): continue + + if line.endswith(":") and line.count(":") == 1: + plugin = line[:-1] + self.accounts[plugin] = {} + + elif line.startswith("@"): + try: + option = line[1:].split() + self.accounts[plugin][name]['options'][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) + except: + pass + + elif ":" in line: + name, sep, pw = line.partition(":") + self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} + + #-------------------------------------------------------------------------- + def saveAccounts(self): + """save all account information""" + + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION) + "\n") + + for plugin, accounts in self.accounts.iteritems(): + f.write("\n") + f.write(plugin+":\n") + + for name,data in accounts.iteritems(): + f.write("\n\t%s:%s\n" % (name,data['password']) ) + if data['options']: + for option, values in data['options'].iteritems(): + f.write("\t@%s %s\n" % (option, " ".join(values))) + + f.close() + chmod(f.name, 0600) + + #-------------------------------------------------------------------------- + def initAccountPlugins(self): + """init names""" + for name in self.core.pluginManager.getAccountPlugins(): + self.accounts[name] = {} + + @lock + def updateAccount(self, plugin , user, password=None, options={}): + """add or update account""" + if plugin in self.accounts: + p = self.getAccountPlugin(plugin) + updated = p.updateAccounts(user, password, options) + #since accounts is a ref in plugin self.accounts doesnt need to be updated here + + self.saveAccounts() + if updated: p.scheduleRefresh(user, force=False) + + @lock + def removeAccount(self, plugin, user): + """remove account""" + + if plugin in self.accounts: + p = self.getAccountPlugin(plugin) + p.removeAccount(user) + + self.saveAccounts() + + @lock + def getAccountInfos(self, force=True, refresh=False): + data = {} + + if refresh: + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + force = False + + for p in self.accounts.keys(): + if self.accounts[p]: + p = self.getAccountPlugin(p) + if p: + data[p.__name__] = p.getAllAccounts(force) + else: # When an account has been skipped, p is None + data[p] = [] + else: + data[p] = [] + e = AccountUpdateEvent() + self.core.pullManager.addEvent(e) + return data + + def sendChange(self): + e = AccountUpdateEvent() + self.core.pullManager.addEvent(e) diff --git a/pyload/manager/CaptchaManager.py b/pyload/manager/CaptchaManager.py new file mode 100644 index 000000000..0ba876ae8 --- /dev/null +++ b/pyload/manager/CaptchaManager.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay, RaNaN +""" + +from time import time +from traceback import print_exc +from threading import Lock + +class CaptchaManager: + def __init__(self, core): + self.lock = Lock() + self.core = core + self.tasks = [] #task store, for outgoing tasks only + + self.ids = 0 #only for internal purpose + + def newTask(self, img, format, file, result_type): + task = CaptchaTask(self.ids, img, format, file, result_type) + self.ids += 1 + return task + + def removeTask(self, task): + self.lock.acquire() + if task in self.tasks: + self.tasks.remove(task) + self.lock.release() + + def getTask(self): + self.lock.acquire() + for task in self.tasks: + if task.status in ("waiting", "shared-user"): + self.lock.release() + return task + self.lock.release() + return None + + def getTaskByID(self, tid): + self.lock.acquire() + for task in self.tasks: + if task.id == str(tid): #task ids are strings + self.lock.release() + return task + self.lock.release() + return None + + def handleCaptcha(self, task): + cli = self.core.isClientConnected() + + if cli: #client connected -> should solve the captcha + task.setWaiting(50) #wait 50 sec for response + + for plugin in self.core.hookManager.activePlugins(): + try: + plugin.newCaptchaTask(task) + except: + if self.core.debug: + print_exc() + + if task.handler or cli: #the captcha was handled + self.tasks.append(task) + return True + + task.error = _("No Client connected for captcha decrypting") + + return False + + +class CaptchaTask: + def __init__(self, id, img, format, file, result_type='textual'): + self.id = str(id) + self.captchaImg = img + self.captchaFormat = format + self.captchaFile = file + self.captchaResultType = result_type + self.handler = [] #the hook plugins that will take care of the solution + self.result = None + self.waitUntil = None + self.error = None #error message + + self.status = "init" + self.data = {} #handler can store data here + + def getCaptcha(self): + return self.captchaImg, self.captchaFormat, self.captchaResultType + + def setResult(self, text): + if self.isTextual(): + self.result = text + if self.isPositional(): + try: + parts = text.split(',') + self.result = (int(parts[0]), int(parts[1])) + except: + self.result = None + + def getResult(self): + try: + res = self.result.encode("utf8", "replace") + except: + res = self.result + + return res + + def getStatus(self): + return self.status + + def setWaiting(self, sec): + """ let the captcha wait secs for the solution """ + self.waitUntil = max(time() + sec, self.waitUntil) + self.status = "waiting" + + def isWaiting(self): + if self.result or self.error or time() > self.waitUntil: + return False + + return True + + def isTextual(self): + """ returns if text is written on the captcha """ + return self.captchaResultType == 'textual' + + def isPositional(self): + """ returns if user have to click a specific region on the captcha """ + return self.captchaResultType == 'positional' + + def setWatingForUser(self, exclusive): + if exclusive: + self.status = "user" + else: + self.status = "shared-user" + + def timedOut(self): + return time() > self.waitUntil + + def invalid(self): + """ indicates the captcha was not correct """ + [x.captchaInvalid(self) for x in self.handler] + + def correct(self): + [x.captchaCorrect(self) for x in self.handler] + + def __str__(self): + return "" % self.id diff --git a/pyload/manager/HookManager.py b/pyload/manager/HookManager.py new file mode 100644 index 000000000..6f5477aeb --- /dev/null +++ b/pyload/manager/HookManager.py @@ -0,0 +1,305 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN, mkaay + @interface-version: 0.1 +""" +import __builtin__ + +import traceback +from thread import start_new_thread +from threading import RLock + +from types import MethodType + +from pyload.manager.thread.PluginThread import HookThread +from pyload.manager.PluginManager import literal_eval +from utils import lock + +class HookManager: + """Manages hooks, delegates and handles Events. + + Every plugin can define events, \ + but some very usefull events are called by the Core. + Contrary to overwriting hook methods you can use event listener, + which provides additional entry point in the control flow. + Only do very short tasks or use threads. + + **Known Events:** + Most hook methods exists as events. These are the additional known events. + + ===================== ============== ================================== + Name Arguments Description + ===================== ============== ================================== + downloadPreparing fid A download was just queued and will be prepared now. + downloadStarts fid A plugin will immediately starts the download afterwards. + linksAdded links, pid Someone just added links, you are able to modify the links. + allDownloadsProcessed Every link was handled, pyload would idle afterwards. + allDownloadsFinished Every download in queue is finished. + unrarFinished folder, fname An Unrar job finished + configChanged The config was changed via the api. + pluginConfigChanged The plugin config changed, due to api or internal process. + ===================== ============== ================================== + + | Notes: + | allDownloadsProcessed is *always* called before allDownloadsFinished. + | configChanged is *always* called before pluginConfigChanged. + + + """ + + def __init__(self, core): + self.core = core + self.config = self.core.config + + __builtin__.hookManager = self #needed to let hooks register themself + + self.log = self.core.log + self.plugins = [] + self.pluginMap = {} + self.methods = {} #dict of names and list of methods usable by rpc + + self.events = {} # contains events + + #registering callback for config event + self.config.pluginCB = MethodType(self.dispatchEvent, "pluginConfigChanged", basestring) + + self.addEvent("pluginConfigChanged", self.manageHooks) + + self.lock = RLock() + self.createIndex() + + def try_catch(func): + def new(*args): + try: + return func(*args) + except Exception, e: + args[0].log.error(_("Error executing hooks: %s") % str(e)) + if args[0].core.debug: + traceback.print_exc() + + return new + + + def addRPC(self, plugin, func, doc): + plugin = plugin.rpartition(".")[2] + doc = doc.strip() if doc else "" + + if plugin in self.methods: + self.methods[plugin][func] = doc + else: + self.methods[plugin] = {func: doc} + + def callRPC(self, plugin, func, args, parse): + if not args: args = tuple() + if parse: + args = tuple([literal_eval(x) for x in args]) + + plugin = self.pluginMap[plugin] + f = getattr(plugin, func) + return f(*args) + + + def createIndex(self): + plugins = [] + + active = [] + deactive = [] + + for pluginname in self.core.pluginManager.hookPlugins: + try: + #hookClass = getattr(plugin, plugin.__name__) + + if self.config.getPlugin(pluginname, "activated"): + pluginClass = self.core.pluginManager.loadClass("hooks", pluginname) + if not pluginClass: continue + + plugin = pluginClass(self.core, self) + plugins.append(plugin) + self.pluginMap[pluginClass.__name__] = plugin + if plugin.isActivated(): + active.append(pluginClass.__name__) + else: + deactive.append(pluginname) + + + except: + self.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) + if self.core.debug: + traceback.print_exc() + + self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active))) + self.log.info(_("Deactivate plugins: %s") % ", ".join(sorted(deactive))) + + self.plugins = plugins + + def manageHooks(self, plugin, name, value): + if name == "activated" and value: + self.activateHook(plugin) + elif name == "activated" and not value: + self.deactivateHook(plugin) + + def activateHook(self, plugin): + + #check if already loaded + for inst in self.plugins: + if inst.__name__ == plugin: + return + + pluginClass = self.core.pluginManager.loadClass("hooks", plugin) + + if not pluginClass: return + + self.log.debug("Plugin loaded: %s" % plugin) + + plugin = pluginClass(self.core, self) + self.plugins.append(plugin) + self.pluginMap[pluginClass.__name__] = plugin + + # call core Ready + start_new_thread(plugin.coreReady, tuple()) + + def deactivateHook(self, plugin): + + hook = None + for inst in self.plugins: + if inst.__name__ == plugin: + hook = inst + + if not hook: return + + self.log.debug("Plugin unloaded: %s" % plugin) + + hook.unload() + + #remove periodic call + self.log.debug("Removed callback %s" % self.core.scheduler.removeJob(hook.cb)) + self.plugins.remove(hook) + del self.pluginMap[hook.__name__] + + + @try_catch + def coreReady(self): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.coreReady() + + self.dispatchEvent("coreReady") + + @try_catch + def coreExiting(self): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.coreExiting() + + self.dispatchEvent("coreExiting") + + @lock + def downloadPreparing(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadPreparing(pyfile) + + self.dispatchEvent("downloadPreparing", pyfile) + + @lock + def downloadFinished(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadFinished(pyfile) + + self.dispatchEvent("downloadFinished", pyfile) + + @lock + @try_catch + def downloadFailed(self, pyfile): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.downloadFailed(pyfile) + + self.dispatchEvent("downloadFailed", pyfile) + + @lock + def packageFinished(self, package): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.packageFinished(package) + + self.dispatchEvent("packageFinished", package) + + @lock + def beforeReconnecting(self, ip): + for plugin in self.plugins: + plugin.beforeReconnecting(ip) + + self.dispatchEvent("beforeReconnecting", ip) + + @lock + def afterReconnecting(self, ip): + for plugin in self.plugins: + if plugin.isActivated(): + plugin.afterReconnecting(ip) + + self.dispatchEvent("afterReconnecting", ip) + + def startThread(self, function, *args, **kwargs): + t = HookThread(self.core.threadManager, function, args, kwargs) + + def activePlugins(self): + """ returns all active plugins """ + return [x for x in self.plugins if x.isActivated()] + + def getAllInfo(self): + """returns info stored by hook plugins""" + info = {} + for name, plugin in self.pluginMap.iteritems(): + if plugin.info: + #copy and convert so str + info[name] = dict([(x, str(y) if not isinstance(y, basestring) else y) for x, y in plugin.info.iteritems()]) + return info + + + def getInfo(self, plugin): + info = {} + if plugin in self.pluginMap and self.pluginMap[plugin].info: + info = dict([(x, str(y) if not isinstance(y, basestring) else y) + for x, y in self.pluginMap[plugin].info.iteritems()]) + + return info + + def addEvent(self, event, func): + """Adds an event listener for event name""" + if event in self.events: + self.events[event].append(func) + else: + self.events[event] = [func] + + def removeEvent(self, event, func): + """removes previously added event listener""" + if event in self.events: + self.events[event].remove(func) + + def dispatchEvent(self, event, *args): + """dispatches event with args""" + if event in self.events: + for f in self.events[event]: + try: + f(*args) + except Exception, e: + self.log.warning("Error calling event handler %s: %s, %s, %s" + % (event, f, args, str(e))) + if self.core.debug: + traceback.print_exc() diff --git a/pyload/manager/PluginManager.py b/pyload/manager/PluginManager.py new file mode 100644 index 000000000..56e59237c --- /dev/null +++ b/pyload/manager/PluginManager.py @@ -0,0 +1,356 @@ +# -*- coding: utf-8 -*- + +import re +import sys + +from itertools import chain +from os import listdir, makedirs +from os.path import isfile, join, exists, abspath +from sys import version_info +from traceback import print_exc + +from SafeEval import const_eval as literal_eval + +from pyload.config.Parser import IGNORE + + +class PluginManager: + ROOT = "pyload.plugins." + USERROOT = "userplugins." + TYPES = ("accounts", "container", "crypter", "hooks", "hoster", "internal", "ocr") + + PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') + VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)') + CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) + DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') + + + def __init__(self, core): + self.core = core + + self.config = core.config + self.log = core.log + + self.plugins = {} + self.createIndex() + + #register for import hook + sys.meta_path.append(self) + + + def createIndex(self): + """create information for all plugins available""" + + sys.path.append(abspath("")) + + if not exists("userplugins"): + makedirs("userplugins") + if not exists(join("userplugins", "__init__.py")): + f = open(join("userplugins", "__init__.py"), "wb") + f.close() + + self.plugins['crypter'] = self.crypterPlugins = self.parse("crypter", pattern=True) + self.plugins['container'] = self.containerPlugins = self.parse("container", pattern=True) + self.plugins['hoster'] = self.hosterPlugins = self.parse("hoster", pattern=True) + + self.plugins['ocr'] = self.captchaPlugins = self.parse("ocr") + self.plugins['accounts'] = self.accountPlugins = self.parse("accounts") + self.plugins['hooks'] = self.hookPlugins = self.parse("hooks") + self.plugins['internal'] = self.internalPlugins = self.parse("internal") + + self.log.debug("created index of plugins") + + def parse(self, folder, pattern=False, home={}): + """ + returns dict with information + home contains parsed plugins from pyload. + + { + name : {path, version, config, (pattern, re), (plugin, class)} + } + + """ + plugins = {} + if home: + pfolder = join("userplugins", folder) + if not exists(pfolder): + makedirs(pfolder) + if not exists(join(pfolder, "__init__.py")): + f = open(join(pfolder, "__init__.py"), "wb") + f.close() + + else: + pfolder = join(pypath, "pyload", "plugins", folder) + + for f in listdir(pfolder): + if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( + "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): + data = open(join(pfolder, f)) + content = data.read() + data.close() + + if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): + continue + elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): + continue + elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): + continue + + name = f[:-3] + if name[-1] == ".": name = name[:-4] + + version = self.VERSION.findall(content) + if version: + version = float(version[0][1]) + else: + version = 0 + + # home contains plugins from pyload root + if home and name in home: + if home[name]['v'] >= version: + continue + + if name in IGNORE or (folder, name) in IGNORE: + continue + + plugins[name] = {} + plugins[name]['v'] = version + + module = f.replace(".pyc", "").replace(".py", "") + + # the plugin is loaded from user directory + plugins[name]['user'] = True if home else False + plugins[name]['name'] = module + + if pattern: + pattern = self.PATTERN.findall(content) + + if pattern: + pattern = pattern[0][1] + else: + pattern = "^unmachtable$" + + plugins[name]['pattern'] = pattern + + try: + plugins[name]['re'] = re.compile(pattern) + except: + self.log.error(_("%s has a invalid pattern.") % name) + + + # internals have no config + if folder == "internal": + self.config.deleteConfig(name) + continue + + config = self.CONFIG.findall(content) + if config: + config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) + desc = self.DESC.findall(content) + desc = desc[0][1] if desc else "" + + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] + + if folder == "hooks": + append = True + for item in config: + if item[0] == "activated": append = False + + # activated flag missing + if append: config.append(["activated", "bool", "Activated", False]) + + try: + self.config.addPluginConfig(name, config, desc) + except: + self.log.error("Invalid config in %s: %s" % (name, config)) + + elif folder == "hooks": #force config creation + desc = self.DESC.findall(content) + desc = desc[0][1] if desc else "" + config = (["activated", "bool", "Activated", False],) + + try: + self.config.addPluginConfig(name, config, desc) + except: + self.log.error("Invalid config in %s: %s" % (name, config)) + + if not home: + temp = self.parse(folder, pattern, plugins) + plugins.update(temp) + + return plugins + + + def parseUrls(self, urls): + """parse plugins for given list of urls""" + + last = None + res = [] # tupels of (url, plugin) + + for url in urls: + if type(url) not in (str, unicode, buffer): continue + found = False + + if last and last[1]['re'].match(url): + res.append((url, last[0])) + continue + + for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), + self.containerPlugins.iteritems()): + if value['re'].match(url): + res.append((url, name)) + last = (name, value) + found = True + break + + if not found: + res.append((url, "BasePlugin")) + + return res + + def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): + for ptype in pluginlist: + if name in self.plugins[ptype]: + return self.plugins[ptype][name], ptype + return None, None + + def getPlugin(self, name, original=False): + """return plugin module from hoster|decrypter|container""" + plugin, type = self.findPlugin(name) + + if not plugin: + self.log.warning("Plugin %s not found." % name) + plugin = self.hosterPlugins['BasePlugin'] + + if "new_module" in plugin and not original: + return plugin['new_module'] + + return self.loadModule(type, name) + + def getPluginName(self, name): + """ used to obtain new name if other plugin was injected""" + plugin, type = self.findPlugin(name) + + if "new_name" in plugin: + return plugin['new_name'] + + return name + + def loadModule(self, type, name): + """ Returns loaded module for plugin + + :param type: plugin type, subfolder of pyload.plugins + :param name: + """ + plugins = self.plugins[type] + if name in plugins: + if "module" in plugins[name]: return plugins[name]['module'] + try: + module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]['name']), globals(), locals(), + plugins[name]['name']) + plugins[name]['module'] = module #cache import, maybe unneeded + return module + except Exception, e: + self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) + if self.core.debug: + print_exc() + + def loadClass(self, type, name): + """Returns the class of a plugin with the same name""" + module = self.loadModule(type, name) + if module: return getattr(module, name) + + def getAccountPlugins(self): + """return list of account plugin names""" + return self.accountPlugins.keys() + + def find_module(self, fullname, path=None): + #redirecting imports if necesarry + if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins + if fullname.startswith(self.USERROOT): user = 1 + else: user = 0 #used as bool and int + + split = fullname.split(".") + if len(split) != 4 - user: return + type, name = split[2 - user:4 - user] + + if type in self.plugins and name in self.plugins[type]: + #userplugin is a newer version + if not user and self.plugins[type][name]['user']: + return self + #imported from userdir, but pyloads is newer + if user and not self.plugins[type][name]['user']: + return self + + + def load_module(self, name, replace=True): + if name not in sys.modules: #could be already in modules + if replace: + if self.ROOT in name: + newname = name.replace(self.ROOT, self.USERROOT) + else: + newname = name.replace(self.USERROOT, self.ROOT) + else: newname = name + + base, plugin = newname.rsplit(".", 1) + + self.log.debug("Redirected import %s -> %s" % (name, newname)) + + module = __import__(newname, globals(), locals(), [plugin]) + #inject under new an old name + sys.modules[name] = module + sys.modules[newname] = module + + return sys.modules[name] + + + def reloadPlugins(self, type_plugins): + """ reload and reindex plugins """ + if not type_plugins: + return None + + self.log.debug("Request reload of plugins: %s" % type_plugins) + + reloaded = [] + + as_dict = {} + for t,n in type_plugins: + if t in ("hooks", "internal"): #: do not reload hooks or internals, because would cause to much side effects + continue + elif t in as_dict: + as_dict[t].append(n) + else: + as_dict[t] = [n] + + for type in as_dict.iterkeys(): + for plugin in as_dict[type]: + if plugin in self.plugins[type] and "module" in self.plugins[type][plugin]: + self.log.debug("Reloading %s" % plugin) + id = (type, plugin) + try: + reload(self.plugins[type][plugin]['module']) + except Exception, e: + self.log.error("Error when reloading %s" % id, str(e)) + continue + else: + reloaded.append(id) + + #index creation + self.plugins['crypter'] = self.crypterPlugins = self.parse("crypter", pattern=True) + self.plugins['container'] = self.containerPlugins = self.parse("container", pattern=True) + self.plugins['hoster'] = self.hosterPlugins = self.parse("hoster", pattern=True) + self.plugins['ocr'] = self.captchaPlugins = self.parse("ocr") + self.plugins['accounts'] = self.accountPlugins = self.parse("accounts") + + if "accounts" in as_dict: #: accounts needs to be reloaded + self.core.accountManager.initPlugins() + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + + return reloaded #: return a list of the plugins successfully reloaded + + def reloadPlugin(self, type_plugin): + """ reload and reindex ONE plugin """ + return True if self.reloadPlugins(type_plugin) else False diff --git a/pyload/manager/RemoteManager.py b/pyload/manager/RemoteManager.py new file mode 100644 index 000000000..e53e317e3 --- /dev/null +++ b/pyload/manager/RemoteManager.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from threading import Thread +from traceback import print_exc + +class BackendBase(Thread): + def __init__(self, manager): + Thread.__init__(self) + self.m = manager + self.core = manager.core + self.enabled = True + self.running = False + + def run(self): + self.running = True + try: + self.serve() + except Exception, e: + self.core.log.error(_("Remote backend error: %s") % e) + if self.core.debug: + print_exc() + finally: + self.running = False + + def setup(self, host, port): + pass + + def checkDeps(self): + return True + + def serve(self): + pass + + def shutdown(self): + pass + + def stop(self): + self.enabled = False# set flag and call shutdowm message, so thread can react + self.shutdown() + + +class RemoteManager: + available = [] + + def __init__(self, core): + self.core = core + self.backends = [] + + if self.core.remote: + self.available.append("ThriftBackend") +# else: +# self.available.append("SocketBackend") + + + def startBackends(self): + host = self.core.config["remote"]["listenaddr"] + port = self.core.config["remote"]["port"] + + for b in self.available: + klass = getattr(__import__("pyload.remote.%s" % b, globals(), locals(), [b], -1), b) + backend = klass(self) + if not backend.checkDeps(): + continue + try: + backend.setup(host, port) + self.core.log.info(_("Starting %(name)s: %(addr)s:%(port)s") % {"name": b, "addr": host, "port": port}) + except Exception, e: + self.core.log.error(_("Failed loading backend %(name)s | %(error)s") % {"name": b, "error": str(e)}) + if self.core.debug: + print_exc() + else: + backend.start() + self.backends.append(backend) + + port += 1 diff --git a/pyload/manager/ThreadManager.py b/pyload/manager/ThreadManager.py new file mode 100644 index 000000000..1073f8040 --- /dev/null +++ b/pyload/manager/ThreadManager.py @@ -0,0 +1,317 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from os.path import exists, join +import re +from subprocess import Popen +from threading import Event, Lock +from time import sleep, time +from traceback import print_exc +from random import choice + +import pycurl + +from pyload.manager.thread import PluginThread +from pyload.datatypes.PyFile import PyFile +from pyload.network.RequestFactory import getURL +from pyload.utils import freeSpace, lock + + +class ThreadManager: + """manages the download threads, assign jobs, reconnect etc""" + + + def __init__(self, core): + """Constructor""" + self.core = core + self.log = core.log + + self.threads = [] # thread list + self.localThreads = [] #hook+decrypter threads + + self.pause = True + + self.reconnecting = Event() + self.reconnecting.clear() + self.downloaded = 0 #number of files downloaded since last cleanup + + self.lock = Lock() + + # some operations require to fetch url info from hoster, so we caching them so it wont be done twice + # contains a timestamp and will be purged after timeout + self.infoCache = {} + + # pool of ids for online check + self.resultIDs = 0 + + # threads which are fetching hoster results + self.infoResults = {} + #timeout for cache purge + self.timestamp = 0 + + pycurl.global_init(pycurl.GLOBAL_DEFAULT) + + for i in range(0, self.core.config.get("download", "max_downloads")): + self.createThread() + + + def createThread(self): + """create a download thread""" + + thread = PluginThread.DownloadThread(self) + self.threads.append(thread) + + def createInfoThread(self, data, pid): + """ + start a thread whichs fetches online status and other infos + data = [ .. () .. ] + """ + self.timestamp = time() + 5 * 60 + + PluginThread.InfoThread(self, data, pid) + + @lock + def createResultThread(self, data, add=False): + """ creates a thread to fetch online status, returns result id """ + self.timestamp = time() + 5 * 60 + + rid = self.resultIDs + self.resultIDs += 1 + + PluginThread.InfoThread(self, data, rid=rid, add=add) + + return rid + + + @lock + def getInfoResult(self, rid): + """returns result and clears it""" + self.timestamp = time() + 5 * 60 + + if rid in self.infoResults: + data = self.infoResults[rid] + self.infoResults[rid] = {} + return data + else: + return {} + + @lock + def setInfoResults(self, rid, result): + self.infoResults[rid].update(result) + + def getActiveFiles(self): + active = [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)] + + for t in self.localThreads: + active.extend(t.getActiveFiles()) + + return active + + def processingIds(self): + """get a id list of all pyfiles processed""" + return [x.id for x in self.getActiveFiles()] + + + def work(self): + """run all task which have to be done (this is for repetivive call by core)""" + try: + self.tryReconnect() + except Exception, e: + self.log.error(_("Reconnect Failed: %s") % str(e) ) + self.reconnecting.clear() + if self.core.debug: + print_exc() + self.checkThreadCount() + + try: + self.assignJob() + except Exception, e: + self.log.warning("Assign job error", e) + if self.core.debug: + print_exc() + + sleep(0.5) + self.assignJob() + #it may be failed non critical so we try it again + + if (self.infoCache or self.infoResults) and self.timestamp < time(): + self.infoCache.clear() + self.infoResults.clear() + self.log.debug("Cleared Result cache") + + #-------------------------------------------------------------------------- + def tryReconnect(self): + """checks if reconnect needed""" + + if not (self.core.config["reconnect"]["activated"] and self.core.api.isTimeReconnect()): + return False + + active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active] + + if not (0 < active.count(True) == len(active)): + return False + + if not exists(self.core.config['reconnect']['method']): + if exists(join(pypath, self.core.config['reconnect']['method'])): + self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method']) + else: + self.core.config["reconnect"]["activated"] = False + self.log.warning(_("Reconnect script not found!")) + return + + self.reconnecting.set() + + #Do reconnect + self.log.info(_("Starting reconnect")) + + while [x.active.plugin.waiting for x in self.threads if x.active].count(True) != 0: + sleep(0.25) + + ip = self.getIP() + + self.core.hookManager.beforeReconnecting(ip) + + self.log.debug("Old IP: %s" % ip) + + try: + reconn = Popen(self.core.config['reconnect']['method'], bufsize=-1, shell=True)#, stdout=subprocess.PIPE) + except: + self.log.warning(_("Failed executing reconnect script!")) + self.core.config["reconnect"]["activated"] = False + self.reconnecting.clear() + if self.core.debug: + print_exc() + return + + reconn.wait() + sleep(1) + ip = self.getIP() + self.core.hookManager.afterReconnecting(ip) + + self.log.info(_("Reconnected, new IP: %s") % ip) + + self.reconnecting.clear() + + def getIP(self): + """retrieve current ip""" + services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"), + ("http://checkip.dyndns.org/",".*Current IP Address: (\S+).*")] + + ip = "" + for i in range(10): + try: + sv = choice(services) + ip = getURL(sv[0]) + ip = re.match(sv[1], ip).group(1) + break + except: + ip = "" + sleep(1) + + return ip + + #-------------------------------------------------------------------------- + def checkThreadCount(self): + """checks if there are need for increasing or reducing thread count""" + + if len(self.threads) == self.core.config.get("download", "max_downloads"): + return True + elif len(self.threads) < self.core.config.get("download", "max_downloads"): + self.createThread() + else: + free = [x for x in self.threads if not x.active] + if free: + free[0].put("quit") + + + def cleanPycurl(self): + """ make a global curl cleanup (currently ununused) """ + if self.processingIds(): + return False + pycurl.global_cleanup() + pycurl.global_init(pycurl.GLOBAL_DEFAULT) + self.downloaded = 0 + self.log.debug("Cleaned up pycurl") + return True + + #-------------------------------------------------------------------------- + def assignJob(self): + """assing a job to a thread if possible""" + + if self.pause or not self.core.api.isTimeDownload(): return + + #if self.downloaded > 20: + # if not self.cleanPyCurl(): return + + free = [x for x in self.threads if not x.active] + + inuse = set([(x.active.pluginname, self.getLimit(x)) for x in self.threads if x.active and x.active.hasPlugin() and x.active.plugin.account]) + inuse = map(lambda x: (x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) ,inuse) + onlimit = [x[0] for x in inuse if x[1] > 0 and x[2] >= x[1]] + + occ = [x.active.pluginname for x in self.threads if x.active and x.active.hasPlugin() and not x.active.plugin.multiDL] + onlimit + + occ.sort() + occ = tuple(set(occ)) + job = self.core.files.getJob(occ) + if job: + try: + job.initPlugin() + except Exception, e: + self.log.critical(str(e)) + print_exc() + job.setStatus("failed") + job.error = str(e) + job.release() + return + + if job.plugin.__type__ == "hoster": + spaceLeft = freeSpace(self.core.config["general"]["download_folder"]) / 1024 / 1024 + if spaceLeft < self.core.config["general"]["min_free_space"]: + self.log.warning(_("Not enough space left on device")) + self.pause = True + + if free and not self.pause: + thread = free[0] + #self.downloaded += 1 + + thread.put(job) + else: + #put job back + if occ not in self.core.files.jobCache: + self.core.files.jobCache[occ] = [] + self.core.files.jobCache[occ].append(job.id) + + #check for decrypt jobs + job = self.core.files.getDecryptJob() + if job: + job.initPlugin() + thread = PluginThread.DecrypterThread(self, job) + + + else: + thread = PluginThread.DecrypterThread(self, job) + + def getLimit(self, thread): + limit = thread.active.plugin.account.getAccountData(thread.active.plugin.user)["options"].get("limitDL", ["0"])[0] + return int(limit) + + def cleanup(self): + """do global cleanup, should be called when finished with pycurl""" + pycurl.global_cleanup() diff --git a/pyload/manager/__init__.py b/pyload/manager/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/manager/event/PullEvents.py b/pyload/manager/event/PullEvents.py new file mode 100644 index 000000000..0739b4ec8 --- /dev/null +++ b/pyload/manager/event/PullEvents.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from time import time +from pyload.utils import uniqify + +class PullManager: + def __init__(self, core): + self.core = core + self.clients = [] + + def newClient(self, uuid): + self.clients.append(Client(uuid)) + + def clean(self): + for n, client in enumerate(self.clients): + if client.lastActive + 30 < time(): + del self.clients[n] + + def getEvents(self, uuid): + events = [] + validUuid = False + for client in self.clients: + if client.uuid == uuid: + client.lastActive = time() + validUuid = True + while client.newEvents(): + events.append(client.popEvent().toList()) + break + if not validUuid: + self.newClient(uuid) + events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] + return uniqify(events) + + def addEvent(self, event): + for client in self.clients: + client.addEvent(event) + +class Client: + def __init__(self, uuid): + self.uuid = uuid + self.lastActive = time() + self.events = [] + + def newEvents(self): + return len(self.events) > 0 + + def popEvent(self): + if not len(self.events): + return None + return self.events.pop(0) + + def addEvent(self, event): + self.events.append(event) + +class UpdateEvent: + def __init__(self, itype, iid, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.destination = destination + + def toList(self): + return ["update", self.destination, self.type, self.id] + +class RemoveEvent: + def __init__(self, itype, iid, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.destination = destination + + def toList(self): + return ["remove", self.destination, self.type, self.id] + +class InsertEvent: + def __init__(self, itype, iid, after, destination): + assert itype == "pack" or itype == "file" + assert destination == "queue" or destination == "collector" + self.type = itype + self.id = iid + self.after = after + self.destination = destination + + def toList(self): + return ["insert", self.destination, self.type, self.id, self.after] + +class ReloadAllEvent: + def __init__(self, destination): + assert destination == "queue" or destination == "collector" + self.destination = destination + + def toList(self): + return ["reload", self.destination] + +class AccountUpdateEvent: + def toList(self): + return ["account"] + +class ConfigUpdateEvent: + def toList(self): + return ["config"] diff --git a/pyload/manager/event/Scheduler.py b/pyload/manager/event/Scheduler.py new file mode 100644 index 000000000..71b5f96af --- /dev/null +++ b/pyload/manager/event/Scheduler.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from time import time +from heapq import heappop, heappush +from thread import start_new_thread +from threading import Lock + +class AlreadyCalled(Exception): + pass + + +class Deferred: + def __init__(self): + self.call = [] + self.result = () + + def addCallback(self, f, *cargs, **ckwargs): + self.call.append((f, cargs, ckwargs)) + + def callback(self, *args, **kwargs): + if self.result: + raise AlreadyCalled + self.result = (args, kwargs) + for f, cargs, ckwargs in self.call: + args += tuple(cargs) + kwargs.update(ckwargs) + f(*args ** kwargs) + + +class Scheduler: + def __init__(self, core): + self.core = core + + self.queue = PriorityQueue() + + def addJob(self, t, call, args=[], kwargs={}, threaded=True): + d = Deferred() + t += time() + j = Job(t, call, args, kwargs, d, threaded) + self.queue.put((t, j)) + return d + + + def removeJob(self, d): + """ + :param d: defered object + :return: if job was deleted + """ + index = -1 + + for i, j in enumerate(self.queue): + if j[1].deferred == d: + index = i + + if index >= 0: + del self.queue[index] + return True + + return False + + def work(self): + while True: + t, j = self.queue.get() + if not j: + break + else: + if t <= time(): + j.start() + else: + self.queue.put((t, j)) + break + + +class Job: + def __init__(self, time, call, args=[], kwargs={}, deferred=None, threaded=True): + self.time = float(time) + self.call = call + self.args = args + self.kwargs = kwargs + self.deferred = deferred + self.threaded = threaded + + def run(self): + ret = self.call(*self.args, **self.kwargs) + if self.deferred is None: + return + else: + self.deferred.callback(ret) + + def start(self): + if self.threaded: + start_new_thread(self.run, ()) + else: + self.run() + + +class PriorityQueue: + """ a non blocking priority queue """ + + def __init__(self): + self.queue = [] + self.lock = Lock() + + def __iter__(self): + return iter(self.queue) + + def __delitem__(self, key): + del self.queue[key] + + def put(self, element): + self.lock.acquire() + heappush(self.queue, element) + self.lock.release() + + def get(self): + """ return element or None """ + self.lock.acquire() + try: + el = heappop(self.queue) + return el + except IndexError: + return None, None + finally: + self.lock.release() diff --git a/pyload/manager/event/__init__.py b/pyload/manager/event/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/manager/thread/PluginThread.py b/pyload/manager/thread/PluginThread.py new file mode 100644 index 000000000..5c274fa46 --- /dev/null +++ b/pyload/manager/thread/PluginThread.py @@ -0,0 +1,675 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from pyload.datatypes.PyFile import PyFile +from pyload.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload +from pyload.utils.packagetools import parseNames +from pyload.utils import safe_join +from pyload.api import OnlineStatus + +class PluginThread(Thread): + """abstract base class for thread types""" + + #-------------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + Thread.__init__(self) + self.setDaemon(True) + self.m = manager #thread manager + + + def writeDebugReport(self, pyfile): + """ writes a + :return: + """ + + dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) + dump = self.getDebugDump(pyfile) + + try: + import zipfile + + zip = zipfile.ZipFile(dump_name, "w") + + for f in listdir(join("tmp", pyfile.pluginname)): + try: + # avoid encoding errors + zip.write(join("tmp", pyfile.pluginname, f), safe_join(pyfile.pluginname, f)) + except: + pass + + info = zipfile.ZipInfo(safe_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) + info.external_attr = 0644 << 16L # change permissions + + zip.writestr(info, dump) + zip.close() + + if not stat(dump_name).st_size: + raise Exception("Empty Zipfile") + + except Exception, e: + self.m.log.debug("Error creating zip file: %s" % e) + + dump_name = dump_name.replace(".zip", ".txt") + f = open(dump_name, "wb") + f.write(dump) + f.close() + + self.m.core.log.info("Debug Report written to %s" % dump_name) + + def getDebugDump(self, pyfile): + dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( + self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version__, format_exc()) + + tb = exc_info()[2] + stack = [] + while tb: + stack.append(tb.tb_frame) + tb = tb.tb_next + + for frame in stack[1:]: + dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno) + + for key, value in frame.f_locals.items(): + dump += "\t%20s = " % key + try: + dump += pformat(value) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + del frame + + del stack #delete it just to be sure... + + dump += "\n\nPLUGIN OBJECT DUMP: \n\n" + + for name in dir(pyfile.plugin): + attr = getattr(pyfile.plugin, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + dump += "\nPYFILE OBJECT DUMP: \n\n" + + for name in dir(pyfile): + attr = getattr(pyfile, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + if pyfile.pluginname in self.m.core.config.plugin: + dump += "\n\nCONFIG: \n\n" + dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" + + return dump + + def clean(self, pyfile): + """ set thread unactive and release pyfile """ + self.active = False + pyfile.release() + + +class DownloadThread(PluginThread): + """thread for downloading files from 'real' hoster plugins""" + + #-------------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + PluginThread.__init__(self, manager) + + self.queue = Queue() # job queue + self.active = False + + self.start() + + #-------------------------------------------------------------------------- + def run(self): + """run method""" + pyfile = None + + while True: + del pyfile + self.active = self.queue.get() + pyfile = self.active + + if self.active == "quit": + self.active = False + self.m.threads.remove(self) + return True + + try: + if not pyfile.hasPlugin(): continue + #this pyfile was deleted while queueing + + pyfile.plugin.checkForSameFiles(starting=True) + self.m.log.info(_("Download starts: %s" % pyfile.name)) + + # start download + self.m.core.hookManager.downloadPreparing(pyfile) + pyfile.plugin.preprocessing(self) + + self.m.log.info(_("Download finished: %s") % pyfile.name) + self.m.core.hookManager.downloadFinished(pyfile) + self.m.core.files.checkPackageFinished(pyfile) + + except NotImplementedError: + self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + pyfile.setStatus("failed") + pyfile.error = "Plugin does not work" + self.clean(pyfile) + continue + + except Abort: + try: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + except: + pass + + pyfile.setStatus("aborted") + + self.clean(pyfile) + continue + + except Reconnect: + self.queue.put(pyfile) + #pyfile.req.clearCookies() + + while self.m.reconnecting.isSet(): + sleep(0.5) + + continue + + except Retry, e: + reason = e.args[0] + self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) + self.queue.put(pyfile) + continue + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % pyfile.name) + elif msg == "temp. offline": + pyfile.setStatus("temp. offline") + self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + self.m.core.hookManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + except error, e: + if len(e.args) == 2: + code, msg = e.args + else: + code = 0 + msg = e.args + + self.m.log.debug("pycurl exception %s: %s" % (code, msg)) + + if code in (7, 18, 28, 52, 56): + self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) + wait = time() + 60 + + pyfile.waitUntil = wait + pyfile.setStatus("waiting") + while time() < wait: + sleep(1) + if pyfile.abort: + break + + if pyfile.abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + self.clean(pyfile) + else: + self.queue.put(pyfile) + + continue + + else: + pyfile.setStatus("failed") + self.m.log.error("pycurl error %s: %s" % (code, msg)) + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.hookManager.downloadFailed(pyfile) + + self.clean(pyfile) + continue + + except SkipDownload, e: + pyfile.setStatus("skipped") + + self.m.log.info( + _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) + + self.clean(pyfile) + + self.m.core.files.checkPackageFinished(pyfile) + + self.active = False + self.m.core.files.save() + + continue + + + except Exception, e: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.hookManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + finally: + self.m.core.files.save() + pyfile.checkIfProcessed() + exc_clear() + + #pyfile.plugin.req.clean() + + self.active = False + pyfile.finishIfDone() + self.m.core.files.save() + + + def put(self, job): + """assing job to thread""" + self.queue.put(job) + + + def stop(self): + """stops the thread""" + self.put("quit") + + +class DecrypterThread(PluginThread): + """thread for decrypting""" + + def __init__(self, manager, pyfile): + """constructor""" + PluginThread.__init__(self, manager) + + self.active = pyfile + manager.localThreads.append(self) + + pyfile.setStatus("decrypting") + + self.start() + + def getActiveFiles(self): + return [self.active] + + def run(self): + """run method""" + + pyfile = self.active + retry = False + + try: + self.m.log.info(_("Decrypting starts: %s") % self.active.name) + self.active.plugin.preprocessing(self) + + except NotImplementedError: + self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) + return + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + self.active.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % self.active.name) + else: + self.active.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": msg}) + self.active.error = msg + + return + + except Abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + return + + except Retry: + self.m.log.info(_("Retrying %s") % self.active.name) + retry = True + return self.run() + + except Exception, e: + self.active.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": str(e)}) + self.active.error = str(e) + + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + return + + + finally: + if not retry: + self.active.release() + self.active = False + self.m.core.files.save() + self.m.localThreads.remove(self) + exc_clear() + + + #self.m.core.hookManager.downloadFinished(pyfile) + + + #self.m.localThreads.remove(self) + #self.active.finishIfDone() + if not retry: + pyfile.delete() + + +class HookThread(PluginThread): + """thread for hooks""" + + #-------------------------------------------------------------------------- + def __init__(self, m, function, args, kwargs): + """Constructor""" + PluginThread.__init__(self, m) + + self.f = function + self.args = args + self.kwargs = kwargs + + self.active = [] + + m.localThreads.append(self) + + self.start() + + def getActiveFiles(self): + return self.active + + def addActive(self, pyfile): + """ Adds a pyfile to active list and thus will be displayed on overview""" + if pyfile not in self.active: + self.active.append(pyfile) + + def finishFile(self, pyfile): + if pyfile in self.active: + self.active.remove(pyfile) + + pyfile.finishIfDone() + + def run(self): + try: + try: + self.kwargs["thread"] = self + self.f(*self.args, **self.kwargs) + except TypeError, e: + #dirty method to filter out exceptions + if "unexpected keyword argument 'thread'" not in e.args[0]: + raise + + del self.kwargs["thread"] + self.f(*self.args, **self.kwargs) + finally: + local = copy(self.active) + for x in local: + self.finishFile(x) + + self.m.localThreads.remove(self) + + +class InfoThread(PluginThread): + def __init__(self, manager, data, pid=-1, rid=-1, add=False): + """Constructor""" + PluginThread.__init__(self, manager) + + self.data = data + self.pid = pid # package id + # [ .. (name, plugin) .. ] + + self.rid = rid #result id + self.add = add #add packages instead of return result + + self.cache = [] #accumulated data + + self.start() + + def run(self): + """run method""" + + plugins = {} + container = [] + + for url, plugin in self.data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + + # filter out container plugins + for name in self.m.core.pluginManager.containerPlugins: + if name in plugins: + container.extend([(name, url) for url in plugins[name]]) + + del plugins[name] + + #directly write to database + if self.pid > -1: + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) + self.m.core.files.save() + + elif self.add: + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateCache(pluginname, result) + + packs = parseNames([(name, url) for name, x, y, url in self.cache]) + + self.m.log.debug("Fetched and generated %d packages" % len(packs)) + + for k, v in packs: + self.m.core.api.addPackage(k, v) + + #empty cache + del self.cache[:] + + else: #post the results + + + for name, url in container: + #attach container content + try: + data = self.decryptContainer(name, url) + except: + print_exc() + self.m.log.error("Could not decrypt container.") + data = [] + + for url, plugin in data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + self.m.infoResults[self.rid] = {} + + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + + #force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateResult(pluginname, result, True) + + self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} + + self.m.timestamp = time() + 5 * 60 + + + def updateDB(self, plugin, result): + self.m.core.files.updateFileInfo(result, self.pid) + + def updateResult(self, plugin, result, force=False): + #parse package name and generate result + #accumulate results + + self.cache.extend(result) + + if len(self.cache) >= 20 or force: + #used for package generating + tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) + for name, size, status, url in self.cache] + + data = parseNames(tmp) + result = {} + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + self.m.setInfoResults(self.rid, result) + + self.cache = [] + + def updateCache(self, plugin, result): + self.cache.extend(result) + + def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): + try: + result = [] #result loaded from cache + process = [] #urls to process + for url in urls: + if url in self.m.infoCache: + result.append(self.m.infoCache[url]) + else: + process.append(url) + + if result: + self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) + cb(pluginname, result) + + if process: + self.m.log.debug("Run Info Fetching for %s" % pluginname) + for result in plugin.getInfo(process): + #result = [ .. (name, size, status, url) .. ] + if not type(result) == list: result = [result] + + for res in result: + self.m.infoCache[res[3]] = res + + cb(pluginname, result) + + self.m.log.debug("Finished Info Fetching for %s" % pluginname) + except Exception, e: + self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % + {"name": pluginname, "err": str(e)}) + if self.m.core.debug: + print_exc() + + # generate default results + if err: + result = [(url, 0, 3, url) for url in urls] + cb(pluginname, result) + + + def decryptContainer(self, plugin, url): + data = [] + # only works on container plugins + + self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) + + # dummy pyfile + pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) + + pyfile.initPlugin() + + # little plugin lifecycle + try: + pyfile.plugin.setup() + pyfile.plugin.loadToDisk() + pyfile.plugin.decrypt(pyfile) + pyfile.plugin.deleteTmp() + + for pack in pyfile.plugin.packages: + pyfile.plugin.urls.extend(pack[1]) + + data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) + + self.m.log.debug("Got %d links." % len(data)) + + except Exception, e: + self.m.log.debug("Pre decrypting error: %s" % str(e)) + finally: + pyfile.release() + + return data diff --git a/pyload/manager/thread/ServerThread.py b/pyload/manager/thread/ServerThread.py new file mode 100644 index 000000000..7de3b1ca1 --- /dev/null +++ b/pyload/manager/thread/ServerThread.py @@ -0,0 +1,108 @@ +from __future__ import with_statement +from os.path import exists + +import os +import threading +import logging + +core = None +setup = None +log = logging.getLogger("log") + +class WebServer(threading.Thread): + def __init__(self, pycore): + global core + threading.Thread.__init__(self) + self.core = pycore + core = pycore + self.running = True + self.server = pycore.config['webinterface']['server'] + self.https = pycore.config['webinterface']['https'] + self.cert = pycore.config["ssl"]["cert"] + self.key = pycore.config["ssl"]["key"] + self.host = pycore.config['webinterface']['host'] + self.port = pycore.config['webinterface']['port'] + + self.setDaemon(True) + + def run(self): + import pyload.webui as webinterface + global webinterface + + reset = False + + if self.https and (not exists(self.cert) or not exists(self.key)): + log.warning(_("SSL certificates not found.")) + self.https = False + + if self.server in ("lighttpd", "nginx"): + log.warning(_("Sorry, we dropped support for starting %s directly within pyLoad") % self.server) + log.warning(_("You can use the threaded server which offers good performance and ssl,")) + log.warning(_("of course you can still use your existing %s with pyLoads fastcgi server") % self.server) + log.warning(_("sample configs are located in the pyload/web/servers directory")) + reset = True + elif self.server == "fastcgi": + try: + import flup + except: + log.warning(_("Can't use %(server)s, python-flup is not installed!") % { + "server": self.server}) + reset = True + + if reset or self.server == "lightweight": + if os.name != "nt": + try: + import bjoern + except Exception, e: + log.error(_("Error importing lightweight server: %s") % e) + log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) + log.warning(_("Copy the boern.so to pyload/lib folder or use setup.py install")) + log.warning(_("Of course you need to be familiar with linux and know how to compile software")) + self.server = "builtin" + else: + self.core.log.info(_("Server set to threaded, due to known performance problems on windows.")) + self.core.config['webinterface']['server'] = "threaded" + self.server = "threaded" + + if self.server == "threaded": + self.start_threaded() + elif self.server == "fastcgi": + self.start_fcgi() + elif self.server == "lightweight": + self.start_lightweight() + else: + self.start_builtin() + + def start_builtin(self): + + if self.https: + log.warning(_("This server offers no SSL, please consider using threaded instead")) + + self.core.log.info(_("Starting builtin webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_simple(host=self.host, port=self.port) + + def start_threaded(self): + if self.https: + self.core.log.info(_("Starting threaded SSL webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + else: + self.cert = "" + self.key = "" + self.core.log.info(_("Starting threaded webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + + webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) + + def start_fcgi(self): + + self.core.log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_fcgi(host=self.host, port=self.port) + + + def start_lightweight(self): + if self.https: + log.warning(_("This server offers no SSL, please consider using threaded instead")) + + self.core.log.info(_("Starting lightweight webserver (bjoern): %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_lightweight(host=self.host, port=self.port) + + def quit(self): + self.running = False diff --git a/pyload/manager/thread/__init__.py b/pyload/manager/thread/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyload/plugins/AccountManager.py b/pyload/plugins/AccountManager.py deleted file mode 100644 index 67909072f..000000000 --- a/pyload/plugins/AccountManager.py +++ /dev/null @@ -1,173 +0,0 @@ -# -*- coding: utf-8 -*- - -from os.path import exists -from shutil import copy - -from threading import Lock - -from pyload.PullEvents import AccountUpdateEvent -from pyload.utils import chmod, lock - -ACC_VERSION = 1 - - -class AccountManager: - """manages all accounts""" - - #-------------------------------------------------------------------------- - def __init__(self, core): - """Constructor""" - - self.core = core - self.lock = Lock() - - self.initPlugins() - self.saveAccounts() # save to add categories to conf - - def initPlugins(self): - self.accounts = {} # key = ( plugin ) - self.plugins = {} - - self.initAccountPlugins() - self.loadAccounts() - - def getAccountPlugin(self, plugin): - """get account instance for plugin or None if anonymous""" - if plugin in self.accounts: - if plugin not in self.plugins: - try: - self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin]) - except TypeError: # The account class no longer exists (blacklisted plugin). Skipping the account to avoid crash - return None - - return self.plugins[plugin] - else: - return None - - def getAccountPlugins(self): - """ get all account instances""" - - plugins = [] - for plugin in self.accounts.keys(): - plugins.append(self.getAccountPlugin(plugin)) - - return plugins - - #-------------------------------------------------------------------------- - def loadAccounts(self): - """loads all accounts available""" - - if not exists("accounts.conf"): - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - - f = open("accounts.conf", "rb") - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - f.close() - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - self.core.log.warning(_("Account settings deleted, due to new config format.")) - return - - plugin = "" - name = "" - - for line in content[1:]: - line = line.strip() - - if not line: continue - if line.startswith("#"): continue - if line.startswith("version"): continue - - if line.endswith(":") and line.count(":") == 1: - plugin = line[:-1] - self.accounts[plugin] = {} - - elif line.startswith("@"): - try: - option = line[1:].split() - self.accounts[plugin][name]['options'][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) - except: - pass - - elif ":" in line: - name, sep, pw = line.partition(":") - self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - - #-------------------------------------------------------------------------- - def saveAccounts(self): - """save all account information""" - - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION) + "\n") - - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin+":\n") - - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data['password']) ) - if data['options']: - for option, values in data['options'].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) - - f.close() - chmod(f.name, 0600) - - #-------------------------------------------------------------------------- - def initAccountPlugins(self): - """init names""" - for name in self.core.pluginManager.getAccountPlugins(): - self.accounts[name] = {} - - @lock - def updateAccount(self, plugin , user, password=None, options={}): - """add or update account""" - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - updated = p.updateAccounts(user, password, options) - #since accounts is a ref in plugin self.accounts doesnt need to be updated here - - self.saveAccounts() - if updated: p.scheduleRefresh(user, force=False) - - @lock - def removeAccount(self, plugin, user): - """remove account""" - - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - p.removeAccount(user) - - self.saveAccounts() - - @lock - def getAccountInfos(self, force=True, refresh=False): - data = {} - - if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - force = False - - for p in self.accounts.keys(): - if self.accounts[p]: - p = self.getAccountPlugin(p) - if p: - data[p.__name__] = p.getAllAccounts(force) - else: # When an account has been skipped, p is None - data[p] = [] - else: - data[p] = [] - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) - return data - - def sendChange(self): - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) diff --git a/pyload/plugins/PluginManager.py b/pyload/plugins/PluginManager.py deleted file mode 100644 index be467b41a..000000000 --- a/pyload/plugins/PluginManager.py +++ /dev/null @@ -1,356 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -import sys - -from itertools import chain -from os import listdir, makedirs -from os.path import isfile, join, exists, abspath -from sys import version_info -from traceback import print_exc - -from SafeEval import const_eval as literal_eval - -from pyload.ConfigParser import IGNORE - - -class PluginManager: - ROOT = "pyload.plugins." - USERROOT = "userplugins." - TYPES = ("accounts", "container", "crypter", "hooks", "hoster", "internal", "ocr") - - PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') - VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)') - CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) - DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') - - - def __init__(self, core): - self.core = core - - self.config = core.config - self.log = core.log - - self.plugins = {} - self.createIndex() - - #register for import hook - sys.meta_path.append(self) - - - def createIndex(self): - """create information for all plugins available""" - - sys.path.append(abspath("")) - - if not exists("userplugins"): - makedirs("userplugins") - if not exists(join("userplugins", "__init__.py")): - f = open(join("userplugins", "__init__.py"), "wb") - f.close() - - self.plugins['crypter'] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins['container'] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins['hoster'] = self.hosterPlugins = self.parse("hoster", pattern=True) - - self.plugins['ocr'] = self.captchaPlugins = self.parse("ocr") - self.plugins['accounts'] = self.accountPlugins = self.parse("accounts") - self.plugins['hooks'] = self.hookPlugins = self.parse("hooks") - self.plugins['internal'] = self.internalPlugins = self.parse("internal") - - self.log.debug("created index of plugins") - - def parse(self, folder, pattern=False, home={}): - """ - returns dict with information - home contains parsed plugins from pyload. - - { - name : {path, version, config, (pattern, re), (plugin, class)} - } - - """ - plugins = {} - if home: - pfolder = join("userplugins", folder) - if not exists(pfolder): - makedirs(pfolder) - if not exists(join(pfolder, "__init__.py")): - f = open(join(pfolder, "__init__.py"), "wb") - f.close() - - else: - pfolder = join(pypath, "pyload", "plugins", folder) - - for f in listdir(pfolder): - if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( - "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - data = open(join(pfolder, f)) - content = data.read() - data.close() - - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): - continue - elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): - continue - elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): - continue - - name = f[:-3] - if name[-1] == ".": name = name[:-4] - - version = self.VERSION.findall(content) - if version: - version = float(version[0][1]) - else: - version = 0 - - # home contains plugins from pyload root - if home and name in home: - if home[name]['v'] >= version: - continue - - if name in IGNORE or (folder, name) in IGNORE: - continue - - plugins[name] = {} - plugins[name]['v'] = version - - module = f.replace(".pyc", "").replace(".py", "") - - # the plugin is loaded from user directory - plugins[name]['user'] = True if home else False - plugins[name]['name'] = module - - if pattern: - pattern = self.PATTERN.findall(content) - - if pattern: - pattern = pattern[0][1] - else: - pattern = "^unmachtable$" - - plugins[name]['pattern'] = pattern - - try: - plugins[name]['re'] = re.compile(pattern) - except: - self.log.error(_("%s has a invalid pattern.") % name) - - - # internals have no config - if folder == "internal": - self.config.deleteConfig(name) - continue - - config = self.CONFIG.findall(content) - if config: - config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] - - if folder == "hooks": - append = True - for item in config: - if item[0] == "activated": append = False - - # activated flag missing - if append: config.append(["activated", "bool", "Activated", False]) - - try: - self.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) - - elif folder == "hooks": #force config creation - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - config = (["activated", "bool", "Activated", False],) - - try: - self.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) - - if not home: - temp = self.parse(folder, pattern, plugins) - plugins.update(temp) - - return plugins - - - def parseUrls(self, urls): - """parse plugins for given list of urls""" - - last = None - res = [] # tupels of (url, plugin) - - for url in urls: - if type(url) not in (str, unicode, buffer): continue - found = False - - if last and last[1]['re'].match(url): - res.append((url, last[0])) - continue - - for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), - self.containerPlugins.iteritems()): - if value['re'].match(url): - res.append((url, name)) - last = (name, value) - found = True - break - - if not found: - res.append((url, "BasePlugin")) - - return res - - def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): - for ptype in pluginlist: - if name in self.plugins[ptype]: - return self.plugins[ptype][name], ptype - return None, None - - def getPlugin(self, name, original=False): - """return plugin module from hoster|decrypter|container""" - plugin, type = self.findPlugin(name) - - if not plugin: - self.log.warning("Plugin %s not found." % name) - plugin = self.hosterPlugins['BasePlugin'] - - if "new_module" in plugin and not original: - return plugin['new_module'] - - return self.loadModule(type, name) - - def getPluginName(self, name): - """ used to obtain new name if other plugin was injected""" - plugin, type = self.findPlugin(name) - - if "new_name" in plugin: - return plugin['new_name'] - - return name - - def loadModule(self, type, name): - """ Returns loaded module for plugin - - :param type: plugin type, subfolder of pyload.plugins - :param name: - """ - plugins = self.plugins[type] - if name in plugins: - if "module" in plugins[name]: return plugins[name]['module'] - try: - module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]['name']), globals(), locals(), - plugins[name]['name']) - plugins[name]['module'] = module #cache import, maybe unneeded - return module - except Exception, e: - self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) - if self.core.debug: - print_exc() - - def loadClass(self, type, name): - """Returns the class of a plugin with the same name""" - module = self.loadModule(type, name) - if module: return getattr(module, name) - - def getAccountPlugins(self): - """return list of account plugin names""" - return self.accountPlugins.keys() - - def find_module(self, fullname, path=None): - #redirecting imports if necesarry - if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins - if fullname.startswith(self.USERROOT): user = 1 - else: user = 0 #used as bool and int - - split = fullname.split(".") - if len(split) != 4 - user: return - type, name = split[2 - user:4 - user] - - if type in self.plugins and name in self.plugins[type]: - #userplugin is a newer version - if not user and self.plugins[type][name]['user']: - return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name]['user']: - return self - - - def load_module(self, name, replace=True): - if name not in sys.modules: #could be already in modules - if replace: - if self.ROOT in name: - newname = name.replace(self.ROOT, self.USERROOT) - else: - newname = name.replace(self.USERROOT, self.ROOT) - else: newname = name - - base, plugin = newname.rsplit(".", 1) - - self.log.debug("Redirected import %s -> %s" % (name, newname)) - - module = __import__(newname, globals(), locals(), [plugin]) - #inject under new an old name - sys.modules[name] = module - sys.modules[newname] = module - - return sys.modules[name] - - - def reloadPlugins(self, type_plugins): - """ reload and reindex plugins """ - if not type_plugins: - return None - - self.log.debug("Request reload of plugins: %s" % type_plugins) - - reloaded = [] - - as_dict = {} - for t,n in type_plugins: - if t in ("hooks", "internal"): #: do not reload hooks or internals, because would cause to much side effects - continue - elif t in as_dict: - as_dict[t].append(n) - else: - as_dict[t] = [n] - - for type in as_dict.iterkeys(): - for plugin in as_dict[type]: - if plugin in self.plugins[type] and "module" in self.plugins[type][plugin]: - self.log.debug("Reloading %s" % plugin) - id = (type, plugin) - try: - reload(self.plugins[type][plugin]['module']) - except Exception, e: - self.log.error("Error when reloading %s" % id, str(e)) - continue - else: - reloaded.append(id) - - #index creation - self.plugins['crypter'] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins['container'] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins['hoster'] = self.hosterPlugins = self.parse("hoster", pattern=True) - self.plugins['ocr'] = self.captchaPlugins = self.parse("ocr") - self.plugins['accounts'] = self.accountPlugins = self.parse("accounts") - - if "accounts" in as_dict: #: accounts needs to be reloaded - self.core.accountManager.initPlugins() - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - - return reloaded #: return a list of the plugins successfully reloaded - - def reloadPlugin(self, type_plugin): - """ reload and reindex ONE plugin """ - return True if self.reloadPlugins(type_plugin) else False diff --git a/pyload/plugins/accounts/BayfilesCom.py b/pyload/plugins/accounts/BayfilesCom.py index a42ac6457..38537da0e 100644 --- a/pyload/plugins/accounts/BayfilesCom.py +++ b/pyload/plugins/accounts/BayfilesCom.py @@ -3,7 +3,7 @@ from time import time from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class BayfilesCom(Account): diff --git a/pyload/plugins/accounts/FastixRu.py b/pyload/plugins/accounts/FastixRu.py index a7b939a2c..69840fa2c 100644 --- a/pyload/plugins/accounts/FastixRu.py +++ b/pyload/plugins/accounts/FastixRu.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class FastixRu(Account): diff --git a/pyload/plugins/accounts/FilecloudIo.py b/pyload/plugins/accounts/FilecloudIo.py index 5e6b001e8..dc764d218 100644 --- a/pyload/plugins/accounts/FilecloudIo.py +++ b/pyload/plugins/accounts/FilecloudIo.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class FilecloudIo(Account): diff --git a/pyload/plugins/accounts/FileserveCom.py b/pyload/plugins/accounts/FileserveCom.py index 211023991..99f4430c2 100644 --- a/pyload/plugins/accounts/FileserveCom.py +++ b/pyload/plugins/accounts/FileserveCom.py @@ -3,7 +3,7 @@ from time import mktime, strptime from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class FileserveCom(Account): diff --git a/pyload/plugins/accounts/FourSharedCom.py b/pyload/plugins/accounts/FourSharedCom.py index a62749c43..217bd2874 100644 --- a/pyload/plugins/accounts/FourSharedCom.py +++ b/pyload/plugins/accounts/FourSharedCom.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class FourSharedCom(Account): diff --git a/pyload/plugins/accounts/FreeWayMe.py b/pyload/plugins/accounts/FreeWayMe.py index baca53cd4..5106067a9 100644 --- a/pyload/plugins/accounts/FreeWayMe.py +++ b/pyload/plugins/accounts/FreeWayMe.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class FreeWayMe(Account): diff --git a/pyload/plugins/accounts/LetitbitNet.py b/pyload/plugins/accounts/LetitbitNet.py index 88e679f8e..bce97d378 100644 --- a/pyload/plugins/accounts/LetitbitNet.py +++ b/pyload/plugins/accounts/LetitbitNet.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -# from pyload.common.json_layer import json_loads, json_dumps +# from pyload.utils import json_loads, json_dumps class LetitbitNet(Account): diff --git a/pyload/plugins/accounts/LinksnappyCom.py b/pyload/plugins/accounts/LinksnappyCom.py index 4ac046988..35fc881b0 100644 --- a/pyload/plugins/accounts/LinksnappyCom.py +++ b/pyload/plugins/accounts/LinksnappyCom.py @@ -3,7 +3,7 @@ from hashlib import md5 from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class LinksnappyCom(Account): diff --git a/pyload/plugins/accounts/MegaDebridEu.py b/pyload/plugins/accounts/MegaDebridEu.py index b449d7246..5ba8d3577 100644 --- a/pyload/plugins/accounts/MegaDebridEu.py +++ b/pyload/plugins/accounts/MegaDebridEu.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class MegaDebridEu(Account): diff --git a/pyload/plugins/accounts/MultiDebridCom.py b/pyload/plugins/accounts/MultiDebridCom.py index fa10c92cc..cca6a9849 100644 --- a/pyload/plugins/accounts/MultiDebridCom.py +++ b/pyload/plugins/accounts/MultiDebridCom.py @@ -3,7 +3,7 @@ from time import time from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class MultiDebridCom(Account): diff --git a/pyload/plugins/accounts/OboomCom.py b/pyload/plugins/accounts/OboomCom.py index 3ef6f9d58..b281a289a 100644 --- a/pyload/plugins/accounts/OboomCom.py +++ b/pyload/plugins/accounts/OboomCom.py @@ -4,7 +4,7 @@ import time from beaker.crypto.pbkdf2 import PBKDF2 -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Account import Account diff --git a/pyload/plugins/accounts/OverLoadMe.py b/pyload/plugins/accounts/OverLoadMe.py index ba5a58158..18a3db645 100644 --- a/pyload/plugins/accounts/OverLoadMe.py +++ b/pyload/plugins/accounts/OverLoadMe.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class OverLoadMe(Account): diff --git a/pyload/plugins/accounts/PremiumizeMe.py b/pyload/plugins/accounts/PremiumizeMe.py index 822ca8db9..ecb03afda 100644 --- a/pyload/plugins/accounts/PremiumizeMe.py +++ b/pyload/plugins/accounts/PremiumizeMe.py @@ -2,7 +2,7 @@ from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class PremiumizeMe(Account): diff --git a/pyload/plugins/accounts/RPNetBiz.py b/pyload/plugins/accounts/RPNetBiz.py index 0e600b4e3..c2b7cad21 100644 --- a/pyload/plugins/accounts/RPNetBiz.py +++ b/pyload/plugins/accounts/RPNetBiz.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class RPNetBiz(Account): diff --git a/pyload/plugins/accounts/RapidgatorNet.py b/pyload/plugins/accounts/RapidgatorNet.py index 391d7ffcb..c8f129c5b 100644 --- a/pyload/plugins/accounts/RapidgatorNet.py +++ b/pyload/plugins/accounts/RapidgatorNet.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class RapidgatorNet(Account): diff --git a/pyload/plugins/accounts/SimplyPremiumCom.py b/pyload/plugins/accounts/SimplyPremiumCom.py index 5958c1f34..3a385c477 100644 --- a/pyload/plugins/accounts/SimplyPremiumCom.py +++ b/pyload/plugins/accounts/SimplyPremiumCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Account import Account diff --git a/pyload/plugins/accounts/UnrestrictLi.py b/pyload/plugins/accounts/UnrestrictLi.py index 39a75f959..6e878b556 100644 --- a/pyload/plugins/accounts/UnrestrictLi.py +++ b/pyload/plugins/accounts/UnrestrictLi.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Account import Account -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class UnrestrictLi(Account): diff --git a/pyload/plugins/crypter/DailymotionBatch.py b/pyload/plugins/crypter/DailymotionBatch.py index d44350c6b..2c818d8bc 100644 --- a/pyload/plugins/crypter/DailymotionBatch.py +++ b/pyload/plugins/crypter/DailymotionBatch.py @@ -4,7 +4,7 @@ import re from urlparse import urljoin -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Crypter import Crypter from pyload.utils import safe_join diff --git a/pyload/plugins/crypter/GooGl.py b/pyload/plugins/crypter/GooGl.py index ae48c61b5..93f3456cc 100644 --- a/pyload/plugins/crypter/GooGl.py +++ b/pyload/plugins/crypter/GooGl.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pyload.plugins.Crypter import Crypter -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class GooGl(Crypter): diff --git a/pyload/plugins/crypter/LinkSaveIn.py b/pyload/plugins/crypter/LinkSaveIn.py index 84dd8172e..4a56606c8 100644 --- a/pyload/plugins/crypter/LinkSaveIn.py +++ b/pyload/plugins/crypter/LinkSaveIn.py @@ -9,7 +9,7 @@ import re from Crypto.Cipher import AES from pyload.plugins.Crypter import Crypter -from pyload.unescape import unescape +from pyload.utils import html_unescape class LinkSaveIn(Crypter): @@ -153,7 +153,7 @@ class LinkSaveIn(Crypter): dlLink = re.search(r'http://linksave\.in/dl-\w+', jseval).group(0) self.logDebug("JsEngine returns value [%s] for redirection link" % dlLink) response = self.load(dlLink) - link = unescape(re.search(r'", self.req.load("http://1kh.de/l/" + id)).group(1)) self.urls.append(new_link) diff --git a/pyload/plugins/crypter/SafelinkingNet.py b/pyload/plugins/crypter/SafelinkingNet.py index 543fb32fb..9c68ba915 100644 --- a/pyload/plugins/crypter/SafelinkingNet.py +++ b/pyload/plugins/crypter/SafelinkingNet.py @@ -6,7 +6,7 @@ from pycurl import FOLLOWLOCATION from BeautifulSoup import BeautifulSoup -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Crypter import Crypter from pyload.plugins.internal.CaptchaService import SolveMedia diff --git a/pyload/plugins/crypter/SerienjunkiesOrg.py b/pyload/plugins/crypter/SerienjunkiesOrg.py index a1745c6c9..713086cb9 100644 --- a/pyload/plugins/crypter/SerienjunkiesOrg.py +++ b/pyload/plugins/crypter/SerienjunkiesOrg.py @@ -8,7 +8,7 @@ from time import sleep from BeautifulSoup import BeautifulSoup from pyload.plugins.Crypter import Crypter -from pyload.unescape import unescape +from pyload.utils import html_unescape class SerienjunkiesOrg(Crypter): @@ -48,7 +48,7 @@ class SerienjunkiesOrg(Crypter): soup = BeautifulSoup(src) packageName = self.pyfile.package().name if self.getConfig("changeNameSJ") == "Show": - found = unescape(soup.find("h2").find("a").string.split(' –')[0]) + found = html_unescape(soup.find("h2").find("a").string.split(' –')[0]) if found: packageName = found @@ -71,7 +71,7 @@ class SerienjunkiesOrg(Crypter): post = soup.find("div", attrs={"class": "post-content"}) ps = post.findAll("p") - seasonName = unescape(soup.find("a", attrs={"rel": "bookmark"}).string).replace("–", "-") + seasonName = html_unescape(soup.find("a", attrs={"rel": "bookmark"}).string).replace("–", "-") groups = {} gid = -1 for p in ps: @@ -79,7 +79,7 @@ class SerienjunkiesOrg(Crypter): var = p.findAll("strong") opts = {"Sprache": "", "Format": ""} for v in var: - n = unescape(v.string).strip() + n = html_unescape(v.string).strip() n = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', n) if n.strip() not in opts: continue @@ -198,7 +198,7 @@ class SerienjunkiesOrg(Crypter): soup = BeautifulSoup(src) post = soup.find("div", attrs={"id": "page_post"}) ps = post.findAll("p") - found = unescape(soup.find("h2").find("a").string.split(' –')[0]) + found = html_unescape(soup.find("h2").find("a").string.split(' –')[0]) if found: seasonName = found @@ -209,7 +209,7 @@ class SerienjunkiesOrg(Crypter): var = p.findAll("strong") opts = {"Sprache": "", "Format": ""} for v in var: - n = unescape(v.string).strip() + n = html_unescape(v.string).strip() n = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', n) if n.strip() not in opts: continue diff --git a/pyload/plugins/crypter/TurbobitNetFolder.py b/pyload/plugins/crypter/TurbobitNetFolder.py index 48b28c28a..c7786b7be 100644 --- a/pyload/plugins/crypter/TurbobitNetFolder.py +++ b/pyload/plugins/crypter/TurbobitNetFolder.py @@ -3,7 +3,7 @@ import re from pyload.plugins.internal.SimpleCrypter import SimpleCrypter -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class TurbobitNetFolder(SimpleCrypter): diff --git a/pyload/plugins/crypter/YoutubeBatch.py b/pyload/plugins/crypter/YoutubeBatch.py index 5b7cb6530..bc72e04ea 100644 --- a/pyload/plugins/crypter/YoutubeBatch.py +++ b/pyload/plugins/crypter/YoutubeBatch.py @@ -4,7 +4,7 @@ import re from urlparse import urljoin -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Crypter import Crypter from pyload.utils import safe_join diff --git a/pyload/plugins/hooks/DeathByCaptcha.py b/pyload/plugins/hooks/DeathByCaptcha.py index 57bf9031f..6db91b8c1 100644 --- a/pyload/plugins/hooks/DeathByCaptcha.py +++ b/pyload/plugins/hooks/DeathByCaptcha.py @@ -9,7 +9,7 @@ from pycurl import FORM_FILE, HTTPHEADER from thread import start_new_thread from time import sleep -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.HTTPRequest import BadHeader from pyload.network.RequestFactory import getRequest from pyload.plugins.Hook import Hook diff --git a/pyload/plugins/hooks/FastixRu.py b/pyload/plugins/hooks/FastixRu.py index 966bc6bd3..9e2abd92a 100644 --- a/pyload/plugins/hooks/FastixRu.py +++ b/pyload/plugins/hooks/FastixRu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/IRCInterface.py b/pyload/plugins/hooks/IRCInterface.py index af8d8fa69..ef1fa2a09 100644 --- a/pyload/plugins/hooks/IRCInterface.py +++ b/pyload/plugins/hooks/IRCInterface.py @@ -10,7 +10,7 @@ from threading import Thread from time import sleep from traceback import print_exc -from pyload.Api import PackageDoesNotExists, FileDoesNotExists +from pyload.api import PackageDoesNotExists, FileDoesNotExists from pyload.network.RequestFactory import getURL from pyload.plugins.Hook import Hook from pyload.utils import formatSize diff --git a/pyload/plugins/hooks/LinksnappyCom.py b/pyload/plugins/hooks/LinksnappyCom.py index f662ae4e9..9086e3c2a 100644 --- a/pyload/plugins/hooks/LinksnappyCom.py +++ b/pyload/plugins/hooks/LinksnappyCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/MegaDebridEu.py b/pyload/plugins/hooks/MegaDebridEu.py index da151f9aa..8c8894c9b 100644 --- a/pyload/plugins/hooks/MegaDebridEu.py +++ b/pyload/plugins/hooks/MegaDebridEu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/MultiDebridCom.py b/pyload/plugins/hooks/MultiDebridCom.py index 7d9b6526a..6aa5ba3eb 100644 --- a/pyload/plugins/hooks/MultiDebridCom.py +++ b/pyload/plugins/hooks/MultiDebridCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/PremiumizeMe.py b/pyload/plugins/hooks/PremiumizeMe.py index 70bc4a0f2..b6d89adb6 100644 --- a/pyload/plugins/hooks/PremiumizeMe.py +++ b/pyload/plugins/hooks/PremiumizeMe.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/RPNetBiz.py b/pyload/plugins/hooks/RPNetBiz.py index e119e6451..b46e326e6 100644 --- a/pyload/plugins/hooks/RPNetBiz.py +++ b/pyload/plugins/hooks/RPNetBiz.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/SimplyPremiumCom.py b/pyload/plugins/hooks/SimplyPremiumCom.py index 8e9bc5e1e..f0df36b22 100644 --- a/pyload/plugins/hooks/SimplyPremiumCom.py +++ b/pyload/plugins/hooks/SimplyPremiumCom.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hooks/UnSkipOnFail.py b/pyload/plugins/hooks/UnSkipOnFail.py index fd3b35a0a..941ce4fc7 100644 --- a/pyload/plugins/hooks/UnSkipOnFail.py +++ b/pyload/plugins/hooks/UnSkipOnFail.py @@ -2,7 +2,7 @@ from os.path import basename -from pyload.PyFile import PyFile +from pyload.datatypes.PyFile import PyFile from pyload.plugins.Hook import Hook from pyload.utils import fs_encode diff --git a/pyload/plugins/hooks/UnrestrictLi.py b/pyload/plugins/hooks/UnrestrictLi.py index 1562bdf24..2ca5ad907 100644 --- a/pyload/plugins/hooks/UnrestrictLi.py +++ b/pyload/plugins/hooks/UnrestrictLi.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.internal.MultiHoster import MultiHoster diff --git a/pyload/plugins/hoster/AlldebridCom.py b/pyload/plugins/hoster/AlldebridCom.py index 1b115f19e..bdb5b1599 100644 --- a/pyload/plugins/hoster/AlldebridCom.py +++ b/pyload/plugins/hoster/AlldebridCom.py @@ -5,7 +5,7 @@ import re from random import randrange from urllib import unquote -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster from pyload.utils import parseFileSize diff --git a/pyload/plugins/hoster/BayfilesCom.py b/pyload/plugins/hoster/BayfilesCom.py index ea4bd3ca5..5906ade75 100644 --- a/pyload/plugins/hoster/BayfilesCom.py +++ b/pyload/plugins/hoster/BayfilesCom.py @@ -4,7 +4,7 @@ import re from time import time -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/plugins/hoster/DailymotionCom.py b/pyload/plugins/hoster/DailymotionCom.py index 0ae4c697b..5692fa652 100644 --- a/pyload/plugins/hoster/DailymotionCom.py +++ b/pyload/plugins/hoster/DailymotionCom.py @@ -2,8 +2,8 @@ import re -from pyload.PyFile import statusMap -from pyload.common.json_layer import json_loads +from pyload.datatypes.PyFile import statusMap +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/DlFreeFr.py b/pyload/plugins/hoster/DlFreeFr.py index 387e11efc..0365754bc 100644 --- a/pyload/plugins/hoster/DlFreeFr.py +++ b/pyload/plugins/hoster/DlFreeFr.py @@ -3,7 +3,7 @@ import pycurl import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.Browser import Browser from pyload.network.CookieJar import CookieJar from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns diff --git a/pyload/plugins/hoster/ExtabitCom.py b/pyload/plugins/hoster/ExtabitCom.py index 38479410e..78172ef02 100644 --- a/pyload/plugins/hoster/ExtabitCom.py +++ b/pyload/plugins/hoster/ExtabitCom.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.hoster.UnrestrictLi import secondsToMidnight from pyload.plugins.internal.CaptchaService import ReCaptcha diff --git a/pyload/plugins/hoster/FastixRu.py b/pyload/plugins/hoster/FastixRu.py index e031e3e55..aa1794047 100644 --- a/pyload/plugins/hoster/FastixRu.py +++ b/pyload/plugins/hoster/FastixRu.py @@ -5,7 +5,7 @@ import re from random import randrange from urllib import unquote -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/FilecloudIo.py b/pyload/plugins/hoster/FilecloudIo.py index 05753a67e..9cf9306d1 100644 --- a/pyload/plugins/hoster/FilecloudIo.py +++ b/pyload/plugins/hoster/FilecloudIo.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.CaptchaService import ReCaptcha from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/plugins/hoster/FilepostCom.py b/pyload/plugins/hoster/FilepostCom.py index ac2ae4845..382971c61 100644 --- a/pyload/plugins/hoster/FilepostCom.py +++ b/pyload/plugins/hoster/FilepostCom.py @@ -4,7 +4,7 @@ import re from time import time -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.CaptchaService import ReCaptcha from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/plugins/hoster/FileserveCom.py b/pyload/plugins/hoster/FileserveCom.py index 15830b759..5892cd96a 100644 --- a/pyload/plugins/hoster/FileserveCom.py +++ b/pyload/plugins/hoster/FileserveCom.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.RequestFactory import getURL from pyload.plugins.Hoster import Hoster from pyload.plugins.Plugin import chunks diff --git a/pyload/plugins/hoster/IfileIt.py b/pyload/plugins/hoster/IfileIt.py index 2707edd5a..fc26e2f23 100644 --- a/pyload/plugins/hoster/IfileIt.py +++ b/pyload/plugins/hoster/IfileIt.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.CaptchaService import ReCaptcha from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/plugins/hoster/LetitbitNet.py b/pyload/plugins/hoster/LetitbitNet.py index 3a8e28a90..a28f06571 100644 --- a/pyload/plugins/hoster/LetitbitNet.py +++ b/pyload/plugins/hoster/LetitbitNet.py @@ -10,7 +10,7 @@ import re from urllib import urlencode, urlopen -from pyload.common.json_layer import json_loads, json_dumps +from pyload.utils import json_loads, json_dumps from pyload.plugins.hoster.UnrestrictLi import secondsToMidnight from pyload.plugins.internal.CaptchaService import ReCaptcha from pyload.plugins.internal.SimpleHoster import SimpleHoster diff --git a/pyload/plugins/hoster/LinksnappyCom.py b/pyload/plugins/hoster/LinksnappyCom.py index aed74d09b..e7cc61391 100644 --- a/pyload/plugins/hoster/LinksnappyCom.py +++ b/pyload/plugins/hoster/LinksnappyCom.py @@ -4,7 +4,7 @@ import re from urlparse import urlsplit -from pyload.common.json_layer import json_loads, json_dumps +from pyload.utils import json_loads, json_dumps from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/MegaDebridEu.py b/pyload/plugins/hoster/MegaDebridEu.py index 6c980009e..ed3747aed 100644 --- a/pyload/plugins/hoster/MegaDebridEu.py +++ b/pyload/plugins/hoster/MegaDebridEu.py @@ -4,7 +4,7 @@ import re from urllib import unquote_plus -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/MegaNz.py b/pyload/plugins/hoster/MegaNz.py index 801df9c9d..2e70b5dc6 100644 --- a/pyload/plugins/hoster/MegaNz.py +++ b/pyload/plugins/hoster/MegaNz.py @@ -9,7 +9,7 @@ from array import array from base64 import standard_b64decode from os import remove -from pyload.common.json_layer import json_loads, json_dumps +from pyload.utils import json_loads, json_dumps from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/MegacrypterCom.py b/pyload/plugins/hoster/MegacrypterCom.py index 0cc17bfe3..cdc019fdf 100644 --- a/pyload/plugins/hoster/MegacrypterCom.py +++ b/pyload/plugins/hoster/MegacrypterCom.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads, json_dumps +from pyload.utils import json_loads, json_dumps from pyload.plugins.hoster.MegaNz import MegaNz diff --git a/pyload/plugins/hoster/MultiDebridCom.py b/pyload/plugins/hoster/MultiDebridCom.py index 765022eef..bd7d9460c 100644 --- a/pyload/plugins/hoster/MultiDebridCom.py +++ b/pyload/plugins/hoster/MultiDebridCom.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/MyvideoDe.py b/pyload/plugins/hoster/MyvideoDe.py index 4ce75b4a2..06cfb9c63 100644 --- a/pyload/plugins/hoster/MyvideoDe.py +++ b/pyload/plugins/hoster/MyvideoDe.py @@ -3,7 +3,7 @@ import re from pyload.plugins.Hoster import Hoster -from pyload.unescape import unescape +from pyload.utils import html_unescape class MyvideoDe(Hoster): @@ -35,7 +35,7 @@ class MyvideoDe(Hoster): def get_file_name(self): file_name_pattern = r"

(.*)

" - return unescape(re.search(file_name_pattern, self.html).group(1).replace("/", "") + '.flv') + return html_unescape(re.search(file_name_pattern, self.html).group(1).replace("/", "") + '.flv') def file_exists(self): self.download_html() diff --git a/pyload/plugins/hoster/OboomCom.py b/pyload/plugins/hoster/OboomCom.py index 04efa31b7..8c0d9d760 100644 --- a/pyload/plugins/hoster/OboomCom.py +++ b/pyload/plugins/hoster/OboomCom.py @@ -5,7 +5,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster from pyload.plugins.internal.CaptchaService import ReCaptcha diff --git a/pyload/plugins/hoster/OverLoadMe.py b/pyload/plugins/hoster/OverLoadMe.py index 8061b2e1d..e27972fc2 100644 --- a/pyload/plugins/hoster/OverLoadMe.py +++ b/pyload/plugins/hoster/OverLoadMe.py @@ -5,7 +5,7 @@ import re from random import randrange from urllib import unquote -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster from pyload.utils import parseFileSize diff --git a/pyload/plugins/hoster/PremiumizeMe.py b/pyload/plugins/hoster/PremiumizeMe.py index 16649f492..45c011084 100644 --- a/pyload/plugins/hoster/PremiumizeMe.py +++ b/pyload/plugins/hoster/PremiumizeMe.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/RPNetBiz.py b/pyload/plugins/hoster/RPNetBiz.py index e305c35ce..8a7bec097 100644 --- a/pyload/plugins/hoster/RPNetBiz.py +++ b/pyload/plugins/hoster/RPNetBiz.py @@ -3,7 +3,7 @@ import re from pyload.plugins.Hoster import Hoster -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads class RPNetBiz(Hoster): diff --git a/pyload/plugins/hoster/RapidgatorNet.py b/pyload/plugins/hoster/RapidgatorNet.py index 46fe285b7..cc13f7097 100644 --- a/pyload/plugins/hoster/RapidgatorNet.py +++ b/pyload/plugins/hoster/RapidgatorNet.py @@ -4,7 +4,7 @@ import re from pycurl import HTTPHEADER -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.network.HTTPRequest import BadHeader from pyload.plugins.hoster.UnrestrictLi import secondsToMidnight from pyload.plugins.internal.CaptchaService import AdsCaptcha, ReCaptcha, SolveMedia diff --git a/pyload/plugins/hoster/RealdebridCom.py b/pyload/plugins/hoster/RealdebridCom.py index a458cc5d0..08f7b9329 100644 --- a/pyload/plugins/hoster/RealdebridCom.py +++ b/pyload/plugins/hoster/RealdebridCom.py @@ -6,7 +6,7 @@ from random import randrange from urllib import quote, unquote from time import time -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster from pyload.utils import parseFileSize diff --git a/pyload/plugins/hoster/RedtubeCom.py b/pyload/plugins/hoster/RedtubeCom.py index 42c24628e..9f5d2d0d9 100644 --- a/pyload/plugins/hoster/RedtubeCom.py +++ b/pyload/plugins/hoster/RedtubeCom.py @@ -3,7 +3,7 @@ import re from pyload.plugins.Hoster import Hoster -from pyload.unescape import unescape +from pyload.utils import html_unescape class RedtubeCom(Hoster): @@ -36,7 +36,7 @@ class RedtubeCom(Hoster): if not self.html: self.download_html() - file_url = unescape(re.search(r'hashlink=(http.*?)"', self.html).group(1)) + file_url = html_unescape(re.search(r'hashlink=(http.*?)"', self.html).group(1)) return file_url diff --git a/pyload/plugins/hoster/UlozTo.py b/pyload/plugins/hoster/UlozTo.py index 2f1fdc595..c3957aaa0 100644 --- a/pyload/plugins/hoster/UlozTo.py +++ b/pyload/plugins/hoster/UlozTo.py @@ -3,7 +3,7 @@ import re import time -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/plugins/hoster/UnrestrictLi.py b/pyload/plugins/hoster/UnrestrictLi.py index 2cad6616f..c0c1e3965 100644 --- a/pyload/plugins/hoster/UnrestrictLi.py +++ b/pyload/plugins/hoster/UnrestrictLi.py @@ -4,7 +4,7 @@ import re from datetime import datetime, timedelta -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/UploadingCom.py b/pyload/plugins/hoster/UploadingCom.py index a7c328eec..882cb863f 100644 --- a/pyload/plugins/hoster/UploadingCom.py +++ b/pyload/plugins/hoster/UploadingCom.py @@ -4,7 +4,7 @@ import re from pycurl import HTTPHEADER -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp diff --git a/pyload/plugins/hoster/XHamsterCom.py b/pyload/plugins/hoster/XHamsterCom.py index 0afb94b74..aa406cc45 100644 --- a/pyload/plugins/hoster/XHamsterCom.py +++ b/pyload/plugins/hoster/XHamsterCom.py @@ -4,7 +4,7 @@ import re from urllib import unquote -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.Hoster import Hoster diff --git a/pyload/plugins/hoster/YibaishiwuCom.py b/pyload/plugins/hoster/YibaishiwuCom.py index b6d06d234..24cd0e9fc 100644 --- a/pyload/plugins/hoster/YibaishiwuCom.py +++ b/pyload/plugins/hoster/YibaishiwuCom.py @@ -2,7 +2,7 @@ import re -from pyload.common.json_layer import json_loads +from pyload.utils import json_loads from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo diff --git a/pyload/remote/ClickAndLoadBackend.py b/pyload/remote/ClickAndLoadBackend.py index 82167f6e3..365364a3b 100644 --- a/pyload/remote/ClickAndLoadBackend.py +++ b/pyload/remote/ClickAndLoadBackend.py @@ -27,7 +27,7 @@ try: except: pass -from RemoteManager import BackendBase +from pyload.manager.RemoteManager import BackendBase core = None js = None diff --git a/pyload/remote/RemoteManager.py b/pyload/remote/RemoteManager.py deleted file mode 100644 index e53e317e3..000000000 --- a/pyload/remote/RemoteManager.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from threading import Thread -from traceback import print_exc - -class BackendBase(Thread): - def __init__(self, manager): - Thread.__init__(self) - self.m = manager - self.core = manager.core - self.enabled = True - self.running = False - - def run(self): - self.running = True - try: - self.serve() - except Exception, e: - self.core.log.error(_("Remote backend error: %s") % e) - if self.core.debug: - print_exc() - finally: - self.running = False - - def setup(self, host, port): - pass - - def checkDeps(self): - return True - - def serve(self): - pass - - def shutdown(self): - pass - - def stop(self): - self.enabled = False# set flag and call shutdowm message, so thread can react - self.shutdown() - - -class RemoteManager: - available = [] - - def __init__(self, core): - self.core = core - self.backends = [] - - if self.core.remote: - self.available.append("ThriftBackend") -# else: -# self.available.append("SocketBackend") - - - def startBackends(self): - host = self.core.config["remote"]["listenaddr"] - port = self.core.config["remote"]["port"] - - for b in self.available: - klass = getattr(__import__("pyload.remote.%s" % b, globals(), locals(), [b], -1), b) - backend = klass(self) - if not backend.checkDeps(): - continue - try: - backend.setup(host, port) - self.core.log.info(_("Starting %(name)s: %(addr)s:%(port)s") % {"name": b, "addr": host, "port": port}) - except Exception, e: - self.core.log.error(_("Failed loading backend %(name)s | %(error)s") % {"name": b, "error": str(e)}) - if self.core.debug: - print_exc() - else: - backend.start() - self.backends.append(backend) - - port += 1 diff --git a/pyload/remote/SocketBackend.py b/pyload/remote/SocketBackend.py index 1a157cf1d..c85e59f42 100644 --- a/pyload/remote/SocketBackend.py +++ b/pyload/remote/SocketBackend.py @@ -2,7 +2,7 @@ import SocketServer -from RemoteManager import BackendBase +from pyload.manager.RemoteManager import BackendBase class RequestHandler(SocketServer.BaseRequestHandler): diff --git a/pyload/remote/ThriftBackend.py b/pyload/remote/ThriftBackend.py index 2c65578be..609a3c158 100644 --- a/pyload/remote/ThriftBackend.py +++ b/pyload/remote/ThriftBackend.py @@ -17,7 +17,7 @@ """ from os.path import exists -from pyload.remote.RemoteManager import BackendBase +from pyload.manager.RemoteManager import BackendBase from thriftbackend.Processor import Processor from thriftbackend.Protocol import ProtocolFactory diff --git a/pyload/setup.py b/pyload/setup.py deleted file mode 100644 index 6a98cc6f3..000000000 --- a/pyload/setup.py +++ /dev/null @@ -1,538 +0,0 @@ -# -*- coding: utf-8 -*- - -import os -import sys - -import pyload.common.pylgettext as gettext - -from getpass import getpass -from os import makedirs -from os.path import abspath, dirname, exists, join -from subprocess import PIPE, call - -from pyload.utils import get_console_encoding, versiontuple - - -class Setup: - """ pyLoads initial setup configuration assistant """ - - def __init__(self, path, config): - self.path = path - self.config = config - self.stdin_encoding = get_console_encoding(sys.stdin.encoding) - - - def start(self): - langs = self.config.getMetaData("general", "language")["type"].split(";") - lang = self.ask(u"Choose setup language", "en", langs) - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), languages=[lang, "en"], fallback=True) - translation.install(True) - - #Input shorthand for yes - self.yes = _("y") - #Input shorthand for no - self.no = _("n") - - # print - # print _("Would you like to configure pyLoad via Webinterface?") - # print _("You need a Browser and a connection to this PC for it.") - # viaweb = self.ask(_("Start initial webinterface for configuration?"), "y", bool=True) - # if viaweb: - # try: - # from pyload.threads import ServerThread - # ServerThread.setup = self - # import pyload.webui as webinterface - # webinterface.run_simple() - # return False - # except Exception, e: - # print "Setup failed with this error: ", e - # print "Falling back to commandline setup." - - print - print - print _("Welcome to the pyLoad Configuration Assistant.") - print _("It will check your system and make a basic setup in order to run pyLoad.") - print - print _("The value in brackets [] always is the default value,") - print _("in case you don't want to change it or you are unsure what to choose, just hit enter.") - print _( - "Don't forget: You can always rerun this assistant with --setup or -s parameter, when you start pyload.py .") - print _("If you have any problems with this assistant hit STRG-C,") - print _("to abort and don't let him start with pyload.py automatically anymore.") - print - print - raw_input(_("When you are ready for system check, hit enter.")) - print - print - - basic, ssl, captcha, web, js = self.system_check() - print - print - - if not basic: - print _("You need pycurl, sqlite and python 2.5, 2.6 or 2.7 to run pyLoad.") - print _("Please correct this and re-run pyLoad.") - print - print _("Setup will now close.") - print - print - raw_input(_("Press Enter to exit.")) - return False - - raw_input(_("System check finished, hit enter to see your status report.")) - print - print - print _("## Status ##") - print - - avail = [] - if self.check_module("Crypto"): - avail.append(_("container decrypting")) - if ssl: - avail.append(_("ssl connection")) - if captcha: - avail.append(_("automatic captcha decryption")) - if web: - avail.append(_("webinterface")) - if js: - avail.append(_("extended Click'N'Load")) - - string = "" - - for av in avail: - string += ", " + av - - print _("AVAILABLE FEATURES:") + string[1:] - print - - if len(avail) < 5: - print _("MISSING FEATURES: ") - - if not self.check_module("Crypto"): - print _("- no py-crypto available") - print _("You need this if you want to decrypt container files.") - print - - if not ssl: - print _("- no SSL available") - print _("This is needed if you want to establish a secure connection to core or webinterface.") - print _("If you only want to access locally to pyLoad ssl is not usefull.") - print - - if not captcha: - print _("- no Captcha Recognition available") - print _("Only needed for some hosters and as freeuser.") - print - - if not js: - print _("- no JavaScript engine found") - print _("You will need this for some Click'N'Load links. Install Spidermonkey, ossp-js, pyv8 or rhino") - print - - print - print _("You can abort the setup now and fix some dependicies if you want.") - - print - con = self.ask(_("Continue with setup?"), self.yes, bool=True) - - if not con: - return False - - print - print - print _("CURRENT CONFIG PATH: %s") % abspath("") - print - print _("NOTE: If you use pyLoad on a server or the home partition lives on an iternal flash it may be a good idea to change it.") - path = self.ask(_("Do you want to change the config path?"), self.no, bool=True) - if path: - print - self.conf_path() - #calls exit when changed - - print - print _("Do you want to configure login data and basic settings?") - print _("This is recommend for first run.") - con = self.ask(_("Make basic setup?"), self.yes, bool=True) - - if con: - print - print - self.conf_basic() - - if ssl: - print - print _("Do you want to configure ssl?") - ssl = self.ask(_("Configure ssl?"), self.no, bool=True) - if ssl: - print - print - self.conf_ssl() - - if web: - print - print _("Do you want to configure webinterface?") - web = self.ask(_("Configure webinterface?"), self.yes, bool=True) - if web: - print - print - self.conf_web() - - print - print - print _("Setup finished successfully!") - print - print - raw_input(_("Hit enter to exit and restart pyLoad.")) - return True - - - def system_check(self): - """ make a systemcheck and return the results""" - - print _("## System Information ##") - print - print _("Platform: %s") % sys.platform - print _("Operating System: %s") % os.name - print _("Python: %s") % sys.version.replace("\n", "") - print - print - - print _("## System Check ##") - print - - if sys.version_info[:2] > (2, 7): - print _("Your python version is to new, Please use Python 2.6/2.7") - python = False - elif sys.version_info[:2] < (2, 5): - print _("Your python version is to old, Please use at least Python 2.5") - python = False - else: - print _("Python Version: OK") - python = True - - curl = self.check_module("pycurl") - self.print_dep("pycurl", curl) - - sqlite = self.check_module("sqlite3") - self.print_dep("sqlite3", sqlite) - - basic = python and curl and sqlite - - print - - crypto = self.check_module("Crypto") - self.print_dep("pycrypto", crypto) - - ssl = self.check_module("OpenSSL") - self.print_dep("py-OpenSSL", ssl) - - print - - pil = self.check_module("PIL.Image") - self.print_dep("PIL/Pillow", pil) - - if os.name == "nt": - tesser = self.check_prog([join(pypath, "tesseract", "tesseract.exe"), "-v"]) - else: - tesser = self.check_prog(["tesseract", "-v"]) - - self.print_dep("tesseract", tesser) - - captcha = pil and tesser - - print - - try: - import jinja2 - - v = jinja2.__version__ - if v and versiontuple(v) < (2, 5, 0): - jinja = False - else: - jinja = True - except: - jinja = False - - jinja = self.print_dep("jinja2", jinja) - - beaker = self.check_module("beaker") - self.print_dep("beaker", beaker) - - bjoern = self.check_module("bjoern") - self.print_dep("bjoern", bjoern) - - web = sqlite and beaker - - from pyload.common import JsEngine - js = True if JsEngine.ENGINE else False - self.print_dep(_("JS engine"), js) - - if not jinja: - print - print - print _("WARNING: Your installed jinja2 version %s seems too old.") % jinja2.__version__ - print _("You can safely continue but if the webinterface is not working,") - print _("please upgrade or uninstall it, because pyLoad self-includes jinja2 libary.") - - return basic, ssl, captcha, web, js - - - def conf_basic(self): - print _("## Basic Setup ##") - - print - print _("The following logindata is valid for CLI and webinterface.") - - from pyload.database import DatabaseBackend - - db = DatabaseBackend(None) - db.setup() - print _("NOTE: Consider a password of 10 or more symbols if you expect to access from outside your local network (ex. internet).") - print - username = self.ask(_("Username"), "User") - password = self.ask("", "", password=True) - db.addUser(username, password) - db.shutdown() - - print - print _("External clients (GUI, CLI or other) need remote access to work over the network.") - print _("However, if you only want to use the webinterface you may disable it to save ram.") - self.config["remote"]["activated"] = self.ask(_("Enable remote access"), self.no, bool=True) - - print - langs = self.config.getMetaData("general", "language") - self.config["general"]["language"] = self.ask(_("Choose pyLoad language"), "en", langs["type"].split(";")) - - print - self.config["general"]["download_folder"] = self.ask(_("Download folder"), "Downloads") - print - self.config["download"]["max_downloads"] = self.ask(_("Max parallel downloads"), "3") - print - reconnect = self.ask(_("Use Reconnect?"), self.no, bool=True) - self.config["reconnect"]["activated"] = reconnect - if reconnect: - self.config["reconnect"]["method"] = self.ask(_("Reconnect script location"), "./reconnect.sh") - - - def conf_web(self): - print _("## Webinterface Setup ##") - - print - self.config["webinterface"]["activated"] = self.ask(_("Activate webinterface?"), self.yes, bool=True) - print - print _("Listen address, if you use 127.0.0.1 or localhost, the webinterface will only accessible locally.") - self.config["webinterface"]["host"] = self.ask(_("Address"), "0.0.0.0") - self.config["webinterface"]["port"] = self.ask(_("Port"), "8000") - print - print _("pyLoad offers several server backends, now following a short explanation.") - print "- builtin:", _("Default server; best choice if you plan to use pyLoad just for you.") - print "- threaded:", _("Support SSL connection and can serve simultaneously more client flawlessly.") - print "- fastcgi:", _( - "Can be used by apache, lighttpd, etc.; needs to be properly configured before.") - if os.name != "nt": - print "- lightweight:", _("Very fast alternative to builtin; requires libev and bjoern packages.") - - print - print _("NOTE: In some rare cases the builtin server is not working, if you notice problems with the webinterface") - print _("come back here and change the builtin server to the threaded one here.") - - if os.name == "nt": - servers = ["builtin", "threaded", "fastcgi"] - default = "threaded" - else: - servers = ["builtin", "threaded", "fastcgi", "lightweight"] - default = "lightweight" if self.check_module("bjoern") else "builtin" - - self.config["webinterface"]["server"] = self.ask(_("Server"), default, servers) - - - def conf_ssl(self): - print _("## SSL Setup ##") - print - print _("Execute these commands from pyLoad config folder to make ssl certificates:") - print - print "openssl genrsa -out ssl.key 1024" - print "openssl req -new -key ssl.key -out ssl.csr" - print "openssl req -days 36500 -x509 -key ssl.key -in ssl.csr > ssl.crt " - print - print _("If you're done and everything went fine, you can activate ssl now.") - - self.config["ssl"]["activated"] = self.ask(_("Activate SSL?"), self.yes, bool=True) - - - def set_user(self): - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), - languages=[self.config["general"]["language"], "en"], fallback=True) - translation.install(True) - - from pyload.database import DatabaseBackend - - db = DatabaseBackend(None) - db.setup() - - noaction = True - try: - while True: - print _("Select action") - print _("1 - Create/Edit user") - print _("2 - List users") - print _("3 - Remove user") - print _("4 - Quit") - action = raw_input("[1]/2/3/4: ") - if not action in ("1", "2", "3", "4"): - continue - elif action == "1": - print - username = self.ask(_("Username"), "User") - password = self.ask("", "", password=True) - db.addUser(username, password) - noaction = False - elif action == "2": - print - print _("Users") - print "-----" - users = db.listUsers() - noaction = False - for user in users: - print user - print "-----" - print - elif action == "3": - print - username = self.ask(_("Username"), "") - if username: - db.removeUser(username) - noaction = False - elif action == "4": - break - finally: - if not noaction: - db.shutdown() - - - def conf_path(self, trans=False): - if trans: - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("setup", join(self.path, "locale"), - languages=[self.config["general"]["language"], "en"], fallback=True) - translation.install(True) - - print _("Setting new config path, current configuration will not be transfered!") - path = self.ask(_("CONFIG PATH"), abspath("")) - try: - path = join(pypath, path) - if not exists(path): - makedirs(path) - f = open(join(pypath, "pyload", "config", "configdir"), "wb") - f.write(path) - f.close() - print - print - print _("pyLoad config path changed, setup will now close!") - print - print - raw_input(_("Press Enter to exit.")) - sys.exit() - except Exception, e: - print _("Setting config path failed: %s") % str(e) - - - def print_dep(self, name, value): - """Print Status of dependency""" - if value: - print _("%s: OK") % name - else: - print _("%s: MISSING") % name - - - def check_module(self, module): - try: - __import__(module) - return True - except: - return False - - - def check_prog(self, command): - pipe = PIPE - try: - call(command, stdout=pipe, stderr=pipe) - return True - except: - return False - - - def ask(self, qst, default, answers=[], bool=False, password=False): - """produce one line to asking for input""" - if answers: - info = "(" - - for i, answer in enumerate(answers): - info += (", " if i != 0 else "") + str((answer == default and "[%s]" % answer) or answer) - - info += ")" - elif bool: - if default == self.yes: - info = "([%s]/%s)" % (self.yes, self.no) - else: - info = "(%s/[%s])" % (self.yes, self.no) - else: - info = "[%s]" % default - - if password: - p1 = True - p2 = False - pwlen = 8 - while p1 != p2: - # getpass(_("Password: ")) will crash on systems with broken locales (Win, NAS) - sys.stdout.write(_("Password: ")) - p1 = getpass("") - - if len(p1) < pwlen: - print _("Password too short! Use at least %s symbols." % pwlen) - continue - elif not p1.isalnum(): - print _("Password must be alphanumeric.") - continue - - sys.stdout.write(_("Password (again): ")) - p2 = getpass("") - - if p1 == p2: - return p1 - else: - print _("Passwords did not match.") - - while True: - try: - input = raw_input(qst + " %s: " % info) - except KeyboardInterrupt: - print "\nSetup interrupted" - sys.exit() - - input = input.decode(self.stdin_encoding) - - if input.strip() == "": - input = default - - if bool: - # yes, true, t are inputs for booleans with value true - if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]: - return True - # no, false, f are inputs for booleans with value false - elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]: - return False - else: - print _("Invalid Input") - continue - - if not answers: - return input - - else: - if input in answers: - return input - else: - print _("Invalid Input") diff --git a/pyload/threads/PluginThread.py b/pyload/threads/PluginThread.py deleted file mode 100644 index ca1d396dc..000000000 --- a/pyload/threads/PluginThread.py +++ /dev/null @@ -1,675 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from Queue import Queue -from threading import Thread -from os import listdir, stat -from os.path import join -from time import sleep, time, strftime, gmtime -from traceback import print_exc, format_exc -from pprint import pformat -from sys import exc_info, exc_clear -from copy import copy -from types import MethodType - -from pycurl import error - -from pyload.PyFile import PyFile -from pyload.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload -from pyload.common.packagetools import parseNames -from pyload.utils import safe_join -from pyload.Api import OnlineStatus - -class PluginThread(Thread): - """abstract base class for thread types""" - - #-------------------------------------------------------------------------- - def __init__(self, manager): - """Constructor""" - Thread.__init__(self) - self.setDaemon(True) - self.m = manager #thread manager - - - def writeDebugReport(self, pyfile): - """ writes a - :return: - """ - - dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) - dump = self.getDebugDump(pyfile) - - try: - import zipfile - - zip = zipfile.ZipFile(dump_name, "w") - - for f in listdir(join("tmp", pyfile.pluginname)): - try: - # avoid encoding errors - zip.write(join("tmp", pyfile.pluginname, f), safe_join(pyfile.pluginname, f)) - except: - pass - - info = zipfile.ZipInfo(safe_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) - info.external_attr = 0644 << 16L # change permissions - - zip.writestr(info, dump) - zip.close() - - if not stat(dump_name).st_size: - raise Exception("Empty Zipfile") - - except Exception, e: - self.m.log.debug("Error creating zip file: %s" % e) - - dump_name = dump_name.replace(".zip", ".txt") - f = open(dump_name, "wb") - f.write(dump) - f.close() - - self.m.core.log.info("Debug Report written to %s" % dump_name) - - def getDebugDump(self, pyfile): - dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( - self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version__, format_exc()) - - tb = exc_info()[2] - stack = [] - while tb: - stack.append(tb.tb_frame) - tb = tb.tb_next - - for frame in stack[1:]: - dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, - frame.f_code.co_filename, - frame.f_lineno) - - for key, value in frame.f_locals.items(): - dump += "\t%20s = " % key - try: - dump += pformat(value) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - del frame - - del stack #delete it just to be sure... - - dump += "\n\nPLUGIN OBJECT DUMP: \n\n" - - for name in dir(pyfile.plugin): - attr = getattr(pyfile.plugin, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - dump += "\nPYFILE OBJECT DUMP: \n\n" - - for name in dir(pyfile): - attr = getattr(pyfile, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - if pyfile.pluginname in self.m.core.config.plugin: - dump += "\n\nCONFIG: \n\n" - dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" - - return dump - - def clean(self, pyfile): - """ set thread unactive and release pyfile """ - self.active = False - pyfile.release() - - -class DownloadThread(PluginThread): - """thread for downloading files from 'real' hoster plugins""" - - #-------------------------------------------------------------------------- - def __init__(self, manager): - """Constructor""" - PluginThread.__init__(self, manager) - - self.queue = Queue() # job queue - self.active = False - - self.start() - - #-------------------------------------------------------------------------- - def run(self): - """run method""" - pyfile = None - - while True: - del pyfile - self.active = self.queue.get() - pyfile = self.active - - if self.active == "quit": - self.active = False - self.m.threads.remove(self) - return True - - try: - if not pyfile.hasPlugin(): continue - #this pyfile was deleted while queueing - - pyfile.plugin.checkForSameFiles(starting=True) - self.m.log.info(_("Download starts: %s" % pyfile.name)) - - # start download - self.m.core.hookManager.downloadPreparing(pyfile) - pyfile.plugin.preprocessing(self) - - self.m.log.info(_("Download finished: %s") % pyfile.name) - self.m.core.hookManager.downloadFinished(pyfile) - self.m.core.files.checkPackageFinished(pyfile) - - except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) - pyfile.setStatus("failed") - pyfile.error = "Plugin does not work" - self.clean(pyfile) - continue - - except Abort: - try: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - except: - pass - - pyfile.setStatus("aborted") - - self.clean(pyfile) - continue - - except Reconnect: - self.queue.put(pyfile) - #pyfile.req.clearCookies() - - while self.m.reconnecting.isSet(): - sleep(0.5) - - continue - - except Retry, e: - reason = e.args[0] - self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) - self.queue.put(pyfile) - continue - - except Fail, e: - msg = e.args[0] - - if msg == "offline": - pyfile.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % pyfile.name) - elif msg == "temp. offline": - pyfile.setStatus("temp. offline") - self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) - else: - pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) - pyfile.error = msg - - self.m.core.hookManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - except error, e: - if len(e.args) == 2: - code, msg = e.args - else: - code = 0 - msg = e.args - - self.m.log.debug("pycurl exception %s: %s" % (code, msg)) - - if code in (7, 18, 28, 52, 56): - self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) - wait = time() + 60 - - pyfile.waitUntil = wait - pyfile.setStatus("waiting") - while time() < wait: - sleep(1) - if pyfile.abort: - break - - if pyfile.abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") - - self.clean(pyfile) - else: - self.queue.put(pyfile) - - continue - - else: - pyfile.setStatus("failed") - self.m.log.error("pycurl error %s: %s" % (code, msg)) - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - self.m.core.hookManager.downloadFailed(pyfile) - - self.clean(pyfile) - continue - - except SkipDownload, e: - pyfile.setStatus("skipped") - - self.m.log.info( - _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) - - self.clean(pyfile) - - self.m.core.files.checkPackageFinished(pyfile) - - self.active = False - self.m.core.files.save() - - continue - - - except Exception, e: - pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) - pyfile.error = str(e) - - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - self.m.core.hookManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - finally: - self.m.core.files.save() - pyfile.checkIfProcessed() - exc_clear() - - #pyfile.plugin.req.clean() - - self.active = False - pyfile.finishIfDone() - self.m.core.files.save() - - - def put(self, job): - """assing job to thread""" - self.queue.put(job) - - - def stop(self): - """stops the thread""" - self.put("quit") - - -class DecrypterThread(PluginThread): - """thread for decrypting""" - - def __init__(self, manager, pyfile): - """constructor""" - PluginThread.__init__(self, manager) - - self.active = pyfile - manager.localThreads.append(self) - - pyfile.setStatus("decrypting") - - self.start() - - def getActiveFiles(self): - return [self.active] - - def run(self): - """run method""" - - pyfile = self.active - retry = False - - try: - self.m.log.info(_("Decrypting starts: %s") % self.active.name) - self.active.plugin.preprocessing(self) - - except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) - return - - except Fail, e: - msg = e.args[0] - - if msg == "offline": - self.active.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % self.active.name) - else: - self.active.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": msg}) - self.active.error = msg - - return - - except Abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") - - return - - except Retry: - self.m.log.info(_("Retrying %s") % self.active.name) - retry = True - return self.run() - - except Exception, e: - self.active.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": str(e)}) - self.active.error = str(e) - - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - return - - - finally: - if not retry: - self.active.release() - self.active = False - self.m.core.files.save() - self.m.localThreads.remove(self) - exc_clear() - - - #self.m.core.hookManager.downloadFinished(pyfile) - - - #self.m.localThreads.remove(self) - #self.active.finishIfDone() - if not retry: - pyfile.delete() - - -class HookThread(PluginThread): - """thread for hooks""" - - #-------------------------------------------------------------------------- - def __init__(self, m, function, args, kwargs): - """Constructor""" - PluginThread.__init__(self, m) - - self.f = function - self.args = args - self.kwargs = kwargs - - self.active = [] - - m.localThreads.append(self) - - self.start() - - def getActiveFiles(self): - return self.active - - def addActive(self, pyfile): - """ Adds a pyfile to active list and thus will be displayed on overview""" - if pyfile not in self.active: - self.active.append(pyfile) - - def finishFile(self, pyfile): - if pyfile in self.active: - self.active.remove(pyfile) - - pyfile.finishIfDone() - - def run(self): - try: - try: - self.kwargs["thread"] = self - self.f(*self.args, **self.kwargs) - except TypeError, e: - #dirty method to filter out exceptions - if "unexpected keyword argument 'thread'" not in e.args[0]: - raise - - del self.kwargs["thread"] - self.f(*self.args, **self.kwargs) - finally: - local = copy(self.active) - for x in local: - self.finishFile(x) - - self.m.localThreads.remove(self) - - -class InfoThread(PluginThread): - def __init__(self, manager, data, pid=-1, rid=-1, add=False): - """Constructor""" - PluginThread.__init__(self, manager) - - self.data = data - self.pid = pid # package id - # [ .. (name, plugin) .. ] - - self.rid = rid #result id - self.add = add #add packages instead of return result - - self.cache = [] #accumulated data - - self.start() - - def run(self): - """run method""" - - plugins = {} - container = [] - - for url, plugin in self.data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - - # filter out container plugins - for name in self.m.core.pluginManager.containerPlugins: - if name in plugins: - container.extend([(name, url) for url in plugins[name]]) - - del plugins[name] - - #directly write to database - if self.pid > -1: - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) - self.m.core.files.save() - - elif self.add: - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) - - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateCache(pluginname, result) - - packs = parseNames([(name, url) for name, x, y, url in self.cache]) - - self.m.log.debug("Fetched and generated %d packages" % len(packs)) - - for k, v in packs: - self.m.core.api.addPackage(k, v) - - #empty cache - del self.cache[:] - - else: #post the results - - - for name, url in container: - #attach container content - try: - data = self.decryptContainer(name, url) - except: - print_exc() - self.m.log.error("Could not decrypt container.") - data = [] - - for url, plugin in data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - self.m.infoResults[self.rid] = {} - - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - - #force to process cache - if self.cache: - self.updateResult(pluginname, [], True) - - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateResult(pluginname, result, True) - - self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} - - self.m.timestamp = time() + 5 * 60 - - - def updateDB(self, plugin, result): - self.m.core.files.updateFileInfo(result, self.pid) - - def updateResult(self, plugin, result, force=False): - #parse package name and generate result - #accumulate results - - self.cache.extend(result) - - if len(self.cache) >= 20 or force: - #used for package generating - tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) - for name, size, status, url in self.cache] - - data = parseNames(tmp) - result = {} - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - self.m.setInfoResults(self.rid, result) - - self.cache = [] - - def updateCache(self, plugin, result): - self.cache.extend(result) - - def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): - try: - result = [] #result loaded from cache - process = [] #urls to process - for url in urls: - if url in self.m.infoCache: - result.append(self.m.infoCache[url]) - else: - process.append(url) - - if result: - self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) - cb(pluginname, result) - - if process: - self.m.log.debug("Run Info Fetching for %s" % pluginname) - for result in plugin.getInfo(process): - #result = [ .. (name, size, status, url) .. ] - if not type(result) == list: result = [result] - - for res in result: - self.m.infoCache[res[3]] = res - - cb(pluginname, result) - - self.m.log.debug("Finished Info Fetching for %s" % pluginname) - except Exception, e: - self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % - {"name": pluginname, "err": str(e)}) - if self.m.core.debug: - print_exc() - - # generate default results - if err: - result = [(url, 0, 3, url) for url in urls] - cb(pluginname, result) - - - def decryptContainer(self, plugin, url): - data = [] - # only works on container plugins - - self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) - - # dummy pyfile - pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) - - pyfile.initPlugin() - - # little plugin lifecycle - try: - pyfile.plugin.setup() - pyfile.plugin.loadToDisk() - pyfile.plugin.decrypt(pyfile) - pyfile.plugin.deleteTmp() - - for pack in pyfile.plugin.packages: - pyfile.plugin.urls.extend(pack[1]) - - data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) - - self.m.log.debug("Got %d links." % len(data)) - - except Exception, e: - self.m.log.debug("Pre decrypting error: %s" % str(e)) - finally: - pyfile.release() - - return data diff --git a/pyload/threads/ServerThread.py b/pyload/threads/ServerThread.py deleted file mode 100644 index 7de3b1ca1..000000000 --- a/pyload/threads/ServerThread.py +++ /dev/null @@ -1,108 +0,0 @@ -from __future__ import with_statement -from os.path import exists - -import os -import threading -import logging - -core = None -setup = None -log = logging.getLogger("log") - -class WebServer(threading.Thread): - def __init__(self, pycore): - global core - threading.Thread.__init__(self) - self.core = pycore - core = pycore - self.running = True - self.server = pycore.config['webinterface']['server'] - self.https = pycore.config['webinterface']['https'] - self.cert = pycore.config["ssl"]["cert"] - self.key = pycore.config["ssl"]["key"] - self.host = pycore.config['webinterface']['host'] - self.port = pycore.config['webinterface']['port'] - - self.setDaemon(True) - - def run(self): - import pyload.webui as webinterface - global webinterface - - reset = False - - if self.https and (not exists(self.cert) or not exists(self.key)): - log.warning(_("SSL certificates not found.")) - self.https = False - - if self.server in ("lighttpd", "nginx"): - log.warning(_("Sorry, we dropped support for starting %s directly within pyLoad") % self.server) - log.warning(_("You can use the threaded server which offers good performance and ssl,")) - log.warning(_("of course you can still use your existing %s with pyLoads fastcgi server") % self.server) - log.warning(_("sample configs are located in the pyload/web/servers directory")) - reset = True - elif self.server == "fastcgi": - try: - import flup - except: - log.warning(_("Can't use %(server)s, python-flup is not installed!") % { - "server": self.server}) - reset = True - - if reset or self.server == "lightweight": - if os.name != "nt": - try: - import bjoern - except Exception, e: - log.error(_("Error importing lightweight server: %s") % e) - log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) - log.warning(_("Copy the boern.so to pyload/lib folder or use setup.py install")) - log.warning(_("Of course you need to be familiar with linux and know how to compile software")) - self.server = "builtin" - else: - self.core.log.info(_("Server set to threaded, due to known performance problems on windows.")) - self.core.config['webinterface']['server'] = "threaded" - self.server = "threaded" - - if self.server == "threaded": - self.start_threaded() - elif self.server == "fastcgi": - self.start_fcgi() - elif self.server == "lightweight": - self.start_lightweight() - else: - self.start_builtin() - - def start_builtin(self): - - if self.https: - log.warning(_("This server offers no SSL, please consider using threaded instead")) - - self.core.log.info(_("Starting builtin webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_simple(host=self.host, port=self.port) - - def start_threaded(self): - if self.https: - self.core.log.info(_("Starting threaded SSL webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - else: - self.cert = "" - self.key = "" - self.core.log.info(_("Starting threaded webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - - webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) - - def start_fcgi(self): - - self.core.log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_fcgi(host=self.host, port=self.port) - - - def start_lightweight(self): - if self.https: - log.warning(_("This server offers no SSL, please consider using threaded instead")) - - self.core.log.info(_("Starting lightweight webserver (bjoern): %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_lightweight(host=self.host, port=self.port) - - def quit(self): - self.running = False diff --git a/pyload/threads/__init__.py b/pyload/threads/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/pyload/unescape.py b/pyload/unescape.py deleted file mode 100644 index 7bcc1aa72..000000000 --- a/pyload/unescape.py +++ /dev/null @@ -1,3 +0,0 @@ -from pyload.utils import html_unescape -#deprecated -unescape = html_unescape diff --git a/pyload/utils.py b/pyload/utils.py deleted file mode 100644 index b9eb8c88e..000000000 --- a/pyload/utils.py +++ /dev/null @@ -1,227 +0,0 @@ -# -*- coding: utf-8 -*- - -""" Store all useful functions here """ - -import os -import sys -import time -import re -from os.path import join -from string import maketrans -from htmlentitydefs import name2codepoint - - -def chmod(*args): - try: - os.chmod(*args) - except: - pass - - -def decode(string): - """ decode string with utf if possible """ - try: - return string.decode("utf8", "replace") - except: - return string - - -def remove_chars(string, repl): - """ removes all chars in repl from string""" - if type(repl) == unicode: - for badc in list(repl): - string = string.replace(badc, "") - return string - else: - if type(string) == str: - return string.translate(maketrans("", ""), repl) - elif type(string) == unicode: - return string.translate(dict([(ord(s), None) for s in repl])) - - -def safe_filename(name): - """ remove bad chars """ - name = name.encode('ascii', 'replace') # Non-ASCII chars usually breaks file saving. Replacing. - if os.name == 'nt': - return remove_chars(name, u'\00\01\02\03\04\05\06\07\10\11\12\13\14\15\16\17\20\21\22\23\24\25\26\27\30\31\32' - u'\33\34\35\36\37/\\?%*:|"<>') - else: - return remove_chars(name, u'\0/\\"') - -#: Deprecated method -def save_path(name): - return safe_filename(name) - - -def safe_join(*args): - """ joins a path, encoding aware """ - return fs_encode(join(*[x if type(x) == unicode else decode(x) for x in args])) - -#: Deprecated method -def save_join(*args): - return safe_join(*args) - - -# File System Encoding functions: -# Use fs_encode before accesing files on disk, it will encode the string properly - -if sys.getfilesystemencoding().startswith('ANSI'): - def fs_encode(string): - try: - string = string.encode('utf-8') - finally: - return string - - fs_decode = decode #decode utf8 - -else: - fs_encode = fs_decode = lambda x: x # do nothing - - -def get_console_encoding(enc): - if os.name == "nt": - if enc == "cp65001": # aka UTF-8 - print "WARNING: Windows codepage 65001 is not supported." - enc = "cp850" - else: - enc = "utf8" - - return enc - - -def compare_time(start, end): - start = map(int, start) - end = map(int, end) - - if start == end: return True - - now = list(time.localtime()[3:5]) - if start < now < end: return True - elif start > end and (now > start or now < end): return True - elif start < now > end < start: return True - else: return False - - -def formatSize(size): - """formats size of bytes""" - size = int(size) - steps = 0 - sizes = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB") - while size > 1000: - size /= 1024.0 - steps += 1 - return "%.2f %s" % (size, sizes[steps]) - - -def formatSpeed(speed): - return formatSize(speed) + "/s" - - -def freeSpace(folder): - if os.name == "nt": - import ctypes - - free_bytes = ctypes.c_ulonglong(0) - ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(folder), None, None, ctypes.pointer(free_bytes)) - return free_bytes.value - else: - s = os.statvfs(folder) - return s.f_bsize * s.f_bavail - - -def fs_bsize(path): - """ get optimal file system buffer size (in bytes) for I/O calls """ - path = fs_encode(path) - - if os.name == "nt": - import ctypes - - drive = "%s\\" % os.path.splitdrive(path)[0] - cluster_sectors, sector_size = ctypes.c_longlong(0) - ctypes.windll.kernel32.GetDiskFreeSpaceW(ctypes.c_wchar_p(drive), ctypes.pointer(cluster_sectors), ctypes.pointer(sector_size), None, None) - return cluster_sectors * sector_size - else: - return os.statvfs(path).f_bsize - - -def uniqify(seq): #: Originally by Dave Kirby - """ removes duplicates from list, preserve order """ - seen = set() - seen_add = seen.add - return [x for x in seq if x not in seen and not seen_add(x)] - - -def parseFileSize(string, unit=None): #returns bytes - if not unit: - m = re.match(r"([\d.,]+) *([a-zA-Z]*)", string.strip().lower()) - if m: - traffic = float(m.group(1).replace(",", ".")) - unit = m.group(2) - else: - return 0 - else: - if isinstance(string, basestring): - traffic = float(string.replace(",", ".")) - else: - traffic = string - - #ignore case - unit = unit.lower().strip() - - if unit in ("eb", "ebyte", "exabyte", "eib", "e"): - traffic *= 1 << 60 - elif unit in ("pb", "pbyte", "petabyte", "pib", "p"): - traffic *= 1 << 50 - elif unit in ("tb", "tbyte", "terabyte", "tib", "t"): - traffic *= 1 << 40 - elif unit in ("gb", "gbyte", "gigabyte", "gib", "g", "gig"): - traffic *= 1 << 30 - elif unit in ("mb", "mbyte", "megabyte", "mib", "m"): - traffic *= 1 << 20 - elif unit in ("kb", "kbyte", "kilobyte", "kib", "k"): - traffic *= 1 << 10 - - return traffic - - -def lock(func): - def new(*args): - #print "Handler: %s args: %s" % (func, args[1:]) - args[0].lock.acquire() - try: - return func(*args) - finally: - args[0].lock.release() - - return new - - -def fixup(m): - text = m.group(0) - if text[:2] == "&#": - # character reference - try: - if text[:3] == "&#x": - return unichr(int(text[3:-1], 16)) - else: - return unichr(int(text[2:-1])) - except ValueError: - pass - else: - # named entity - try: - name = text[1:-1] - text = unichr(name2codepoint[name]) - except KeyError: - pass - - return text # leave as is - - -def html_unescape(text): - """Removes HTML or XML character references and entities from a text string""" - return re.sub("&#?\w+;", fixup, text) - - -def versiontuple(v): #: By kindall (http://stackoverflow.com/a/11887825) - return tuple(map(int, (v.split(".")))) diff --git a/pyload/utils/JsEngine.py b/pyload/utils/JsEngine.py new file mode 100644 index 000000000..46789f64d --- /dev/null +++ b/pyload/utils/JsEngine.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from imp import find_module +from os.path import join, exists +from urllib import quote + + +ENGINE = "" + +DEBUG = False +JS = False +PYV8 = False +RHINO = False + + +if not ENGINE: + try: + import subprocess + + subprocess.Popen(["js", "-v"], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + p = subprocess.Popen(["js", "-e", "print(23+19)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + #integrity check + if out.strip() == "42": + ENGINE = "js" + JS = True + except: + pass + +if not ENGINE or DEBUG: + try: + find_module("PyV8") + ENGINE = "pyv8" + PYV8 = True + except: + pass + +if not ENGINE or DEBUG: + try: + path = "" #path where to find rhino + + if exists("/usr/share/java/js.jar"): + path = "/usr/share/java/js.jar" + elif exists("js.jar"): + path = "js.jar" + elif exists(join(pypath, "js.jar")): #may raises an exception, but js.jar wasnt found anyway + path = join(pypath, "js.jar") + + if not path: + raise Exception + + import subprocess + + p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", "print(23+19)"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + #integrity check + if out.strip() == "42": + ENGINE = "rhino" + RHINO = True + except: + pass + +class JsEngine: + def __init__(self): + self.engine = ENGINE + self.init = False + + def __nonzero__(self): + return False if not ENGINE else True + + def eval(self, script): + if not self.init: + if ENGINE == "pyv8" or (DEBUG and PYV8): + import PyV8 + global PyV8 + + self.init = True + + if type(script) == unicode: + script = script.encode("utf8") + + if not ENGINE: + raise Exception("No JS Engine") + + if not DEBUG: + if ENGINE == "pyv8": + return self.eval_pyv8(script) + elif ENGINE == "js": + return self.eval_js(script) + elif ENGINE == "rhino": + return self.eval_rhino(script) + else: + results = [] + if PYV8: + res = self.eval_pyv8(script) + print "PyV8:", res + results.append(res) + if JS: + res = self.eval_js(script) + print "JS:", res + results.append(res) + if RHINO: + res = self.eval_rhino(script) + print "Rhino:", res + results.append(res) + + warning = False + for x in results: + for y in results: + if x != y: + warning = True + + if warning: print "### WARNING ###: Different results" + + return results[0] + + def eval_pyv8(self, script): + rt = PyV8.JSContext() + rt.enter() + return rt.eval(script) + + def eval_js(self, script): + script = "print(eval(unescape('%s')))" % quote(script) + p = subprocess.Popen(["js", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) + out, err = p.communicate() + res = out.strip() + return res + + def eval_rhino(self, script): + script = "print(eval(unescape('%s')))" % quote(script) + p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", script], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) + out, err = p.communicate() + res = out.strip() + return res.decode("utf8").encode("ISO-8859-1") + + def error(self): + return _("No js engine detected, please install either Spidermonkey, ossp-js, pyv8 or rhino") diff --git a/pyload/utils/__init__.py b/pyload/utils/__init__.py new file mode 100644 index 000000000..b9eb8c88e --- /dev/null +++ b/pyload/utils/__init__.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- + +""" Store all useful functions here """ + +import os +import sys +import time +import re +from os.path import join +from string import maketrans +from htmlentitydefs import name2codepoint + + +def chmod(*args): + try: + os.chmod(*args) + except: + pass + + +def decode(string): + """ decode string with utf if possible """ + try: + return string.decode("utf8", "replace") + except: + return string + + +def remove_chars(string, repl): + """ removes all chars in repl from string""" + if type(repl) == unicode: + for badc in list(repl): + string = string.replace(badc, "") + return string + else: + if type(string) == str: + return string.translate(maketrans("", ""), repl) + elif type(string) == unicode: + return string.translate(dict([(ord(s), None) for s in repl])) + + +def safe_filename(name): + """ remove bad chars """ + name = name.encode('ascii', 'replace') # Non-ASCII chars usually breaks file saving. Replacing. + if os.name == 'nt': + return remove_chars(name, u'\00\01\02\03\04\05\06\07\10\11\12\13\14\15\16\17\20\21\22\23\24\25\26\27\30\31\32' + u'\33\34\35\36\37/\\?%*:|"<>') + else: + return remove_chars(name, u'\0/\\"') + +#: Deprecated method +def save_path(name): + return safe_filename(name) + + +def safe_join(*args): + """ joins a path, encoding aware """ + return fs_encode(join(*[x if type(x) == unicode else decode(x) for x in args])) + +#: Deprecated method +def save_join(*args): + return safe_join(*args) + + +# File System Encoding functions: +# Use fs_encode before accesing files on disk, it will encode the string properly + +if sys.getfilesystemencoding().startswith('ANSI'): + def fs_encode(string): + try: + string = string.encode('utf-8') + finally: + return string + + fs_decode = decode #decode utf8 + +else: + fs_encode = fs_decode = lambda x: x # do nothing + + +def get_console_encoding(enc): + if os.name == "nt": + if enc == "cp65001": # aka UTF-8 + print "WARNING: Windows codepage 65001 is not supported." + enc = "cp850" + else: + enc = "utf8" + + return enc + + +def compare_time(start, end): + start = map(int, start) + end = map(int, end) + + if start == end: return True + + now = list(time.localtime()[3:5]) + if start < now < end: return True + elif start > end and (now > start or now < end): return True + elif start < now > end < start: return True + else: return False + + +def formatSize(size): + """formats size of bytes""" + size = int(size) + steps = 0 + sizes = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB") + while size > 1000: + size /= 1024.0 + steps += 1 + return "%.2f %s" % (size, sizes[steps]) + + +def formatSpeed(speed): + return formatSize(speed) + "/s" + + +def freeSpace(folder): + if os.name == "nt": + import ctypes + + free_bytes = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(folder), None, None, ctypes.pointer(free_bytes)) + return free_bytes.value + else: + s = os.statvfs(folder) + return s.f_bsize * s.f_bavail + + +def fs_bsize(path): + """ get optimal file system buffer size (in bytes) for I/O calls """ + path = fs_encode(path) + + if os.name == "nt": + import ctypes + + drive = "%s\\" % os.path.splitdrive(path)[0] + cluster_sectors, sector_size = ctypes.c_longlong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceW(ctypes.c_wchar_p(drive), ctypes.pointer(cluster_sectors), ctypes.pointer(sector_size), None, None) + return cluster_sectors * sector_size + else: + return os.statvfs(path).f_bsize + + +def uniqify(seq): #: Originally by Dave Kirby + """ removes duplicates from list, preserve order """ + seen = set() + seen_add = seen.add + return [x for x in seq if x not in seen and not seen_add(x)] + + +def parseFileSize(string, unit=None): #returns bytes + if not unit: + m = re.match(r"([\d.,]+) *([a-zA-Z]*)", string.strip().lower()) + if m: + traffic = float(m.group(1).replace(",", ".")) + unit = m.group(2) + else: + return 0 + else: + if isinstance(string, basestring): + traffic = float(string.replace(",", ".")) + else: + traffic = string + + #ignore case + unit = unit.lower().strip() + + if unit in ("eb", "ebyte", "exabyte", "eib", "e"): + traffic *= 1 << 60 + elif unit in ("pb", "pbyte", "petabyte", "pib", "p"): + traffic *= 1 << 50 + elif unit in ("tb", "tbyte", "terabyte", "tib", "t"): + traffic *= 1 << 40 + elif unit in ("gb", "gbyte", "gigabyte", "gib", "g", "gig"): + traffic *= 1 << 30 + elif unit in ("mb", "mbyte", "megabyte", "mib", "m"): + traffic *= 1 << 20 + elif unit in ("kb", "kbyte", "kilobyte", "kib", "k"): + traffic *= 1 << 10 + + return traffic + + +def lock(func): + def new(*args): + #print "Handler: %s args: %s" % (func, args[1:]) + args[0].lock.acquire() + try: + return func(*args) + finally: + args[0].lock.release() + + return new + + +def fixup(m): + text = m.group(0) + if text[:2] == "&#": + # character reference + try: + if text[:3] == "&#x": + return unichr(int(text[3:-1], 16)) + else: + return unichr(int(text[2:-1])) + except ValueError: + pass + else: + # named entity + try: + name = text[1:-1] + text = unichr(name2codepoint[name]) + except KeyError: + pass + + return text # leave as is + + +def html_unescape(text): + """Removes HTML or XML character references and entities from a text string""" + return re.sub("&#?\w+;", fixup, text) + + +def versiontuple(v): #: By kindall (http://stackoverflow.com/a/11887825) + return tuple(map(int, (v.split(".")))) diff --git a/pyload/utils/packagetools.py b/pyload/utils/packagetools.py new file mode 100644 index 000000000..d5ab4d182 --- /dev/null +++ b/pyload/utils/packagetools.py @@ -0,0 +1,136 @@ +# JDownloader/src/jd/controlling/LinkGrabberPackager.java + +import re +from urlparse import urlparse + +def matchFirst(string, *args): + """ matches against list of regexp and returns first match""" + for patternlist in args: + for pattern in patternlist: + r = pattern.search(string) + if r is not None: + name = r.group(1) + return name + + return string + + +def parseNames(files): + """ Generates packages names from name, data lists + + :param files: list of (name, data) + :return: packagenames mapt to data lists (eg. urls) + """ + packs = {} + + endings = "\\.(3gp|7zip|7z|abr|ac3|aiff|aifc|aif|ai|au|avi|bin|bz2|cbr|cbz|ccf|cue|cvd|chm|dta|deb|divx|djvu|dlc|dmg|doc|docx|dot|eps|exe|ff|flv|f4v|gsd|gif|gz|iwd|iso|ipsw|java|jar|jpg|jpeg|jdeatme|load|mws|mw|m4v|m4a|mkv|mp2|mp3|mp4|mov|movie|mpeg|mpe|mpg|msi|msu|msp|nfo|npk|oga|ogg|ogv|otrkey|pkg|png|pdf|pptx|ppt|pps|ppz|pot|psd|qt|rmvb|rm|rar|ram|ra|rev|rnd|r\\d+|rpm|run|rsdf|rtf|sh(!?tml)|srt|snd|sfv|swf|tar|tif|tiff|ts|txt|viv|vivo|vob|wav|wmv|xla|xls|xpi|zeno|zip|z\\d+|_[_a-z]{2}|\\d+$)" + + rarPats = [re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I), + re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I), + re.compile("(.*)\\.rar$", re.I), + re.compile("(.*)\\.r\\d+$", re.I), + re.compile("(.*)(\\.|_|-)\\d+$", re.I)] + + zipPats = [re.compile("(.*)\\.zip$", re.I), + re.compile("(.*)\\.z\\d+$", re.I), + re.compile("(?is).*\\.7z\\.[\\d]+$", re.I), + re.compile("(.*)\\.a.$", re.I)] + + ffsjPats = [re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)"), + re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I)] + + iszPats = [re.compile("(.*)\\.isz$", re.I), + re.compile("(.*)\\.i\\d{2}$", re.I)] + + pat1 = re.compile("(\\.?CD\\d+)", re.I) + pat2 = re.compile("(\\.?part\\d+)", re.I) + + pat3 = re.compile("(.+)[\\.\\-_]+$") + pat4 = re.compile("(.+)\\.\\d+\\.xtm$") + + for file, url in files: + patternMatch = False + + if file is None: + continue + + # remove trailing / + name = file.rstrip('/') + + # extract last path part .. if there is a path + split = name.rsplit("/", 1) + if len(split) > 1: + name = split.pop(1) + + #check if a already existing package may be ok for this file + # found = False + # for pack in packs: + # if pack in file: + # packs[pack].append(url) + # found = True + # break + # + # if found: continue + + # unrar pattern, 7zip/zip and hjmerge pattern, isz pattern, FFSJ pattern + before = name + name = matchFirst(name, rarPats, zipPats, iszPats, ffsjPats) + if before != name: + patternMatch = True + + # xtremsplit pattern + r = pat4.search(name) + if r is not None: + name = r.group(1) + + # remove part and cd pattern + r = pat1.search(name) + if r is not None: + name = name.replace(r.group(0), "") + patternMatch = True + + r = pat2.search(name) + if r is not None: + name = name.replace(r.group(0), "") + patternMatch = True + + # additional checks if extension pattern matched + if patternMatch: + # remove extension + index = name.rfind(".") + if index <= 0: + index = name.rfind("_") + if index > 0: + length = len(name) - index + if length <= 4: + name = name[:-length] + + # remove endings like . _ - + r = pat3.search(name) + if r is not None: + name = r.group(1) + + # replace . and _ with space + name = name.replace(".", " ") + name = name.replace("_", " ") + + name = name.strip() + else: + name = "" + + # fallback: package by hoster + if not name: + name = urlparse(file).hostname + if name: name = name.replace("www.", "") + + # fallback : default name + if not name: + name = "unknown" + + # build mapping + if name in packs: + packs[name].append(url) + else: + packs[name] = [url] + + return packs diff --git a/pyload/utils/pylgettext.py b/pyload/utils/pylgettext.py new file mode 100644 index 000000000..cab631cf4 --- /dev/null +++ b/pyload/utils/pylgettext.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +from gettext import * + +_searchdirs = None + +origfind = find + +def setpaths(pathlist): + global _searchdirs + if isinstance(pathlist, list): + _searchdirs = pathlist + else: + _searchdirs = list(pathlist) + + +def addpath(path): + global _searchdirs + if _searchdirs is None: + _searchdirs = list(path) + else: + if path not in _searchdirs: + _searchdirs.append(path) + + +def delpath(path): + global _searchdirs + if _searchdirs is not None: + if path in _searchdirs: + _searchdirs.remove(path) + + +def clearpath(): + global _searchdirs + if _searchdirs is not None: + _searchdirs = None + + +def find(domain, localedir=None, languages=None, all=False): + if _searchdirs is None: + return origfind(domain, localedir, languages, all) + searches = [localedir] + _searchdirs + results = list() + for dir in searches: + res = origfind(domain, dir, languages, all) + if all is False: + results.append(res) + else: + results.extend(res) + if all is False: + results = filter(lambda x: x is not None, results) + if len(results) == 0: + return None + else: + return results[0] + else: + return results + +#Is there a smarter/cleaner pythonic way for this? +translation.func_globals['find'] = find diff --git a/pyload/webui/__init__.py b/pyload/webui/__init__.py index c421b632e..6173c57df 100644 --- a/pyload/webui/__init__.py +++ b/pyload/webui/__init__.py @@ -17,7 +17,7 @@ """ import sys -import pyload.common.pylgettext as gettext +import pyload.utils.pylgettext as gettext import os from os.path import join, abspath, dirname, exists @@ -40,7 +40,7 @@ from middlewares import StripPathMiddleware, GZipMiddleWare, PrefixMiddleware SETUP = None PYLOAD = None -from pyload.threads import ServerThread +from pyload.manager.thread import ServerThread if not ServerThread.core: if ServerThread.setup: @@ -52,7 +52,7 @@ else: PYLOAD = ServerThread.core.api config = ServerThread.core.config -from pyload.common.JsEngine import JsEngine +from pyload.utils.JsEngine import JsEngine JS = JsEngine() diff --git a/pyload/webui/app/api.py b/pyload/webui/app/api.py index 7050b78dc..286061c2a 100644 --- a/pyload/webui/app/api.py +++ b/pyload/webui/app/api.py @@ -9,9 +9,9 @@ from bottle import route, request, response, HTTPError from utils import toDict, set_session from pyload.webui import PYLOAD -from pyload.common.json_layer import json +from pyload.utils import json from SafeEval import const_eval as literal_eval -from pyload.Api import BaseObject +from pyload.api import BaseObject # json encoder that accepts TBase objects class TBaseEncoder(json.JSONEncoder): diff --git a/pyload/webui/app/utils.py b/pyload/webui/app/utils.py index 895696c19..d5fa66a35 100644 --- a/pyload/webui/app/utils.py +++ b/pyload/webui/app/utils.py @@ -22,7 +22,7 @@ from bottle import request, HTTPError, redirect, ServerAdapter from pyload.webui import env, THEME -from pyload.Api import has_permission, PERMS, ROLE +from pyload.api import has_permission, PERMS, ROLE def render_to_response(file, args={}, proc=[]): for p in proc: diff --git a/scripts/clonedigger.sh b/scripts/clonedigger.sh deleted file mode 100644 index 358c1b68b..000000000 --- a/scripts/clonedigger.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -PYLOAD="../pyload" # Check pyload directory -clonedigger -o cpd.xml --cpd-output --fast --ignore-dir="lib" --ignore-dir="remote" $PYLOAD diff --git a/scripts/code_analysis.sh b/scripts/code_analysis.sh deleted file mode 100644 index c853652bf..000000000 --- a/scripts/code_analysis.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -PYLOAD="../pyload" # Check pyload directory - -echo "Running sloccount ..." -REPORT="sloccount.sc" -sloccount --duplicates --wide --details $PYLOAD > $REPORT && echo "Done. Report saved to $REPORT" - -echo "Running pep8 ..." -REPORT="pep8.txt" -pep8 $PYLOAD > $REPORT && echo "Done. Report saved to $REPORT" - -echo "Running pylint ..." -REPORT="pylint.txt" -pylint --reports=no $PYLOAD > $REPORT && echo "Done. Report saved to $REPORT" - -#echo "Running pyflakes ..." -#REPORT="pyflakes.txt" -#{ - # pyflakes to pylint syntak -# find $PYLOAD -type f -name "*.py" | egrep -v '^\./lib' | xargs pyflakes > pyflakes.log || : - # Filter warnings and strip ./ from path -# cat pyflakes.log | awk -F\: '{printf "%s:%s: [E]%s\n", $1, $2, $3}' | grep -i -E -v "'_'|pypath|webinterface|pyreq|addonmanager" > $REPORT -# sed -i 's/^.\///g' $REPORT -#} && echo "Done. Report saved to $REPORT" diff --git a/tests/APIExerciser.py b/tests/APIExerciser.py new file mode 100644 index 000000000..886c72a4a --- /dev/null +++ b/tests/APIExerciser.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- + +import string +from threading import Thread +from random import choice, random, sample, randint +from time import time, sleep +from math import floor +import gc + +from traceback import print_exc, format_exc + +from pyload.remote.thriftbackend.ThriftClient import ThriftClient, Destination + +def createURLs(): + """ create some urls, some may fail """ + urls = [] + for x in range(0, randint(20, 100)): + name = "DEBUG_API" + if randint(0, 5) == 5: + name = "" #this link will fail + + urls.append(name + "".join(sample(string.ascii_letters, randint(10, 20)))) + + return urls + +AVOID = (0, 3, 8) + +idPool = 0 +sumCalled = 0 + + +def startApiExerciser(core, n): + for i in range(n): + APIExerciser(core).start() + +class APIExerciser(Thread): + + + def __init__(self, core, thrift=False, user=None, pw=None): + global idPool + + Thread.__init__(self) + self.setDaemon(True) + self.core = core + self.count = 0 #number of methods + self.time = time() + + if thrift: + self.api = ThriftClient(user=user, password=pw) + else: + self.api = core.api + + + self.id = idPool + + idPool += 1 + + #self.start() + + def run(self): + + self.core.log.info("API Excerciser started %d" % self.id) + + out = open("error.log", "ab") + #core errors are not logged of course + out.write("\n" + "Starting\n") + out.flush() + + while True: + try: + self.testAPI() + except Exception: + self.core.log.error("Excerciser %d throw an execption" % self.id) + print_exc() + out.write(format_exc() + 2 * "\n") + out.flush() + + if not self.count % 100: + self.core.log.info("Exerciser %d tested %d api calls" % (self.id, self.count)) + if not self.count % 1000: + out.flush() + + if not sumCalled % 1000: #not thread safe + self.core.log.info("Exercisers tested %d api calls" % sumCalled) + persec = sumCalled / (time() - self.time) + self.core.log.info("Approx. %.2f calls per second." % persec) + self.core.log.info("Approx. %.2f ms per call." % (1000 / persec)) + self.core.log.info("Collected garbage: %d" % gc.collect()) + + + #sleep(random() / 500) + + def testAPI(self): + global sumCalled + + m = ["statusDownloads", "statusServer", "addPackage", "getPackageData", "getFileData", "deleteFiles", + "deletePackages", "getQueue", "getCollector", "getQueueData", "getCollectorData", "isCaptchaWaiting", + "getCaptchaTask", "stopAllDownloads", "getAllInfo", "getServices" , "getAccounts", "getAllUserData"] + + method = choice(m) + #print "Testing:", method + + if hasattr(self, method): + res = getattr(self, method)() + else: + res = getattr(self.api, method)() + + self.count += 1 + sumCalled += 1 + + #print res + + def addPackage(self): + name = "".join(sample(string.ascii_letters, 10)) + urls = createURLs() + + self.api.addPackage(name, urls, choice([Destination.Queue, Destination.Collector])) + + + def deleteFiles(self): + info = self.api.getQueueData() + if not info: return + + pack = choice(info) + fids = pack.links + + if len(fids): + fids = [f.fid for f in sample(fids, randint(1, max(len(fids) / 2, 1)))] + self.api.deleteFiles(fids) + + + def deletePackages(self): + info = choice([self.api.getQueue(), self.api.getCollector()]) + if not info: return + + pids = [p.pid for p in info] + if len(pids): + pids = sample(pids, randint(1, max(floor(len(pids) / 2.5), 1))) + self.api.deletePackages(pids) + + def getFileData(self): + info = self.api.getQueueData() + if info: + p = choice(info) + if p.links: + self.api.getFileData(choice(p.links).fid) + + def getPackageData(self): + info = self.api.getQueue() + if info: + self.api.getPackageData(choice(info).pid) + + def getAccounts(self): + self.api.getAccounts(False) + + def getCaptchaTask(self): + self.api.getCaptchaTask(False) diff --git a/tests/clonedigger.sh b/tests/clonedigger.sh new file mode 100644 index 000000000..358c1b68b --- /dev/null +++ b/tests/clonedigger.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +PYLOAD="../pyload" # Check pyload directory +clonedigger -o cpd.xml --cpd-output --fast --ignore-dir="lib" --ignore-dir="remote" $PYLOAD diff --git a/tests/code_analysis.sh b/tests/code_analysis.sh new file mode 100644 index 000000000..c853652bf --- /dev/null +++ b/tests/code_analysis.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +PYLOAD="../pyload" # Check pyload directory + +echo "Running sloccount ..." +REPORT="sloccount.sc" +sloccount --duplicates --wide --details $PYLOAD > $REPORT && echo "Done. Report saved to $REPORT" + +echo "Running pep8 ..." +REPORT="pep8.txt" +pep8 $PYLOAD > $REPORT && echo "Done. Report saved to $REPORT" + +echo "Running pylint ..." +REPORT="pylint.txt" +pylint --reports=no $PYLOAD > $REPORT && echo "Done. Report saved to $REPORT" + +#echo "Running pyflakes ..." +#REPORT="pyflakes.txt" +#{ + # pyflakes to pylint syntak +# find $PYLOAD -type f -name "*.py" | egrep -v '^\./lib' | xargs pyflakes > pyflakes.log || : + # Filter warnings and strip ./ from path +# cat pyflakes.log | awk -F\: '{printf "%s:%s: [E]%s\n", $1, $2, $3}' | grep -i -E -v "'_'|pypath|webinterface|pyreq|addonmanager" > $REPORT +# sed -i 's/^.\///g' $REPORT +#} && echo "Done. Report saved to $REPORT" diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 000000000..a4229cfe6 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +import APIExerciser + +from nose.tools import nottest + + +class TestApi: + + def __init__(self): + self.api = APIExerciser.APIExerciser(None, True, "TestUser", "pwhere") + + def test_login(self): + assert self.api.api.login("crapp", "wrong pw") is False + + #takes really long, only test when needed + @nottest + def test_random(self): + + for i in range(0, 100): + self.api.testAPI() diff --git a/tests/test_json.py b/tests/test_json.py new file mode 100644 index 000000000..320a42d4f --- /dev/null +++ b/tests/test_json.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +from urllib import urlencode +from urllib2 import urlopen, HTTPError +from json import loads + +from logging import log + +url = "http://localhost:8001/api/%s" + +class TestJson: + + def call(self, name, post=None): + if not post: post = {} + post["session"] = self.key + u = urlopen(url % name, data=urlencode(post)) + return loads(u.read()) + + def setUp(self): + u = urlopen(url % "login", data=urlencode({"username": "TestUser", "password": "pwhere"})) + self.key = loads(u.read()) + assert self.key is not False + + def test_wronglogin(self): + u = urlopen(url % "login", data=urlencode({"username": "crap", "password": "wrongpw"})) + assert loads(u.read()) is False + + def test_access(self): + try: + urlopen(url % "getServerVersion") + except HTTPError, e: + assert e.code == 403 + else: + assert False + + def test_status(self): + ret = self.call("statusServer") + log(1, str(ret)) + assert "pause" in ret + assert "queue" in ret + + def test_unknown_method(self): + try: + self.call("notExisting") + except HTTPError, e: + assert e.code == 404 + else: + assert False -- cgit v1.2.3