diff options
Diffstat (limited to 'pavement.py')
| -rw-r--r-- | pavement.py | 437 |
1 files changed, 252 insertions, 185 deletions
diff --git a/pavement.py b/pavement.py index ac9a6fa1a..ea6775b1a 100644 --- a/pavement.py +++ b/pavement.py @@ -1,84 +1,44 @@ # -*- coding: utf-8 -*- - from paver.easy import * -from paver.setuputils import setup from paver.doctools import cog +import fnmatch + +# patch to let it support list of patterns +def new_fnmatch(self, pattern): + if type(pattern) == list: + for p in pattern: + if fnmatch.fnmatch(self.name, p): + return True + return False + else: + return fnmatch.fnmatch(self.name, pattern) + + +path.fnmatch = new_fnmatch + +import os import sys +import shutil import re -from urllib import urlretrieve -from subprocess import call, Popen, PIPE -from zipfile import ZipFile +from glob import glob +from tempfile import mkdtemp +from subprocess import call, Popen PROJECT_DIR = path(__file__).dirname() sys.path.append(PROJECT_DIR) -options = environment.options -path('pyload').mkdir() - -extradeps = [] -if sys.version_info <= (2, 5): - extradeps += 'simplejson' - -setup( - name="pyload", - version="0.4.9", - description='Fast, lightweight and full featured download manager.', - long_description=open(PROJECT_DIR / "README").read(), - keywords = ('pyload', 'download-manager', 'one-click-hoster', 'download'), - url="http://pyload.org", - download_url='http://pyload.org/download', - license='GPL v3', - author="pyLoad Team", - author_email="support@pyload.org", - platforms = ('Any',), - #package_dir={'pyload': 'src'}, - packages=['pyload'], - #package_data=find_package_data(), - #data_files=[], - include_package_data=True, - exclude_package_data={'pyload': ['docs*', 'scripts*', 'tests*']}, #exluced from build but not from sdist - # 'bottle >= 0.10.0' not in list, because its small and contain little modifications - install_requires=['thrift >= 0.8.0', 'jinja2', 'pycurl', 'Beaker', 'BeautifulSoup>=3.2, <3.3'] + extradeps, - extras_require={ - 'SSL': ["pyOpenSSL"], - 'DLC': ['pycrypto'], - 'lightweight webserver': ['bjoern'], - 'RSS plugins': ['feedparser'], - }, - #setup_requires=["setuptools_hg"], - entry_points={ - 'console_scripts': [ - 'pyLoadCore = pyLoadCore:main', - 'pyLoadCli = pyLoadCli:main' - ]}, - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Topic :: Internet :: WWW/HTTP", - "Environment :: Console", - "Environment :: Web Environment", - "Intended Audience :: End Users/Desktop", - "License :: OSI Approved :: GNU General Public License (GPL)", - "Operating System :: OS Independent", - "Programming Language :: Python :: 2" - ] -) +from pyload import __version__ +options = environment.options options( sphinx=Bunch( builddir="_build", sourcedir="" ), - get_source=Bunch( - src="https://bitbucket.org/spoob/pyload/get/tip.zip", - rev=None, - clean=False - ), - thrift=Bunch( - path="../thrift/trunk/compiler/cpp/thrift", - gen="" + apitypes=Bunch( + path="thrift", ), virtualenv=Bunch( dir="env", @@ -86,165 +46,212 @@ options( virtual="virtualenv2", ), cog=Bunch( - pattern="*.py", + pattern=["*.py", "*.rst"], ) ) # xgettext args -xargs = ["--from-code=utf-8", "--copyright-holder=pyLoad Team", "--package-name=pyLoad", - "--package-version=%s" % options.version, "--msgid-bugs-address='bugs@pyload.org'"] +xargs = ["--language=Python", "--add-comments=L10N", + "--from-code=utf-8", "--copyright-holder=pyLoad Team", "--package-name=pyload", + "--package-version=%s" % __version__, "--msgid-bugs-address='bugs@pyload.org'"] + +# Modules replace rules +module_replace = [ + ('from module.plugins.Hoster import Hoster', 'from pyload.plugins.Hoster import Hoster'), + ('from module.plugins.Hook import threaded, Expose, Hook', + 'from pyload.plugins.Addon import threaded, Expose, Addon'), + ('from module.plugins.Hook import Hook', 'from pyload.plugins.Addon import Addon'), + ('from module.common.json_layer import json_loads, json_dumps', 'from pyload.utils import json_loads, json_dumps'), + ('from module.common.json_layer import json_loads', 'from pyload.utils import json_loads'), + ('from module.common.json_layer import json_dumps', 'from pyload.utils import json_dumps'), + ('from module.utils import parseFileSize', 'from pyload.utils import parseFileSize'), + ('from module.utils import save_join, save_path', + 'from pyload.utils.fs import save_join, save_filename as save_path'), + ('from module.utils import save_join, fs_encode', 'from pyload.utils.fs import save_join, fs_encode'), + ('from module.utils import save_join', 'from pyload.utils.fs import save_join'), + ('from module.utils import fs_encode', 'from pyload.utils.fs import fs_encode'), + ('from module.unescape import unescape', 'from pyload.utils import html_unescape as unescape'), + ('from module.lib.BeautifulSoup import BeautifulSoup', 'from BeautifulSoup import BeautifulSoup'), + ('from module.lib import feedparser', 'import feedparser'), + ('self.account.getAccountInfo(self.user, ', 'self.account.getAccountData('), + ('self.account.getAccountInfo(self.user)', 'self.account.getAccountData()'), + ('self.account.accounts[self.user]["password"]', 'self.account.password'), + ("self.account.accounts[self.user]['password']", 'self.account.password'), + ('from module.', 'from pyload.') # This should be always the last one +] + @task @needs('cog') def html(): """Build html documentation""" - module = path("docs") / "module" + module = path("docs") / "pyload" module.rmtree() call_task('paver.doctools.html') @task @cmdopts([ - ('src=', 's', 'Url to source'), - ('rev=', 'r', "HG revision"), - ("clean", 'c', 'Delete old source folder') + ('path=', 'p', 'Thrift path'), ]) -def get_source(options): - """ Downloads pyload source from bitbucket tip or given rev""" - if options.rev: options.url = "https://bitbucket.org/spoob/pyload/get/%s.zip" % options.rev +def apitypes(options): + """ Generate data types stubs """ - pyload = path("pyload") + outdir = PROJECT_DIR / "pyload" / "remote" - if len(pyload.listdir()) and not options.clean: - return - elif pyload.exists(): - pyload.rmtree() + if (outdir / "gen-py").exists(): + (outdir / "gen-py").rmtree() - urlretrieve(options.src, "pyload_src.zip") - zip = ZipFile("pyload_src.zip") - zip.extractall() - path("pyload_src.zip").remove() + cmd = [options.apitypes.path, "-strict", "-o", outdir, "--gen", "py:slots,dynamic", outdir / "pyload.thrift"] - folder = [x for x in path(".").dirs() if x.name.startswith("spoob-pyload-")][0] - folder.move(pyload) + print "running", cmd - change_mode(pyload, 0644) - change_mode(pyload, 0755, folder=True) + p = Popen(cmd) + p.communicate() - for file in pyload.files(): - if file.name.endswith(".py"): - file.chmod(0755) + (outdir / "thriftgen").rmtree() + (outdir / "gen-py").move(outdir / "thriftgen") - (pyload / ".hgtags").remove() - (pyload / ".hgignore").remove() - #(pyload / "docs").rmtree() + #create light ttypes + from pyload.remote.create_apitypes import main - f = open(pyload / "__init__.py", "wb") - f.close() + main() + from pyload.remote.create_jstypes import main - #options.setup.packages = find_packages() - #options.setup.package_data = find_package_data() + main() @task -@needs('clean', 'generate_setup', 'minilib', 'get_source', 'setuptools.command.sdist') -def sdist(): - """ Build source code package with distutils """ +def webapp(): + """ Builds the pyload web app. Nodejs and npm must be installed """ + + os.chdir(PROJECT_DIR / "pyload" / "web") + + # Preserve exit codes + ret = call(["npm", "install", "--no-color"]) + if ret: + exit(ret) + ret = call(["bower", "install", "--no-color"]) + if ret: + exit(ret) + ret = call(["bower", "update", "--no-color"]) + if ret: + exit(ret) + ret = call(["grunt", "--no-color"]) + if ret: + exit(ret) @task -@cmdopts([ - ('path=', 'p', 'Thrift path'), - ('gen=', 'g', "Extra --gen option") -]) -def thrift(options): - """ Generate Thrift stubs """ - - print "add import for TApplicationException manually as long it is not fixed" - - outdir = path("module") / "remote" / "thriftbackend" - (outdir / "gen-py").rmtree() +def generate_locale(): + """ Generates localisation files """ - cmd = [options.thrift.path, "-strict", "-o", outdir, "--gen", "py:slots,dynamic", outdir / "pyload.thrift"] + EXCLUDE = ["pyload/lib", "pyload/cli", "pyload/setup", "pyload/plugins", "Setup.py"] - if options.gen: - cmd.insert(len(cmd) - 1, "--gen") - cmd.insert(len(cmd) - 1, options.gen) + makepot("core", path("pyload"), EXCLUDE) + makepot("plugins", path("pyload") / "plugins") + makepot("setup", "", [], includes="./pyload/setup/Setup.py\n") + makepot("cli", path("pyload") / "cli", []) + makepot("webUI", path("pyload") / "web" / "app", ["components", "vendor", "gettext"], endings=[".js", ".html"], + xxargs="--language=Python --force-po".split(" ")) - print "running", cmd + makehtml("webUI", path("pyload") / "web" / "app" / "templates") - p = Popen(cmd) - p.communicate() + path("includes.txt").remove() - (outdir / "thriftgen").rmtree() - (outdir / "gen-py").move(outdir / "thriftgen") + print "Locale generated" - #create light ttypes - from module.remote.socketbackend.create_ttypes import main - main() @task -def compile_js(): - """ Compile .coffee files to javascript""" - - root = path("module") / "web" / "media" / "js" - for f in root.glob("*.coffee"): - print "generate", f - coffee = Popen(["coffee", "-cbs"], stdin=open(f, "rb"), stdout=PIPE) - yui = Popen(["yuicompressor", "--type", "js"], stdin=coffee.stdout, stdout=PIPE) - coffee.stdout.close() - content = yui.communicate()[0] - with open(root / f.name.replace(".coffee", ".js"), "wb") as js: - js.write("{% autoescape true %}\n") - js.write(content) - js.write("\n{% endautoescape %}") +@cmdopts([ + ('key=', 'k', 'api key') +]) +def upload_translations(options): + """ Uploads the locale files to translation server """ + tmp = path(mkdtemp()) + + shutil.copy('locale/crowdin.yaml', tmp) + os.mkdir(tmp / 'pyLoad') + for f in glob('locale/*.pot'): + if os.path.isfile(f): + shutil.copy(f, tmp / 'pyLoad') + + config = tmp / 'crowdin.yaml' + content = open(config, 'rb').read() + content = content.format(key=options.key, tmp=tmp) + f = open(config, 'wb') + f.write(content) + f.close() + call(['crowdin-cli', '-c', config, 'upload', 'source']) -@task -def generate_locale(): - """ Generates localisation files """ + shutil.rmtree(tmp) - EXCLUDE = ["BeautifulSoup.py", "module/gui", "module/cli", "web/locale", "web/ajax", "web/cnl", "web/pyload", - "setup.py"] - makepot("core", path("module"), EXCLUDE, "./pyLoadCore.py\n") + print "Translations uploaded" - makepot("gui", path("module") / "gui", [], includes="./pyLoadGui.py\n") - makepot("cli", path("module") / "cli", [], includes="./pyLoadCli.py\n") - makepot("setup", "", [], includes="./module/setup.py\n") - EXCLUDE = ["ServerThread.py", "web/media/default"] - - # strings from js files - strings = set() +@task +@cmdopts([ + ('key=', 'k', 'api key') +]) +def download_translations(options): + """ Downloads the translated files from translation server """ + tmp = path(mkdtemp()) + + shutil.copy('locale/crowdin.yaml', tmp) + os.mkdir(tmp / 'pyLoad') + for f in glob('locale/*.pot'): + if os.path.isfile(f): + shutil.copy(f, tmp / 'pyLoad') + + config = tmp / 'crowdin.yaml' + content = open(config, 'rb').read() + content = content.format(key=options.key, tmp=tmp) + f = open(config, 'wb') + f.write(content) + f.close() - for fi in path("module/web").walkfiles(): - if not fi.name.endswith(".js") and not fi.endswith(".coffee"): continue - with open(fi, "rb") as c: - content = c.read() + call(['crowdin-cli', '-c', config, 'download']) - strings.update(re.findall(r"_\s*\(\s*\"([^\"]+)", content)) - strings.update(re.findall(r"_\s*\(\s*\'([^\']+)", content)) + for language in (tmp / 'pyLoad').listdir(): + if not language.isdir(): + continue - trans = path("module") / "web" / "translations.js" + target = path('locale') / language.basename() + print "Copy language %s" % target + if target.exists(): + shutil.rmtree(target) - with open(trans, "wb") as js: - for s in strings: - js.write('_("%s")\n' % s) + shutil.copytree(language, target) - makepot("django", path("module/web"), EXCLUDE, "./%s\n" % trans.relpath(), [".py", ".html"], ["--language=Python"]) + shutil.rmtree(tmp) - trans.remove() - path("includes.txt").remove() +@task +def compile_translations(): + """ Compile PO files to MO """ + for language in path('locale').listdir(): + if not language.isdir(): + continue - print "Locale generated" + for f in glob(language / 'LC_MESSAGES' / '*.po'): + print "Compiling %s" % f + call(['msgfmt', '-o', f.replace('.po', '.mo'), f]) @task def tests(): - call(["nosetests2"]) + """ Run complete test suite """ + call(["tests/run_pyload.sh"]) + call(["tests/nosetests.sh"]) + call(["tests/quit_pyload.sh"]) + @task +@cmdopts([ + ('virtual=', 'v', 'virtualenv path'), + ('python=', 'p', 'python path') +]) def virtualenv(options): """Setup virtual environment""" if path(options.dir).exists(): @@ -263,28 +270,41 @@ def clean_env(): @task -@needs('generate_setup', 'minilib', 'get_source', 'virtualenv') -def env_install(): - """Install pyLoad into the virtualenv""" - venv = options.virtualenv - call([path(venv.dir) / "bin" / "easy_install", "."]) - - -@task def clean(): """Cleans build directories""" path("build").rmtree() path("dist").rmtree() +@task +def replace_module_imports(): + """Replace imports from stable syntax to master""" + for root, dirnames, filenames in os.walk('pyload/plugins'): + for filename in fnmatch.filter(filenames, '*.py'): + path = os.path.join(root, filename) + f = open(path, 'r') + content = f.read() + f.close() + for rule in module_replace: + content = content.replace(rule[0], rule[1]) + if '/addons/' in path: + content = content.replace('(Hook):', '(Addon):') + elif '/accounts/' in path: + content = content.replace('self.accounts[user]["password"]', 'self.password') + content = content.replace("self.accounts[user]['password']", 'self.password') + f = open(path, 'w') + f.write(content) + f.close() + + #helper functions -def walk_trans(path, EXCLUDE, endings=[".py"]): +def walk_trans(path, excludes, endings=[".py"]): result = "" for f in path.walkfiles(): - if [True for x in EXCLUDE if x in f.dirname().relpath()]: continue - if f.name in EXCLUDE: continue + if [True for x in excludes if x in f.dirname().relpath()]: continue + if f.name in excludes: continue for e in endings: if f.name.endswith(e): @@ -318,15 +338,62 @@ def makepot(domain, p, excludes=[], includes="", endings=[".py"], xxargs=[]): with open("locale/%s.pot" % domain, "wb") as f: f.write(content) +def makehtml(domain, p): + """ Parses entries from html and append them to existing pot file""" + + pot = path("locale") / "%s.pot" % domain + + with open(pot, 'rb') as f: + content = f.readlines() + + msgids = {} + # parse existing ids and line + for i, line in enumerate(content): + if line.startswith("msgid "): + msgid = line[6:-1].strip('"') + msgids[msgid] = i + + # TODO: parses only n=2 plural + single = re.compile(r'\{\{ ?(?:gettext|_) "((?:\\.|[^"\\])*)" ?\}\}') + plural = re.compile(r'\{\{ ?(?:ngettext) *"((?:\\.|[^"\\])*)" *"((?:\\.|[^"\\])*)"') + + for f in p.walkfiles(): + if not f.endswith("html"): continue + with open(f, "rb") as html: + for i, line in enumerate(html.readlines()): + key = None + nmessage = plural.search(line) + message = single.search(line) + if nmessage: + key = nmessage.group(1) + keyp = nmessage.group(2) + + if key not in msgids: + content.append("\n") + content.append('msgid "%s"\n' % key) + content.append('msgid_plural "%s"\n' % keyp) + content.append('msgstr[0] ""\n') + content.append('msgstr[1] ""\n') + msgids[key] = len(content) - 4 + + + elif message: + key = message.group(1) + + if key not in msgids: + content.append("\n") + content.append('msgid "%s"\n' % key) + content.append('msgstr ""\n') + msgids[key] = len(content) - 2 + + if key: + content.insert(msgids[key], "#: %s:%d\n" % (f, i)) + msgids[key] += 1 + + + with open(pot, 'wb') as f: + f.writelines(content) -def change_owner(dir, uid, gid): - for p in dir.walk(): - p.chown(uid, gid) + print "Parsed html files" -def change_mode(dir, mode, folder=False): - for p in dir.walk(): - if folder and p.isdir(): - p.chmod(mode) - elif p.isfile() and not folder: - p.chmod(mode) |