diff options
| -rw-r--r-- | module/network/Browser.py | 22 | ||||
| -rw-r--r-- | module/network/HTTPChunk.py | 2 | ||||
| -rw-r--r-- | module/network/HTTPRequest.py | 10 | ||||
| -rw-r--r-- | module/network/RequestFactory.py | 6 | ||||
| -rw-r--r-- | module/plugins/hooks/ExtractArchive.py | 20 | ||||
| -rw-r--r-- | module/plugins/hoster/BasePlugin.py | 78 | ||||
| -rw-r--r-- | module/plugins/hoster/DepositfilesCom.py | 46 | ||||
| -rw-r--r-- | module/plugins/hoster/HellshareCz.py | 4 | ||||
| -rw-r--r-- | module/plugins/hoster/HotfileCom.py | 9 | ||||
| -rw-r--r-- | module/plugins/internal/UnRar.py | 6 | ||||
| -rwxr-xr-x | module/remote/thriftbackend/thriftgen/pyload/Pyload-remote | 2 | ||||
| -rw-r--r-- | module/remote/thriftbackend/thriftgen/pyload/Pyload.py | 17 | ||||
| -rw-r--r-- | systemCheck.py | 2 | ||||
| -rw-r--r--[-rwxr-xr-x] | testlinks.txt | 0 |
14 files changed, 154 insertions, 70 deletions
diff --git a/module/network/Browser.py b/module/network/Browser.py index 822e2ed6d..23cf7666b 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -8,7 +8,6 @@ from HTTPDownload import HTTPDownload class Browser(object): - __slots__ = ("log", "options", "bucket", "cj", "_size", "http", "dl") def __init__(self, bucket=None, options={}): @@ -20,9 +19,14 @@ class Browser(object): self.cj = None # needs to be setted later self._size = 0 - self.http = HTTPRequest(self.cj, options) + self.renewHTTPRequest() self.dl = None + + def renewHTTPRequest(self): + if hasattr(self, "http"): self.http.close() + self.http = HTTPRequest(self.cj, self.options) + def setLastURL(self, val): self.http.lastURL = val @@ -80,7 +84,7 @@ class Browser(object): """ this can also download ftp """ self._size = 0 self.dl = HTTPDownload(url, filename, get, post, self.lastEffectiveURL if ref else None, - self.cj if cookies else None, self.bucket, self.options, progressNotify, disposition) + self.cj if cookies else None, self.bucket, self.options, progressNotify, disposition) name = self.dl.download(chunks, resume) self._size = self.dl.size @@ -96,6 +100,18 @@ class Browser(object): """ add a header to the request """ self.http.putHeader(name, value) + def addAuth(self, pwd): + """Adds user and pw for http auth + + :param pwd: string, user:password + """ + self.options["auth"] = pwd + self.renewHTTPRequest() #we need a new request + + def removeAuth(self): + if "auth" in self.options: del self.options["auth"] + self.renewHTTPRequest() + def clearHeaders(self): self.http.clearHeaders() diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index 680b982d3..69eedb19c 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -137,7 +137,7 @@ class HTTPChunk(HTTPRequest): self.fp = None #file handle self.initHandle() - self.setInterface(self.p.options["interface"], self.p.options["proxies"], self.p.options["ipv6"]) + self.setInterface(self.p.options) self.BOMChecked = False # check and remove byte order mark diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index ffe5d1873..6672a58e6 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -55,7 +55,7 @@ class HTTPRequest(): self.headers = [] #temporary request header self.initHandle() - self.setInterface(options["interface"], options["proxies"], options["ipv6"]) + self.setInterface(options) self.c.setopt(pycurl.WRITEFUNCTION, self.write) self.c.setopt(pycurl.HEADERFUNCTION, self.writeHeader) @@ -89,7 +89,10 @@ class HTTPRequest(): "Keep-Alive: 300", "Expect:"]) - def setInterface(self, interface, proxy, ipv6=False): + def setInterface(self, options): + + interface, proxy, ipv6 = options["interface"], options["proxies"], options["ipv6"] + if interface and interface.lower() != "none": self.c.setopt(pycurl.INTERFACE, str(interface)) @@ -112,6 +115,9 @@ class HTTPRequest(): else: self.c.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) + if "auth" in options: + self.c.setopt(pycurl.USERPWD, str(options["auth"])) + def addCookies(self): """ put cookies from curl handle to cj """ if self.cj: diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 774249a70..5b1528281 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -54,9 +54,11 @@ class RequestFactory(): self.lock.release() return req - def getHTTPRequest(self): + def getHTTPRequest(self, **kwargs): """ returns a http request, dont forget to close it ! """ - return HTTPRequest(CookieJar(None), self.getOptions()) + options = self.getOptions() + options.update(kwargs) # submit kwargs as additional options + return HTTPRequest(CookieJar(None), options) def getURL(self, *args, **kwargs): """ see HTTPRequest for argument list """ diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 54dd6ff2d..0a70da417 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -3,8 +3,8 @@ import sys import os -from os import remove, chmod -from os.path import exists, basename, isfile, isdir +from os import remove, chmod, makedirs +from os.path import exists, basename, isfile, isdir, join from traceback import print_exc from copy import copy @@ -50,7 +50,7 @@ if os.name != "nt": from grp import getgrnam from module.plugins.Hook import Hook, threaded, Expose -from module.utils import save_join +from module.utils import save_join, fs_encode class ArchiveError(Exception): @@ -77,6 +77,7 @@ class ExtractArchive(Hook): ("overwrite", "bool", "Overwrite files", True), ("passwordfile", "file", "password file", "unrar_passwords.txt"), ("deletearchive", "bool", "Delete archives when done", False), + ("subfolder", "bool", "Create subfolder for each package", False), ("destination", "folder", "Extract files to", ""), ("queue", "bool", "Wait for all downloads to be fninished", True), ("renice", "int", "CPU Priority", 0), ] @@ -151,7 +152,7 @@ class ExtractArchive(Hook): #iterate packages -> plugins -> targets for pid in ids: p = self.core.files.getPackage(pid) - self.logInfo(_("Extract package %s") % p.name) + self.logInfo(_("Check package %s") % p.name) if not p: continue # determine output folder @@ -159,8 +160,15 @@ class ExtractArchive(Hook): # force trailing slash if self.getConfig("destination") and self.getConfig("destination").lower() != "none": - if exists(self.getConfig("destination")): - out = save_join(self.getConfig("destination"), "") + + out = save_join(dl, p.folder, self.getConfig("destination"), "") + #relative to package folder if destination is relative, otherwise absolute path overwrites them + + if self.getConf("subfolder"): + out = join(out, fs_encode(p.folder)) + + if not exists(out): + makedirs(out) files_ids = [(save_join(dl, p.folder, x["name"]), x["id"]) for x in p.getChildren().itervalues()] diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index ed11c378b..15e35ce24 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - -from module.plugins.Hoster import Hoster -from module.utils import html_unescape - from urlparse import urlparse from re import search from urllib import unquote +from module.network.HTTPRequest import BadHeader +from module.plugins.Hoster import Hoster +from module.utils import html_unescape, removeChars + class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" @@ -41,31 +41,49 @@ class BasePlugin(Hoster): # # return if pyfile.url.startswith("http"): - header = self.load(pyfile.url, just_header = True) - #self.logDebug(header) - - if 'location' in header: - self.logDebug("Location: " + header['location']) - url = unquote(header['location']) - else: - url = pyfile.url - - name = html_unescape(urlparse(url).path.split("/")[-1]) - - if 'content-disposition' in header: - self.logDebug("Content-Disposition: " + header['content-disposition']) - m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", header['content-disposition']) - if m: - disp = m.groupdict() - self.logDebug(disp) - if not disp['enc']: disp['enc'] = 'utf-8' - name = disp['name'].replace('"', "").replace("'", "").replace(";", "").strip() - name = unicode(unquote(name), disp['enc']) - - if not name: self.offline() - pyfile.name = name - self.logDebug("Filename: %s" % pyfile.name) - self.download(url, disposition=True) + + try: + self.downloadFile(pyfile) + except BadHeader, e: + if e.code in (401, 403): + self.logDebug("Auth required") + + pwd = pyfile.package().password.strip() + if ":" not in pwd: + self.fail(_("Authorization required (username:password)")) + + self.req.addAuth(pwd) + self.downloadFile(pyfile) + else: + raise else: - self.fail("No Plugin matched and not a downloadable url.")
\ No newline at end of file + self.fail("No Plugin matched and not a downloadable url.") + + + def downloadFile(self, pyfile): + header = self.load(pyfile.url, just_header = True) + #self.logDebug(header) + + if 'location' in header: + self.logDebug("Location: " + header['location']) + url = unquote(header['location']) + else: + url = pyfile.url + + name = html_unescape(urlparse(url).path.split("/")[-1]) + + if 'content-disposition' in header: + self.logDebug("Content-Disposition: " + header['content-disposition']) + m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", header['content-disposition']) + if m: + disp = m.groupdict() + self.logDebug(disp) + if not disp['enc']: disp['enc'] = 'utf-8' + name = removeChars(disp['name'], "\"';").strip() + name = unicode(unquote(name), disp['enc']) + + if not name: name = url + pyfile.name = name + self.logDebug("Filename: %s" % pyfile.name) + self.download(url, disposition=True)
\ No newline at end of file diff --git a/module/plugins/hoster/DepositfilesCom.py b/module/plugins/hoster/DepositfilesCom.py index 1fba3bad5..81e6aa4d6 100644 --- a/module/plugins/hoster/DepositfilesCom.py +++ b/module/plugins/hoster/DepositfilesCom.py @@ -5,6 +5,7 @@ import re import urllib from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL +from module.plugins.ReCaptcha import ReCaptcha def getInfo(urls): result = [] @@ -29,13 +30,15 @@ class DepositfilesCom(Hoster): __name__ = "DepositfilesCom" __type__ = "hoster" __pattern__ = r"http://[\w\.]*?depositfiles\.com(/\w{1,3})?/files/[\w]+" - __version__ = "0.33" + __version__ = "0.34" __description__ = """Depositfiles.com Download Hoster""" __author_name__ = ("spoob", "zoidberg") __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz") FILE_INFO_PATTERN = r'File name: <b title="([^"]+)">.*\s*<span class="nowrap">File size: <b>([0-9.]+) (KB|MB|GB)</b>' FILE_OFFLINE_PATTERN = r'<span class="html_download_api-not_exists"></span>' + RECAPTCHA_PATTERN = r"Recaptcha.create\('([^']+)', this\);" + DOWNLOAD_LINK_PATTERN = r'<form action="(http://.+?\.depositfiles.com/.+?)" method="get"' def setup(self): self.resumeDownload = self.multiDL = True if self.account else False @@ -96,27 +99,42 @@ class DepositfilesCom(Hoster): found = re.search(r"var fid = '(\w+)';", self.html) if not found: self.retry(wait_time=5) - fid = found.group(1) - self.logDebug ("FID: %s" % fid) + params = {'fid' : found.group(1)} + self.logDebug ("FID: %s" % params['fid']) - self.wait() - - #form = re.search(r"\$\('#download_container'\)\.load\('([^']+)", self.html) + captcha_key = None + found = re.search(self.RECAPTCHA_PATTERN, self.html) + if found: captcha_key = found.group(1) + self.logDebug ("CAPTCHA_KEY: %s" % captcha_key) - #self.html = self.load("http://depositfiles.com/"+ form.group(1)) + self.wait() + recaptcha = ReCaptcha(self) + + for i in range(5): + self.html = self.load("http://depositfiles.com/get_file.php", get = params) + if '<input type=button value="Continue" onclick="check_recaptcha' in self.html: + if not captcha_key: self.fail('Parse error (Captcha key)') + if 'response' in params: self.invalidCaptcha() + params['challenge'], params['response'] = recaptcha.challenge(captcha_key) + self.logDebug(params) + continue + + found = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) + if found: + if 'response' in params: self.correctCaptcha() + link = urllib.unquote(found.group(1)) + self.logDebug ("LINK: %s" % link) + break + else: + self.fail('Parse error (Download link)') + else: + self.fail('No valid captcha response received') - self.html = self.load("http://depositfiles.com/get_file.php?fid=" + fid) - link = urllib.unquote(re.search('<form action="(http://.+?\.depositfiles.com/.+?)" method="get"', self.html).group(1)) - self.logDebug ("LINK: %s" % link) try: self.download(link) except: self.retry(wait_time = 60) - #wait_time = int(re.search(r'<span id="download_waiter_remain">(.*?)</span>', self.html).group(1)) - #self.setWait(wait_time) - #self.log.debug("DepositFiles.com: Waiting %d seconds." % wait_time) - def handlePremium(self): link = urllib.unquote(re.search('<div id="download_url">\s*<a href="(http://.+?\.depositfiles.com/.+?)"', self.html).group(1)) self.download(link)
\ No newline at end of file diff --git a/module/plugins/hoster/HellshareCz.py b/module/plugins/hoster/HellshareCz.py index 7986ecf11..8c90e8099 100644 --- a/module/plugins/hoster/HellshareCz.py +++ b/module/plugins/hoster/HellshareCz.py @@ -48,7 +48,7 @@ class HellshareCz(Hoster): __name__ = "HellshareCz" __type__ = "hoster" __pattern__ = r"http://(.*\.)*hellshare\.(cz|com|sk|hu)/.*" - __version__ = "0.70" + __version__ = "0.71" __description__ = """Hellshare.cz""" __author_name__ = ("zoidberg") @@ -97,7 +97,7 @@ class HellshareCz(Hoster): def handleFree(self): # hellshare is very generous - if 'You exceeded your today's limit for free download. You can download only 1 files per 24 hours.' in self.html: + if "You exceeded your today's limit for free download. You can download only 1 files per 24 hours." in self.html: t = datetime.datetime.today().replace(hour=1, minute=0, second=0) + datetime.timedelta( days=1) - datetime.datetime.today() self.setWait(t.seconds, True) diff --git a/module/plugins/hoster/HotfileCom.py b/module/plugins/hoster/HotfileCom.py index 96927a9fc..9c056d899 100644 --- a/module/plugins/hoster/HotfileCom.py +++ b/module/plugins/hoster/HotfileCom.py @@ -37,6 +37,8 @@ class HotfileCom(Hoster): __author_name__ = ("sitacuisses","spoob","mkaay") __author_mail__ = ("sitacuisses@yhoo.de","spoob@pyload.org","mkaay@mkaay.de") + FILE_OFFLINE_PATTERN = r'File is removed' + def setup(self): self.html = [None, None] self.wantReconnect = False @@ -73,7 +75,10 @@ class HotfileCom(Hoster): if not self.account: self.downloadHTML() - + + if self.FILE_OFFLINE_PATTERN in self.html[0]: + self.offline() + self.setWait(self.getWaitTime()) self.wait() @@ -131,3 +136,5 @@ class HotfileCom(Hoster): self.wantReconnect = True return waittime return 65 + else: + self.fail("Don't know how long to wait. Cannot proceed.") diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 7ba1d01ba..1943f69e0 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -18,16 +18,14 @@ """ import os +import re from os.path import join from glob import glob from subprocess import Popen, PIPE - from module.plugins.hooks.ExtractArchive import AbtractExtractor from module.utils import save_join, decode -import re - class UnRar(AbtractExtractor): __name__ = "UnRar" __version__ = "0.1" @@ -138,7 +136,7 @@ class UnRar(AbtractExtractor): def getDeleteFiles(self): if ".part" in self.file: - return glob(self.file.replace("0", "*").replace("1", "*")) + return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.IGNORECASE)) return [self.file] def listContent(self): diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote index e2e56d5ca..854b1589e 100755 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote @@ -74,7 +74,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print ' void orderPackage(PackageID pid, i16 position)' print ' void orderFile(FileID fid, i16 position)' print ' void setPackageData(PackageID pid, data)' - print ' void deleteFinished()' + print ' deleteFinished()' print ' void restartFailed()' print ' bool isCaptchaWaiting()' print ' CaptchaTask getCaptchaTask(bool exclusive)' diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py index 677e4afe2..a1bc63f75 100644 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py @@ -1873,7 +1873,7 @@ class Client(Iface): def deleteFinished(self, ): self.send_deleteFinished() - self.recv_deleteFinished() + return self.recv_deleteFinished() def send_deleteFinished(self, ): self._oprot.writeMessageBegin('deleteFinished', TMessageType.CALL, self._seqid) @@ -1892,7 +1892,9 @@ class Client(Iface): result = deleteFinished_result() result.read(self._iprot) self._iprot.readMessageEnd() - return + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "deleteFinished failed: unknown result"); def restartFailed(self, ): self.send_restartFailed() @@ -3085,7 +3087,7 @@ class Processor(Iface, TProcessor): args.read(iprot) iprot.readMessageEnd() result = deleteFinished_result() - self._handler.deleteFinished() + result.success = self._handler.deleteFinished() oprot.writeMessageBegin("deleteFinished", TMessageType.REPLY, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -4904,13 +4906,22 @@ class deleteFinished_args(TBase): class deleteFinished_result(TBase): + """ + Attributes: + - success + """ __slots__ = [ + 'success', ] thrift_spec = ( + (0, TType.LIST, 'success', (TType.I32,None), None, ), # 0 ) + def __init__(self, success=None,): + self.success = success + class restartFailed_args(TBase): diff --git a/systemCheck.py b/systemCheck.py index 667a1a53c..60fe0313b 100644 --- a/systemCheck.py +++ b/systemCheck.py @@ -139,4 +139,4 @@ def main(): if __name__ == "__main__": main() - input("Press Enter to Exit.") + raw_input("Press Enter to Exit.") diff --git a/testlinks.txt b/testlinks.txt index 428cf63ea..428cf63ea 100755..100644 --- a/testlinks.txt +++ b/testlinks.txt |