From 04038a2cf0c4c2d9cc9a0c8e8bf9beb6426afae8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 10 Jul 2014 03:02:26 +0200 Subject: Use parseError instead PluginParseError + unified all download pattern attributes as LINK_PATTERN + removed some old patterns (not used anymore) + other code cosmetics --- module/plugins/hoster/BasePlugin.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 970ac8081..68b98e0a7 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -18,6 +18,7 @@ class BasePlugin(Hoster): __author_name__ = "RaNaN" __author_mail__ = "RaNaN@pyload.org" + def setup(self): self.chunkLimit = -1 self.resumeDownload = True -- cgit v1.2.3 From 7b8c458cca7d21a029620f98e453f746fce69cd1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 14 Jul 2014 16:10:01 +0200 Subject: Prefer single quote for dict key name --- module/plugins/hoster/BasePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 68b98e0a7..9206aaabd 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -56,7 +56,7 @@ class BasePlugin(Hoster): if server in servers: self.logDebug("Logging on to %s" % server) - self.req.addAuth(account.accounts[server]["password"]) + self.req.addAuth(account.accounts[server]['password']) else: for pwd in pyfile.package().password.splitlines(): if ":" in pwd: -- cgit v1.2.3 From ba916633f2bedb04c7358000b91aed69f52e8e43 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 1 Aug 2014 19:35:59 +0200 Subject: Remove trailing whitespaces + remove license headers + import urllib methods directly + sort and fix key attributes + use save_join instead join + sort some import declarations + other minor code cosmetics --- module/plugins/hoster/BasePlugin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 9206aaabd..839c4100a 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -from urlparse import urlparse from re import match, search from urllib import unquote +from urlparse import urlparse from module.network.HTTPRequest import BadHeader from module.plugins.Hoster import Hoster @@ -12,8 +12,10 @@ from module.utils import html_unescape, remove_chars class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __pattern__ = r'^unmatchable$' __version__ = "0.19" + + __pattern__ = r'^unmatchable$' + __description__ = """Base Plugin when any other didnt fit""" __author_name__ = "RaNaN" __author_mail__ = "RaNaN@pyload.org" -- cgit v1.2.3 From bd642f4597382aedc4c2f1dd28795cc15f5477cd Mon Sep 17 00:00:00 2001 From: William Pickering Date: Sat, 30 Aug 2014 19:27:13 +0200 Subject: [BasePlugin] Don't unquote url header --- module/plugins/hoster/BasePlugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 839c4100a..54d789054 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -12,7 +12,7 @@ from module.utils import html_unescape, remove_chars class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.19" + __version__ = "0.20" __pattern__ = r'^unmatchable$' @@ -88,7 +88,7 @@ class BasePlugin(Hoster): self.logDebug("Location: " + header['location']) base = match(r'https?://[^/]+', url).group(0) if header['location'].startswith("http"): - url = unquote(header['location']) + url = header['location'] elif header['location'].startswith("/"): url = base + unquote(header['location']) else: -- cgit v1.2.3 From 03374192ff2a74fea3c7a2201c3ef9ef929e3a39 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 14 Sep 2014 16:50:20 +0200 Subject: [BasePlugin] Little code cosmetics --- module/plugins/hoster/BasePlugin.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 54d789054..573288ade 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -from re import match, search +import re + from urllib import unquote from urlparse import urlparse @@ -25,10 +26,11 @@ class BasePlugin(Hoster): self.chunkLimit = -1 self.resumeDownload = True + def process(self, pyfile): """main function""" - #debug part, for api exerciser + #: debug part, for api exerciser if pyfile.url.startswith("DEBUG_API"): self.multiDL = False return @@ -74,6 +76,7 @@ class BasePlugin(Hoster): else: self.fail("No Plugin matched and not a downloadable url.") + def downloadFile(self, pyfile): url = pyfile.url @@ -86,7 +89,7 @@ class BasePlugin(Hoster): if 'location' in header: self.logDebug("Location: " + header['location']) - base = match(r'https?://[^/]+', url).group(0) + base = re.match(r'https?://[^/]+', url).group(0) if header['location'].startswith("http"): url = header['location'] elif header['location'].startswith("/"): @@ -100,7 +103,7 @@ class BasePlugin(Hoster): if 'content-disposition' in header: self.logDebug("Content-Disposition: " + header['content-disposition']) - m = search("filename(?P=|\*=(?P.+)'')(?P.*)", header['content-disposition']) + m = re.search("filename(?P=|\*=(?P.+)'')(?P.*)", header['content-disposition']) if m: disp = m.groupdict() self.logDebug(disp) -- cgit v1.2.3 From b0868ae6446078bacf1635dde5e4ab316b4a94cb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 7 Oct 2014 18:57:59 +0200 Subject: New __authors__ key replaces __author_name__ and __author_mail__ + Whitespaces and EOF fixup --- module/plugins/hoster/BasePlugin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 573288ade..41569db1d 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -18,8 +18,7 @@ class BasePlugin(Hoster): __pattern__ = r'^unmatchable$' __description__ = """Base Plugin when any other didnt fit""" - __author_name__ = "RaNaN" - __author_mail__ = "RaNaN@pyload.org" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] def setup(self): -- cgit v1.2.3 From ae7a7e66981456e5bbe2b54006d79b6f907be7a4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 8 Oct 2014 20:18:13 +0200 Subject: Add __license__ key attribute to plugins --- module/plugins/hoster/BasePlugin.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 41569db1d..ffc1a66af 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -18,6 +18,7 @@ class BasePlugin(Hoster): __pattern__ = r'^unmatchable$' __description__ = """Base Plugin when any other didnt fit""" + __license__ = "GPLv3" __authors__ = [("RaNaN", "RaNaN@pyload.org")] -- cgit v1.2.3 From 1c4bf83881d2a22da3773666a580d51f6b57bfd1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Oct 2014 03:07:21 +0200 Subject: Avoid gettext conflict due variable `_` --- module/plugins/hoster/BasePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index ffc1a66af..573134cb9 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -80,7 +80,7 @@ class BasePlugin(Hoster): def downloadFile(self, pyfile): url = pyfile.url - for _ in xrange(5): + for _i in xrange(5): header = self.load(url, just_header=True) # self.load does not raise a BadHeader on 404 responses, do it here -- cgit v1.2.3 From 9f2ebe486a3e155fb6a60e07cccb77ab6a772eb2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 02:31:54 +0200 Subject: Extend translation support in plugins + a lot of code cosmetics and typo fixes --- module/plugins/hoster/BasePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 573134cb9..af8718817 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -74,7 +74,7 @@ class BasePlugin(Hoster): raise else: - self.fail("No Plugin matched and not a downloadable url.") + self.fail(_("No Plugin matched and not a downloadable url")) def downloadFile(self, pyfile): -- cgit v1.2.3 From 34984dae733c3f3d47b41a0acfba3724d53c65a1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 16:52:10 +0100 Subject: Code cosmetics: plugin class attributes --- module/plugins/hoster/BasePlugin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index af8718817..7c6a6c102 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -11,15 +11,15 @@ from module.utils import html_unescape, remove_chars class BasePlugin(Hoster): - __name__ = "BasePlugin" - __type__ = "hoster" + __name__ = "BasePlugin" + __type__ = "hoster" __version__ = "0.20" __pattern__ = r'^unmatchable$' __description__ = """Base Plugin when any other didnt fit""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org")] + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] def setup(self): @@ -35,7 +35,7 @@ class BasePlugin(Hoster): self.multiDL = False return - # self.__name__ = "NetloadIn" + # self.__name__ = "NetloadIn" # pyfile.name = "test" # self.html = self.load("http://localhost:9000/short") # self.download("http://localhost:9000/short") -- cgit v1.2.3 From 2b8bfcfed0dd32b79a8a3ba95cb84adb5ab22811 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 17:40:39 +0100 Subject: Code cosmetics: use self.core.api instead self.api (for now) --- module/plugins/hoster/BasePlugin.py | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 7c6a6c102..63714c437 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -35,17 +35,6 @@ class BasePlugin(Hoster): self.multiDL = False return - # self.__name__ = "NetloadIn" - # pyfile.name = "test" - # self.html = self.load("http://localhost:9000/short") - # self.download("http://localhost:9000/short") - # self.api = self.load("http://localhost:9000/short") - # self.decryptCaptcha("http://localhost:9000/captcha") - # - # if pyfile.url == "79": - # self.core.api.addPackage("test", [str(i) for i in xrange(80)], 1) - # - # return if pyfile.url.startswith("http"): try: -- cgit v1.2.3 From e2e96af3b35192107d32cc48d25c6c2aa6f822ba Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 01:27:42 +0100 Subject: [BasePlugin] Improve a bit --- module/plugins/hoster/BasePlugin.py | 85 ++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 35 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 63714c437..27e8d050a 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -6,20 +6,27 @@ from urllib import unquote from urlparse import urlparse from module.network.HTTPRequest import BadHeader +from module.plugins.internal.SimpleHoster import create_getInfo from module.plugins.Hoster import Hoster -from module.utils import html_unescape, remove_chars +from module.utils import remove_chars class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.20" + __version__ = "0.21" __pattern__ = r'^unmatchable$' __description__ = """Base Plugin when any other didnt fit""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org")] + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")]] + + + @classmethod + def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10 + return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url or ""} def setup(self): @@ -30,40 +37,42 @@ class BasePlugin(Hoster): def process(self, pyfile): """main function""" - #: debug part, for api exerciser - if pyfile.url.startswith("DEBUG_API"): - self.multiDL = False - return + self.getInfo(pyfile.url) if pyfile.url.startswith("http"): + for _i in xrange(2): + try: + self.downloadFile(pyfile) - try: - self.downloadFile(pyfile) - except BadHeader, e: - if e.code in (401, 403): - self.logDebug("Auth required") + except BadHeader, e: + if e.code is 404: + self.offline() - account = self.core.accountManager.getAccountPlugin('Http') - servers = [x['login'] for x in account.getAllAccounts()] - server = urlparse(pyfile.url).netloc + elif e.code in (401, 403): + self.logDebug("Auth required") - if server in servers: - self.logDebug("Logging on to %s" % server) - self.req.addAuth(account.accounts[server]['password']) - else: - for pwd in pyfile.package().password.splitlines(): - if ":" in pwd: - self.req.addAuth(pwd.strip()) - break - else: - self.fail(_("Authorization required (username:password)")) + account = self.core.accountManager.getAccountPlugin('Http') + servers = [x['login'] for x in account.getAllAccounts()] + server = urlparse(pyfile.url).netloc - self.downloadFile(pyfile) + if server in servers: + self.logDebug("Logging on to %s" % server) + self.req.addAuth(account.accounts[server]['password']) + else: + for pwd in pyfile.package().password.splitlines(): + if ":" in pwd: + self.req.addAuth(pwd.strip()) + break + else: + self.fail(_("Authorization required (username:password)")) + else: + self.fail(e) else: - raise - + break + else: + self.fail(_("No file downloaded")) #@TODO: Move to hoster class (check if self.lastDownload) in 0.4.10 else: - self.fail(_("No Plugin matched and not a downloadable url")) + self.fail(_("No plugin matched")) def downloadFile(self, pyfile): @@ -78,31 +87,37 @@ class BasePlugin(Hoster): if 'location' in header: self.logDebug("Location: " + header['location']) + base = re.match(r'https?://[^/]+', url).group(0) + if header['location'].startswith("http"): url = header['location'] + elif header['location'].startswith("/"): url = base + unquote(header['location']) + else: url = '%s/%s' % (base, unquote(header['location'])) else: break - name = html_unescape(unquote(urlparse(url).path.split("/")[-1])) - if 'content-disposition' in header: self.logDebug("Content-Disposition: " + header['content-disposition']) + m = re.search("filename(?P=|\*=(?P.+)'')(?P.*)", header['content-disposition']) if m: disp = m.groupdict() + self.logDebug(disp) + if not disp['enc']: disp['enc'] = 'utf-8' + name = remove_chars(disp['name'], "\"';").strip() name = unicode(unquote(name), disp['enc']) - if not name: - name = url - pyfile.name = name - self.logDebug("Filename: %s" % pyfile.name) + pyfile.name = name + + self.logDebug("Filename changed to: " + name) + self.download(url, disposition=True) -- cgit v1.2.3 From f47b649a005045e62bc34720c0d39c4cfb8edaa3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 12:54:15 +0100 Subject: [BasePlugin] Fix typo --- module/plugins/hoster/BasePlugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 27e8d050a..faf40dc41 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -14,14 +14,14 @@ from module.utils import remove_chars class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.21" + __version__ = "0.22" __pattern__ = r'^unmatchable$' __description__ = """Base Plugin when any other didnt fit""" __license__ = "GPLv3" __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com")]] + ("Walter Purcaro", "vuolter@gmail.com")] @classmethod -- cgit v1.2.3 From f5236b83ac0381747e39542069b5769861229a4e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 17:54:43 +0100 Subject: [BasePlugin] Fix filename recognition --- module/plugins/hoster/BasePlugin.py | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index faf40dc41..bdd7f782d 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -8,13 +8,12 @@ from urlparse import urlparse from module.network.HTTPRequest import BadHeader from module.plugins.internal.SimpleHoster import create_getInfo from module.plugins.Hoster import Hoster -from module.utils import remove_chars class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.22" + __version__ = "0.23" __pattern__ = r'^unmatchable$' @@ -37,7 +36,7 @@ class BasePlugin(Hoster): def process(self, pyfile): """main function""" - self.getInfo(pyfile.url) + pyfile.name = self.getInfo(pyfile.url)['name'] if pyfile.url.startswith("http"): for _i in xrange(2): @@ -74,6 +73,9 @@ class BasePlugin(Hoster): else: self.fail(_("No plugin matched")) + # if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": + # self.fail(_("Empty file")) + def downloadFile(self, pyfile): url = pyfile.url @@ -101,23 +103,4 @@ class BasePlugin(Hoster): else: break - if 'content-disposition' in header: - self.logDebug("Content-Disposition: " + header['content-disposition']) - - m = re.search("filename(?P=|\*=(?P.+)'')(?P.*)", header['content-disposition']) - if m: - disp = m.groupdict() - - self.logDebug(disp) - - if not disp['enc']: - disp['enc'] = 'utf-8' - - name = remove_chars(disp['name'], "\"';").strip() - name = unicode(unquote(name), disp['enc']) - - pyfile.name = name - - self.logDebug("Filename changed to: " + name) - self.download(url, disposition=True) -- cgit v1.2.3 From 089802454ba94ceb5439f124e4e4d96243eb88e6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 29 Nov 2014 23:06:13 +0100 Subject: [BasePlugin] Improve downloadFile routine --- module/plugins/hoster/BasePlugin.py | 89 +++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 43 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index bdd7f782d..01a234d3b 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -13,7 +13,7 @@ from module.plugins.Hoster import Hoster class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.23" + __version__ = "0.24" __pattern__ = r'^unmatchable$' @@ -25,7 +25,7 @@ class BasePlugin(Hoster): @classmethod def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10 - return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url or ""} + return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3 if url else 1, 'url': url or ""} def setup(self): @@ -38,40 +38,40 @@ class BasePlugin(Hoster): pyfile.name = self.getInfo(pyfile.url)['name'] - if pyfile.url.startswith("http"): - for _i in xrange(2): - try: - self.downloadFile(pyfile) + if not pyfile.url.startswith("http"): + self.fail(_("No plugin matched")) + + for _i in xrange(5): + try: + self.downloadFile(pyfile) - except BadHeader, e: - if e.code is 404: - self.offline() + except BadHeader, e: + if e.code is 404: + self.offline() - elif e.code in (401, 403): - self.logDebug("Auth required") + elif e.code in (401, 403): + self.logDebug("Auth required") - account = self.core.accountManager.getAccountPlugin('Http') - servers = [x['login'] for x in account.getAllAccounts()] - server = urlparse(pyfile.url).netloc + account = self.core.accountManager.getAccountPlugin('Http') + servers = [x['login'] for x in account.getAllAccounts()] + server = urlparse(pyfile.url).netloc - if server in servers: - self.logDebug("Logging on to %s" % server) - self.req.addAuth(account.accounts[server]['password']) - else: - for pwd in pyfile.package().password.splitlines(): - if ":" in pwd: - self.req.addAuth(pwd.strip()) - break - else: - self.fail(_("Authorization required (username:password)")) + if server in servers: + self.logDebug("Logging on to %s" % server) + self.req.addAuth(account.accounts[server]['password']) else: - self.fail(e) + for pwd in pyfile.package().password.splitlines(): + if ":" in pwd: + self.req.addAuth(pwd.strip()) + break + else: + self.fail(_("Authorization required (username:password)")) else: - break + self.fail(e) else: - self.fail(_("No file downloaded")) #@TODO: Move to hoster class (check if self.lastDownload) in 0.4.10 + break else: - self.fail(_("No plugin matched")) + self.fail(_("No file downloaded")) #@TODO: Move to hoster class (check if self.lastDownload) in 0.4.10 # if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": # self.fail(_("Empty file")) @@ -80,27 +80,30 @@ class BasePlugin(Hoster): def downloadFile(self, pyfile): url = pyfile.url - for _i in xrange(5): - header = self.load(url, just_header=True) + for i in xrange(1, 7): #@TODO: retrieve the pycurl.MAXREDIRS value set by req + header = self.load(url, ref=True, cookies=True, just_header=True, decode=True) - # self.load does not raise a BadHeader on 404 responses, do it here - if 'code' in header and header['code'] == 404: - raise BadHeader(404) + if 'location' not in header or not header['location']: + if 'code' in header and header['code'] not in (200, 201, 203, 206): + self.fail(_("File not found"), _("HTTP status code: %d") % header['code']) + else: + break - if 'location' in header: - self.logDebug("Location: " + header['location']) + location = header['location'] - base = re.match(r'https?://[^/]+', url).group(0) + self.logDebug("Redirect #%d to: %s" % (i, location)) - if header['location'].startswith("http"): - url = header['location'] + base = re.match(r'https?://[^/]+', url).group(0) - elif header['location'].startswith("/"): - url = base + unquote(header['location']) + if location.startswith("http"): + url = location + + elif location.startswith("/"): + url = base + unquote(location) - else: - url = '%s/%s' % (base, unquote(header['location'])) else: - break + url = "%s/%s" % (base, unquote(location)) + else: + self.fail(_("Too many redirects")) self.download(url, disposition=True) -- cgit v1.2.3 From 2512e1f1bf655c987828698d69790d8f6f9bebe8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 1 Dec 2014 18:19:47 +0100 Subject: [BasePlugin] Improve some routines --- module/plugins/hoster/BasePlugin.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 01a234d3b..0b1888e3b 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -3,7 +3,7 @@ import re from urllib import unquote -from urlparse import urlparse +from urlparse import urljoin, urlparse from module.network.HTTPRequest import BadHeader from module.plugins.internal.SimpleHoster import create_getInfo @@ -13,7 +13,7 @@ from module.plugins.Hoster import Hoster class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.24" + __version__ = "0.25" __pattern__ = r'^unmatchable$' @@ -25,7 +25,7 @@ class BasePlugin(Hoster): @classmethod def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10 - return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3 if url else 1, 'url': url or ""} + return {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3 if url else 1, 'url': unquote(url) or ""} def setup(self): @@ -50,7 +50,7 @@ class BasePlugin(Hoster): self.offline() elif e.code in (401, 403): - self.logDebug("Auth required") + self.logDebug("Auth required", "Received HTTP status code: %d" % e.code) account = self.core.accountManager.getAccountPlugin('Http') servers = [x['login'] for x in account.getAllAccounts()] @@ -65,16 +65,16 @@ class BasePlugin(Hoster): self.req.addAuth(pwd.strip()) break else: - self.fail(_("Authorization required (username:password)")) + self.fail(_("Authorization required")) else: self.fail(e) else: break else: - self.fail(_("No file downloaded")) #@TODO: Move to hoster class (check if self.lastDownload) in 0.4.10 + self.fail(_("No file downloaded")) #@TODO: Move to hoster class in 0.4.10 - # if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": - # self.fail(_("Empty file")) + if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": #@TODO: Move to hoster in 0.4.10 + self.fail(_("Empty file")) def downloadFile(self, pyfile): @@ -85,7 +85,8 @@ class BasePlugin(Hoster): if 'location' not in header or not header['location']: if 'code' in header and header['code'] not in (200, 201, 203, 206): - self.fail(_("File not found"), _("HTTP status code: %d") % header['code']) + self.logDebug("Received HTTP status code: %d" % header['code']) + self.fail(_("File not found")) else: break @@ -93,17 +94,13 @@ class BasePlugin(Hoster): self.logDebug("Redirect #%d to: %s" % (i, location)) - base = re.match(r'https?://[^/]+', url).group(0) - - if location.startswith("http"): + if urlparse(location).scheme: url = location - - elif location.startswith("/"): - url = base + unquote(location) - else: - url = "%s/%s" % (base, unquote(location)) + p = urlparse(url) + base = "%s://%s" % (p.scheme, p.netloc) + url = urljoin(base, location) else: self.fail(_("Too many redirects")) - self.download(url, disposition=True) + self.download(unquote(url), disposition=True) -- cgit v1.2.3 From ca0ce20d5a805080f69dad4c24997ace1958bbcb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 6 Dec 2014 17:18:40 +0100 Subject: Plugin code cosmetics --- module/plugins/hoster/BasePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 0b1888e3b..c8d632dc7 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -60,7 +60,7 @@ class BasePlugin(Hoster): self.logDebug("Logging on to %s" % server) self.req.addAuth(account.accounts[server]['password']) else: - for pwd in pyfile.package().password.splitlines(): + for pwd in self.getPassword().splitlines(): if ":" in pwd: self.req.addAuth(pwd.strip()) break -- cgit v1.2.3 From 0f530be2642c63759cb6156866f1d4ab119760da Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 12 Dec 2014 20:33:34 +0100 Subject: Fix missing create_getInfo --- module/plugins/hoster/BasePlugin.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index c8d632dc7..7b59303ef 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -104,3 +104,6 @@ class BasePlugin(Hoster): self.fail(_("Too many redirects")) self.download(unquote(url), disposition=True) + + +getInfo = create_getInfo(BasePlugin) -- cgit v1.2.3 From 6325eda4e8c142edd11c747f7a9d4a3fa975c494 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 20 Dec 2014 14:24:13 +0100 Subject: Fix password retrieving in some plugins --- module/plugins/hoster/BasePlugin.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'module/plugins/hoster/BasePlugin.py') diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 7b59303ef..d0d8e7cc8 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -13,7 +13,7 @@ from module.plugins.Hoster import Hoster class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.25" + __version__ = "0.26" __pattern__ = r'^unmatchable$' @@ -60,10 +60,9 @@ class BasePlugin(Hoster): self.logDebug("Logging on to %s" % server) self.req.addAuth(account.accounts[server]['password']) else: - for pwd in self.getPassword().splitlines(): - if ":" in pwd: - self.req.addAuth(pwd.strip()) - break + pwd = self.getPassword() + if ':' in pwd: + self.req.addAuth(pwd) else: self.fail(_("Authorization required")) else: -- cgit v1.2.3