diff options
| author | 2015-10-18 19:08:02 +0200 | |
|---|---|---|
| committer | 2015-10-18 19:08:02 +0200 | |
| commit | b8e23365241337ebb8a42a0d93cbc67afbbee480 (patch) | |
| tree | 5293dadec7c4aecf0fe32d9f6986c9987c1fbed9 /module/plugins/internal | |
| parent | [Hoster] Fix download routine + new method is_download (diff) | |
| download | pyload-b8e23365241337ebb8a42a0d93cbc67afbbee480.tar.xz | |
[Plugin] Minimize code + spare fixes
Diffstat (limited to 'module/plugins/internal')
| -rw-r--r-- | module/plugins/internal/MultiHoster.py | 56 | ||||
| -rw-r--r-- | module/plugins/internal/Plugin.py | 312 | ||||
| -rw-r--r-- | module/plugins/internal/SimpleHoster.py | 117 | 
3 files changed, 66 insertions, 419 deletions
| diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 5655571b8..a31a6843b 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -13,11 +13,11 @@ class MultiHoster(SimpleHoster):      __status__  = "testing"      __pattern__ = r'^unmatchable$' -    __config__  = [("activated"       , "bool", "Activated"                                 , True), -                   ("use_premium"     , "bool", "Use premium account if available"          , True), -                   ("fallback_premium", "bool", "Fallback to free download if premium fails", True), -                   ("chk_filesize"    , "bool", "Check file size"                           , True), -                   ("revertfailed"    , "bool", "Revert to standard download if fails"      , True)] +    __config__  = [("activated"   , "bool", "Activated"                                 , True), +                   ("use_premium" , "bool", "Use premium account if available"          , True), +                   ("fallback"    , "bool", "Fallback to free download if premium fails", True), +                   ("chk_filesize", "bool", "Check file size"                           , True), +                   ("revertfailed", "bool", "Revert to standard download if fails"      , True)]      __description__ = """Multi hoster plugin"""      __license__     = "GPLv3" @@ -66,47 +66,13 @@ class MultiHoster(SimpleHoster):          self.direct_dl = direct_dl -    def process(self, pyfile): +    def _process(self, thread):          try: -            self.prepare() -            self.check_info()  #@TODO: Remove in 0.4.10 +            super(MultiHoster, self)._process(thread) -            if self.direct_dl: -                self.log_info(_("Looking for direct download link...")) -                self.handle_direct(pyfile) - -                if self.link or was_downloaded(): -                    self.log_info(_("Direct download link detected")) -                else: -                    self.log_info(_("Direct download link not found")) - -            if not self.link and not self.last_download: -                self.preload() - -                self.check_errors() -                self.check_status(getinfo=False) - -                if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): -                    self.log_info(_("Processing as premium download...")) -                    self.handle_premium(pyfile) - -                elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()): -                    self.log_info(_("Processing as free download...")) -                    self.handle_free(pyfile) - -            if not self.last_download: -                self.log_info(_("Downloading file...")) -                self.download(self.link, disposition=self.DISPOSITION) - -            self.check_download() - -        except Fail, e:  #@TODO: Move to PluginThread in 0.4.10 -            if self.premium: -                self.log_warning(_("Premium download failed")) -                self.restart(premium=False) - -            elif self.get_config("revertfailed", True) and \ -                 self.pyload.pluginManager.hosterPlugins[self.classname].get('new_module'): +        except Fail, e: +            if self.get_config("revertfailed", True) and \ +               self.pyload.pluginManager.hosterPlugins[self.classname].get('new_module'):                  hdict = self.pyload.pluginManager.hosterPlugins[self.classname]                  tmp_module = hdict['new_module'] @@ -122,7 +88,7 @@ class MultiHoster(SimpleHoster):                  self.restart(_("Revert to original hoster plugin"))              else: -                raise Fail(encode(e))  #@TODO: Remove `encode` in 0.4.10 +                raise Fail(e)      def handle_premium(self, pyfile): diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index f5db49d8b..0b5561df8 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -2,284 +2,19 @@  from __future__ import with_statement -import datetime  import inspect  import os -import re -import sys -import time -import traceback -import urllib -import urlparse - -import pycurl  if os.name is not "nt":      import grp      import pwd -from module.common.json_layer import json_dumps, json_loads -from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip  #@TODO: Remove in 0.4.10 -from module.utils import (fs_encode, fs_decode, get_console_encoding, html_unescape, -                          parseFileSize as parse_size, save_join as fs_join) - - -#@TODO: Move to utils in 0.4.10 -def isiterable(obj): -    return hasattr(obj, "__iter__") - - -#@TODO: Move to utils in 0.4.10 -def decode(string, encoding=None): -    """Encoded string (default to UTF-8) -> unicode string""" -    if type(string) is str: -        try: -            res = unicode(string, encoding or "utf-8") - -        except UnicodeDecodeError, e: -            if encoding: -                raise UnicodeDecodeError(e) - -            encoding = get_console_encoding(sys.stdout.encoding) -            res = unicode(string, encoding) - -    elif type(string) is unicode: -        res = string - -    else: -        res = unicode(string) - -    return res - - -#@TODO: Remove in 0.4.10 -def _decode(*args, **kwargs): -    return decode(*args, **kwargs) - - -#@TODO: Move to utils in 0.4.10 -def encode(string, encoding=None, decoding=None): -    """Unicode or decoded string -> encoded string (default to UTF-8)""" -    if type(string) is unicode: -        res = string.encode(encoding or "utf-8") - -    elif type(string) is str: -        res = encode(decode(string, decoding), encoding) - -    else: -        res = str(string) - -    return res - - -#@TODO: Move to utils in 0.4.10 -def exists(path): -    if os.path.exists(path): -        if os.name is "nt": -            dir, name = os.path.split(path.rstrip(os.sep)) -            return name in os.listdir(dir) -        else: -            return True -    else: -        return False - - -def fixurl(url, unquote=None): -    old = url -    url = urllib.unquote(url) - -    if unquote is None: -        unquote = url is old - -    url = html_unescape(decode(url).decode('unicode-escape')) -    url = re.sub(r'(?<!:)/{2,}', '/', url).strip().lstrip('.') - -    if not unquote: -        url = urllib.quote(url) - -    return url - - -def parse_name(string): -    path  = fixurl(decode(string), unquote=False) -    url_p = urlparse.urlparse(path.rstrip('/')) -    name  = (url_p.path.split('/')[-1] or -             url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or -             url_p.netloc.split('.', 1)[0]) - -    return urllib.unquote(name) - - -#@TODO: Move to utils in 0.4.10 -def str2int(string): -    try: -        return int(string) -    except: -        pass - -    ones = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", -            "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", -            "sixteen", "seventeen", "eighteen", "nineteen"] -    tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", -            "eighty", "ninety"] - -    o_tuple = [(w, i) for i, w in enumerate(ones)] -    t_tuple = [(w, i * 10) for i, w in enumerate(tens)] - -    numwords = dict(o_tuple + t_tuple) -    tokens   = re.split(r"[\s\-]+", string.lower()) - -    try: -        return sum(numwords[word] for word in tokens) -    except: -        return 0 - - -def parse_time(string): -    if re.search("da(il)?y|today", string): -        seconds = seconds_to_midnight() - -    else: -        regex   = re.compile(r'(\d+| (?:this|an?) )\s*(hr|hour|min|sec|)', re.I) -        seconds = sum((int(v) if v.strip() not in ("this", "a", "an") else 1) * -                      {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, '': 1}[u.lower()] -                      for v, u in regex.findall(string)) -    return seconds - - -#@TODO: Move to utils in 0.4.10 -def timestamp(): -    return int(time.time() * 1000) - - -#@TODO: Move to utils in 0.4.10 -def which(program): -    """ -    Works exactly like the unix command which -    Courtesy of http://stackoverflow.com/a/377028/675646 -    """ -    isExe = lambda x: os.path.isfile(x) and os.access(x, os.X_OK) - -    fpath, fname = os.path.split(program) - -    if fpath: -        if isExe(program): -            return program -    else: -        for path in os.environ['PATH'].split(os.pathsep): -            exe_file = os.path.join(path.strip('"'), program) -            if isExe(exe_file): -                return exe_file - - -#@TODO: Move to utils in 0.4.10 -def format_exc(frame=None): -    """ -    Format call-stack and display exception information (if availible) -    """ -    exception_info = sys.exc_info() -    callstack_list = traceback.extract_stack(frame) -    callstack_list = callstack_list[:-1] - -    exception_desc = "" -    if exception_info[0] is not None: -        exception_callstack_list = traceback.extract_tb(exception_info[2]) -        if callstack_list[-1][0] == exception_callstack_list[0][0]:  #Does this exception belongs to us? -            callstack_list = callstack_list[:-1] -            callstack_list.extend(exception_callstack_list) -            exception_desc = "".join(traceback.format_exception_only(exception_info[0], exception_info[1])) - -    traceback_str = "Traceback (most recent call last):\n" -    traceback_str += "".join(traceback.format_list(callstack_list)) -    traceback_str += exception_desc -    traceback_str = traceback_str[:-1]  #Removing the last '\n' -    return traceback_str - -def seconds_to_nexthour(strict=False): -    now      = datetime.datetime.today() -    nexthour = now.replace(minute=0 if strict else 1, second=0, microsecond=0) + datetime.timedelta(hours=1) -    return (nexthour - now).seconds - - -def seconds_to_midnight(utc=None, strict=False): -    if utc is None: -        now = datetime.datetime.today() -    else: -        now = datetime.datetime.utcnow() + datetime.timedelta(hours=utc) - -    midnight = now.replace(hour=0, minute=0 if strict else 1, second=0, microsecond=0) + datetime.timedelta(days=1) - -    return (midnight - now).seconds - - -def replace_patterns(string, ruleslist): -    for r in ruleslist: -        rf, rt = r -        string = re.sub(rf, rt, string) -    return string - - -#@TODO: Remove in 0.4.10 and fix CookieJar.setCookie -def set_cookie(cj, domain, name, value): -    return cj.setCookie(domain, name, encode(value)) - - -def set_cookies(cj, cookies): -    for cookie in cookies: -        if isinstance(cookie, tuple) and len(cookie) == 3: -            set_cookie(cj, *cookie) - - -def parse_html_tag_attr_value(attr_name, tag): -    m = re.search(r"%s\s*=\s*([\"']?)((?<=\")[^\"]+|(?<=')[^']+|[^>\s\"'][^>\s]*)\1" % attr_name, tag, re.I) -    return m.group(2) if m else None - - -def parse_html_form(attr_str, html, input_names={}): -    for form in re.finditer(r"(?P<TAG><form[^>]*%s[^>]*>)(?P<CONTENT>.*?)</?(form|body|html)[^>]*>" % attr_str, -                            html, re.I | re.S): -        inputs = {} -        action = parse_html_tag_attr_value("action", form.group('TAG')) - -        for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.I | re.S): -            name = parse_html_tag_attr_value("name", inputtag.group(1)) -            if name: -                value = parse_html_tag_attr_value("value", inputtag.group(1)) -                if not value: -                    inputs[name] = inputtag.group(3) or "" -                else: -                    inputs[name] = value - -        if not input_names: -            #: No attribute check -            return action, inputs -        else: -            #: Check input attributes -            for key, val in input_names.items(): -                if key in inputs: -                    if isinstance(val, basestring) and inputs[key] is val: -                        continue -                    elif isinstance(val, tuple) and inputs[key] in val: -                        continue -                    elif hasattr(val, "search") and re.match(val, inputs[key]): -                        continue -                    else: -                        break  #: Attibute value does not match -                else: -                    break  #: Attibute name does not match -            else: -                return action, inputs  #: Passed attribute check - -    return {}, None  #: No matching form found +import pycurl +import module.plugins.internal.utils as utils -#@TODO: Move to utils in 0.4.10 -def chunks(iterable, size): -    it   = iter(iterable) -    item = list(islice(it, size)) -    while item: -        yield item -        item = list(islice(it, size)) +from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip  #@TODO: Remove in 0.4.10 +from module.plugins.internal.utils import *  class Plugin(object): @@ -336,38 +71,37 @@ class Plugin(object):      def log_debug(self, *args, **kwargs):          self._log("debug", self.__type__, self.__name__, args) -        if self.pyload.debug and kwargs.get('trace', False): -            self.log_exc("debug") +        if self.pyload.debug and kwargs.get('trace'): +            self.print_exc()      def log_info(self, *args, **kwargs):          self._log("info", self.__type__, self.__name__, args) -        if kwargs.get('trace', False): -            self.log_exc("info") +        if self.pyload.debug and kwargs.get('trace'): +            self.print_exc()      def log_warning(self, *args, **kwargs):          self._log("warning", self.__type__, self.__name__, args) -        if kwargs.get('trace', False): -            self.log_exc("warning") +        if self.pyload.debug and kwargs.get('trace'): +            self.print_exc()      def log_error(self, *args, **kwargs):          self._log("error", self.__type__, self.__name__, args) -        if kwargs.get('trace', False): -            self.log_exc("error") +        if self.pyload.debug and kwargs.get('trace', True): +            self.print_exc()      def log_critical(self, *args, **kwargs):          self._log("critical", self.__type__, self.__name__, args)          if kwargs.get('trace', True): -            self.log_exc("critical") +            self.print_exc() -    def log_exc(self, level): +    def print_exc(self):          frame = inspect.currentframe() -        log = getattr(self.pyload.log, level) -        log(format_exc(frame.f_back)) +        print format_exc(frame.f_back)          del frame @@ -427,7 +161,7 @@ class Plugin(object):          Saves a value persistently to the database          """          value = map(decode, value) if isiterable(value) else decode(value) -        entry = json_dumps(value).encode('base64') +        entry = json.dumps(value).encode('base64')          self.pyload.db.setStorage(self.classname, key, entry) @@ -441,12 +175,12 @@ class Plugin(object):              if entry is None:                  value = default              else: -                value = json_loads(entry.decode('base64')) +                value = json.loads(entry.decode('base64'))          else:              if not entry:                  value = default              else: -                value = dict((k, json_loads(v.decode('base64'))) for k, v in value.items()) +                value = dict((k, json.loads(v.decode('base64'))) for k, v in value.items())          return value @@ -506,8 +240,8 @@ class Plugin(object):              req.http.c.setopt(pycurl.FOLLOWLOCATION, 1)          elif type(redirect) is int: -            req.http.c.setopt(pycurl.MAXREDIRS, -                              self.get_config("maxredirs", 5, plugin="UserAgentSwitcher")) +            maxredirs = self.get_config("maxredirs", default=5, plugin="UserAgentSwitcher") +            req.http.c.setopt(pycurl.MAXREDIRS, maxredirs)          #@TODO: Move to network in 0.4.10          if decode: @@ -515,7 +249,7 @@ class Plugin(object):          #@TODO: Move to network in 0.4.10          if isinstance(decode, basestring): -            html = _decode(html, decode)  #@NOTE: Use `utils.decode()` in 0.4.10 +            html = utils.decode(html, decode)          self.last_html = html @@ -544,13 +278,15 @@ class Plugin(object):          else:              #@TODO: Move to network in 0.4.10              header = {'code': req.code} +              for line in html.splitlines():                  line = line.strip()                  if not line or ":" not in line:                      continue                  key, none, value = line.partition(":") -                key = key.strip().lower() + +                key   = key.strip().lower()                  value = value.strip()                  if key in header: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index a4e01249e..4ab305f20 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -9,8 +9,10 @@ import time  from module.network.HTTPRequest import BadHeader  from module.network.RequestFactory import getURL as get_url  from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, encode, parse_name, parse_size, parse_time, replace_patterns, seconds_to_midnight, set_cookie, set_cookies -from module.utils import fixup, fs_encode +from module.plugins.internal.Plugin import Fail +from module.plugins.internal.utils import (encode, fixup, parse_name, parse_size, +                                           parse_time, replace_patterns, seconds_to_midnight, +                                           set_cookie, set_cookies)  class SimpleHoster(Hoster): @@ -20,10 +22,11 @@ class SimpleHoster(Hoster):      __status__  = "testing"      __pattern__ = r'^unmatchable$' -    __config__  = [("activated"       , "bool", "Activated"                                 , True), -                   ("use_premium"     , "bool", "Use premium account if available"          , True), -                   ("fallback_premium", "bool", "Fallback to free download if premium fails", True), -                   ("chk_filesize"    , "bool", "Check file size"                           , True)] +    __config__  = [("activated"   , "bool", "Activated"                                        , True), +                   ("use_premium" , "bool", "Use premium account if available"                 , True), +                   ("fallback"    , "bool", "Fallback to free download if premium fails"       , True), +                   ("chk_filesize", "bool", "Check file size"                                  , True), +                   ("max_wait"    , "int" , "Reconnect if waiting time is greater than minutes", 10  )]      __description__ = """Simple hoster plugin"""      __license__     = "GPLv3" @@ -133,7 +136,6 @@ class SimpleHoster(Hoster):      @classmethod      def get_info(cls, url="", html=""):          info = super(SimpleHoster, cls).get_info(url) -          info.update(cls.api_info(url))          if not html and info['status'] is not 2: @@ -148,7 +150,7 @@ class SimpleHoster(Hoster):                  except BadHeader, e:                      info['error'] = "%d: %s" % (e.code, e.content) -                    if e.code is 404: +                    if e.code in (404, 410):                          info['status'] = 1                      elif e.code is 503: @@ -200,7 +202,8 @@ class SimpleHoster(Hoster):      def setup(self): -        self.resume_download = self.multiDL = self.premium +        self.multiDL         = self.premium +        self.resume_download = self.premium      def prepare(self): @@ -249,7 +252,6 @@ class SimpleHoster(Hoster):      def process(self, pyfile):          self.prepare() -        self.check_info()  #@TODO: Remove in 0.4.10          if self.leech_dl:              self.log_info(_("Processing as debrid download...")) @@ -271,8 +273,8 @@ class SimpleHoster(Hoster):              if not self.link and not self.last_download:                  self.preload() -                if self.info.get('status', 3) is 3:  #@TODO: Recheck in 0.4.10 -                    self.check_info() +                if self.info.get('status', 3) is not 2: +                    self.grab_info()                  if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()):                      self.log_info(_("Processing as premium download...")) @@ -286,12 +288,15 @@ class SimpleHoster(Hoster):              self.log_info(_("Downloading file..."))              self.download(self.link, disposition=self.DISPOSITION) + +    def _check_download(self): +        super(SimpleHoster, self)._check_download()          self.check_download()      def check_download(self): -        self.log_info(_("Checking downloaded file...")) -        self.log_debug("Using default check rules...") +        self.log_debug("Performing default check rules...") +          for r, p in self.FILE_ERRORS:              errmsg = self.check_file({r: re.compile(p)})              if errmsg is not None: @@ -308,12 +313,12 @@ class SimpleHoster(Hoster):                  self.restart(errmsg)          else:              if self.CHECK_FILE: -                self.log_debug("Using custom check rules...") -                with open(fs_encode(self.last_download), "rb") as f: +                self.log_debug("Performing custom check rules...") + +                with open(encode(self.last_download), "rb") as f:                      self.html = f.read(1048576)  #@TODO: Recheck in 0.4.10 -                self.check_errors() -        self.log_info(_("No errors found")) +                self.check_errors()      def check_errors(self): @@ -346,7 +351,7 @@ class SimpleHoster(Hoster):                  self.log_warning(errmsg)                  wait_time = parse_time(errmsg) -                self.wait(wait_time, reconnect=wait_time > 300) +                self.wait(wait_time, reconnect=wait_time > self.get_config("max_wait", 10) * 60)                  self.restart(_("Download limit exceeded"))          if self.HAPPY_HOUR_PATTERN and re.search(self.HAPPY_HOUR_PATTERN, self.html): @@ -369,7 +374,7 @@ class SimpleHoster(Hoster):                  if re.search('limit|wait|slot', errmsg, re.I):                      wait_time = parse_time(errmsg) -                    self.wait(wait_time, reconnect=wait_time > 300) +                    self.wait(wait_time, reconnect=wait_time > self.get_config("max_wait", 10) * 60)                      self.restart(_("Download limit exceeded"))                  elif re.search('country|ip|region|nation', errmsg, re.I): @@ -410,80 +415,20 @@ class SimpleHoster(Hoster):                      waitmsg = m.group(0).strip()                  wait_time = parse_time(waitmsg) -                self.wait(wait_time, reconnect=wait_time > 300) +                self.wait(wait_time, reconnect=wait_time > self.get_config("max_wait", 10) * 60)          self.info.pop('error', None) -    def check_status(self, getinfo=True): -        if not self.info or getinfo: -            self.log_info(_("Updating file info...")) -            old_info = self.info.copy() -            self.info.update(self.get_info(self.pyfile.url, self.html)) -            self.log_debug("File info: %s" % self.info) -            self.log_debug("Previous file info: %s" % old_info) - -        try: -            status = self.info['status'] or 14 - -            if status is 1: -                self.offline() - -            elif status is 6: -                self.temp_offline() - -            elif status is 8: -                self.fail() - -        finally: -            self.log_info(_("File status: ") + self.pyfile.getStatusName()) - - -    def check_name_size(self, getinfo=True): -        if not self.info or getinfo: -            self.log_info(_("Updating file info...")) -            old_info = self.info.copy() -            self.info.update(self.get_info(self.pyfile.url, self.html)) -            self.log_debug("File info: %s" % self.info) -            self.log_debug("Previous file info: %s" % old_info) - -        name = self.info.get('name') -        size = self.info.get('size') - -        if name and name is not self.info.get('url'): -            self.pyfile.name = name -        else: -            name = self.pyfile.name - -        if size > 0: -            self.pyfile.size = int(self.info['size'])  #@TODO: Fix int conversion in 0.4.10 -        else: -            size = self.pyfile.size - -        self.log_info(_("File name: ") + name) -        self.log_info(_("File size: %s bytes") % size or "N/D") - - -    #@TODO: Rewrite in 0.4.10 -    def check_info(self): -        self.check_name_size() - -        if self.html: -            self.check_errors() -            self.check_name_size() - -        self.check_status(getinfo=False) - -      #: Deprecated method (Remove in 0.4.10)      def get_fileInfo(self): -        self.info = {} -        self.check_info() +        self.info.clear() +        self.grab_info()          return self.info      def handle_direct(self, pyfile): -        self.link = self.direct_link(pyfile.url, self.resume_download) +        self.link = self.is_download(pyfile.url)      def handle_multi(self, pyfile):  #: Multi-hoster handler @@ -492,7 +437,7 @@ class SimpleHoster(Hoster):      def handle_free(self, pyfile):          if not self.LINK_FREE_PATTERN: -            self.log_error(_("Free download not implemented")) +            self.log_warning(_("Free download not implemented"))          m = re.search(self.LINK_FREE_PATTERN, self.html)          if m is None: @@ -503,7 +448,7 @@ class SimpleHoster(Hoster):      def handle_premium(self, pyfile):          if not self.LINK_PREMIUM_PATTERN: -            self.log_error(_("Premium download not implemented")) +            self.log_warning(_("Premium download not implemented"))              self.restart(premium=False)          m = re.search(self.LINK_PREMIUM_PATTERN, self.html) | 
