new ul plugin

6 files changed, 118 insertions, 112 deletions

diff --git a/module/Utils.py b/module/Utils.py
index 6f889118d..3d344a147 100644
--- a/module/Utils.py
+++ b/module/Utils.py
@@ -5,6 +5,7 @@
 import os
 import sys
 import time
+import re
 from os.path import join
 
 def chmod(*args):
@@ -84,5 +85,22 @@ def uniqify(seq, idfun=None):
         result.append(item)
     return result
 
+def parseFileSize(string): #returns bytes
+    string = string.strip().lower()
+    p = re.compile(r"(\d+[\.,]\d+)(.*)")
+    m = p.match(string)
+    if m:
+        traffic = float(m.group(1).replace(",", "."))
+        unit = m.group(2).strip()
+        if unit in ("gb", "gig", "gbyte", "gigabyte", "gib"):
+            traffic *= 1 << 30
+        elif unit in ("mb", "mbyte", "megabyte", "mib"):
+            traffic *= 1 << 20
+        elif unit in ("kb", "kib", "kilobyte", "kbyte"):
+            traffic *= 1 << 10
+        return traffic
+
+    return 0
+
 if __name__ == "__main__":
     print freeSpace(".")
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index 6c8c47fb7..dcc51c516 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -90,6 +90,7 @@ class Plugin(object):
 
         self.ocr = None  # captcha reader instance
         self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) # account handler instance
+        self.premium = False
         if self.account and not self.account.canUse(): self.account = None
         if self.account:
             self.user, data = self.account.selectAccount()
diff --git a/module/plugins/accounts/UploadedTo.py b/module/plugins/accounts/UploadedTo.py
index 42dcb2d29..e0782e3e4 100644
--- a/module/plugins/accounts/UploadedTo.py
+++ b/module/plugins/accounts/UploadedTo.py
@@ -19,7 +19,7 @@
 
 from module.plugins.Account import Account
 import re
-from time import strptime, mktime
+from time import time
 
 class UploadedTo(Account):
     __name__ = "UploadedTo"
@@ -30,16 +30,28 @@ class UploadedTo(Account):
     __author_mail__ = ("mkaay@mkaay.de")
     
     def loadAccountInfo(self, user, req):
-        html = req.load("http://uploaded.to/?setlang=en", cookies=True)
-        raw_traffic = re.search(r"Traffic left: </span><span class=.*?>(.*?)</span>", html).group(1)
-        raw_valid = re.search(r"Valid until: </span> <span class=.*?>(.*?)</span>", html).group(1)
-        traffic = int(self.parseTraffic(raw_traffic))
-        validuntil = int(mktime(strptime(raw_valid.strip(), "%d-%m-%Y %H:%M")))
-    
-        tmp =  {"validuntil":validuntil, "trafficleft":traffic, "maxtraffic":50*1024*1024}
-        return tmp
+        html = req.load("http://uploaded.to/me", cookies=True)
+        open("ul.html", "wb").write(html)
+
+        premium = '<a href="me#premium"><em>Premium</em>' in html
+
+        if premium:
+            raw_traffic = re.search(r'<th colspan="2"><b class="cB">([^<]+)', html).group(1)
+            raw_valid = re.search(r"<td>Duration:</td>\s*<th>([^<]+)", html, re.MULTILINE).group(1)
+            raw_valid = re.findall(r"\d+", raw_valid)
+
+            traffic = int(self.parseTraffic(raw_traffic))
+
+            validuntil = time() + 24 * 60 * 60 * int(raw_valid[0]) + 60 * 60 * int(raw_valid[1])
+
+            return {"validuntil":validuntil, "trafficleft":traffic, "maxtraffic":50*1024*1024}
+        else:
+            return {"premium" : False}
 
     def login(self, user, data, req):
-        page = req.load("http://uploaded.to/login", post={ "email" : user, "password" : data["password"]})
-        if "Login failed!" in page:
+        page = req.load("http://uploaded.to/io/login", post={ "id" : user, "pw" : data["password"], "_" : ""})
+        if "User and password do not match!" in page:
             self.wrongPassword()
+
+        req.load("http://uploaded.to/language/en")
+        req.cj.setCookie(".uploaded.to", "lang", "en")
diff --git a/module/plugins/hoster/UploadedTo.py b/module/plugins/hoster/UploadedTo.py
index aa595be2c..a725bb980 100644
--- a/module/plugins/hoster/UploadedTo.py
+++ b/module/plugins/hoster/UploadedTo.py
@@ -5,6 +5,7 @@ from module.plugins.Hoster import Hoster
 from module.network.RequestFactory import getURL
 from module.plugins.Plugin import chunks
 from module.plugins.ReCaptcha import ReCaptcha
+from module.utils import parseFileSize
 
 def getInfo(urls):
     pattern = re.compile(UploadedTo.__pattern__)
@@ -33,126 +34,95 @@ class UploadedTo(Hoster):
     
     def setup(self):
         self.html = None
-        self.api_data = None
+        self.data = {}
         self.multiDL = False
         self.url = False
         if self.account:
             self.multiDL = True
             self.chunkLimit = -1
             self.resumeDownload = True
-        
+
+
+        self.pyfile.url = self.cleanUrl(self.pyfile.url)
+        self.fileID = re.search(self.__pattern__, self.pyfile.url).group(1)
 
     def process(self, pyfile):
-        self.url = self.get_file_url()
         self.req.cj.setCookie("uploaded.to", "lang", "en")
-        self.download_html()
 
-        if not self.file_exists():
+        self.html = self.load(self.pyfile.url, cookies=False, utf8=True)
+
+        if re.search(r"(File doesn't exist)", self.html) is not None:
             self.offline()
 
-        self.download_api_data()
+        self.parseInfos()
+        pyfile.name = self.data["name"]
 
         # self.pyfile.name = self.get_file_name()
+        self.premium = self.account.getAccountInfo(self.user)
 
-        if self.account:
-            info = self.account.getAccountInfo(self.user, True)
-            self.log.debug("%(name)s: Use Premium Account (%(left)sGB left)" % {"name" :self.__name__, "left" : info["trafficleft"]/1024/1024})
-            if self.api_data["size"]/1024 > info["trafficleft"]:
-                self.log.info(_("%s: Not enough traffic left" % self.__name__))
-                self.account.empty()
-                self.resetAccount()
-                self.fail(_("Traffic exceeded"))
-            else:
-                self.url = self.get_file_url()
-                pyfile.name = self.get_file_name()
-                self.download(self.url+"?redirect", cookies=True)
-
-            return True
-
-        wait = self.get_waiting_time()
-        if wait:
-            self.setWait(wait, True)
-            self.wait()
-            self.process(pyfile)
-            return
+        if self.account and self.premium:
+            self.handlePremium()
         else:
-            self.setWait(30, False)
+            self.handleFree()
 
-        time = re.search(r'name="time" value="([^"]+)', self.html).group(1)
-        time_secure = re.search(r'name="time_secure" value="([^"]+)', self.html).group(1)
-        file_password = re.search(r'name="file_password" value="([^"]*)', self.html).group(1)
 
-        challenge = re.search(r"recaptcha/api/challenge\?k=([0-9A-Za-z]+)", self.html)
+    def handlePremium(self):
+        info = self.account.getAccountInfo(self.user, True)
+        self.log.debug("%(name)s: Use Premium Account (%(left)sGB left)" % {"name" :self.__name__, "left" : info["trafficleft"]/1024/1024})
+        if self.data["size"]/1024 > info["trafficleft"]:
+            self.log.info(_("%s: Not enough traffic left" % self.__name__))
+            self.account.empty()
+            self.resetAccount()
+            self.fail(_("Traffic exceeded"))
 
-        options = {"time": time, "time_secure": time_secure, "file_password": file_password}
+        html = self.load(self.pyfile.url, utf8=True)
+        url = re.search(r'action="(http://.*\.uploaded.to/dl\?id=[^"]+)', html)
+        url = url.group(1)
+        self.download(url)
 
-        if challenge:
-            self.req.lastURL = str(self.url)
-            re_captcha = ReCaptcha(self)
-            challenge, result = re_captcha.challenge(challenge.group(1))
-            options["recaptcha_challenge_field"] = challenge
-            options["recaptcha_response_field"] = result
 
-        self.wait()
+    def handleFree(self):
 
-        pyfile.name = self.get_file_name()
+        wait = re.search(r"Current waiting period: <span>(\d+)</span> seconds", self.html).group(1)
+        self.setWait(wait)
 
-        self.download(self.url, post=options)
+        js = self.load("http://uploaded.to/js/download.js")
 
-        check = self.checkDownload({"wrong_captcha": "Wrong captcha."})
-        if check == "wrong_captcha":
-            self.invalidCaptcha()
-            self.process(pyfile)
+        challengeId = re.search(r'Recaptcha\.create\("([^"]+)', js)
+
+        url = "http://uploaded.to/io/ticket/captcha/%s" % self.fileID
+        downloadURL = ""
+
+        for i in range(5):
+            self.req.lastURL = str(self.url)
+            re_captcha = ReCaptcha(self)
+            challenge, result = re_captcha.challenge(challengeId.group(1))
+            options = {"recaptcha_challenge_field" : challenge, "recaptcha_response_field": result}
+            self.wait()
+
+            result = self.load(url, post=options)
+            self.log.debug("UploadedTo result: %s" % result)
+
+            if "limit-dl" in result:
+                self.setWait(30 * 60, True)
+                self.wait()
+                self.retry()
+            elif 'err:"captcha"' in result:
+                self.invalidCaptcha()
+            elif "type:'download'" in result:
+                downloadURL = re.search("url:'([^']+)", result).group(1)
+                break
+                
+        if not downloadURL:
+            self.fail("No Download url retrieved")
+
+        self.download(downloadURL)
         
-    def download_api_data(self, force=False):
-        if self.api_data and not force:
-            return
-        match = re.compile(self.__pattern__).search(self.pyfile.url)
-        if match:
-            src = self.load("http://uploaded.to/api/file", cookies=False, get={"id": match.group(1).split("/")[0]}).decode("utf8", "ignore")
-            if not src.find("404 Not Found"):
-                return
-            self.api_data = {}
-            lines = src.splitlines()
-            self.log.debug("Uploaded API: %s" % lines)
-            self.api_data["filename"] = lines[0]
-            self.api_data["size"] = int(lines[1]) # in bytes
-            self.api_data["checksum"] = lines[2] #sha1
-
-    def download_html(self):
-        self.html = self.load(self.pyfile.url, cookies=False).decode("utf8", "ignore")
-
-    def get_waiting_time(self):
-        try:
-            wait_minutes = re.search(r"Or wait ([\d\-]+) minutes", self.html).group(1)
-            if int(wait_minutes) < 0: wait_minutes = 1            
-            self.wantReconnect = True
-            return 60 * int(wait_minutes)
-        except:
-            return 0
-
-    def get_file_url(self):
-        return self.cleanUrl(self.pyfile.url)
-
-    def get_file_name(self):
-        try:
-            if self.api_data and self.api_data["filename"]:
-                return self.api_data["filename"]
-            file_name = re.search(r"<td><b>\s+(.+)\s", self.html).group(1)
-            file_suffix = re.search(r"</td><td>(\..+)</td></tr>", self.html)
-            if not file_suffix:
-                return file_name
-            return file_name + file_suffix.group(1)
-        except:
-            return self.pyfile.url.split('/')[-1]
-
-    def file_exists(self):
-        if re.search(r"(File doesn't exist)", self.html) is not None:
-            return False
-        else:
-            return True
+    def parseInfos(self):
+        match = re.search(r'id="filename">([^<]+)</a><br /><small>([^<]+)', self.html)
+        self.data = {"name" : match.group(1),
+                         "size": parseFileSize(match.group(2))}
 
-    
     def cleanUrl(self, url):
         url = url.replace("ul.to/", "uploaded.to/file/")
         url = url.replace("/?id=", "/file/")
diff --git a/module/setup.py b/module/setup.py
index 920323adb..75884ea13 100644
--- a/module/setup.py
+++ b/module/setup.py
@@ -154,7 +154,7 @@ class Setup():
         
         
         print ""
-        print _("Do you want to configure basic settings?")
+        print _("Do you want to configure login data and basic settings?")
         print _("This is recommend for first run.")
         con = self.ask(_("Make basic setup?"), "y", bool=True)
 
@@ -239,8 +239,9 @@ class Setup():
         try:
             import jinja2
             if jinja2.__version__ and "unknown" not in jinja2.__version__ and not jinja2.__version__.startswith("2.5"): #@TODO: could be to new aswell
-                print _("Your installed jinja2 version %s is too old.") % jinja2.__version__
-                print _("Please upgrade or deinstall it, pyLoad includes a sufficient jinja2 libary.")
+                print _("Your installed jinja2 version %s seems too old.") % jinja2.__version__
+                print _("You can safely continue but if the webinterface is not working,")
+                print _("please upgrade or deinstall it, pyLoad includes a sufficient jinja2 libary.")
                 print 
                 jinja = False
         except :
@@ -252,7 +253,7 @@ class Setup():
         self.print_dep("beaker", beaker)
 
 
-        web = sqlite and jinja and beaker
+        web = sqlite and beaker
 
         from module import JsEngine
 
@@ -302,7 +303,11 @@ class Setup():
         print _("Listen address, if you use 127.0.0.1 or localhost, the webinterface will only accessible locally.")
         self.config["webinterface"]["host"] = self.ask(_("Address"), "0.0.0.0")
         self.config["webinterface"]["port"] = self.ask(_("Port"), "8000")
-        #@TODO setup for additional webservers
+        print ""
+        print _("In some rare cases the builtin server is not working, if you notice problems with the webinterface")
+        print _("change the builtin server to the threaded one here.")
+
+        self.config["webinterface"]["server"] = self.ask(_("Server"), "builtin", ["builtin", "threaded"])
 
     def conf_ssl(self):
         print ""
diff --git a/module/web/json_app.py b/module/web/json_app.py
index ac1f3ec9c..df58238ec 100644
--- a/module/web/json_app.py
+++ b/module/web/json_app.py
@@ -211,7 +211,7 @@ def link_order(ids):
 @route("/json/add_package", method="POST")
 @login_required('add')
 def add_package():
-    name = request.forms.get("add_name", "New Package")
+    name = request.forms.get("add_name", "New Package").strip()
     queue = int(request.forms['add_dest'])
     links = request.forms['add_links'].decode("utf8", "ignore")
     links = links.split("\n")
@@ -220,7 +220,7 @@ def add_package():
     try:
         f = request.files['add_file']
 
-        if name == "New Package":
+        if not name or name == "New Package":
             name = f.name
 
         fpath = join(PYLOAD.get_conf_val("general", "download_folder"), "tmp_" + f.filename)