diff options
Diffstat (limited to 'pyload')
35 files changed, 623 insertions, 449 deletions
diff --git a/pyload/AccountManager.py b/pyload/AccountManager.py index b9f1536d9..e2a1b28a6 100644 --- a/pyload/AccountManager.py +++ b/pyload/AccountManager.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- ############################################################################### -# Copyright(c) 2008-2013 pyLoad Team +# Copyright(c) 2008-2014 pyLoad Team # http://www.pyload.org # # This file is part of pyLoad. @@ -13,9 +13,10 @@ # # Subjected to the terms and conditions in LICENSE # -# @author: RaNaN, mkaay +# @author: RaNaN ############################################################################### + from threading import Lock from random import choice @@ -81,45 +82,55 @@ class AccountManager: data = [] for plugin, accounts in self.accounts.iteritems(): data.extend( - [(plugin, acc.loginname, acc.owner, 1 if acc.activated else 0, 1 if acc.shared else 0, acc.password, - json.dumps(acc.options)) for acc in + [(acc.loginname, 1 if acc.activated else 0, 1 if acc.shared else 0, acc.password, + json.dumps(acc.options), acc.aid) for acc in accounts]) self.core.db.saveAccounts(data) - def getAccount(self, plugin, loginname, user=None): + def getAccount(self, aid, plugin, user=None): """ Find a account by specific user (if given) """ if plugin in self.accounts: for acc in self.accounts[plugin]: - if acc.loginname == loginname and (not user or acc.owner == user.true_primary): + if acc.aid == aid and (not user or acc.owner == user.true_primary): return acc @lock - def updateAccount(self, plugin, loginname, password, user): + def createAccount(self, plugin, loginname, password, uid): + """ Creates a new account """ + + aid = self.core.db.createAccount(plugin, loginname, password, uid) + info = AccountInfo(aid, plugin, loginname, uid, activated=True) + account = self._createAccount(info, password, {}) + account.scheduleRefresh() + self.saveAccounts() + + self.core.eventManager.dispatchEvent("account:created", account.toInfoData()) + return account + + @lock + def updateAccount(self, aid, plugin, loginname, password, user): """add or update account""" - account = self.getAccount(plugin, loginname, user) - if account: - if account.setPassword(password): - self.saveAccounts() - account.scheduleRefresh(force=True) - else: - info = AccountInfo(plugin, loginname, user.true_primary, activated=True) - account = self._createAccount(info, password, {}) - account.scheduleRefresh() + account = self.getAccount(aid, plugin, user) + if not account: + return + + if account.setLogin(loginname, password): self.saveAccounts() + account.scheduleRefresh(force=True) self.core.eventManager.dispatchEvent("account:updated", account.toInfoData()) return account @lock - def removeAccount(self, plugin, loginname, uid): + def removeAccount(self, aid, plugin, uid): """remove account""" if plugin in self.accounts: for acc in self.accounts[plugin]: # admins may delete accounts - if acc.loginname == loginname and (not uid or acc.owner == uid): + if acc.aid == aid and (not uid or acc.owner == uid): self.accounts[plugin].remove(acc) - self.core.db.removeAccount(plugin, loginname) - self.core.evm.dispatchEvent("account:deleted", plugin, loginname) + self.core.db.removeAccount(aid) + self.core.evm.dispatchEvent("account:deleted", aid, user=uid) break @lock diff --git a/pyload/Core.py b/pyload/Core.py index 6f3893481..04a77f45f 100644 --- a/pyload/Core.py +++ b/pyload/Core.py @@ -388,7 +388,7 @@ class Core(object): self.interactionManager = self.im = InteractionManager(self) self.accountManager = AccountManager(self) self.threadManager = ThreadManager(self) - self.downloadManager = DownloadManager(self) + self.downloadManager = self.dlm = DownloadManager(self) self.addonManager = AddonManager(self) self.remoteManager = RemoteManager(self) @@ -438,7 +438,8 @@ class Core(object): self.log.info(_("Restarting failed downloads...")) self.api.restartFailed() - self.threadManager.pause = False + # start downloads + self.dlm.paused = False self.running = True self.addonManager.activateAddons() @@ -474,11 +475,15 @@ class Core(object): self.removeLogger() _exit(0) # TODO check exits codes, clean exit is still blocked + try: + self.downloadManager.work() + self.threadManager.work() + self.interactionManager.work() + self.scheduler.work() + except Exception, e: + self.log.critical(_("Critical error: ") + str(e)) + self.print_exc() - self.downloadManager.work() - self.threadManager.work() - self.interactionManager.work() - self.scheduler.work() def setupDB(self): from database import DatabaseBackend @@ -586,10 +591,8 @@ class Core(object): pass # TODO: quit webserver? # self.webserver.quit() - for thread in self.threadManager.threads: - thread.put("quit") - - self.api.stopAllDownloads() + self.dlm.abort() + self.dlm.shutdown() self.addonManager.deactivateAddons() except: diff --git a/pyload/DownloadManager.py b/pyload/DownloadManager.py index 706f9afeb..04c9f66df 100644 --- a/pyload/DownloadManager.py +++ b/pyload/DownloadManager.py @@ -16,18 +16,33 @@ # @author: RaNaN ############################################################################### +from collections import defaultdict from threading import Event +from time import sleep +from random import sample +from subprocess import call + from ReadWriteLock import ReadWriteLock -from utils import lock, read_lock, primary_uid +from Api import DownloadStatus as DS + +from datatypes.PyFile import PyFile + +from utils import lock, read_lock +from utils.fs import exists, join, free_space + +from network import get_ip + from threads.DownloadThread import DownloadThread from threads.DecrypterThread import DecrypterThread + class DownloadManager: """ Schedules and manages download and decrypter jobs. """ def __init__(self, core): self.core = core + self.log = core.log #: won't start download when true self.paused = True @@ -36,8 +51,10 @@ class DownloadManager: self.free = [] #: a thread that in working must have a pyfile as active attribute self.working = [] + #: holds the decrypter threads + self.decrypter = [] - #: indicates when reconnect has occured + #: indicates when reconnect has occurred self.reconnecting = Event() self.reconnecting.clear() @@ -47,24 +64,233 @@ class DownloadManager: def done(self, thread): """ Switch thread from working to free state """ self.working.remove(thread) - self.free.append(thread) + # only download threads will be re-used + if isinstance(thread, DownloadThread): + self.free.append(thread) + thread.isWorking.clear() + + @lock + def stop(self, thread): + """ Removes a thread from all lists """ + + if thread in self.free: + self.free.remove(thread) + + if thread in self.working: + self.working.remove(thread) + + @lock + def startDownloadThread(self, info): + """ Use a free dl thread or create a new one """ + if self.free: + thread = self.free[0] + del self.free[0] + else: + thread = DownloadThread(self) + + thread.put(PyFile.fromInfoData(self.core.files, info)) + + # wait until it picked up the task + thread.isWorking.wait() + self.working.append(thread) + + @lock + def startDecrypterThread(self, info): + """ Start decrypting of entered data, all links in one package are accumulated to one thread.""" + self.decrypter.append(DecrypterThread(self, [(info.plugin, info.url)], info.pid)) @read_lock - def activeDownloads(self, user): + def activeDownloads(self, uid=None): """ retrieve pyfiles of running downloads """ - uid = primary_uid(user) - return [x.active for x in self.working if uid is None or x.active.owner == uid] + return [x.active for x in self.working + if uid is None or x.active.owner == uid] - def getProgressList(self, user): + @read_lock + def getProgressList(self, uid): """ Progress of all running downloads """ - return [p.getProgressInfo() for p in self.activeDownloads(user)] + # decrypter progress could be none + return filter(lambda x: x is not None, + [p.getProgress() for p in self.working + self.decrypter + if uid is None or p.owner == uid]) - def canDownload(self, user): - """ check if a user is eligible to start a new download """ + def processingIds(self): + """get a id list of all pyfiles processed""" + return [x.id for x in self.activeDownloads(None)] + @read_lock def abort(self): """ Cancels all downloads """ + # TODO: may dead lock + for t in self.working: + t.active.abortDownload() + + @read_lock + def shutdown(self): + """ End all threads """ + for thread in self.working + self.free: + thread.put("quit") def work(self): """ main routine that does the periodical work """ + self.tryReconnect() + + if free_space(self.core.config["general"]["download_folder"]) / 1024 / 1024 < \ + self.core.config["general"]["min_free_space"]: + self.log.warning(_("Not enough space left on device")) + self.paused = True + + if self.paused or not self.core.api.isTimeDownload(): + return False + + # at least one thread want reconnect and we are supposed to wait + if self.core.config['reconnect']['wait'] and self.wantReconnect() > 1: + return False + + self.assignJobs() + + def assignJobs(self): + """ Load jobs from db and try to assign them """ + + limit = self.core.config['download']['max_downloads'] - len(self.activeDownloads()) + slots = self.getRemainingPluginSlots() + occ = tuple([plugin for plugin, v in slots.iteritems() if v == 0]) + jobs = self.core.files.getJobs(occ) + + # map plugin to list of jobs + plugins = defaultdict(list) + + for uid, info in jobs.items(): + # check the quota of each user and filter + quota = self.core.api.calcQuota(uid) + if -1 < quota < info.size: + del jobs[uid] + + plugins[info.download.plugin].append(info) + + for plugin, jobs in plugins.iteritems(): + # we know exactly the number of remaining jobs + # or only can start one job if limit is not known + to_schedule = slots[plugin] if plugin in slots else 1 + # start all chosen jobs + for job in self.chooseJobs(jobs, to_schedule): + # if the job was started the limit will be reduced + if self.startJob(job, limit): + limit -= 1 + + def chooseJobs(self, jobs, k): + """ make a fair choice of which k jobs to start """ + # TODO: prefer admins, make a fairer choice? + if k >= len(jobs): + return jobs + + return sample(jobs, k) + + def startJob(self, info, limit): + """ start a download or decrypter thread with given file info """ + + plugin = self.core.pluginManager.findPlugin(info.download.plugin) + # this plugin does not exits + if plugin is None: + self.log.error(_("Plugin '%s' does not exists") % info.download.plugin) + self.core.files.setDownloadStatus(info.fid, DS.Failed) + return False + + if plugin == "hoster": + # this job can't be started + if limit == 0: + return False + + self.startDownloadThread(info) + return True + + elif plugin == "crypter": + self.startDecrypterThread(info) + else: + self.log.error(_("Plugin type '%s' is can be used for downloading") % plugin) + + return False + + @read_lock + def tryReconnect(self): + """checks if reconnect needed""" + + if not self.core.config["reconnect"]["activated"] or not self.core.api.isTimeReconnect(): + return False + + # only reconnect when all threads are ready + if not (0 < self.wantReconnect() == len(self.working)): + return False + + if not exists(self.core.config['reconnect']['method']): + if exists(join(pypath, self.core.config['reconnect']['method'])): + self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method']) + else: + self.core.config["reconnect"]["activated"] = False + self.log.warning(_("Reconnect script not found!")) + return + + self.reconnecting.set() + + self.log.info(_("Starting reconnect")) + + # wait until all thread got the event + while [x.active.plugin.waiting for x in self.working].count(True) != 0: + sleep(0.25) + + old_ip = get_ip() + + self.core.evm.dispatchEvent("reconnect:before", old_ip) + self.log.debug("Old IP: %s" % old_ip) + + try: + call(self.core.config['reconnect']['method'], shell=True) + except: + self.log.warning(_("Failed executing reconnect script!")) + self.core.config["reconnect"]["activated"] = False + self.reconnecting.clear() + self.core.print_exc() + return + + sleep(1) + ip = get_ip() + self.core.evm.dispatchEvent("reconnect:after", ip) + + if not old_ip or old_ip == ip: + self.log.warning(_("Reconnect not successful")) + else: + self.log.info(_("Reconnected, new IP: %s") % ip) + + self.reconnecting.clear() + + @read_lock + def wantReconnect(self): + """ number of downloads that are waiting for reconnect """ + active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.working] + return active.count(True) + + @read_lock + def getRemainingPluginSlots(self): + """ dict of plugin names mapped to remaining dls """ + occ = defaultdict(lambda: -1) + # decrypter are treated as occupied + for p in self.decrypter: + progress = p.getProgressInfo() + if progress: + occ[progress.plugin] = 0 + + # get all default dl limits + for t in self.working: + if not t.active.hasPlugin(): continue + limit = t.active.plugin.getDownloadLimit() + if limit < 0: continue + occ[t.active.pluginname] = limit + + # subtract with running downloads + for t in self.working: + if not t.active.hasPlugin(): continue + plugin = t.active.pluginname + if plugin in occ: + occ[plugin] -= 1 + + return occ
\ No newline at end of file diff --git a/pyload/FileManager.py b/pyload/FileManager.py index 2edf81bfc..9702307a0 100644 --- a/pyload/FileManager.py +++ b/pyload/FileManager.py @@ -35,6 +35,7 @@ def invalidate(func): return new + class FileManager: """Handles all request made to obtain information, modify status or other request for links or packages""" @@ -49,9 +50,10 @@ class FileManager: # translations self.statusMsg = [_("none"), _("offline"), _("online"), _("queued"), _("paused"), - _("finished"), _("skipped"), _("failed"), _("starting"),_("waiting"), + _("finished"), _("skipped"), _("failed"), _("starting"), _("waiting"), _("downloading"), _("temp. offline"), _("aborted"), _("not possible"), _("missing"), - _("file mismatch"), _("decrypting"), _("processing"), _("custom"), _("unknown")] + _("file mismatch"), _("occupied"), _("decrypting"), _("processing"), _("custom"), + _("unknown")] self.files = {} # holds instances for files self.packages = {} # same for packages @@ -93,7 +95,7 @@ class FileManager: @invalidate def addLinks(self, data, pid, owner): - """Add links, data = (plugin, url) tuple. Internal method should use API.""" + """Add links, data = (url, plugin) tuple. Internal method should use API.""" self.db.addLinks(data, pid, owner) self.evm.dispatchEvent("package:updated", pid) @@ -102,7 +104,7 @@ class FileManager: def addPackage(self, name, folder, root, password, site, comment, paused, owner): """Adds a package to database""" pid = self.db.addPackage(name, folder, root, password, site, comment, - PackageStatus.Paused if paused else PackageStatus.Ok, owner) + PackageStatus.Paused if paused else PackageStatus.Ok, owner) p = self.db.getPackageInfo(pid) self.evm.dispatchEvent("package:inserted", pid, p.root, p.packageorder) @@ -250,46 +252,14 @@ class FileManager: @lock - def getJob(self, occ): - """get suitable job""" - - #TODO only accessed by one thread, should not need a lock - #TODO needs to be approved for new database - #TODO clean mess - #TODO improve selection of valid jobs - - if occ in self.jobCache: - if self.jobCache[occ]: - id = self.jobCache[occ].pop() - if id == "empty": - pyfile = None - self.jobCache[occ].append("empty") - else: - pyfile = self.getFile(id) - else: - jobs = self.db.getJob(occ) - jobs.reverse() - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - self.jobCache[occ].extend(jobs) - pyfile = self.getFile(self.jobCache[occ].pop()) - - else: - self.jobCache = {} #better not caching to much - jobs = self.db.getJob(occ) - jobs.reverse() - self.jobCache[occ] = jobs + def getJobs(self, occ): - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - pyfile = self.getFile(self.jobCache[occ].pop()) + # load jobs with file info + if occ not in self.jobCache: + self.jobCache[occ] = dict([(k, self.getFileInfo(fid)) for k, fid + in self.db.getJobs(occ).iteritems()]) - - return pyfile + return self.jobCache[occ] def getDownloadStats(self, user=None): """ return number of downloads """ @@ -346,7 +316,6 @@ class FileManager: if fid in self.core.threadManager.processingIds(): f.abortDownload() - self.db.deleteFile(fid, f.fileorder, f.packageid) self.releaseFile(fid) @@ -377,6 +346,17 @@ class FileManager: self.evm.dispatchEvent("file:updated", pyfile) @invalidate + @read_lock + def setDownloadStatus(self, fid, status): + """ sets a download status for a file """ + if fid in self.files: + self.files[fid].setStatus(status) + else: + self.db.setDownloadStatus(fid, status) + + self.evm.dispatchEvent("file:updated", fid) + + @invalidate def updatePackage(self, pypack): """updates a package""" self.db.updatePackage(pypack) @@ -488,7 +468,7 @@ class FileManager: raise Exception("Tried to reorder non continuous block of files") # minimum fileorder - f = reduce(lambda x,y: x if x.fileorder < y.fileorder else y, files) + f = reduce(lambda x, y: x if x.fileorder < y.fileorder else y, files) order = f.fileorder self.db.orderFiles(pid, fids, order, position) @@ -507,12 +487,12 @@ class FileManager: elif f.fileorder < position: for pyfile in self.files.itervalues(): if pyfile.packageid != f.package or pyfile.fileorder < 0: continue - if position >= pyfile.fileorder >= f.fileorder+diff: + if position >= pyfile.fileorder >= f.fileorder + diff: pyfile.fileorder -= diff for i, fid in enumerate(fids): if fid in self.files: - self.files[fid].fileorder = position -diff + i + 1 + self.files[fid].fileorder = position - diff + i + 1 self.db.commit() diff --git a/pyload/InitHomeDir.py b/pyload/InitHomeDir.py index a68e1a197..51dfc7686 100644 --- a/pyload/InitHomeDir.py +++ b/pyload/InitHomeDir.py @@ -59,6 +59,7 @@ else: __builtin__.homedir = homedir configdir = None +final = False args = " ".join(argv) # dirty method to set configdir from commandline arguments if "--configdir=" in args: @@ -83,9 +84,15 @@ if not configdir: configname = ".pyload" if platform in ("posix", "linux2", "darwin") else "pyload" configdir = path.join(homedir, configname + dev) -def init_dir(other_path=None): + +def init_dir(other_path=None, no_change=False): # switch to pyload home directory, or path at other_path global configdir + global final + + if final: return + + if no_change: final = True if other_path is not None: configdir = join(pypath, other_path) diff --git a/pyload/PluginManager.py b/pyload/PluginManager.py index f3d2b999d..389eb86a2 100644 --- a/pyload/PluginManager.py +++ b/pyload/PluginManager.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- ############################################################################### -# Copyright(c) 2008-2013 pyLoad Team +# Copyright(c) 2008-2014 pyLoad Team # http://www.pyload.org # # This file is part of pyLoad. @@ -12,7 +12,7 @@ # # Subjected to the terms and conditions in LICENSE # -# @author: RaNaN, mkaay +# @author: RaNaN ############################################################################### import sys @@ -132,6 +132,10 @@ class PluginManager: return res["hoster"], res["crypter"] + def findPlugin(self, name): + """ Finds the type to a plugin name """ + return self.loader.findPlugin(name) + def getPlugin(self, plugin, name): """ Retrieves the plugin tuple for a single plugin or none """ return self.loader.getPlugin(plugin, name) diff --git a/pyload/api/AccountApi.py b/pyload/api/AccountApi.py index d4b39c12b..6b89a2aad 100644 --- a/pyload/api/AccountApi.py +++ b/pyload/api/AccountApi.py @@ -27,12 +27,12 @@ class AccountApi(ApiComponent): return [acc.toInfoData() for acc in accounts] @RequirePerm(Permission.Accounts) - def getAccountInfo(self, plugin, loginname, refresh=False): + def getAccountInfo(self, aid, plugin, refresh=False): """ Returns :class:`AccountInfo` for a specific account :param refresh: reload account info """ - account = self.core.accountManager.getAccount(plugin, loginname) + account = self.core.accountManager.getAccount(aid, plugin) # Admins can see and refresh accounts if not account or (self.primaryUID and self.primaryUID != account.owner): @@ -45,20 +45,27 @@ class AccountApi(ApiComponent): return account.toInfoData() @RequirePerm(Permission.Accounts) - def updateAccount(self, plugin, loginname, password): - """Creates an account if not existent or updates the password + def createAccount(self, plugin, loginname, password): + """ Creates a new account - :return: newly created or updated account info + :return class:`AccountInfo` """ - # TODO: None pointer - return self.core.accountManager.updateAccount(plugin, loginname, password, self.user).toInfoData() + return self.core.accountManager.createAccount(plugin, loginname, password, self.user.true_primary).toInfoData() + + @RequirePerm(Permission.Accounts) + def updateAccount(self, aid, plugin, loginname, password): + """Updates loginname and password of an existent account + + :return: updated account info + """ + return self.core.accountManager.updateAccount(aid, plugin, loginname, password, self.user).toInfoData() @RequirePerm(Permission.Accounts) def updateAccountInfo(self, account): """ Update account settings from :class:`AccountInfo` """ - inst = self.core.accountManager.getAccount(account.plugin, account.loginname, self.user) - if not account: + inst = self.core.accountManager.getAccount(account.aid, account.plugin, self.user) + if not inst: return inst.activated = to_bool(account.activated) @@ -72,7 +79,7 @@ class AccountApi(ApiComponent): :param account: :class:`ÀccountInfo` instance """ - self.core.accountManager.removeAccount(account.plugin, account.loginname, self.primaryUID) + self.core.accountManager.removeAccount(account.aid, account.plugin, self.primaryUID) if Api.extend(AccountApi): diff --git a/pyload/api/CoreApi.py b/pyload/api/CoreApi.py index 187286b48..b15272196 100644 --- a/pyload/api/CoreApi.py +++ b/pyload/api/CoreApi.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from pyload.Api import Api, RequirePerm, Permission, ServerStatus, Interaction +from pyload.Api import Api, RequirePerm, Permission, StatusInfo, Interaction from pyload.utils.fs import join, free_space, exists from pyload.utils import compare_time @@ -37,24 +37,24 @@ class CoreApi(ApiComponent): return "%s://%%s:%d" % (ws, self.core.config['webUI']['wsPort']) @RequirePerm(Permission.All) - def getServerStatus(self): + def getStatusInfo(self): """Some general information about the current status of pyLoad. - :return: `ServerStatus` + :return: `StatusInfo` """ queue = self.core.files.getQueueStats(self.primaryUID) total = self.core.files.getDownloadStats(self.primaryUID) - serverStatus = ServerStatus(0, + serverStatus = StatusInfo(0, total[0], queue[0], total[1], queue[1], self.isInteractionWaiting(Interaction.All), - not self.core.threadManager.pause and self.isTimeDownload(), - self.core.threadManager.pause, - self.core.config['reconnect']['activated'] and self.isTimeReconnect()) + not self.core.dlm.paused and self.isTimeDownload(), + self.core.dlm.paused, + self.core.config['reconnect']['activated'] and self.isTimeReconnect(), + self.getQuota()) - - for pyfile in self.core.threadManager.getActiveDownloads(self.primaryUID): + for pyfile in self.core.dlm.activeDownloads(self.primaryUID): serverStatus.speed += pyfile.getSpeed() #bytes/s return serverStatus @@ -65,23 +65,24 @@ class CoreApi(ApiComponent): :rtype: list of :class:`ProgressInfo` """ - return self.core.threadManager.getProgressList(self.primaryUID) + return self.core.dlm.getProgressList(self.primaryUID) +\ + self.core.threadManager.getProgressList(self.primaryUID) def pauseServer(self): """Pause server: It won't start any new downloads, but nothing gets aborted.""" - self.core.threadManager.pause = True + self.core.dlm.paused = True def unpauseServer(self): """Unpause server: New Downloads will be started.""" - self.core.threadManager.pause = False + self.core.dlm.paused = False def togglePause(self): """Toggle pause state. :return: new pause state """ - self.core.threadManager.pause ^= True - return self.core.threadManager.pause + self.core.dlm.paused ^= True + return self.core.dlm.paused def toggleReconnect(self): """Toggle reconnect activation. diff --git a/pyload/api/DownloadApi.py b/pyload/api/DownloadApi.py index b29f9c06c..71d112e44 100644 --- a/pyload/api/DownloadApi.py +++ b/pyload/api/DownloadApi.py @@ -148,10 +148,7 @@ class DownloadApi(ApiComponent): @RequirePerm(Permission.Modify) def stopAllDownloads(self): """Aborts all running downloads.""" - - pyfiles = self.core.files.cachedFiles() - for pyfile in pyfiles: - pyfile.abortDownload() + self.core.dlm.abort() @RequirePerm(Permission.Modify) def stopDownloads(self, fids): diff --git a/pyload/api/StatisticsApi.py b/pyload/api/StatisticsApi.py new file mode 100644 index 000000000..d313e4d0e --- /dev/null +++ b/pyload/api/StatisticsApi.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from pyload.Api import Api, RequirePerm, Permission + +from ApiComponent import ApiComponent + +CACHE = {} +QUOTA_UNLIMITED = -1 + + +class StatisticsApi(ApiComponent): + """ Retrieve download statistics and quota """ + + def recordDownload(self, pyfile): + """ Add download record to the statistics """ + del CACHE[:] + + def calcQuota(self, uid): + return QUOTA_UNLIMITED + + def getQuota(self): + """ Number of bytes the user has left for download """ + return self.calcQuota(self.user.true_primary) + + +if Api.extend(StatisticsApi): + del StatisticsApi
\ No newline at end of file diff --git a/pyload/api/__init__.py b/pyload/api/__init__.py index a2b292a27..e25c82a52 100644 --- a/pyload/api/__init__.py +++ b/pyload/api/__init__.py @@ -1,5 +1,5 @@ __all__ = ["CoreApi", "ConfigApi", "DownloadApi", "DownloadPreparingApi", "FileApi", - "UserInteractionApi", "AccountApi", "AddonApi", "UserApi"] + "UserInteractionApi", "AccountApi", "AddonApi", "UserApi", "StatisticsApi"] # Import all components # from .import * diff --git a/pyload/config/default.py b/pyload/config/default.py index 1ad164c31..d4ac84c9b 100644 --- a/pyload/config/default.py +++ b/pyload/config/default.py @@ -38,10 +38,9 @@ def make_config(config): [ ("language", "en;de;fr;it;es;nl;sv;ru;pl;cs;sr;pt_BR", _("Language"), "en"), ("download_folder", "folder", _("Download Folder"), "Downloads"), - ("checksum", "bool", _("Use Checksum"), False), ("folder_per_package", "bool", _("Create folder for each package"), True), ("debug_mode", "bool", _("Debug Mode"), False), - ("min_free_space", "int", _("Min Free Space (MB)"), 200), + ("min_free_space", "int", _("Min Free Space (MB)"), 512), ("renice", "int", _("CPU Priority"), 0), ]) @@ -79,20 +78,23 @@ def make_config(config): config.addConfigSection("reconnect", _("Reconnect"), _("Description"), _("Long description"), [ - ("endTime", "time", _("End"), "0:00"), ("activated", "bool", _("Use Reconnect"), False), ("method", "str", _("Method"), "./reconnect.sh"), + ("wait", "str", _("Wait for reconnect"), False), ("startTime", "time", _("Start"), "0:00"), + ("endTime", "time", _("End"), "0:00"), ]) config.addConfigSection("download", _("Download"), _("Description"), _("Long description"), [ - ("max_downloads", "int", _("Max Parallel Downloads"), 3), - ("limit_speed", "bool", _("Limit Download Speed"), False), + ("max_downloads", "int", _("Max parallel downloads"), 3), + ("wait_downloads", "int", _("Start downloads while waiting"), 2), + ("limit_speed", "bool", _("Limit download speed"), False), ("interface", "str", _("Download interface to bind (ip or Name)"), ""), ("skip_existing", "bool", _("Skip already existing files"), False), - ("max_speed", "int", _("Max Download Speed in kb/s"), -1), + ("max_speed", "int", _("Max download speed in kb/s"), -1), ("ipv6", "bool", _("Allow IPv6"), False), + ("ssl", "bool", _("Prefer SSL downloads"), True), ("chunks", "int", _("Max connections for one download"), 3), ("restart_failed", "bool", _("Restart failed downloads on startup"), False), ]) diff --git a/pyload/database/AccountDatabase.py b/pyload/database/AccountDatabase.py index 3ca841fbc..e432af192 100644 --- a/pyload/database/AccountDatabase.py +++ b/pyload/database/AccountDatabase.py @@ -7,23 +7,33 @@ from pyload.database import DatabaseMethods, queue, async class AccountMethods(DatabaseMethods): @queue def loadAccounts(self): - self.c.execute('SELECT plugin, loginname, owner, activated, shared, password, options FROM accounts') + self.c.execute('SELECT aid, plugin, loginname, owner, activated, shared, password, options FROM accounts') - return [(AccountInfo(r[0], r[1], r[2], activated=r[3] is 1, shared=r[4] is 1), r[5], r[6]) for r in self.c] + return [(AccountInfo(r[0], r[1], r[2], r[3], activated=r[4] is 1, shared=r[5] is 1), r[6], r[7]) for r in + self.c] + + @queue + def createAccount(self, plugin, loginname, password, owner): + self.c.execute('INSERT INTO accounts(plugin, loginname, password, owner) VALUES(?,?,?,?)', + (plugin, loginname, password, owner)) + + return self.c.lastrowid @async def saveAccounts(self, data): - self.c.executemany( - 'INSERT INTO accounts(plugin, loginname, owner, activated, shared, password, options) VALUES(?,?,?,?,?,?,?)', + 'UPDATE accounts SET ' + 'loginname=?, activated=?, shared=?, password=?, options=? ' + 'WHERE aid=?', data) @async - def removeAccount(self, plugin, loginname): - self.c.execute('DELETE FROM accounts WHERE plugin=? AND loginname=?', (plugin, loginname)) + def removeAccount(self, aid): + self.c.execute('DELETE FROM accounts WHERE aid=?', (aid,)) @queue def purgeAccounts(self): self.c.execute('DELETE FROM accounts') + AccountMethods.register()
\ No newline at end of file diff --git a/pyload/database/DatabaseBackend.py b/pyload/database/DatabaseBackend.py index df8c6e704..1bdcdc582 100644 --- a/pyload/database/DatabaseBackend.py +++ b/pyload/database/DatabaseBackend.py @@ -30,7 +30,7 @@ except: import sqlite3 DB = None -DB_VERSION = 6 +DB_VERSION = 7 def set_DB(db): @@ -298,6 +298,7 @@ class DatabaseBackend(Thread): ) self.c.execute('CREATE INDEX IF NOT EXISTS "file_index" ON files(package, owner)') self.c.execute('CREATE INDEX IF NOT EXISTS "file_owner" ON files(owner)') + self.c.execute('CREATE INDEX IF NOT EXISTS "file_plugin" ON files(plugin)') self.c.execute( 'CREATE TRIGGER IF NOT EXISTS "insert_file" AFTER INSERT ON "files"' @@ -366,30 +367,32 @@ class DatabaseBackend(Thread): self.c.execute( 'CREATE TABLE IF NOT EXISTS "accounts" (' + '"aid" INTEGER PRIMARY KEY AUTOINCREMENT, ' '"plugin" TEXT NOT NULL, ' '"loginname" TEXT NOT NULL, ' - '"owner" INTEGER NOT NULL DEFAULT -1, ' + '"owner" INTEGER NOT NULL, ' '"activated" INTEGER NOT NULL DEFAULT 1, ' '"password" TEXT DEFAULT "", ' '"shared" INTEGER NOT NULL DEFAULT 0, ' '"options" TEXT DEFAULT "", ' - 'FOREIGN KEY(owner) REFERENCES users(uid), ' - 'PRIMARY KEY (plugin, loginname, owner) ON CONFLICT REPLACE' + 'FOREIGN KEY(owner) REFERENCES users(uid)' ')' ) + self.c.execute('CREATE INDEX IF NOT EXISTS "accounts_login" ON accounts(plugin, loginname)') + self.c.execute( 'CREATE TABLE IF NOT EXISTS "stats" (' + '"id" INTEGER PRIMARY KEY AUTOINCREMENT, ' '"user" INTEGER NOT NULL, ' '"plugin" TEXT NOT NULL, ' '"time" INTEGER NOT NULL, ' '"premium" INTEGER DEFAULT 0 NOT NULL, ' '"amount" INTEGER DEFAULT 0 NOT NULL, ' - 'FOREIGN KEY(user) REFERENCES users(uid), ' - 'PRIMARY KEY(user, plugin, time)' + 'FOREIGN KEY(user) REFERENCES users(uid)' ')' ) - self.c.execute('CREATE INDEX IF NOT EXISTS "stats_time" ON stats(time)') + self.c.execute('CREATE INDEX IF NOT EXISTS "stats_time" ON stats(user, time)') #try to lower ids self.c.execute('SELECT max(fid) FROM files') diff --git a/pyload/database/FileDatabase.py b/pyload/database/FileDatabase.py index 219b16663..e6e051a92 100644 --- a/pyload/database/FileDatabase.py +++ b/pyload/database/FileDatabase.py @@ -314,6 +314,10 @@ class FileMethods(DatabaseMethods): f.hash, f.status, f.error, f.fid)) @async + def setDownloadStatus(self, fid, status): + self.c.execute('UPDATE files SET dlstatus=? WHERE fid=?', (status, fid)) + + @async def updatePackage(self, p): self.c.execute( 'UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, tags=?, status=?, shared=? WHERE pid=?', @@ -383,22 +387,28 @@ class FileMethods(DatabaseMethods): # status -> queued self.c.execute('UPDATE files SET status=3 WHERE package=?', (pid,)) - - # TODO: multi user approach @queue - def getJob(self, occ): + def getJobs(self, occ): """return pyfile ids, which are suitable for download and don't use a occupied plugin""" cmd = "(%s)" % ", ".join(["'%s'" % x for x in occ]) - #TODO - # dlstatus in online, queued | package status = ok - cmd = ("SELECT f.fid FROM files as f INNER JOIN packages as p ON f.package=p.pid " - "WHERE f.plugin NOT IN %s AND f.dlstatus IN (2,3) AND p.status=0 " - "ORDER BY p.packageorder ASC, f.fileorder ASC LIMIT 5") % cmd + # dlstatus in online, queued, occupied | package status = ok + cmd = ("SELECT f.owner, f.fid FROM files as f INNER JOIN packages as p ON f.package=p.pid " + "WHERE f.owner=? AND f.plugin NOT IN %s AND f.dlstatus IN (2,3,16) AND p.status=0 " + "ORDER BY p.packageorder ASC, f.fileorder ASC LIMIT 1") % cmd + - self.c.execute(cmd) + self.c.execute("SELECT uid FROM users") + uids = self.c.fetchall() + jobs = {} + # get jobs for all uids + for uid in uids: + self.c.execute(cmd, uid) + r = self.c.fetchone() + if r: + jobs[r[0]] = r[1] - return [x[0] for x in self.c] + return jobs @queue def getUnfinished(self, pid): diff --git a/pyload/database/StatisticDatabase.py b/pyload/database/StatisticDatabase.py index d5f9658f2..5dd5ec7ed 100644 --- a/pyload/database/StatisticDatabase.py +++ b/pyload/database/StatisticDatabase.py @@ -3,11 +3,10 @@ from pyload.database import DatabaseMethods, queue, async, inner -# TODO class StatisticMethods(DatabaseMethods): - pass - + def addEntry(self, user, plugin, premium, amount): + pass StatisticMethods.register()
\ No newline at end of file diff --git a/pyload/database/UserDatabase.py b/pyload/database/UserDatabase.py index 8d8381a40..14b4ae40c 100644 --- a/pyload/database/UserDatabase.py +++ b/pyload/database/UserDatabase.py @@ -44,6 +44,14 @@ class UserMethods(DatabaseMethods): self.c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) @queue + def addDebugUser(self, uid): + # just add a user with uid to db + try: + self.c.execute('INSERT INTO users (uid, name, password) VALUES (?, ?, ?)', (uid, "debugUser", random_salt())) + except: + pass + + @queue def getUserData(self, name=None, uid=None): qry = ('SELECT uid, name, email, role, permission, folder, traffic, dllimit, dlquota, ' 'hddquota, user, template FROM "users" WHERE ') diff --git a/pyload/datatypes/PyFile.py b/pyload/datatypes/PyFile.py index 18ac06c50..b83a057aa 100644 --- a/pyload/datatypes/PyFile.py +++ b/pyload/datatypes/PyFile.py @@ -41,10 +41,11 @@ statusMap = { "not possible": 13, "missing": 14, "file mismatch": 15, - "decrypting": 16, - "processing": 17, - "custom": 18, - "unknown": 19, + "occupied": 16, + "decrypting": 17, + "processing": 18, + "custom": 19, + "unknown": 20, } @@ -205,7 +206,7 @@ class PyFile(object): def abortDownload(self): """abort pyfile if possible""" # TODO: abort timeout, currently dead locks - while self.id in self.m.core.threadManager.processingIds(): + while self.id in self.m.core.dlm.processingIds(): self.abort = True if self.plugin and self.plugin.req: self.plugin.req.abort() @@ -225,7 +226,7 @@ class PyFile(object): def finishIfDone(self): """set status to finish and release file if every thread is finished with it""" - if self.id in self.m.core.threadManager.processingIds(): + if self.id in self.m.core.dlm.processingIds(): return False self.setStatus("finished") diff --git a/pyload/datatypes/User.py b/pyload/datatypes/User.py index 645fd0983..fbfb24378 100644 --- a/pyload/datatypes/User.py +++ b/pyload/datatypes/User.py @@ -54,6 +54,9 @@ class User(UserData): def isAdmin(self): return self.hasRole(Role.Admin) + def isOwner(self, obj): + return self.primary is None or obj.owner == self.true_primary + @property def primary(self): """ Primary user id, Internal user handle used for most operations diff --git a/pyload/network/__init__.py b/pyload/network/__init__.py index 8b1378917..1abafc327 100644 --- a/pyload/network/__init__.py +++ b/pyload/network/__init__.py @@ -1 +1,30 @@ +# -*- coding: utf-8 -*- +import re +from random import choice +from time import sleep + +from RequestFactory import getURL + + +def get_ip(n=10): + """retrieve current ip. try n times for n seconds""" + services = [ + ("http://checkip.dyndns.org", r".*Current IP Address: (\S+)</body>.*"), + ("http://myexternalip.com/raw", r"(\S+)"), + ("http://icanhazip.com", r"(\S+)"), + ("http://ifconfig.me/ip", r"(\S+)") + ] + + ip = "" + for i in range(n): + try: + sv = choice(services) + ip = getURL(sv[0]) + ip = re.match(sv[1], ip).group(1) + break + except: + ip = "" + sleep(1) + + return ip diff --git a/pyload/plugins/Account.py b/pyload/plugins/Account.py index cbf545611..e81609971 100644 --- a/pyload/plugins/Account.py +++ b/pyload/plugins/Account.py @@ -42,14 +42,15 @@ class Account(Base): @classmethod def fromInfoData(cls, m, info, password, options): - return cls(m, info.loginname, info.owner, + return cls(m, info.aid, info.loginname, info.owner, True if info.activated else False, True if info.shared else False, password, options) __type__ = "account" - def __init__(self, manager, loginname, owner, activated, shared, password, options): + def __init__(self, manager, aid, loginname, owner, activated, shared, password, options): Base.__init__(self, manager.core, owner) + self.aid = aid self.loginname = loginname self.owner = owner self.activated = activated @@ -74,7 +75,7 @@ class Account(Base): self.init() def toInfoData(self): - info = AccountInfo(self.__name__, self.loginname, self.owner, self.valid, self.validuntil, self.trafficleft, + info = AccountInfo(self.aid, self.__name__, self.loginname, self.owner, self.valid, self.validuntil, self.trafficleft, self.maxtraffic, self.premium, self.activated, self.shared, self.options) info.config = [ConfigItem(name, item.label, item.description, item.input, @@ -158,13 +159,14 @@ class Account(Base): self.maxtraffic = Account.maxtraffic self.premium = Account.premium - def setPassword(self, password): - """ updates the password and returns true if anything changed """ + def setLogin(self, loginname, password): + """ updates the loginname and password and returns true if anything changed """ - if password != self.password: + if password != self.password or loginname != self.loginname: self.login_ts = 0 self.valid = True #set valid, so the login will be retried + self.loginname = loginname self.password = password return True diff --git a/pyload/plugins/addons/MultiHoster.py b/pyload/plugins/addons/MultiHoster.py index 871defb05..ff5da32ae 100644 --- a/pyload/plugins/addons/MultiHoster.py +++ b/pyload/plugins/addons/MultiHoster.py @@ -90,7 +90,7 @@ class MultiHoster(Addon, PluginMatcher): @AddEventListener(["account:deleted", "account:updated"]) - def refreshAccounts(self, plugin=None, loginname=None): + def refreshAccounts(self, plugin=None, loginname=None, user=None): self.logDebug("Re-checking accounts") self.plugins = {} diff --git a/pyload/remote/apitypes.py b/pyload/remote/apitypes.py index ab68116c3..4c66800c6 100644 --- a/pyload/remote/apitypes.py +++ b/pyload/remote/apitypes.py @@ -36,10 +36,11 @@ class DownloadStatus: NotPossible = 13 Missing = 14 FileMismatch = 15 - Decrypting = 16 - Processing = 17 - Custom = 18 - Unknown = 19 + Occupied = 16 + Decrypting = 17 + Processing = 18 + Custom = 19 + Unknown = 20 class FileStatus: Ok = 0 @@ -55,13 +56,15 @@ class InputType: Textbox = 5 Password = 6 Time = 7 - Bool = 8 - Click = 9 - Select = 10 - Multiple = 11 - List = 12 - PluginList = 13 - Table = 14 + TimeSpan = 8 + ByteSize = 9 + Bool = 10 + Click = 11 + Select = 12 + Multiple = 13 + List = 14 + PluginList = 15 + Table = 16 class Interaction: All = 0 @@ -109,9 +112,10 @@ class Role: User = 1 class AccountInfo(BaseObject): - __slots__ = ['plugin', 'loginname', 'owner', 'valid', 'validuntil', 'trafficleft', 'maxtraffic', 'premium', 'activated', 'shared', 'config'] + __slots__ = ['aid', 'plugin', 'loginname', 'owner', 'valid', 'validuntil', 'trafficleft', 'maxtraffic', 'premium', 'activated', 'shared', 'config'] - def __init__(self, plugin=None, loginname=None, owner=None, valid=None, validuntil=None, trafficleft=None, maxtraffic=None, premium=None, activated=None, shared=None, config=None): + def __init__(self, aid=None, plugin=None, loginname=None, owner=None, valid=None, validuntil=None, trafficleft=None, maxtraffic=None, premium=None, activated=None, shared=None, config=None): + self.aid = aid self.plugin = plugin self.loginname = loginname self.owner = owner @@ -322,20 +326,6 @@ class ProgressInfo(BaseObject): self.type = type self.download = download -class ServerStatus(BaseObject): - __slots__ = ['speed', 'linkstotal', 'linksqueue', 'sizetotal', 'sizequeue', 'notifications', 'paused', 'download', 'reconnect'] - - def __init__(self, speed=None, linkstotal=None, linksqueue=None, sizetotal=None, sizequeue=None, notifications=None, paused=None, download=None, reconnect=None): - self.speed = speed - self.linkstotal = linkstotal - self.linksqueue = linksqueue - self.sizetotal = sizetotal - self.sizequeue = sizequeue - self.notifications = notifications - self.paused = paused - self.download = download - self.reconnect = reconnect - class ServiceDoesNotExist(ExceptionObject): __slots__ = ['plugin', 'func'] @@ -349,6 +339,21 @@ class ServiceException(ExceptionObject): def __init__(self, msg=None): self.msg = msg +class StatusInfo(BaseObject): + __slots__ = ['speed', 'linkstotal', 'linksqueue', 'sizetotal', 'sizequeue', 'notifications', 'paused', 'download', 'reconnect', 'quota'] + + def __init__(self, speed=None, linkstotal=None, linksqueue=None, sizetotal=None, sizequeue=None, notifications=None, paused=None, download=None, reconnect=None, quota=None): + self.speed = speed + self.linkstotal = linkstotal + self.linksqueue = linksqueue + self.sizetotal = sizetotal + self.sizequeue = sizequeue + self.notifications = notifications + self.paused = paused + self.download = download + self.reconnect = reconnect + self.quota = quota + class TreeCollection(BaseObject): __slots__ = ['root', 'files', 'packages'] @@ -402,6 +407,8 @@ class Iface(object): pass def checkLinks(self, links): pass + def createAccount(self, plugin, loginname, password): + pass def createPackage(self, name, folder, root, password, site, comment, paused): pass def deleteConfig(self, plugin): @@ -420,7 +427,7 @@ class Iface(object): pass def generatePackages(self, links): pass - def getAccountInfo(self, plugin, loginname, refresh): + def getAccountInfo(self, aid, plugin, refresh): pass def getAccountTypes(self): pass @@ -464,10 +471,12 @@ class Iface(object): pass def getProgressInfo(self): pass - def getServerStatus(self): + def getQuota(self): pass def getServerVersion(self): pass + def getStatusInfo(self): + pass def getUserData(self): pass def getWSAddress(self): @@ -538,7 +547,7 @@ class Iface(object): pass def unpauseServer(self): pass - def updateAccount(self, plugin, loginname, password): + def updateAccount(self, aid, plugin, loginname, password): pass def updateAccountInfo(self, account): pass diff --git a/pyload/remote/apitypes_debug.py b/pyload/remote/apitypes_debug.py index d5e5f36a0..a30009bad 100644 --- a/pyload/remote/apitypes_debug.py +++ b/pyload/remote/apitypes_debug.py @@ -19,7 +19,7 @@ enums = [ ] classes = { - 'AccountInfo' : [basestring, basestring, int, bool, int, int, int, bool, bool, bool, (list, ConfigItem)], + 'AccountInfo' : [int, basestring, basestring, int, bool, int, int, int, bool, bool, bool, (list, ConfigItem)], 'AddonInfo' : [basestring, basestring, basestring], 'AddonService' : [basestring, basestring, basestring, (list, basestring), bool, int], 'ConfigHolder' : [basestring, basestring, basestring, basestring, (list, ConfigItem), (None, (list, AddonInfo))], @@ -39,9 +39,9 @@ classes = { 'PackageInfo' : [int, basestring, basestring, int, int, basestring, basestring, basestring, int, (list, basestring), int, bool, int, PackageStats, (list, int), (list, int)], 'PackageStats' : [int, int, int, int], 'ProgressInfo' : [basestring, basestring, basestring, int, int, int, int, int, (None, DownloadProgress)], - 'ServerStatus' : [int, int, int, int, int, bool, bool, bool, bool], 'ServiceDoesNotExist' : [basestring, basestring], 'ServiceException' : [basestring], + 'StatusInfo' : [int, int, int, int, int, bool, bool, bool, bool, int], 'TreeCollection' : [PackageInfo, (dict, int, FileInfo), (dict, int, PackageInfo)], 'UserData' : [int, basestring, basestring, int, int, basestring, int, int, basestring, int, int, basestring], 'UserDoesNotExist' : [basestring], @@ -57,6 +57,7 @@ methods = { 'checkContainer': OnlineCheck, 'checkHTML': OnlineCheck, 'checkLinks': OnlineCheck, + 'createAccount': AccountInfo, 'createPackage': int, 'deleteConfig': None, 'deleteFiles': bool, @@ -88,8 +89,9 @@ methods = { 'getPackageInfo': PackageInfo, 'getPluginConfig': (list, ConfigInfo), 'getProgressInfo': (list, ProgressInfo), - 'getServerStatus': ServerStatus, + 'getQuota': int, 'getServerVersion': basestring, + 'getStatusInfo': StatusInfo, 'getUserData': UserData, 'getWSAddress': basestring, 'invokeAddon': basestring, diff --git a/pyload/remote/pyload.thrift b/pyload/remote/pyload.thrift index a9431ea7c..9d400c4e2 100644 --- a/pyload/remote/pyload.thrift +++ b/pyload/remote/pyload.thrift @@ -5,6 +5,7 @@ typedef i32 PackageID typedef i32 ResultID typedef i32 InteractionID typedef i32 UserID +typedef i32 AccountID typedef i64 UTCDate typedef i64 ByteCount typedef list<string> LinkList @@ -28,6 +29,7 @@ enum DownloadStatus { NotPossible, Missing, FileMismatch, + Occupied, Decrypting, Processing, Custom, @@ -71,7 +73,6 @@ enum PackageStatus { // types for user interaction // some may only be place holder currently not supported // also all input - output combination are not reasonable, see InteractionManager for further info -// Todo: how about: time, ip, s.o. enum InputType { NA, Text, @@ -81,6 +82,8 @@ enum InputType { Textbox, Password, Time, + TimeSpan, + ByteSize, // size in bytes Bool, // confirm like, yes or no dialog Click, // for positional captchas Select, // select from list @@ -217,7 +220,7 @@ struct LinkStatus { 6: optional string hash } -struct ServerStatus { +struct StatusInfo { 1: ByteCount speed, 2: i16 linkstotal, 3: i16 linksqueue, @@ -227,6 +230,7 @@ struct ServerStatus { 7: bool paused, 8: bool download, 9: bool reconnect, + 10: ByteCount quota } struct InteractionTask { @@ -300,17 +304,18 @@ struct UserData { } struct AccountInfo { - 1: PluginName plugin, - 2: string loginname, - 3: UserID owner, - 4: bool valid, - 5: UTCDate validuntil, - 6: ByteCount trafficleft, - 7: ByteCount maxtraffic, - 8: bool premium, - 9: bool activated, - 10: bool shared, - 11: list <ConfigItem> config, + 1: AccountID aid, + 2: PluginName plugin, + 3: string loginname, + 4: UserID owner, + 5: bool valid, + 6: UTCDate validuntil, + 7: ByteCount trafficleft, + 8: ByteCount maxtraffic, + 9: bool premium, + 10: bool activated, + 11: bool shared, + 13: list <ConfigItem> config, } struct OnlineCheck { @@ -363,7 +368,7 @@ service Pyload { string getServerVersion(), string getWSAddress(), - ServerStatus getServerStatus(), + StatusInfo getStatusInfo(), list<ProgressInfo> getProgressInfo(), list<string> getLog(1: i32 offset), @@ -500,9 +505,10 @@ service Pyload { list<string> getAccountTypes(), list<AccountInfo> getAccounts(), - AccountInfo getAccountInfo(1: PluginName plugin, 2: string loginname, 3: bool refresh), + AccountInfo getAccountInfo(1: AccountID aid, 2: PluginName plugin, 3: bool refresh), - AccountInfo updateAccount(1: PluginName plugin, 2: string loginname, 3: string password), + AccountInfo createAccount(1: PluginName plugin, 2: string loginname, 3: string password), + AccountInfo updateAccount(1:AccountID aid, 2: PluginName plugin, 3: string loginname, 4: string password), void updateAccountInfo(1: AccountInfo account), void removeAccount(1: AccountInfo account), @@ -542,6 +548,15 @@ service Pyload { JSONString invokeAddonHandler(1: PluginName plugin, 2: string func, 3: PackageID pid_or_fid) throws (1: ServiceDoesNotExist e, 2: ServiceException ex), + + /////////////////////// + // Statistics Api + /////////////////////// + + ByteCount getQuota(), + + + /////////////////////// // Media finder /////////////////////// diff --git a/pyload/remote/wsbackend/AsyncHandler.py b/pyload/remote/wsbackend/AsyncHandler.py index c7a26cd6b..bf931d70d 100644 --- a/pyload/remote/wsbackend/AsyncHandler.py +++ b/pyload/remote/wsbackend/AsyncHandler.py @@ -171,12 +171,8 @@ class AsyncHandler(AbstractHandler): pass if req.t <= time(): - # TODO: server status is not enough - # modify core api to include progress? think of other needed information to show - # eta is quite wrong currently - # notifications - self.send(req, self.api.getServerStatus()) - self.send(req, self.api.getProgressInfo()) + self.send(req, req.api.getStatusInfo()) + self.send(req, req.api.getProgressInfo()) # update time for next update req.t = time() + req.interval
\ No newline at end of file diff --git a/pyload/threads/DecrypterThread.py b/pyload/threads/DecrypterThread.py index 9f796da22..419f153a2 100644 --- a/pyload/threads/DecrypterThread.py +++ b/pyload/threads/DecrypterThread.py @@ -23,7 +23,6 @@ class DecrypterThread(BaseThread): # holds the progress, while running self.progress = None - self.m.addThread(self) self.start() def getProgress(self): diff --git a/pyload/threads/DownloadThread.py b/pyload/threads/DownloadThread.py index d1672531b..b8f7e4965 100644 --- a/pyload/threads/DownloadThread.py +++ b/pyload/threads/DownloadThread.py @@ -18,6 +18,7 @@ @author: RaNaN """ +from threading import Event from Queue import Queue from time import sleep, time from traceback import print_exc @@ -37,6 +38,9 @@ class DownloadThread(BaseThread): """Constructor""" BaseThread.__init__(self, manager) + self.isWorking = Event() + self.isWorking.clear() + self.queue = Queue() # job queue self.active = None @@ -53,12 +57,19 @@ class DownloadThread(BaseThread): if self.active == "quit": self.active = None - self.m.threads.remove(self) + self.m.stop(self) return True try: - if not pyfile.hasPlugin(): continue + pyfile.initPlugin() + + # after initialization the thread is fully ready + self.isWorking.set() + #this pyfile was deleted while queuing + # TODO: what will happen with new thread manager? + #if not pyfile.hasPlugin(): continue + pyfile.plugin.checkForSameFiles(starting=True) self.log.info(_("Download starts: %s" % pyfile.name)) @@ -204,7 +215,9 @@ class DownloadThread(BaseThread): self.core.files.save() pyfile.checkIfProcessed() exc_clear() - + # manager could still be waiting for it + self.isWorking.set() + self.m.done(self) #pyfile.plugin.req.clean() diff --git a/pyload/threads/ThreadManager.py b/pyload/threads/ThreadManager.py index 298b0402d..f6cb3daea 100644 --- a/pyload/threads/ThreadManager.py +++ b/pyload/threads/ThreadManager.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- ############################################################################### -# Copyright(c) 2008-2013 pyLoad Team +# Copyright(c) 2008-2014 pyLoad Team # http://www.pyload.org # # This file is part of pyLoad. @@ -16,22 +16,11 @@ # @author: RaNaN ############################################################################### -from os.path import exists, join -import re -from subprocess import Popen -from threading import Event, RLock -from time import sleep, time -from traceback import print_exc -from random import choice +from threading import RLock +from time import time -from pyload.datatypes.PyFile import PyFile from pyload.datatypes.OnlineCheck import OnlineCheck -from pyload.network.RequestFactory import getURL -from pyload.utils import lock, uniqify, to_list -from pyload.utils.fs import free_space - -from DecrypterThread import DecrypterThread -from DownloadThread import DownloadThread +from pyload.utils import lock, to_list from InfoThread import InfoThread @@ -44,13 +33,6 @@ class ThreadManager: self.log = core.log self.threads = [] # thread list - self.localThreads = [] #addon+decrypter threads - - self.pause = True - - self.reconnecting = Event() - self.reconnecting.clear() - self.downloaded = 0 #number of files downloaded since last cleanup self.lock = RLock() @@ -67,24 +49,15 @@ class ThreadManager: # timeout for cache purge self.timestamp = 0 - for i in range(self.core.config.get("download", "max_downloads")): - self.createThread() - - def createThread(self): - """create a download thread""" - - thread = DownloadThread(self) - self.threads.append(thread) - @lock def addThread(self, thread): - self.localThreads.append(thread) + self.threads.append(thread) @lock def removeThread(self, thread): """ Remove a thread from the local list """ - if thread in self.localThreads: - self.localThreads.remove(thread) + if thread in self.threads: + self.threads.remove(thread) @lock def createInfoThread(self, data, pid): @@ -108,11 +81,6 @@ class ThreadManager: return rid @lock - def createDecryptThread(self, data, pid): - """ Start decrypting of entered data, all links in one package are accumulated to one thread.""" - if data: DecrypterThread(self, data, pid) - - @lock def getInfoResult(self, rid): return self.infoResults.get(rid) @@ -120,14 +88,10 @@ class ThreadManager: self.core.evm.dispatchEvent("linkcheck:updated", oc.rid, result, owner=oc.owner) oc.update(result) - def getActiveDownloads(self, user=None): - # TODO: user context - return [x.active for x in self.threads if x.active and isinstance(x.active, PyFile)] - def getProgressList(self, user=None): info = [] - for thread in self.threads + self.localThreads: + for thread in self.threads: # skip if not belong to current user if user is not None and thread.owner != user: continue @@ -136,38 +100,8 @@ class ThreadManager: return info - def getActiveFiles(self): - active = self.getActiveDownloads() - - for t in self.localThreads: - active.extend(t.getActiveFiles()) - - return active - - def processingIds(self): - """get a id list of all pyfiles processed""" - return [x.id for x in self.getActiveFiles()] - def work(self): """run all task which have to be done (this is for repetitive call by core)""" - try: - self.tryReconnect() - except Exception, e: - self.log.error(_("Reconnect Failed: %s") % str(e)) - self.reconnecting.clear() - self.core.print_exc() - - self.checkThreadCount() - - try: - self.assignJob() - except Exception, e: - self.log.warning("Assign job error", e) - self.core.print_exc() - - sleep(0.5) - self.assignJob() - #it may be failed non critical so we try it again if self.infoCache and self.timestamp < time(): self.infoCache.clear() @@ -176,141 +110,3 @@ class ThreadManager: for rid in self.infoResults.keys(): if self.infoResults[rid].isStale(): del self.infoResults[rid] - - def tryReconnect(self): - """checks if reconnect needed""" - - if not (self.core.config["reconnect"]["activated"] and self.core.api.isTimeReconnect()): - return False - - active = [x.active.plugin.wantReconnect and x.active.plugin.waiting for x in self.threads if x.active] - - if not (0 < active.count(True) == len(active)): - return False - - if not exists(self.core.config['reconnect']['method']): - if exists(join(pypath, self.core.config['reconnect']['method'])): - self.core.config['reconnect']['method'] = join(pypath, self.core.config['reconnect']['method']) - else: - self.core.config["reconnect"]["activated"] = False - self.log.warning(_("Reconnect script not found!")) - return - - self.reconnecting.set() - - #Do reconnect - self.log.info(_("Starting reconnect")) - - while [x.active.plugin.waiting for x in self.threads if x.active].count(True) != 0: - sleep(0.25) - - ip = self.getIP() - - self.core.evm.dispatchEvent("reconnect:before", ip) - - self.log.debug("Old IP: %s" % ip) - - try: - reconn = Popen(self.core.config['reconnect']['method'], bufsize=-1, shell=True)#, stdout=subprocess.PIPE) - except: - self.log.warning(_("Failed executing reconnect script!")) - self.core.config["reconnect"]["activated"] = False - self.reconnecting.clear() - self.core.print_exc() - return - - reconn.wait() - sleep(1) - ip = self.getIP() - self.core.evm.dispatchEvent("reconnect:after", ip) - - self.log.info(_("Reconnected, new IP: %s") % ip) - - self.reconnecting.clear() - - def getIP(self): - """retrieve current ip""" - services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"), - ("http://checkip.dyndns.org/", ".*Current IP Address: (\S+)</body>.*")] - - ip = "" - for i in range(10): - try: - sv = choice(services) - ip = getURL(sv[0]) - ip = re.match(sv[1], ip).group(1) - break - except: - ip = "" - sleep(1) - - return ip - - def checkThreadCount(self): - """checks if there is a need for increasing or reducing thread count""" - - if len(self.threads) == self.core.config.get("download", "max_downloads"): - return True - elif len(self.threads) < self.core.config.get("download", "max_downloads"): - self.createThread() - else: - free = [x for x in self.threads if not x.active] - if free: - free[0].put("quit") - - - def cleanPycurl(self): - """ make a global curl cleanup (currently unused) """ - if self.processingIds(): - return False - import pycurl - - pycurl.global_cleanup() - pycurl.global_init(pycurl.GLOBAL_DEFAULT) - self.downloaded = 0 - self.log.debug("Cleaned up pycurl") - return True - - - def assignJob(self): - """assign a job to a thread if possible""" - - if self.pause or not self.core.api.isTimeDownload(): return - - #if self.downloaded > 20: - # if not self.cleanPyCurl(): return - - free = [x for x in self.threads if not x.active] - - inuse = [(x.active.pluginname, x.active.plugin.getDownloadLimit()) for x in self.threads if - x.active and x.active.hasPlugin()] - inuse = [(x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) for x in - inuse] - occ = tuple(sorted(uniqify([x[0] for x in inuse if 0 < x[1] <= x[2]]))) - - job = self.core.files.getJob(occ) - if job: - try: - job.initPlugin() - except Exception, e: - self.log.critical(str(e)) - print_exc() - job.setStatus("failed") - job.error = str(e) - job.release() - return - - spaceLeft = free_space(self.core.config["general"]["download_folder"]) / 1024 / 1024 - if spaceLeft < self.core.config["general"]["min_free_space"]: - self.log.warning(_("Not enough space left on device")) - self.pause = True - - if free and not self.pause: - thread = free[0] - #self.downloaded += 1 - thread.put(job) - else: - #put job back - if occ not in self.core.files.jobCache: - self.core.files.jobCache[occ] = [] - self.core.files.jobCache[occ].append(job.id) diff --git a/pyload/utils/PluginLoader.py b/pyload/utils/PluginLoader.py index 57a899e39..743f0e537 100644 --- a/pyload/utils/PluginLoader.py +++ b/pyload/utils/PluginLoader.py @@ -40,6 +40,7 @@ class BaseAttributes(defaultdict): return getattr(Base, attr) + class LoaderFactory: """ Container for multiple plugin loaders """ @@ -62,6 +63,15 @@ class LoaderFactory: if l2 is not loader: l2.removePlugin(plugin_type, plugin, info.version) + def findPlugin(self, name): + """ Finds a plugin type for given name """ + for loader in self.loader: + for t in loader.TYPES: + if loader.hasPlugin(t, name): + return t + + return None + def getPlugin(self, plugin, name): """ retrieve a plugin from an available loader """ for loader in self.loader: @@ -79,7 +89,7 @@ class PluginLoader: SINGLE = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?<!")"(?!")|\').*(?:(?<!")"(?!")|\'))', re.I) # finds the beginning of a expression that could span multiple lines - MULTI = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(\(|\{|\[|"{3})',re.I) + MULTI = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(\(|\{|\[|"{3})', re.I) # closing symbols MULTI_MATCH = { @@ -185,14 +195,14 @@ class PluginLoader: #TODO: strings must be parsed too, otherwise breaks very easily for i in xrange(m.end(2), len(content) - size + 1): - if content[i:i+size] == endchar: + if content[i:i + size] == endchar: # closing char seen and match now complete if stack == 0: endpos = i break else: stack -= 1 - elif content[i:i+size] == char: + elif content[i:i + size] == char: stack += 1 # in case the end was not found match will be empty diff --git a/pyload/web/app/scripts/models/Account.js b/pyload/web/app/scripts/models/Account.js index 26241d8e3..b9fc40036 100644 --- a/pyload/web/app/scripts/models/Account.js +++ b/pyload/web/app/scripts/models/Account.js @@ -3,9 +3,10 @@ define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', './ConfigIt return Backbone.Model.extend({ - idAttribute: 'loginname', + idAttribute: 'aid', defaults: { + aid: null, plugin: null, loginname: null, owner: -1, @@ -53,14 +54,15 @@ define(['jquery', 'backbone', 'underscore', 'app', 'utils/apitypes', './ConfigIt var refresh = _.has(options, 'refresh') && options.refresh; options = App.apiRequest('getAccountInfo', {plugin: this.get('plugin'), - loginname: this.get('loginname'), refresh: refresh}, options); + aid: this.get('aid'), refresh: refresh}, options); return Backbone.Model.prototype.fetch.call(this, options); }, setPassword: function(password, options) { options = App.apiRequest('updateAccount', - {plugin: this.get('plugin'), loginname: this.get('loginname'), password: password}, options); + {aid: this.get('aid'), + plugin: this.get('plugin'), loginname: this.get('loginname'), password: password}, options); return $.ajax(options); }, diff --git a/pyload/web/app/scripts/models/ServerStatus.js b/pyload/web/app/scripts/models/StatusInfo.js index 59739b41e..8712defa7 100644 --- a/pyload/web/app/scripts/models/ServerStatus.js +++ b/pyload/web/app/scripts/models/StatusInfo.js @@ -13,7 +13,8 @@ define(['jquery', 'backbone', 'underscore'], notifications: -1, paused: false, download: false, - reconnect: false + reconnect: false, + quota: -1 }, // Model Constructor @@ -23,7 +24,7 @@ define(['jquery', 'backbone', 'underscore'], fetch: function(options) { options || (options = {}); - options.url = 'api/getServerStatus'; + options.url = 'api/getStatusInfo'; return Backbone.Model.prototype.fetch.call(this, options); }, diff --git a/pyload/web/app/scripts/utils/apitypes.js b/pyload/web/app/scripts/utils/apitypes.js index fc92425de..88123f7ea 100644 --- a/pyload/web/app/scripts/utils/apitypes.js +++ b/pyload/web/app/scripts/utils/apitypes.js @@ -4,9 +4,9 @@ define([], function() { 'use strict'; return { DownloadState: {'Failed': 3, 'All': 0, 'Unmanaged': 4, 'Finished': 1, 'Unfinished': 2}, - DownloadStatus: {'NotPossible': 13, 'FileMismatch': 15, 'Downloading': 10, 'Missing': 14, 'NA': 0, 'Processing': 17, 'Waiting': 9, 'Decrypting': 16, 'Paused': 4, 'Failed': 7, 'Finished': 5, 'Skipped': 6, 'Unknown': 19, 'Aborted': 12, 'Online': 2, 'TempOffline': 11, 'Offline': 1, 'Custom': 18, 'Starting': 8, 'Queued': 3}, + DownloadStatus: {'NotPossible': 13, 'FileMismatch': 15, 'Downloading': 10, 'Missing': 14, 'NA': 0, 'Processing': 18, 'Waiting': 9, 'Decrypting': 17, 'Paused': 4, 'Failed': 7, 'Finished': 5, 'Skipped': 6, 'Unknown': 20, 'Aborted': 12, 'Online': 2, 'Starting': 8, 'TempOffline': 11, 'Offline': 1, 'Custom': 19, 'Occupied': 16, 'Queued': 3}, FileStatus: {'Remote': 2, 'Ok': 0, 'Missing': 1}, - InputType: {'PluginList': 13, 'Multiple': 11, 'Int': 2, 'NA': 0, 'Time': 7, 'List': 12, 'Bool': 8, 'File': 3, 'Text': 1, 'Table': 14, 'Folder': 4, 'Password': 6, 'Click': 9, 'Select': 10, 'Textbox': 5}, + InputType: {'PluginList': 15, 'Multiple': 13, 'TimeSpan': 8, 'Int': 2, 'ByteSize': 9, 'Time': 7, 'List': 14, 'Textbox': 5, 'Bool': 10, 'File': 3, 'NA': 0, 'Table': 16, 'Folder': 4, 'Password': 6, 'Click': 11, 'Select': 12, 'Text': 1}, Interaction: {'Captcha': 2, 'All': 0, 'Query': 4, 'Notification': 1}, MediaType: {'All': 0, 'Audio': 2, 'Image': 4, 'Executable': 64, 'Other': 1, 'Video': 8, 'Document': 16, 'Archive': 32}, PackageStatus: {'Paused': 1, 'Remote': 3, 'Folder': 2, 'Ok': 0}, diff --git a/pyload/web/app/scripts/views/accounts/accountModal.js b/pyload/web/app/scripts/views/accounts/accountModal.js index 31e05dff6..11eed1355 100644 --- a/pyload/web/app/scripts/views/accounts/accountModal.js +++ b/pyload/web/app/scripts/views/accounts/accountModal.js @@ -59,7 +59,7 @@ define(['jquery', 'underscore', 'app', 'views/abstract/modalView', 'hbs!tpl/dial password = this.$('#password').val(), self = this; - $.ajax(App.apiRequest('updateAccount', { + $.ajax(App.apiRequest('createAccount', { plugin: plugin, loginname: login, password: password }, { success: function(data) { App.vent.trigger('account:updated', data); diff --git a/pyload/web/app/scripts/views/headerView.js b/pyload/web/app/scripts/views/headerView.js index d4d07ac39..a12248a7a 100644 --- a/pyload/web/app/scripts/views/headerView.js +++ b/pyload/web/app/scripts/views/headerView.js @@ -1,8 +1,8 @@ -define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus', +define(['jquery', 'underscore', 'backbone', 'app', 'models/StatusInfo', 'views/progressView', 'views/notificationView', 'helpers/formatSize', 'hbs!tpl/header/layout', 'hbs!tpl/header/status', 'hbs!tpl/header/progressbar', 'hbs!tpl/header/progressSup', 'hbs!tpl/header/progressSub' , 'flot'], function( - $, _, Backbone, App, ServerStatus, ProgressView, NotificationView, formatSize, template, templateStatus, templateProgress, templateSup, templateSub) { + $, _, Backbone, App, StatusInfo, ProgressView, NotificationView, formatSize, template, templateStatus, templateProgress, templateSup, templateSub) { 'use strict'; // Renders the header with all information return Backbone.Marionette.ItemView.extend({ @@ -47,7 +47,7 @@ define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus', this.model = App.user; - this.status = new ServerStatus(); + this.status = new StatusInfo(); this.listenTo(this.status, 'change', this.update); this.listenTo(App.progressList, 'add', function(model) { @@ -194,7 +194,7 @@ define(['jquery', 'underscore', 'backbone', 'app', 'models/ServerStatus', var data = JSON.parse(evt.data); if (data === null) return; - if (data['@class'] === 'ServerStatus') { + if (data['@class'] === 'StatusInfo') { this.status.set(data); // There tasks at the server, but not in queue: so fetch them |