From 5499be89203a18ca61a21cfc7266cf0f4ebe6547 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 15 Dec 2011 23:18:21 +0100 Subject: refractoring --- module/database/FileDatabase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 357cd766d..895e0de65 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -22,7 +22,7 @@ from threading import RLock from time import time from module.utils import formatSize, lock -from module.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent +from module.interaction.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent from module.PyPackage import PyPackage from module.PyFile import PyFile from module.database import style, DatabaseBackend -- cgit v1.2.3 From c9a359b75905d4910e42127cef8cbe62d4a13657 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 20 Dec 2011 21:35:43 +0100 Subject: some bugfixes --- module/database/FileDatabase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 895e0de65..4084c46f7 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -340,7 +340,7 @@ class FileHandler: if "decrypt" in self.jobCache: return None - plugins = self.core.pluginManager.crypterPlugins.keys() + self.core.pluginManager.containerPlugins.keys() + plugins = self.core.pluginManager.getPlugins("crypter").keys() + self.core.pluginManager.getPlugins("container").keys() plugins = str(tuple(plugins)) jobs = self.db.getPluginJob(plugins) -- cgit v1.2.3 From 958bf611f5d9d117f19f824990ec6fd6b537e967 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 22 Dec 2011 23:45:38 +0100 Subject: accountmanager v2, delete your accounts.conf and re-enter them in pyload, new nice debug functions, try core.shell() and core.breakpoint() --- module/database/FileDatabase.py | 64 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 4084c46f7..b5c386802 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -25,7 +25,7 @@ from module.utils import formatSize, lock from module.interaction.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent from module.PyPackage import PyPackage from module.PyFile import PyFile -from module.database import style, DatabaseBackend +from module.database import DatabaseBackend, queue, async, inner try: from pysqlite2 import dbapi2 as sqlite3 @@ -574,25 +574,25 @@ class FileHandler: self.db.restartFailed() class FileMethods(): - @style.queue + @queue def filecount(self, queue): """returns number of files in queue""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue, )) return self.c.fetchone()[0] - @style.queue + @queue def queuecount(self, queue): """ number of files in queue not finished yet""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0,4)", (queue, )) return self.c.fetchone()[0] - @style.queue + @queue def processcount(self, queue, fid): """ number of files which have to be proccessed """ self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2,3,5,7,12) AND l.id != ?", (queue, str(fid))) return self.c.fetchone()[0] - @style.inner + @inner def _nextPackageOrder(self, queue=0): self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) max = self.c.fetchone()[0] @@ -601,7 +601,7 @@ class FileMethods(): else: return 0 - @style.inner + @inner def _nextFileOrder(self, package): self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) max = self.c.fetchone()[0] @@ -610,13 +610,13 @@ class FileMethods(): else: return 0 - @style.queue + @queue def addLink(self, url, name, plugin, package): order = self._nextFileOrder(package) self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', (url, name, plugin, package, order)) return self.c.lastrowid - @style.queue + @queue def addLinks(self, links, package): """ links is a list of tupels (url,plugin)""" order = self._nextFileOrder(package) @@ -624,27 +624,27 @@ class FileMethods(): links = [(x[0], x[0], x[1], package, o) for x, o in zip(links, orders)] self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) - @style.queue + @queue def addPackage(self, name, folder, queue): order = self._nextPackageOrder(queue) self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', (name, folder, queue, order)) return self.c.lastrowid - @style.queue + @queue def deletePackage(self, p): self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue)) - @style.queue + @queue def deleteLink(self, f): self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid))) - @style.queue + @queue def getAllLinks(self, q): """return information about all links in queue q @@ -677,7 +677,7 @@ class FileMethods(): return data - @style.queue + @queue def getAllPackages(self, q): """return information about packages in queue q (only useful in get all data) @@ -714,7 +714,7 @@ class FileMethods(): return data - @style.queue + @queue def getLinkData(self, id): """get link information as dict""" self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE id=?', (str(id), )) @@ -738,7 +738,7 @@ class FileMethods(): return data - @style.queue + @queue def getPackageData(self, id): """get data about links for a package""" self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id), )) @@ -762,15 +762,15 @@ class FileMethods(): return data - @style.async + @async def updateLink(self, f): self.c.execute('UPDATE links SET url=?,name=?,size=?,status=?,error=?,package=? WHERE id=?', (f.url, f.name, f.size, f.status, f.error, str(f.packageid), str(f.id))) - @style.queue + @queue def updatePackage(self, p): self.c.execute('UPDATE packages SET name=?,folder=?,site=?,password=?,queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) - @style.queue + @queue def updateLinkInfo(self, data): """ data is list of tupels (name, size, status, url) """ self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1,2,3,14)', data) @@ -780,7 +780,7 @@ class FileMethods(): ids.append(int(r[0])) return ids - @style.queue + @queue def reorderPackage(self, p, position, noMove=False): if position == -1: position = self._nextPackageOrder(p.queue) @@ -792,7 +792,7 @@ class FileMethods(): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) - @style.queue + @queue def reorderLink(self, f, position): """ reorder link with f as dict for pyfile """ if f["order"] > position: @@ -803,20 +803,20 @@ class FileMethods(): self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"])) - @style.queue + @queue def clearPackageOrder(self, p): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) - @style.async + @async def restartFile(self, id): self.c.execute('UPDATE links SET status=3,error="" WHERE id=?', (str(id),)) - @style.async + @async def restartPackage(self, id): self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) - @style.queue + @queue def getPackage(self, id): """return package instance from id""" self.c.execute("SELECT name,folder,site,password,queue,packageorder FROM packages WHERE id=?", (str(id), )) @@ -825,7 +825,7 @@ class FileMethods(): return PyPackage(self.manager, id, * r) #---------------------------------------------------------------------- - @style.queue + @queue def getFile(self, id): """return link instance from id""" self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id), )) @@ -834,7 +834,7 @@ class FileMethods(): return PyFile(self.manager, id, * r) - @style.queue + @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" @@ -854,7 +854,7 @@ class FileMethods(): return [x[0] for x in self.c] - @style.queue + @queue def getPluginJob(self, plugins): """returns pyfile ids with suited plugins""" cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins @@ -863,29 +863,29 @@ class FileMethods(): return [x[0] for x in self.c] - @style.queue + @queue def getUnfinished(self, pid): """return list of max length 3 ids with pyfiles in package not finished or processed""" self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) return [r[0] for r in self.c] - @style.queue + @queue def deleteFinished(self): self.c.execute("DELETE FROM links WHERE status IN (0,4)") self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") - @style.queue + @queue def restartFailed(self): self.c.execute("UPDATE links SET status=3,error='' WHERE status IN (8, 9)") - @style.queue + @queue def findDuplicates(self, id, folder, filename): """ checks if filename exists with different id and same package """ self.c.execute("SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", (folder, id, filename)) return self.c.fetchone() - @style.queue + @queue def purgeLinks(self): self.c.execute("DELETE FROM links;") self.c.execute("DELETE FROM packages;") -- cgit v1.2.3 From d35c003cc53d4723d1dfe0d81eeb9bea78cee594 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 31 Dec 2011 16:01:24 +0100 Subject: new crypter plugin API, now decrypting possible for now. --- module/database/FileDatabase.py | 151 +++++++++++----------------------------- 1 file changed, 41 insertions(+), 110 deletions(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index b5c386802..abe7c8fc9 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -22,7 +22,6 @@ from threading import RLock from time import time from module.utils import formatSize, lock -from module.interaction.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent from module.PyPackage import PyPackage from module.PyFile import PyFile from module.database import DatabaseBackend, queue, async, inner @@ -40,11 +39,12 @@ class FileHandler: def __init__(self, core): """Constructor""" self.core = core + self.ev = None #event manager, set later # translations self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), _("downloading"), _("processing"), _("unknown")] - self.cache = {} #holds instances for files + self.cache = {} # holds instances for files self.packageCache = {} # same for packages #@TODO: purge the cache @@ -54,14 +54,12 @@ class FileHandler: #self.lock._Verbose__verbose = True self.filecount = -1 # if an invalid value is set get current value from db - self.queuecount = -1 #number of package to be loaded - self.unchanged = False #determines if any changes was made since last call + self.queuecount = -1 # number of package to be loaded self.db = self.core.db def change(func): def new(*args): - args[0].unchanged = False args[0].filecount = -1 args[0].queuecount = -1 args[0].jobCache = {} @@ -118,31 +116,23 @@ class FileHandler: @lock @change - def addLinks(self, urls, package): - """adds links""" - - self.core.hookManager.dispatchEvent("linksAdded", urls, package) - - data = self.core.pluginManager.parseUrls(urls) - + def addLinks(self, data, package): + """Add links, data = (plugin, url) tuple. Internal method you should use API.""" self.db.addLinks(data, package) - self.core.threadManager.createInfoThread(data, package) + self.ev.dispatchEvent("packageUpdated", package) - #@TODO change from reloadAll event to package update event - self.core.pullManager.addEvent(ReloadAllEvent("collector")) - #---------------------------------------------------------------------- @lock @change - def addPackage(self, name, folder, queue=0): + def addPackage(self, name, folder, queue=0, password=""): """adds a package, default to link collector""" - lastID = self.db.addPackage(name, folder, queue) - p = self.db.getPackage(lastID) - e = InsertEvent("pack", lastID, p.order, "collector" if not queue else "queue") - self.core.pullManager.addEvent(e) - return lastID + pid = self.db.addPackage(name, folder, queue, password) + p = self.db.getPackage(pid) + + self.ev.dispatchEvent("packageInserted", pid, p.queue, p.order) + return pid + - #---------------------------------------------------------------------- @lock @change def deletePackage(self, id): @@ -156,7 +146,6 @@ class FileHandler: oldorder = p.order queue = p.queue - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") pyfiles = self.cache.values() @@ -166,8 +155,7 @@ class FileHandler: pyfile.release() self.db.deletePackage(p) - self.core.pullManager.addEvent(e) - self.core.hookManager.dispatchEvent("packageDeleted", id) + self.ev.dispatchEvent("packageDeleted", id) if id in self.packageCache: del self.packageCache[id] @@ -178,7 +166,7 @@ class FileHandler: pack.order -= 1 pack.notifyChange() - #---------------------------------------------------------------------- + @lock @change def deleteLink(self, id): @@ -189,8 +177,6 @@ class FileHandler: return None pid = f.packageid - e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") - oldorder = f.order if id in self.core.threadManager.processingIds(): @@ -201,7 +187,7 @@ class FileHandler: self.db.deleteLink(f) - self.core.pullManager.addEvent(e) + self.ev.dispatchEvent("linkDeleted", id, pid) p = self.getPackage(pid) if not len(p.getChildren()): @@ -213,35 +199,26 @@ class FileHandler: pyfile.order -= 1 pyfile.notifyChange() - #---------------------------------------------------------------------- def releaseLink(self, id): """removes pyfile from cache""" if id in self.cache: del self.cache[id] - #---------------------------------------------------------------------- def releasePackage(self, id): """removes package from cache""" if id in self.packageCache: del self.packageCache[id] - #---------------------------------------------------------------------- def updateLink(self, pyfile): """updates link""" self.db.updateLink(pyfile) + self.ev.dispatchEvent("linkUpdated", pyfile.id, pyfile.packageid) - e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") - self.core.pullManager.addEvent(e) - - #---------------------------------------------------------------------- def updatePackage(self, pypack): """updates a package""" self.db.updatePackage(pypack) + self.ev.dispatchEvent("packageUpdated", pypack.id) - e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") - self.core.pullManager.addEvent(e) - - #---------------------------------------------------------------------- def getPackage(self, id): """return package instance""" @@ -250,7 +227,6 @@ class FileHandler: else: return self.db.getPackage(id) - #---------------------------------------------------------------------- def getPackageData(self, id): """returns dict with package information""" pack = self.getPackage(id) @@ -274,7 +250,7 @@ class FileHandler: return pack - #---------------------------------------------------------------------- + def getFileData(self, id): """returns dict with file information""" if id in self.cache: @@ -282,7 +258,7 @@ class FileHandler: return self.db.getLinkData(id) - #---------------------------------------------------------------------- + def getFile(self, id): """returns pyfile instance""" if id in self.cache: @@ -290,7 +266,7 @@ class FileHandler: else: return self.db.getFile(id) - #---------------------------------------------------------------------- + @lock def getJob(self, occ): """get suitable job""" @@ -334,21 +310,6 @@ class FileHandler: #pyfile = self.getFile(self.jobCache[occ].pop()) return pyfile - @lock - def getDecryptJob(self): - """return job for decrypting""" - if "decrypt" in self.jobCache: - return None - - plugins = self.core.pluginManager.getPlugins("crypter").keys() + self.core.pluginManager.getPlugins("container").keys() - plugins = str(tuple(plugins)) - - jobs = self.db.getPluginJob(plugins) - if jobs: - return self.getFile(jobs[0]) - else: - self.jobCache["decrypt"] = "empty" - return None def getFileCount(self): """returns number of files""" @@ -405,8 +366,7 @@ class FileHandler: if id in self.packageCache: self.packageCache[id].setFinished = False - e = UpdateEvent("pack", id, "collector" if not self.getPackage(id).queue else "queue") - self.core.pullManager.addEvent(e) + self.ev.dispatchEvent("packageUpdated", id) @lock @change @@ -420,9 +380,8 @@ class FileHandler: self.db.restartFile(id) + self.ev.dispatchEvent("linkUpdated", id) - e = UpdateEvent("file", id, "collector" if not self.getFile(id).package().queue else "queue") - self.core.pullManager.addEvent(e) @lock @change @@ -431,17 +390,10 @@ class FileHandler: p = self.db.getPackage(id) oldorder = p.order + p.queue = queue - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - self.db.clearPackageOrder(p) - - p = self.db.getPackage(id) - - p.queue = queue self.db.updatePackage(p) - self.db.reorderPackage(p, -1, True) packs = self.packageCache.values() @@ -452,37 +404,34 @@ class FileHandler: self.db.commit() self.releasePackage(id) - p = self.getPackage(id) - - e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) + + self.ev.dispatchEvent("packageDeleted", id) + self.ev.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change def reorderPackage(self, id, position): p = self.getPackage(id) - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) self.db.reorderPackage(p, position) packs = self.packageCache.values() for pack in packs: if pack.queue != p.queue or pack.order < 0 or pack == p: continue if p.order > position: - if pack.order >= position and pack.order < p.order: + if position <= pack.order < p.order: pack.order += 1 pack.notifyChange() elif p.order < position: - if pack.order <= position and pack.order > p.order: + if position >= pack.order > p.order: pack.order -= 1 pack.notifyChange() p.order = position self.db.commit() - e = InsertEvent("pack", id, position, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) + self.ev.dispatchEvent("packageDeleted", id) + self.ev.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change @@ -490,20 +439,17 @@ class FileHandler: f = self.getFileData(id) f = f[id] - e = RemoveEvent("file", id, "collector" if not self.getPackage(f["package"]).queue else "queue") - self.core.pullManager.addEvent(e) - self.db.reorderLink(f, position) pyfiles = self.cache.values() for pyfile in pyfiles: if pyfile.packageid != f["package"] or pyfile.order < 0: continue if f["order"] > position: - if pyfile.order >= position and pyfile.order < f["order"]: + if position <= pyfile.order < f["order"]: pyfile.order += 1 pyfile.notifyChange() elif f["order"] < position: - if pyfile.order <= position and pyfile.order > f["order"]: + if position >= pyfile.order > f["order"]: pyfile.order -= 1 pyfile.notifyChange() @@ -512,15 +458,14 @@ class FileHandler: self.db.commit() - e = InsertEvent("file", id, position, "collector" if not self.getPackage(f["package"]).queue else "queue") - self.core.pullManager.addEvent(e) + self.ev.dispatchEvent("packageUpdated", f["package"]) + @change def updateFileInfo(self, data, pid): """ updates file info (name, size, status, url)""" ids = self.db.updateLinkInfo(data) - e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue") - self.core.pullManager.addEvent(e) + self.ev.dispatchEvent("packageUpdated", pid) def checkPackageFinished(self, pyfile): """ checks if package is finished and calls hookmanager """ @@ -625,9 +570,9 @@ class FileMethods(): self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) @queue - def addPackage(self, name, folder, queue): + def addPackage(self, name, folder, queue, password): order = self._nextPackageOrder(queue) - self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', (name, folder, queue, order)) + self.c.execute('INSERT INTO packages(name, folder, queue, packageorder, password) VALUES(?,?,?,?,?)', (name, folder, queue, order, password)) return self.c.lastrowid @queue @@ -824,7 +769,7 @@ class FileMethods(): if not r: return None return PyPackage(self.manager, id, * r) - #---------------------------------------------------------------------- + @queue def getFile(self, id): """return link instance from id""" @@ -837,28 +782,14 @@ class FileMethods(): @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" - - #@TODO improve this hardcoded method - pre = "('DLC', 'LinkList', 'SerienjunkiesOrg', 'CCF', 'RSDF')" #plugins which are processed in collector - cmd = "(" for i, item in enumerate(occ): if i: cmd += ", " cmd += "'%s'" % item - - cmd += ")" - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) - - self.c.execute(cmd) # very bad! - - return [x[0] for x in self.c] - - @queue - def getPluginJob(self, plugins): - """returns pyfile ids with suited plugins""" - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins + cmd += ")" + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=1 AND l.plugin NOT IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % cmd self.c.execute(cmd) # very bad! return [x[0] for x in self.c] -- cgit v1.2.3 From 35742c2cb023ac49ab3056752d2040cdb030cc2b Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 1 Jan 2012 13:36:59 +0100 Subject: Happy new Year ! --- module/database/FileDatabase.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index abe7c8fc9..116f2b02b 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -190,8 +190,7 @@ class FileHandler: self.ev.dispatchEvent("linkDeleted", id, pid) p = self.getPackage(pid) - if not len(p.getChildren()): - p.delete() + p.deleteIfEmpty() pyfiles = self.cache.values() for pyfile in pyfiles: -- cgit v1.2.3 From 18466eb7f8f3cd4ca9a0824074d2ff454939fce6 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Wed, 4 Jan 2012 17:23:13 +0100 Subject: some fixes --- module/database/FileDatabase.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 116f2b02b..23e657ee4 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -39,7 +39,7 @@ class FileHandler: def __init__(self, core): """Constructor""" self.core = core - self.ev = None #event manager, set later + self.evm = core.eventManager # translations self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), _("downloading"), _("processing"), _("unknown")] @@ -119,7 +119,7 @@ class FileHandler: def addLinks(self, data, package): """Add links, data = (plugin, url) tuple. Internal method you should use API.""" self.db.addLinks(data, package) - self.ev.dispatchEvent("packageUpdated", package) + self.evm.dispatchEvent("packageUpdated", package) @lock @@ -129,7 +129,7 @@ class FileHandler: pid = self.db.addPackage(name, folder, queue, password) p = self.db.getPackage(pid) - self.ev.dispatchEvent("packageInserted", pid, p.queue, p.order) + self.evm.dispatchEvent("packageInserted", pid, p.queue, p.order) return pid @@ -155,7 +155,7 @@ class FileHandler: pyfile.release() self.db.deletePackage(p) - self.ev.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageDeleted", id) if id in self.packageCache: del self.packageCache[id] @@ -187,7 +187,7 @@ class FileHandler: self.db.deleteLink(f) - self.ev.dispatchEvent("linkDeleted", id, pid) + self.evm.dispatchEvent("linkDeleted", id, pid) p = self.getPackage(pid) p.deleteIfEmpty() @@ -211,12 +211,12 @@ class FileHandler: def updateLink(self, pyfile): """updates link""" self.db.updateLink(pyfile) - self.ev.dispatchEvent("linkUpdated", pyfile.id, pyfile.packageid) + self.evm.dispatchEvent("linkUpdated", pyfile.id, pyfile.packageid) def updatePackage(self, pypack): """updates a package""" self.db.updatePackage(pypack) - self.ev.dispatchEvent("packageUpdated", pypack.id) + self.evm.dispatchEvent("packageUpdated", pypack.id) def getPackage(self, id): """return package instance""" @@ -365,7 +365,7 @@ class FileHandler: if id in self.packageCache: self.packageCache[id].setFinished = False - self.ev.dispatchEvent("packageUpdated", id) + self.evm.dispatchEvent("packageUpdated", id) @lock @change @@ -379,7 +379,7 @@ class FileHandler: self.db.restartFile(id) - self.ev.dispatchEvent("linkUpdated", id) + self.evm.dispatchEvent("linkUpdated", id) @lock @@ -404,8 +404,8 @@ class FileHandler: self.db.commit() self.releasePackage(id) - self.ev.dispatchEvent("packageDeleted", id) - self.ev.dispatchEvent("packageInserted", id, p.queue, p.order) + self.evm.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change @@ -429,8 +429,8 @@ class FileHandler: p.order = position self.db.commit() - self.ev.dispatchEvent("packageDeleted", id) - self.ev.dispatchEvent("packageInserted", id, p.queue, p.order) + self.evm.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change @@ -457,14 +457,14 @@ class FileHandler: self.db.commit() - self.ev.dispatchEvent("packageUpdated", f["package"]) + self.evm.dispatchEvent("packageUpdated", f["package"]) @change def updateFileInfo(self, data, pid): """ updates file info (name, size, status, url)""" ids = self.db.updateLinkInfo(data) - self.ev.dispatchEvent("packageUpdated", pid) + self.evm.dispatchEvent("packageUpdated", pid) def checkPackageFinished(self, pyfile): """ checks if package is finished and calls hookmanager """ -- cgit v1.2.3 From d347b354b169044fabd0684f304f1777ae5b75d8 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 21:59:28 +0100 Subject: show empty packages --- module/database/FileDatabase.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 23e657ee4..eb76f468b 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -636,7 +636,7 @@ class FileMethods(): } """ self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ - FROM packages p JOIN pstats s ON p.id = s.id \ + FROM packages p LEFT OUTER JOIN pstats s ON p.id = s.id \ WHERE p.queue=? ORDER BY p.packageorder', str(q)) data = {} @@ -649,10 +649,10 @@ class FileMethods(): 'password': r[4], 'queue': r[5], 'order': r[6], - 'sizetotal': int(r[7]), - 'sizedone': r[8] if r[8] else 0, #these can be None + 'sizetotal': int(r[7]) if r[7] else 0, + 'sizedone': int(r[8]) if r[8] else 0, #these can be None 'linksdone': r[9] if r[9] else 0, - 'linkstotal': r[10], + 'linkstotal': r[10] if r[10] else 0, 'links': {} } -- cgit v1.2.3 From 4df2b77fdf42046fe19bd371be7c7255986b5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 6 Mar 2012 13:36:39 +0100 Subject: renamed hooks to addons, new filemanager and database, many new api methods you will loose ALL your LINKS, webinterface will NOT work --- module/database/FileDatabase.py | 1033 ++++++++++----------------------------- 1 file changed, 270 insertions(+), 763 deletions(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index eb76f468b..08b18765d 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- + """ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -14,781 +16,331 @@ along with this program; if not, see . @author: RaNaN - @author: mkaay """ +from new_collections import OrderedDict +from module.Api import DownloadInfo, LinkStatus, FileInfo, PackageInfo, PackageStats +from module.database import DatabaseMethods, queue, async, inner -from threading import RLock -from time import time - -from module.utils import formatSize, lock -from module.PyPackage import PyPackage -from module.PyFile import PyFile -from module.database import DatabaseBackend, queue, async, inner - -try: - from pysqlite2 import dbapi2 as sqlite3 -except: - import sqlite3 - - -class FileHandler: - """Handles all request made to obtain information, - modify status or other request for links or packages""" - - def __init__(self, core): - """Constructor""" - self.core = core - self.evm = core.eventManager - - # translations - self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), _("downloading"), _("processing"), _("unknown")] - - self.cache = {} # holds instances for files - self.packageCache = {} # same for packages - #@TODO: purge the cache - - self.jobCache = {} - - self.lock = RLock() #@TODO should be a Lock w/o R - #self.lock._Verbose__verbose = True - - self.filecount = -1 # if an invalid value is set get current value from db - self.queuecount = -1 # number of package to be loaded - - self.db = self.core.db - - def change(func): - def new(*args): - args[0].filecount = -1 - args[0].queuecount = -1 - args[0].jobCache = {} - return func(*args) - return new - - #---------------------------------------------------------------------- - def save(self): - """saves all data to backend""" - self.db.commit() - - #---------------------------------------------------------------------- - def syncSave(self): - """saves all data to backend and waits until all data are written""" - pyfiles = self.cache.values() - for pyfile in pyfiles: - pyfile.sync() - - pypacks = self.packageCache.values() - for pypack in pypacks: - pypack.sync() - - self.db.syncSave() - - @lock - def getCompleteData(self, queue=1): - """gets a complete data representation""" - - data = self.db.getAllLinks(queue) - packs = self.db.getAllPackages(queue) - - data.update([(x.id, x.toDbDict()[x.id]) for x in self.cache.values()]) - - for x in self.packageCache.itervalues(): - if x.queue != queue or x.id not in packs: continue - packs[x.id].update(x.toDict()[x.id]) - - for key, value in data.iteritems(): - if value["package"] in packs: - packs[value["package"]]["links"][key] = value - - return packs - - @lock - def getInfoData(self, queue=1): - """gets a data representation without links""" - - packs = self.db.getAllPackages(queue) - for x in self.packageCache.itervalues(): - if x.queue != queue or x.id not in packs: continue - packs[x.id].update(x.toDict()[x.id]) - - return packs - - @lock - @change - def addLinks(self, data, package): - """Add links, data = (plugin, url) tuple. Internal method you should use API.""" - self.db.addLinks(data, package) - self.evm.dispatchEvent("packageUpdated", package) - - - @lock - @change - def addPackage(self, name, folder, queue=0, password=""): - """adds a package, default to link collector""" - pid = self.db.addPackage(name, folder, queue, password) - p = self.db.getPackage(pid) - - self.evm.dispatchEvent("packageInserted", pid, p.queue, p.order) - return pid - - - @lock - @change - def deletePackage(self, id): - """delete package and all contained links""" - - p = self.getPackage(id) - if not p: - if id in self.packageCache: del self.packageCache[id] - return - - oldorder = p.order - queue = p.queue - - - pyfiles = self.cache.values() - - for pyfile in pyfiles: - if pyfile.packageid == id: - pyfile.abortDownload() - pyfile.release() - - self.db.deletePackage(p) - self.evm.dispatchEvent("packageDeleted", id) - - if id in self.packageCache: - del self.packageCache[id] - - packs = self.packageCache.values() - for pack in packs: - if pack.queue == queue and pack.order > oldorder: - pack.order -= 1 - pack.notifyChange() - - - @lock - @change - def deleteLink(self, id): - """deletes links""" - - f = self.getFile(id) - if not f: - return None - - pid = f.packageid - oldorder = f.order - - if id in self.core.threadManager.processingIds(): - self.cache[id].abortDownload() - - if id in self.cache: - del self.cache[id] - - self.db.deleteLink(f) - - self.evm.dispatchEvent("linkDeleted", id, pid) - - p = self.getPackage(pid) - p.deleteIfEmpty() - - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid == pid and pyfile.order > oldorder: - pyfile.order -= 1 - pyfile.notifyChange() - - def releaseLink(self, id): - """removes pyfile from cache""" - if id in self.cache: - del self.cache[id] - - def releasePackage(self, id): - """removes package from cache""" - if id in self.packageCache: - del self.packageCache[id] - - def updateLink(self, pyfile): - """updates link""" - self.db.updateLink(pyfile) - self.evm.dispatchEvent("linkUpdated", pyfile.id, pyfile.packageid) - - def updatePackage(self, pypack): - """updates a package""" - self.db.updatePackage(pypack) - self.evm.dispatchEvent("packageUpdated", pypack.id) - - def getPackage(self, id): - """return package instance""" - - if id in self.packageCache: - return self.packageCache[id] - else: - return self.db.getPackage(id) - - def getPackageData(self, id): - """returns dict with package information""" - pack = self.getPackage(id) - - if not pack: - return None - - pack = pack.toDict()[id] - - data = self.db.getPackageData(id) - - tmplist = [] - - cache = self.cache.values() - for x in cache: - if int(x.toDbDict()[x.id]["package"]) == int(id): - tmplist.append((x.id, x.toDbDict()[x.id])) - data.update(tmplist) - - pack["links"] = data - - return pack - - - def getFileData(self, id): - """returns dict with file information""" - if id in self.cache: - return self.cache[id].toDbDict() - - return self.db.getLinkData(id) - - - def getFile(self, id): - """returns pyfile instance""" - if id in self.cache: - return self.cache[id] - else: - return self.db.getFile(id) - - - @lock - def getJob(self, occ): - """get suitable job""" - - #@TODO clean mess - #@TODO improve selection of valid jobs - - if occ in self.jobCache: - if self.jobCache[occ]: - id = self.jobCache[occ].pop() - if id == "empty": - pyfile = None - self.jobCache[occ].append("empty") - else: - pyfile = self.getFile(id) - else: - jobs = self.db.getJob(occ) - jobs.reverse() - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - self.jobCache[occ].extend(jobs) - pyfile = self.getFile(self.jobCache[occ].pop()) - - else: - self.jobCache = {} #better not caching to much - jobs = self.db.getJob(occ) - jobs.reverse() - self.jobCache[occ] = jobs - - if not jobs: - self.jobCache[occ].append("empty") - pyfile = None - else: - pyfile = self.getFile(self.jobCache[occ].pop()) - - #@TODO: maybe the new job has to be approved... - - - #pyfile = self.getFile(self.jobCache[occ].pop()) - return pyfile - - - def getFileCount(self): - """returns number of files""" - - if self.filecount == -1: - self.filecount = self.db.filecount(1) - - return self.filecount - - def getQueueCount(self, force=False): - """number of files that have to be processed""" - if self.queuecount == -1 or force: - self.queuecount = self.db.queuecount(1) - - return self.queuecount - - def checkAllLinksFinished(self): - """checks if all files are finished and dispatch event""" - - if not self.getQueueCount(True): - self.core.hookManager.dispatchEvent("allDownloadsFinished") - self.core.log.debug("All downloads finished") - return True - - return False - - def checkAllLinksProcessed(self, fid): - """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" - - # reset count so statistic will update (this is called when dl was processed) - self.resetCount() - - if not self.db.processcount(1, fid): - self.core.hookManager.dispatchEvent("allDownloadsProcessed") - self.core.log.debug("All downloads processed") - return True - - return False - - def resetCount(self): - self.queuecount = -1 - - @lock - @change - def restartPackage(self, id): - """restart package""" - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid == id: - self.restartFile(pyfile.id) - - self.db.restartPackage(id) - - if id in self.packageCache: - self.packageCache[id].setFinished = False - - self.evm.dispatchEvent("packageUpdated", id) - - @lock - @change - def restartFile(self, id): - """ restart file""" - if id in self.cache: - self.cache[id].status = 3 - self.cache[id].name = self.cache[id].url - self.cache[id].error = "" - self.cache[id].abortDownload() - - - self.db.restartFile(id) - self.evm.dispatchEvent("linkUpdated", id) - - - @lock - @change - def setPackageLocation(self, id, queue): - """push package to queue""" - - p = self.db.getPackage(id) - oldorder = p.order - p.queue = queue +default = PackageStats(0, 0, 0, 0) - self.db.clearPackageOrder(p) - self.db.updatePackage(p) - self.db.reorderPackage(p, -1, True) - - packs = self.packageCache.values() - for pack in packs: - if pack.queue != queue and pack.order > oldorder: - pack.order -= 1 - pack.notifyChange() - - self.db.commit() - self.releasePackage(id) - - self.evm.dispatchEvent("packageDeleted", id) - self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) - - @lock - @change - def reorderPackage(self, id, position): - p = self.getPackage(id) - - self.db.reorderPackage(p, position) - - packs = self.packageCache.values() - for pack in packs: - if pack.queue != p.queue or pack.order < 0 or pack == p: continue - if p.order > position: - if position <= pack.order < p.order: - pack.order += 1 - pack.notifyChange() - elif p.order < position: - if position >= pack.order > p.order: - pack.order -= 1 - pack.notifyChange() - - p.order = position - self.db.commit() - - self.evm.dispatchEvent("packageDeleted", id) - self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) - - @lock - @change - def reorderFile(self, id, position): - f = self.getFileData(id) - f = f[id] - - self.db.reorderLink(f, position) - - pyfiles = self.cache.values() - for pyfile in pyfiles: - if pyfile.packageid != f["package"] or pyfile.order < 0: continue - if f["order"] > position: - if position <= pyfile.order < f["order"]: - pyfile.order += 1 - pyfile.notifyChange() - elif f["order"] < position: - if position >= pyfile.order > f["order"]: - pyfile.order -= 1 - pyfile.notifyChange() - - if id in self.cache: - self.cache[id].order = position - - self.db.commit() - - self.evm.dispatchEvent("packageUpdated", f["package"]) - - - @change - def updateFileInfo(self, data, pid): - """ updates file info (name, size, status, url)""" - ids = self.db.updateLinkInfo(data) - self.evm.dispatchEvent("packageUpdated", pid) - - def checkPackageFinished(self, pyfile): - """ checks if package is finished and calls hookmanager """ - - ids = self.db.getUnfinished(pyfile.packageid) - if not ids or (pyfile.id in ids and len(ids) == 1): - if not pyfile.package().setFinished: - self.core.log.info(_("Package finished: %s") % pyfile.package().name) - self.core.hookManager.packageFinished(pyfile.package()) - pyfile.package().setFinished = True - - - def reCheckPackage(self, pid): - """ recheck links in package """ - data = self.db.getPackageData(pid) - - urls = [] - - for pyfile in data.itervalues(): - if pyfile["status"] not in (0, 12, 13): - urls.append((pyfile["url"], pyfile["plugin"])) - - self.core.threadManager.createInfoThread(urls, pid) - - @lock - @change - def deleteFinishedLinks(self): - """ deletes finished links and packages, return deleted packages """ - - old_packs = self.getInfoData(0) - old_packs.update(self.getInfoData(1)) - - self.db.deleteFinished() - - new_packs = self.db.getAllPackages(0) - new_packs.update(self.db.getAllPackages(1)) - #get new packages only from db - - deleted = [] - for id in old_packs.iterkeys(): - if id not in new_packs: - deleted.append(id) - self.deletePackage(int(id)) - - return deleted - - @lock - @change - def restartFailed(self): - """ restart all failed links """ - self.db.restartFailed() - -class FileMethods(): +class FileMethods(DatabaseMethods): @queue - def filecount(self, queue): - """returns number of files in queue""" - self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue, )) + def filecount(self): + """returns number of files""" + self.c.execute("SELECT COUNT(*) FROM files") return self.c.fetchone()[0] @queue - def queuecount(self, queue): + def queuecount(self): """ number of files in queue not finished yet""" - self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0,4)", (queue, )) + # status not in NA, finished, skipped + self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus NOT IN (0,5,6)") return self.c.fetchone()[0] @queue - def processcount(self, queue, fid): + def processcount(self, fid): """ number of files which have to be proccessed """ - self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2,3,5,7,12) AND l.id != ?", (queue, str(fid))) + # status in online, queued, starting, waiting, downloading + self.c.execute("SELECT COUNT(*) FROM files as WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (str(fid), )) return self.c.fetchone()[0] - @inner - def _nextPackageOrder(self, queue=0): - self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) - max = self.c.fetchone()[0] - if max is not None: - return max + 1 - else: - return 0 - - @inner - def _nextFileOrder(self, package): - self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) - max = self.c.fetchone()[0] - if max is not None: - return max + 1 - else: - return 0 - @queue def addLink(self, url, name, plugin, package): - order = self._nextFileOrder(package) - self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', (url, name, plugin, package, order)) + # mark filestatus initially as missing, dlstatus - queued + self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', + (url, name, plugin, package)) return self.c.lastrowid - @queue + @async def addLinks(self, links, package): - """ links is a list of tupels (url,plugin)""" - order = self._nextFileOrder(package) - orders = [order + x for x in range(len(links))] - links = [(x[0], x[0], x[1], package, o) for x, o in zip(links, orders)] - self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) + """ links is a list of tupels (url, plugin)""" + links = [(x[0], x[0], x[1], package) for x in links] + self.c.executemany('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', links) @queue - def addPackage(self, name, folder, queue, password): - order = self._nextPackageOrder(queue) - self.c.execute('INSERT INTO packages(name, folder, queue, packageorder, password) VALUES(?,?,?,?,?)', (name, folder, queue, order, password)) + def addFile(self, name, size, media, package): + # filestatus - ok, dl status NA + self.c.execute('INSERT INTO files(name, size, media, package) VALUES(?,?,?,?)', + (name, size, media, package)) return self.c.lastrowid @queue - def deletePackage(self, p): + def addPackage(self, name, folder, root, password, site, comment, status): + self.c.execute('INSERT INTO packages(name, folder, root, password, site, comment, status) VALUES(?,?,?,?,?,?,?)' + , + (name, folder, root, password, site, comment, status)) + return self.c.lastrowid - self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) - self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue)) + @async + def deletePackage(self, pid): + # order updated by trigger + self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) - @queue - def deleteLink(self, f): + @async + def deleteFile(self, fid, order, package): + """ To delete a file order and package of it is needed """ + self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) + self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', + (order, package)) - self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) - self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid))) + @async + def addCollector(self, plugin, package, data): + """ fill collector, data as (name, size, status,[ hash,] url) list """ + if data and len(data[0]) == 4: + data = [(r[0], r[1], r[2], r[3], plugin, package) for r in data] + else: + data = [(r[0], r[1], r[2], r[4], plugin, package) for r in data] + self.c.executemany("INSERT INTO collector(name, size, status, url, plugin, packagename) VALUES (?,?,?,?,?,?)", + data) - @queue - def getAllLinks(self, q): - """return information about all links in queue q + @async + def deleteCollector(self, package=None, url=None): + qry = 'DELETE FROM collector' + if package: + self.c.execute(qry + " WHERE packagename=?", (package,)) + elif url: + self.c.execute(qry + " WHERE url=?", (url,)) + else: + self.c.execute(qry) - q0 queue - q1 collector + @queue + def getCollector(self, package=None): + """ get collector data, optionally filtered by package """ + qry = 'SELECT url, name, plugin, size, status, packagename FROM collector' + if package: + self.c.execute(qry + " WHERE packagename=?", (package,)) + else: + self.c.execute(qry) - format: + return [LinkStatus(*r) for r in self.c] - { - id: {'name': name, ... 'package': id }, ... - } + @queue + def getAllFiles(self, package=None, search=None, unfinished=False): + """ Return dict with file information + :param package: optional package to filter out + :param search: or search string for file name + :param unfinished: filter by dlstatus not finished """ - self.c.execute('SELECT l.id,l.url,l.name,l.size,l.status,l.error,l.plugin,l.package,l.linkorder FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? ORDER BY l.linkorder', (q,)) - data = {} + qry = ('SELECT fid, name, size, status, media, added, fileorder, ' + 'url, plugin, hash, dlstatus, error, package FROM files') + + if unfinished: + qry += ' WHERE dlstatus NOT IN (0, 5, 6)' + + if package is not None: + qry += ' AND' if unfinished else ' WHERE' + self.c.execute(qry + ' package=? ORDER BY package, fileorder', (package,)) + elif search is not None: + qry += ' AND' if unfinished else ' WHERE' + search = "%%%s%%" % search.strip("%") + self.c.execute(qry + ' name LIKE ? ORDER BY package, fileorder', (search,)) + + else: + self.c.execute(qry) + + data = OrderedDict() for r in self.c: - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': r[6], - 'package': r[7], - 'order': r[8], - } + f = FileInfo(r[0], r[1], r[12], r[2], r[3], r[4], r[5], r[6]) + if r[10] > 0: # dl status != NA + f.download = DownloadInfo(r[7], r[8], r[9], r[10], self.manager.statusMsg[r[10]], r[11]) + + data[r[0]] = f return data @queue - def getAllPackages(self, q): - """return information about packages in queue q - (only useful in get all data) + def getAllPackages(self, root=None): + """ Return dict with package information + + :param root: optional root to filter + """ + qry = ('SELECT pid, name, folder, root, site, comment, password, added, status, packageorder ' + 'FROM packages%s ORDER BY root, packageorder') - q0 queue - q1 collector + if root is None: + stats = self.getPackageStats() + self.c.execute(qry % "") + else: + stats = self.getPackageStats(root=root) + self.c.execute(qry % ' WHERE root=? OR pid=?', (root, root)) - format: + data = OrderedDict() + for r in self.c: + data[r[0]] = PackageInfo( + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], stats.get(r[0], default) + ) - { - id: {'name': name ... 'links': {} }, ... - } - """ - self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ - FROM packages p LEFT OUTER JOIN pstats s ON p.id = s.id \ - WHERE p.queue=? ORDER BY p.packageorder', str(q)) + return data + + @inner + def getPackageStats(self, pid=None, root=None): + qry = ("SELECT p.pid, SUM(f.size) AS sizetotal, COUNT(f.fid) AS linkstotal, sizedone, linksdone " + "FROM packages p JOIN files f ON p.pid = f.package AND f.dlstatus > 0 %(sub)s LEFT OUTER JOIN " + "(SELECT p.pid AS pid, SUM(f.size) AS sizedone, COUNT(f.fid) AS linksdone " + "FROM packages p JOIN files f ON p.pid = f.package %(sub)s AND f.dlstatus in (5,6) GROUP BY p.pid) s ON s.pid = p.pid " + "GROUP BY p.pid") + + # status in (finished, skipped, processing) + + if root is not None: + self.c.execute(qry % {"sub": "AND (p.root=:root OR p.pid=:root)"}, locals()) + elif pid is not None: + self.c.execute(qry % {"sub": "AND p.pid=:pid"}, locals()) + else: + self.c.execute(qry % {"sub": ""}) data = {} for r in self.c: - data[r[0]] = { - 'id': r[0], - 'name': r[1], - 'folder': r[2], - 'site': r[3], - 'password': r[4], - 'queue': r[5], - 'order': r[6], - 'sizetotal': int(r[7]) if r[7] else 0, - 'sizedone': int(r[8]) if r[8] else 0, #these can be None - 'linksdone': r[9] if r[9] else 0, - 'linkstotal': r[10] if r[10] else 0, - 'links': {} - } + data[r[0]] = PackageStats( + r[2] if r[2] else 0, + r[4] if r[4] else 0, + int(r[1]) if r[1] else 0, + int(r[3]) if r[3] else 0, + ) return data - + @queue - def getLinkData(self, id): - """get link information as dict""" - self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE id=?', (str(id), )) - data = {} + def getStatsForPackage(self, pid): + return self.getPackageStats(pid=pid)[pid] + + @queue + def getFileInfo(self, fid, force=False): + """get data for specific file""" + self.c.execute('SELECT fid, name, size, status, media, added, fileorder, ' + 'url, plugin, hash, dlstatus, error, package FROM files ' + 'WHERE fid=?', (fid,)) r = self.c.fetchone() if not r: return None - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': r[6], - 'package': r[7], - 'order': r[8], - } + else: + f = FileInfo(r[0], r[1], r[12], r[2], r[3], r[4], r[5], r[6]) + if r[10] > 0 or force: + f.download = DownloadInfo(r[7], r[8], r[9], r[10], self.manager.statusMsg[r[10]], r[11]) - return data + return f @queue - def getPackageData(self, id): - """get data about links for a package""" - self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id), )) + def getPackageInfo(self, pid, stats=True): + """get data for specific package, optional with package stats""" + if stats: + stats = self.getPackageStats(pid=pid) - data = {} - for r in self.c: - data[r[0]] = { - 'id': r[0], - 'url': r[1], - 'name': r[2], - 'size': r[3], - 'format_size': formatSize(r[3]), - 'status': r[4], - 'statusmsg': self.manager.statusMsg[r[4]], - 'error': r[5], - 'plugin': r[6], - 'package': r[7], - 'order': r[8], - } + self.c.execute('SELECT pid, name, folder, root, site, comment, password, added, status, packageorder ' + 'FROM packages WHERE pid=?', (pid,)) - return data + r = self.c.fetchone() + if not r: + return None + else: + return PackageInfo( + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], stats.get(r[0], default) if stats else None + ) + @async + def updateLinkInfo(self, data): + """ data is list of tupels (name, size, status,[ hash,] url)""" + if data and len(data[0]) == 4: + self.c.executemany('UPDATE files SET name=?, size=?, dlstatus=? WHERE url=? AND dlstatus IN (0,1,2,3,14)', + data) + else: + self.c.executemany( + 'UPDATE files SET name=?, size=?, dlstatus=?, hash=? WHERE url=? AND dlstatus IN (0,1,2,3,14)', data) @async - def updateLink(self, f): - self.c.execute('UPDATE links SET url=?,name=?,size=?,status=?,error=?,package=? WHERE id=?', (f.url, f.name, f.size, f.status, f.error, str(f.packageid), str(f.id))) + def updateFile(self, f): + self.c.execute('UPDATE files SET name=?, size=?, status=?,' + 'media=?, url=?, hash=?, dlstatus=?, error=? WHERE fid=?', + (f.name, f.size, f.filestatus, f.media, f.url, + f.hash, f.status, f.error, f.fid)) - @queue + @async def updatePackage(self, p): - self.c.execute('UPDATE packages SET name=?,folder=?,site=?,password=?,queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) - - @queue - def updateLinkInfo(self, data): - """ data is list of tupels (name, size, status, url) """ - self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1,2,3,14)', data) - ids = [] - self.c.execute('SELECT id FROM links WHERE url IN (\'%s\')' % "','".join([x[3] for x in data])) - for r in self.c: - ids.append(int(r[0])) - return ids - - @queue - def reorderPackage(self, p, position, noMove=False): - if position == -1: - position = self._nextPackageOrder(p.queue) - if not noMove: - if p.order > position: - self.c.execute('UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND queue=? AND packageorder >= 0', (position, p.order, p.queue)) - elif p.order < position: - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND queue=? AND packageorder >= 0', (position, p.order, p.queue)) - - self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) - - @queue - def reorderLink(self, f, position): - """ reorder link with f as dict for pyfile """ - if f["order"] > position: - self.c.execute('UPDATE links SET linkorder=linkorder+1 WHERE linkorder >= ? AND linkorder < ? AND package=?', (position, f["order"], f["package"])) - elif f["order"] < position: - self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder <= ? AND linkorder > ? AND package=?', (position, f["order"], f["package"])) - - self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"])) - - - @queue - def clearPackageOrder(self, p): - self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) - self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) - + self.c.execute('UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, status=? WHERE pid=?', + (p.name, p.folder, p.site, p.comment, p.password, p.status, p.pid)) + @async - def restartFile(self, id): - self.c.execute('UPDATE links SET status=3,error="" WHERE id=?', (str(id),)) + def orderPackage(self, pid, root, oldorder, order): + if oldorder > order: # package moved upwards + self.c.execute( + 'UPDATE packages SET packageorder=packageorder+1 WHERE packageorder >= ? AND packageorder < ? AND root=? AND packageorder >= 0' + , (order, oldorder, root)) + elif oldorder < order: # moved downwards + self.c.execute( + 'UPDATE packages SET packageorder=packageorder-1 WHERE packageorder <= ? AND packageorder > ? AND root=? AND packageorder >= 0' + , (order, oldorder, root)) + + self.c.execute('UPDATE packages SET packageorder=? WHERE pid=?', (order, pid)) @async - def restartPackage(self, id): - self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) - - @queue - def getPackage(self, id): - """return package instance from id""" - self.c.execute("SELECT name,folder,site,password,queue,packageorder FROM packages WHERE id=?", (str(id), )) + def orderFiles(self, pid, fids, oldorder, order): + diff = len(fids) + data = [] + + if oldorder > order: # moved upwards + self.c.execute('UPDATE files SET fileorder=fileorder+? WHERE fileorder >= ? AND fileorder < ? AND package=?' + , (diff, order, oldorder, pid)) + data = [(order + i, fid) for i, fid in enumerate(fids)] + elif oldorder < order: + self.c.execute( + 'UPDATE files SET fileorder=fileorder-? WHERE fileorder <= ? AND fileorder >= ? AND package=?' + , (diff, order, oldorder + diff, pid)) + data = [(order - diff + i + 1, fid) for i, fid in enumerate(fids)] + + self.c.executemany('UPDATE files SET fileorder=? WHERE fid=?', data) + + @async + def moveFiles(self, pid, fids, package): + self.c.execute('SELECT max(fileorder) FROM files WHERE package=?', (package,)) r = self.c.fetchone() - if not r: return None - return PyPackage(self.manager, id, * r) + order = (r[0] if r[0] else 0) + 1 + self.c.execute('UPDATE files SET fileorder=fileorder-? WHERE fileorder > ? AND package=?', + (len(fids), order, pid)) - @queue - def getFile(self, id): - """return link instance from id""" - self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id), )) + data = [(package, order + i, fid) for i, fid in enumerate(fids)] + self.c.executemany('UPDATE files SET package=?, fileorder=? WHERE fid=?', data) + + @async + def movePackage(self, root, order, pid, dpid): + self.c.execute('SELECT max(packageorder) FROM packages WHERE root=?', (dpid,)) r = self.c.fetchone() - if not r: return None - return PyFile(self.manager, id, * r) + max = (r[0] if r[0] else 0) + 1 + print max + self.c.execute('SELECT pid, packageorder FROM packages WHERE root=?', (dpid,)) + for r in self.c: + print r + + self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND root=?', + (order, root)) + + self.c.execute('UPDATE packages SET root=?, packageorder=? WHERE pid=?', (dpid, max, pid)) + + @async + def restartFile(self, fid): + # status -> queued + self.c.execute('UPDATE files SET dlstatus=3, error="" WHERE fid=?', (fid,)) + + @async + def restartPackage(self, pid): + # status -> queued + self.c.execute('UPDATE files SET status=3 WHERE package=?', (pid,)) @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" - cmd = "(" - for i, item in enumerate(occ): - if i: cmd += ", " - cmd += "'%s'" % item + cmd = "(%s)" % ", ".join(["'%s'" % x for x in occ]) + #TODO - cmd += ")" + # dlstatus in online, queued | package status = ok + cmd = ("SELECT f.fid FROM files as f INNER JOIN packages as p ON f.package=p.pid " + "WHERE f.plugin NOT IN %s AND f.dlstatus IN (2,3) AND p.status=0 " + "ORDER BY p.packageorder ASC, f.fileorder ASC LIMIT 5") % cmd - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=1 AND l.plugin NOT IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % cmd self.c.execute(cmd) # very bad! return [x[0] for x in self.c] @@ -796,79 +348,34 @@ class FileMethods(): @queue def getUnfinished(self, pid): """return list of max length 3 ids with pyfiles in package not finished or processed""" - - self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) - return [r[0] for r in self.c] - @queue - def deleteFinished(self): - self.c.execute("DELETE FROM links WHERE status IN (0,4)") - self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") + # status in finished, skipped, processing + self.c.execute("SELECT fid FROM files WHERE package=? AND dlstatus NOT IN (5, 6, 14) LIMIT 3", (pid,)) + return [r[0] for r in self.c] @queue def restartFailed(self): - self.c.execute("UPDATE links SET status=3,error='' WHERE status IN (8, 9)") + # status=queued, where status in failed, aborted, temp offline + self.c.execute("UPDATE files SET dlstatus=3, error='' WHERE dlstatus IN (7, 11, 12)") @queue def findDuplicates(self, id, folder, filename): """ checks if filename exists with different id and same package """ - self.c.execute("SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", (folder, id, filename)) + # TODO + self.c.execute( + "SELECT l.plugin FROM files f INNER JOIN packages as p ON f.package=p.pid AND p.folder=? WHERE f.fid!=? AND l.status=0 AND l.name=?" + , (folder, id, filename)) return self.c.fetchone() @queue def purgeLinks(self): - self.c.execute("DELETE FROM links;") - self.c.execute("DELETE FROM packages;") - -DatabaseBackend.registerSub(FileMethods) - -if __name__ == "__main__": - - pypath = "." - _ = lambda x: x - - db = FileHandler(None) - - #p = PyFile(db, 5) - #sleep(0.1) - - a = time() - - #print db.addPackage("package", "folder" , 1) - - pack = db.db.addPackage("package", "folder", 1) - - updates = [] - - - for x in range(0, 200): - x = str(x) - db.db.addLink("http://somehost.com/hoster/file/download?file_id=" + x, x, "BasePlugin", pack) - updates.append(("new name" + x, 0, 3, "http://somehost.com/hoster/file/download?file_id=" + x)) - - - for x in range(0, 100): - updates.append(("unimportant%s" % x, 0, 3, "a really long non existent url%s" % x)) - - db.db.commit() - - b = time() - print "adding 200 links, single sql execs, no commit", b-a - - print db.getCompleteData(1) - - c = time() - - - db.db.updateLinkInfo(updates) - - d = time() - - print "updates", d-c - - print db.getCompleteData(1) - - - e = time() - - print "complete data", e-d + # fstatus = missing + self.c.execute("DELETE FROM files WHERE status == 1") + + @queue + def purgeAll(self): # only used for debugging + self.c.execute("DELETE FROM packages") + self.c.execute("DELETE FROM files") + self.c.execute("DELETE FROM collector") + +FileMethods.register() \ No newline at end of file -- cgit v1.2.3 From b40b32ee05f611323a7827fad2a25fa0a28dcb24 Mon Sep 17 00:00:00 2001 From: X3n0m0rph59 Date: Sun, 22 Apr 2012 19:56:17 +0200 Subject: a huge pile of spelling fixes --- module/database/FileDatabase.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 08b18765d..19dca84c7 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -40,27 +40,27 @@ class FileMethods(DatabaseMethods): @queue def processcount(self, fid): - """ number of files which have to be proccessed """ + """ number of files which have to be processed """ # status in online, queued, starting, waiting, downloading self.c.execute("SELECT COUNT(*) FROM files as WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (str(fid), )) return self.c.fetchone()[0] @queue def addLink(self, url, name, plugin, package): - # mark filestatus initially as missing, dlstatus - queued + # mark file status initially as missing, dlstatus - queued self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', (url, name, plugin, package)) return self.c.lastrowid @async def addLinks(self, links, package): - """ links is a list of tupels (url, plugin)""" + """ links is a list of tuples (url, plugin)""" links = [(x[0], x[0], x[1], package) for x in links] self.c.executemany('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', links) @queue def addFile(self, name, size, media, package): - # filestatus - ok, dl status NA + # file status - ok, dl status NA self.c.execute('INSERT INTO files(name, size, media, package) VALUES(?,?,?,?)', (name, size, media, package)) return self.c.lastrowid @@ -225,7 +225,7 @@ class FileMethods(DatabaseMethods): @queue def getPackageInfo(self, pid, stats=True): - """get data for specific package, optional with package stats""" + """get data for a specific package, optionally with package stats""" if stats: stats = self.getPackageStats(pid=pid) @@ -242,7 +242,7 @@ class FileMethods(DatabaseMethods): @async def updateLinkInfo(self, data): - """ data is list of tupels (name, size, status,[ hash,] url)""" + """ data is list of tuples (name, size, status,[ hash,] url)""" if data and len(data[0]) == 4: self.c.executemany('UPDATE files SET name=?, size=?, dlstatus=? WHERE url=? AND dlstatus IN (0,1,2,3,14)', data) -- cgit v1.2.3 From 84efa9d5ccd46a0adddc256a5eba4f8e33c76afd Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 7 May 2012 18:42:29 +0200 Subject: new multiuser api methods --- module/database/FileDatabase.py | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 19dca84c7..b783d15d9 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -85,36 +85,20 @@ class FileMethods(DatabaseMethods): (order, package)) @async - def addCollector(self, plugin, package, data): - """ fill collector, data as (name, size, status,[ hash,] url) list """ - if data and len(data[0]) == 4: - data = [(r[0], r[1], r[2], r[3], plugin, package) for r in data] - else: - data = [(r[0], r[1], r[2], r[4], plugin, package) for r in data] - - self.c.executemany("INSERT INTO collector(name, size, status, url, plugin, packagename) VALUES (?,?,?,?,?,?)", - data) - - @async - def deleteCollector(self, package=None, url=None): - qry = 'DELETE FROM collector' - if package: - self.c.execute(qry + " WHERE packagename=?", (package,)) - elif url: - self.c.execute(qry + " WHERE url=?", (url,)) - else: - self.c.execute(qry) + def saveCollector(self, owner, data): + """ simply save the json string to database """ + self.c.execute("INSERT INTO collector(owner, data) VALUES (?,?)", (owner, data)) @queue - def getCollector(self, package=None): - """ get collector data, optionally filtered by package """ - qry = 'SELECT url, name, plugin, size, status, packagename FROM collector' - if package: - self.c.execute(qry + " WHERE packagename=?", (package,)) - else: - self.c.execute(qry) + def retrieveCollector(self, owner): + """ retrieve the saved string """ + self.c.execute('SELECT data FROM collector owner=?', (owner,)) + return self.c.fetchone()[0] - return [LinkStatus(*r) for r in self.c] + @async + def deleteCollector(self, owner): + """ drop saved user collector """ + self.c.execute('DELETE FROM collector WHERE owner=?', (owner,)) @queue def getAllFiles(self, package=None, search=None, unfinished=False): -- cgit v1.2.3 From 829244a6140763712d50ed046c33f415f2b04301 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 15 May 2012 19:22:34 +0200 Subject: some multiuser db changes --- module/database/FileDatabase.py | 184 +++++++++++++++++++++++----------------- 1 file changed, 105 insertions(+), 79 deletions(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index b783d15d9..80da775c7 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -1,88 +1,97 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" +############################################################################### +# Copyright(c) 2008-2012 pyLoad Team +# http://www.pyload.org +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Subjected to the terms and conditions in LICENSE +# +# @author: RaNaN +############################################################################### + from new_collections import OrderedDict -from module.Api import DownloadInfo, LinkStatus, FileInfo, PackageInfo, PackageStats +from module.Api import DownloadInfo, FileInfo, PackageInfo, PackageStats from module.database import DatabaseMethods, queue, async, inner -default = PackageStats(0, 0, 0, 0) +zero_stats = PackageStats(0, 0, 0, 0) class FileMethods(DatabaseMethods): @queue - def filecount(self): + def filecount(self, user=None): """returns number of files""" self.c.execute("SELECT COUNT(*) FROM files") return self.c.fetchone()[0] @queue - def queuecount(self): + def queuecount(self, user=None): """ number of files in queue not finished yet""" # status not in NA, finished, skipped self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus NOT IN (0,5,6)") return self.c.fetchone()[0] @queue - def processcount(self, fid): + def processcount(self, fid, user=None): """ number of files which have to be processed """ # status in online, queued, starting, waiting, downloading self.c.execute("SELECT COUNT(*) FROM files as WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (str(fid), )) return self.c.fetchone()[0] + # TODO: think about multiuser side effects on *count methods + @queue - def addLink(self, url, name, plugin, package): + def addLink(self, url, name, plugin, package, owner): # mark file status initially as missing, dlstatus - queued - self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', - (url, name, plugin, package)) + self.c.execute('INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', + (url, name, plugin, package, owner)) return self.c.lastrowid @async - def addLinks(self, links, package): + def addLinks(self, links, package, owner): """ links is a list of tuples (url, plugin)""" - links = [(x[0], x[0], x[1], package) for x in links] - self.c.executemany('INSERT INTO files(url, name, plugin, status, dlstatus, package) VALUES(?,?,?,1,3,?)', links) + links = [(x[0], x[0], x[1], package, owner) for x in links] + self.c.executemany('INSERT INTO files(url, name, plugin, status, dlstatus, package, owner) VALUES(?,?,?,1,3,?,?)', + links) @queue - def addFile(self, name, size, media, package): + def addFile(self, name, size, media, package, owner): # file status - ok, dl status NA - self.c.execute('INSERT INTO files(name, size, media, package) VALUES(?,?,?,?)', - (name, size, media, package)) + self.c.execute('INSERT INTO files(name, size, media, package, owner) VALUES(?,?,?,?,?)', + (name, size, media, package, owner)) return self.c.lastrowid @queue - def addPackage(self, name, folder, root, password, site, comment, status): - self.c.execute('INSERT INTO packages(name, folder, root, password, site, comment, status) VALUES(?,?,?,?,?,?,?)' - , - (name, folder, root, password, site, comment, status)) + def addPackage(self, name, folder, root, password, site, comment, status, owner): + self.c.execute( + 'INSERT INTO packages(name, folder, root, password, site, comment, status, owner) VALUES(?,?,?,?,?,?,?,?)' + , (name, folder, root, password, site, comment, status, owner)) return self.c.lastrowid @async - def deletePackage(self, pid): - # order updated by trigger - self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) + def deletePackage(self, pid, owner=None): + # order updated by trigge + if owner is None: + self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) + else: + self.c.execute('DELETE FROM packages WHERE pid=? AND owner=?', (pid, owner)) @async - def deleteFile(self, fid, order, package): + def deleteFile(self, fid, order, package, owner=None): """ To delete a file order and package of it is needed """ - self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) - self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', - (order, package)) + if owner is None: + self.c.execute('DELETE FROM files WHERE fid=?', (fid,)) + self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=?', + (order, package)) + else: + self.c.execute('DELETE FROM files WHERE fid=? AND owner=?', (fid, owner)) + self.c.execute('UPDATE files SET fileorder=fileorder-1 WHERE fileorder > ? AND package=? AND owner=?', + (order, package, owner)) @async def saveCollector(self, owner, data): @@ -92,8 +101,10 @@ class FileMethods(DatabaseMethods): @queue def retrieveCollector(self, owner): """ retrieve the saved string """ - self.c.execute('SELECT data FROM collector owner=?', (owner,)) - return self.c.fetchone()[0] + self.c.execute('SELECT data FROM collector WHERE owner=?', (owner,)) + r = self.c.fetchone() + if not r: return None + return r[0] @async def deleteCollector(self, owner): @@ -101,66 +112,81 @@ class FileMethods(DatabaseMethods): self.c.execute('DELETE FROM collector WHERE owner=?', (owner,)) @queue - def getAllFiles(self, package=None, search=None, unfinished=False): + def getAllFiles(self, package=None, search=None, unfinished=False, owner=None): """ Return dict with file information :param package: optional package to filter out :param search: or search string for file name :param unfinished: filter by dlstatus not finished + :param owner: only specific owner """ - qry = ('SELECT fid, name, size, status, media, added, fileorder, ' - 'url, plugin, hash, dlstatus, error, package FROM files') + qry = ('SELECT fid, name, owner, size, status, media, added, fileorder, ' + 'url, plugin, hash, dlstatus, error, package FROM files WHERE ') + + arg = [] if unfinished: - qry += ' WHERE dlstatus NOT IN (0, 5, 6)' + qry += 'dlstatus NOT IN (0, 5, 6) AND ' + if owner is not None: + qry += 'owner=? AND ' + arg.append(owner) if package is not None: - qry += ' AND' if unfinished else ' WHERE' - self.c.execute(qry + ' package=? ORDER BY package, fileorder', (package,)) - elif search is not None: - qry += ' AND' if unfinished else ' WHERE' + arg.append(package) + qry += 'package=? AND ' + if search is not None: search = "%%%s%%" % search.strip("%") - self.c.execute(qry + ' name LIKE ? ORDER BY package, fileorder', (search,)) + arg.append(search) + qry += "name LIKE ? " - else: - self.c.execute(qry) + # make qry valid + if qry.endswith("WHERE "): qry = qry[:-6] + if qry.endswith("AND "): qry = qry[:-4] + + self.c.execute(qry + "ORDER BY package, fileorder", arg) data = OrderedDict() for r in self.c: - f = FileInfo(r[0], r[1], r[12], r[2], r[3], r[4], r[5], r[6]) - if r[10] > 0: # dl status != NA - f.download = DownloadInfo(r[7], r[8], r[9], r[10], self.manager.statusMsg[r[10]], r[11]) + f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) + if r[11] > 0: # dl status != NA + f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) data[r[0]] = f return data @queue - def getAllPackages(self, root=None): + def getAllPackages(self, root=None, owner=None): """ Return dict with package information :param root: optional root to filter """ - qry = ('SELECT pid, name, folder, root, site, comment, password, added, status, packageorder ' + qry = ('SELECT pid, name, folder, root, owner, site, comment, password, added, status, packageorder ' 'FROM packages%s ORDER BY root, packageorder') if root is None: - stats = self.getPackageStats() - self.c.execute(qry % "") + stats = self.getPackageStats(owner=owner) + if owner is None: + self.c.execute(qry % "") + else: + self.c.execute(qry % " WHERE owner=?", (owner,)) else: - stats = self.getPackageStats(root=root) - self.c.execute(qry % ' WHERE root=? OR pid=?', (root, root)) + stats = self.getPackageStats(root=root, owner=owner) + if owner is None: + self.c.execute(qry % ' WHERE root=? OR pid=?', (root, root)) + else: + self.c.execute(qry % ' WHERE (root=? OR pid=?) AND owner=?', (root, root, owner)) data = OrderedDict() for r in self.c: data[r[0]] = PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], stats.get(r[0], default) + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], stats.get(r[0], zero_stats) ) return data @inner - def getPackageStats(self, pid=None, root=None): + def getPackageStats(self, pid=None, root=None, owner=None): qry = ("SELECT p.pid, SUM(f.size) AS sizetotal, COUNT(f.fid) AS linkstotal, sizedone, linksdone " "FROM packages p JOIN files f ON p.pid = f.package AND f.dlstatus > 0 %(sub)s LEFT OUTER JOIN " "(SELECT p.pid AS pid, SUM(f.size) AS sizedone, COUNT(f.fid) AS linksdone " @@ -173,6 +199,8 @@ class FileMethods(DatabaseMethods): self.c.execute(qry % {"sub": "AND (p.root=:root OR p.pid=:root)"}, locals()) elif pid is not None: self.c.execute(qry % {"sub": "AND p.pid=:pid"}, locals()) + elif owner is not None: + self.c.execute(qry % {"sub": "AND p.owner=:owner"}, locals()) else: self.c.execute(qry % {"sub": ""}) @@ -193,17 +221,17 @@ class FileMethods(DatabaseMethods): @queue def getFileInfo(self, fid, force=False): - """get data for specific file""" - self.c.execute('SELECT fid, name, size, status, media, added, fileorder, ' + """get data for specific file, when force is true download info will be appended""" + self.c.execute('SELECT fid, name, owner, size, status, media, added, fileorder, ' 'url, plugin, hash, dlstatus, error, package FROM files ' 'WHERE fid=?', (fid,)) r = self.c.fetchone() if not r: return None else: - f = FileInfo(r[0], r[1], r[12], r[2], r[3], r[4], r[5], r[6]) - if r[10] > 0 or force: - f.download = DownloadInfo(r[7], r[8], r[9], r[10], self.manager.statusMsg[r[10]], r[11]) + f = FileInfo(r[0], r[1], r[13], r[2], r[3], r[4], r[5], r[6], r[7]) + if r[11] > 0 or force: + f.download = DownloadInfo(r[8], r[9], r[10], r[11], self.manager.statusMsg[r[11]], r[12]) return f @@ -213,7 +241,7 @@ class FileMethods(DatabaseMethods): if stats: stats = self.getPackageStats(pid=pid) - self.c.execute('SELECT pid, name, folder, root, site, comment, password, added, status, packageorder ' + self.c.execute('SELECT pid, name, folder, root, owner, site, comment, password, added, status, packageorder ' 'FROM packages WHERE pid=?', (pid,)) r = self.c.fetchone() @@ -221,11 +249,11 @@ class FileMethods(DatabaseMethods): return None else: return PackageInfo( - r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], stats.get(r[0], default) if stats else None + r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], stats.get(r[0], zero_stats) if stats else None ) @async - def updateLinkInfo(self, data): + def updateLinkInfo(self, data, owner): """ data is list of tuples (name, size, status,[ hash,] url)""" if data and len(data[0]) == 4: self.c.executemany('UPDATE files SET name=?, size=?, dlstatus=? WHERE url=? AND dlstatus IN (0,1,2,3,14)', @@ -246,6 +274,7 @@ class FileMethods(DatabaseMethods): self.c.execute('UPDATE packages SET name=?, folder=?, site=?, comment=?, password=?, status=? WHERE pid=?', (p.name, p.folder, p.site, p.comment, p.password, p.status, p.pid)) + # TODO: most modifying methods needs owner argument to avoid checking beforehand @async def orderPackage(self, pid, root, oldorder, order): if oldorder > order: # package moved upwards @@ -293,11 +322,6 @@ class FileMethods(DatabaseMethods): self.c.execute('SELECT max(packageorder) FROM packages WHERE root=?', (dpid,)) r = self.c.fetchone() max = (r[0] if r[0] else 0) + 1 - print max - - self.c.execute('SELECT pid, packageorder FROM packages WHERE root=?', (dpid,)) - for r in self.c: - print r self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND root=?', (order, root)) @@ -314,6 +338,8 @@ class FileMethods(DatabaseMethods): # status -> queued self.c.execute('UPDATE files SET status=3 WHERE package=?', (pid,)) + + # TODO: multi user approach @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" @@ -338,7 +364,7 @@ class FileMethods(DatabaseMethods): return [r[0] for r in self.c] @queue - def restartFailed(self): + def restartFailed(self, owner): # status=queued, where status in failed, aborted, temp offline self.c.execute("UPDATE files SET dlstatus=3, error='' WHERE dlstatus IN (7, 11, 12)") -- cgit v1.2.3 From cbd4f4b5375f89362733e10a9b98e5a74c2a5734 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Fri, 31 Aug 2012 23:25:26 +0200 Subject: first js models/views --- module/database/FileDatabase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 80da775c7..ab681dc7f 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -40,7 +40,7 @@ class FileMethods(DatabaseMethods): def processcount(self, fid, user=None): """ number of files which have to be processed """ # status in online, queued, starting, waiting, downloading - self.c.execute("SELECT COUNT(*) FROM files as WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (str(fid), )) + self.c.execute("SELECT COUNT(*) FROM files WHERE dlstatus IN (2,3,8,9,10) AND fid != ?", (fid, )) return self.c.fetchone()[0] # TODO: think about multiuser side effects on *count methods -- cgit v1.2.3 From 560958b70043ea5b7e0e32d41cb51bd44696d775 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 9 Sep 2012 15:39:50 +0200 Subject: new config api --- module/database/FileDatabase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/database/FileDatabase.py') diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index ab681dc7f..e065b84e2 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -75,7 +75,7 @@ class FileMethods(DatabaseMethods): @async def deletePackage(self, pid, owner=None): - # order updated by trigge + # order updated by trigger, as well as links deleted if owner is None: self.c.execute('DELETE FROM packages WHERE pid=?', (pid,)) else: -- cgit v1.2.3