diff options
Diffstat (limited to 'pyload/database')
| -rw-r--r-- | pyload/database/Backend.py | 43 | ||||
| -rw-r--r-- | pyload/database/File.py | 124 | ||||
| -rw-r--r-- | pyload/database/Storage.py | 6 | ||||
| -rw-r--r-- | pyload/database/User.py | 16 |
4 files changed, 189 insertions, 0 deletions
diff --git a/pyload/database/Backend.py b/pyload/database/Backend.py index 4b63dd284..48c1fcb0d 100644 --- a/pyload/database/Backend.py +++ b/pyload/database/Backend.py @@ -22,34 +22,49 @@ class style(object): db = None @classmethod + + def setDB(cls, db): cls.db = db @classmethod + + def inner(cls, f): @staticmethod + + def x(*args, **kwargs): if cls.db: return f(cls.db, *args, **kwargs) return x @classmethod + + def queue(cls, f): @staticmethod + + def x(*args, **kwargs): if cls.db: return cls.db.queue(f, *args, **kwargs) return x @classmethod + + def async(cls, f): @staticmethod + + def x(*args, **kwargs): if cls.db: return cls.db.async(f, *args, **kwargs) return x class DatabaseJob(object): + def __init__(self, f, *args, **kwargs): self.done = Event() @@ -63,6 +78,7 @@ class DatabaseJob(object): # import inspect # self.frame = inspect.currentframe() + def __repr__(self): from os.path import basename frame = self.frame.f_back @@ -75,6 +91,7 @@ class DatabaseJob(object): return "DataBase Job %s:%s\n%sResult: %s" % (self.f.__name__, self.args[1:], output, self.result) + def processJob(self): try: self.result = self.f(*self.args, **self.kwargs) @@ -89,11 +106,14 @@ class DatabaseJob(object): finally: self.done.set() + def wait(self): self.done.wait() class DatabaseBackend(Thread): subs = [] + + def __init__(self, core): Thread.__init__(self) self.setDaemon(True) @@ -105,10 +125,12 @@ class DatabaseBackend(Thread): style.setDB(self) + def setup(self): self.start() self.setuplock.wait() + def run(self): """main loop, which executes commands""" convert = self._checkVersion() #returns None or current version @@ -137,10 +159,13 @@ class DatabaseBackend(Thread): j.processJob() @style.queue + + def shutdown(self): self.conn.commit() self.jobs.put("quit") + def _checkVersion(self): """ check db version and delete it if needed""" if not exists("files.version"): @@ -165,6 +190,7 @@ class DatabaseBackend(Thread): f.close() return v + def _convertDB(self, v): try: getattr(self, "_convertV%i" % v)() @@ -176,6 +202,7 @@ class DatabaseBackend(Thread): #convert scripts start----------------------------------------------------- + def _convertV2(self): self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') try: @@ -184,6 +211,7 @@ class DatabaseBackend(Thread): print "Database was converted from v2 to v3." self._convertV3() + def _convertV3(self): self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') try: @@ -193,6 +221,7 @@ class DatabaseBackend(Thread): #convert scripts end------------------------------------------------------- + def _createTables(self): """create tables for database""" @@ -249,26 +278,35 @@ class DatabaseBackend(Thread): self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users) move("pyload.db", "pyload.old.db") + def createCursor(self): return self.conn.cursor() @style.async + + def commit(self): self.conn.commit() @style.queue + + def syncSave(self): self.conn.commit() @style.async + + def rollback(self): self.conn.rollback() + def async(self, f, *args, **kwargs): args = (self,) + args job = DatabaseJob(f, *args, **kwargs) self.jobs.put(job) + def queue(self, f, *args, **kwargs): args = (self,) + args job = DatabaseJob(f, *args, **kwargs) @@ -277,13 +315,18 @@ class DatabaseBackend(Thread): return job.result @classmethod + + def registerSub(cls, klass): cls.subs.append(klass) @classmethod + + def unregisterSub(cls, klass): cls.subs.remove(klass) + def __getattr__(self, attr): for sub in DatabaseBackend.subs: if hasattr(sub, attr): diff --git a/pyload/database/File.py b/pyload/database/File.py index 857da1ff9..0ff520623 100644 --- a/pyload/database/File.py +++ b/pyload/database/File.py @@ -42,7 +42,10 @@ class FileHandler(object): self.db = self.core.db + def change(func): + + def new(*args): args[0].unchanged = False args[0].filecount = -1 @@ -51,12 +54,16 @@ class FileHandler(object): return func(*args) return new + #-------------------------------------------------------------------------- + def save(self): """saves all data to backend""" self.db.commit() + #-------------------------------------------------------------------------- + def syncSave(self): """saves all data to backend and waits until all data are written""" pyfiles = self.cache.values() @@ -70,6 +77,8 @@ class FileHandler(object): self.db.syncSave() @lock + + def getCompleteData(self, queue=1): """gets a complete data representation""" @@ -89,6 +98,8 @@ class FileHandler(object): return packs @lock + + def getInfoData(self, queue=1): """gets a data representation without links""" @@ -101,6 +112,8 @@ class FileHandler(object): @lock @change + + def addLinks(self, urls, package): """adds links""" @@ -114,7 +127,9 @@ class FileHandler(object): #@TODO: change from reloadAll event to package update event self.core.pullManager.addEvent(ReloadAllEvent("collector")) + #-------------------------------------------------------------------------- + @lock @change def addPackage(self, name, folder, queue=0): @@ -125,7 +140,9 @@ class FileHandler(object): self.core.pullManager.addEvent(e) return lastID + #-------------------------------------------------------------------------- + @lock @change def deletePackage(self, id): @@ -161,7 +178,9 @@ class FileHandler(object): pack.order -= 1 pack.notifyChange() + #-------------------------------------------------------------------------- + @lock @change def deleteLink(self, id): @@ -197,18 +216,24 @@ class FileHandler(object): pyfile.notifyChange() #-------------------------------------------------------------------------- + + def releaseLink(self, id): """removes pyfile from cache""" if id in self.cache: del self.cache[id] + #-------------------------------------------------------------------------- + def releasePackage(self, id): """removes package from cache""" if id in self.packageCache: del self.packageCache[id] + #-------------------------------------------------------------------------- + def updateLink(self, pyfile): """updates link""" self.db.updateLink(pyfile) @@ -216,7 +241,9 @@ class FileHandler(object): e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") self.core.pullManager.addEvent(e) + #-------------------------------------------------------------------------- + def updatePackage(self, pypack): """updates a package""" self.db.updatePackage(pypack) @@ -224,7 +251,9 @@ class FileHandler(object): e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") self.core.pullManager.addEvent(e) + #-------------------------------------------------------------------------- + def getPackage(self, id): """return package instance""" @@ -233,7 +262,9 @@ class FileHandler(object): else: return self.db.getPackage(id) + #-------------------------------------------------------------------------- + def getPackageData(self, id): """returns dict with package information""" pack = self.getPackage(id) @@ -257,7 +288,9 @@ class FileHandler(object): return pack + #-------------------------------------------------------------------------- + def getFileData(self, id): """returns dict with file information""" if id in self.cache: @@ -265,7 +298,9 @@ class FileHandler(object): return self.db.getLinkData(id) + #-------------------------------------------------------------------------- + def getFile(self, id): """returns pyfile instance""" if id in self.cache: @@ -273,7 +308,9 @@ class FileHandler(object): else: return self.db.getFile(id) + #-------------------------------------------------------------------------- + @lock def getJob(self, occ): """get suitable job""" @@ -317,6 +354,8 @@ class FileHandler(object): return pyfile @lock + + def getDecryptJob(self): """return job for decrypting""" if "decrypt" in self.jobCache: @@ -332,6 +371,7 @@ class FileHandler(object): self.jobCache["decrypt"] = "empty" return None + def getFileCount(self): """returns number of files""" @@ -340,6 +380,7 @@ class FileHandler(object): return self.filecount + def getQueueCount(self, force=False): """number of files that have to be processed""" if self.queuecount == -1 or force: @@ -347,6 +388,7 @@ class FileHandler(object): return self.queuecount + def checkAllLinksFinished(self): """checks if all files are finished and dispatch event""" @@ -357,6 +399,7 @@ class FileHandler(object): return False + def checkAllLinksProcessed(self, fid): """checks if all files was processed and pyload would idle now, needs fid which will be ignored when counting""" @@ -370,11 +413,14 @@ class FileHandler(object): return False + def resetCount(self): self.queuecount = -1 @lock @change + + def restartPackage(self, id): """restart package""" pyfiles = self.cache.values() @@ -392,6 +438,8 @@ class FileHandler(object): @lock @change + + def restartFile(self, id): """ restart file""" if id in self.cache: @@ -407,6 +455,8 @@ class FileHandler(object): @lock @change + + def setPackageLocation(self, id, queue): """push package to queue""" @@ -440,6 +490,8 @@ class FileHandler(object): @lock @change + + def reorderPackage(self, id, position): p = self.getPackage(id) @@ -467,6 +519,8 @@ class FileHandler(object): @lock @change + + def reorderFile(self, id, position): f = self.getFileData(id) f = f[id] @@ -497,12 +551,15 @@ class FileHandler(object): self.core.pullManager.addEvent(e) @change + + def updateFileInfo(self, data, pid): """ updates file info (name, size, status, url)""" ids = self.db.updateLinkInfo(data) e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue") self.core.pullManager.addEvent(e) + def checkPackageFinished(self, pyfile): """ checks if package is finished and calls AddonManager """ @@ -513,6 +570,7 @@ class FileHandler(object): self.core.addonManager.packageFinished(pyfile.package()) pyfile.package().setFinished = True + def reCheckPackage(self, pid): """ recheck links in package """ data = self.db.getPackageData(pid) @@ -527,6 +585,8 @@ class FileHandler(object): @lock @change + + def deleteFinishedLinks(self): """ deletes finished links and packages, return deleted packages """ @@ -549,6 +609,8 @@ class FileHandler(object): @lock @change + + def restartFailed(self): """ restart all failed links """ self.db.restartFailed() @@ -556,24 +618,32 @@ class FileHandler(object): class FileMethods(object): @style.queue + + def filecount(self, queue): """returns number of files in queue""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue,)) return self.c.fetchone()[0] @style.queue + + def queuecount(self, queue): """ number of files in queue not finished yet""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0, 4)", (queue,)) return self.c.fetchone()[0] @style.queue + + def processcount(self, queue, fid): """ number of files which have to be proccessed """ self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2, 3, 5, 7, 12) AND l.id != ?", (queue, str(fid))) return self.c.fetchone()[0] @style.inner + + def _nextPackageOrder(self, queue=0): self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) max = self.c.fetchone()[0] @@ -583,6 +653,8 @@ class FileMethods(object): return 0 @style.inner + + def _nextFileOrder(self, package): self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) max = self.c.fetchone()[0] @@ -592,12 +664,16 @@ class FileMethods(object): return 0 @style.queue + + def addLink(self, url, name, plugin, package): order = self._nextFileOrder(package) self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', (url, name, ".".join(plugintype, pluginname), package, order)) return self.c.lastrowid @style.queue + + def addLinks(self, links, package): """ links is a list of tupels (url, plugin)""" order = self._nextFileOrder(package) @@ -606,23 +682,31 @@ class FileMethods(object): self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) @style.queue + + def addPackage(self, name, folder, queue): order = self._nextPackageOrder(queue) self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', (name, folder, queue, order)) return self.c.lastrowid @style.queue + + def deletePackage(self, p): self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue)) @style.queue + + def deleteLink(self, f): self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid))) @style.queue + + def getAllLinks(self, q): """return information about all links in queue q @@ -656,6 +740,8 @@ class FileMethods(object): return data @style.queue + + def getAllPackages(self, q): """return information about packages in queue q (only useful in get all data) @@ -693,6 +779,8 @@ class FileMethods(object): return data @style.queue + + def getLinkData(self, id): """get link information as dict""" self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?', (str(id),)) @@ -717,6 +805,8 @@ class FileMethods(object): return data @style.queue + + def getPackageData(self, id): """get data about links for a package""" self.c.execute('SELECT id, url, name, size, status, error, plugin, package, linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id),)) @@ -740,14 +830,20 @@ class FileMethods(object): return data @style.async + + def updateLink(self, f): self.c.execute('UPDATE links SET url=?, name=?, size=?, status=?, error=?, package=? WHERE id=?', (f.url, f.name, f.size, f.status, str(f.error), str(f.packageid), str(f.id))) @style.queue + + def updatePackage(self, p): self.c.execute('UPDATE packages SET name=?, folder=?, site=?, password=?, queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) @style.queue + + def updateLinkInfo(self, data): """ data is list of tupels (name, size, status, url) """ self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1, 2, 3, 14)', data) @@ -758,6 +854,8 @@ class FileMethods(object): return ids @style.queue + + def reorderPackage(self, p, position, noMove=False): if position == -1: position = self._nextPackageOrder(p.queue) @@ -770,6 +868,8 @@ class FileMethods(object): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) @style.queue + + def reorderLink(self, f, position): """ reorder link with f as dict for pyfile """ if f["order"] > position: @@ -780,19 +880,27 @@ class FileMethods(object): self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"])) @style.queue + + def clearPackageOrder(self, p): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) @style.async + + def restartFile(self, id): self.c.execute('UPDATE links SET status=3, error="" WHERE id=?', (str(id),)) @style.async + + def restartPackage(self, id): self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) @style.queue + + def getPackage(self, id): """return package instance from id""" self.c.execute("SELECT name, folder, site, password, queue, packageorder FROM packages WHERE id=?", (str(id),)) @@ -800,7 +908,9 @@ class FileMethods(object): if not r: return None return PyPackage(self.manager, id, * r) + #-------------------------------------------------------------------------- + @style.queue def getFile(self, id): """return link instance from id""" @@ -812,6 +922,8 @@ class FileMethods(object): return PyFile(self.manager, id, * r) @style.queue + + def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" @@ -832,6 +944,8 @@ class FileMethods(object): return [x[0] for x in self.c] @style.queue + + def getPluginJob(self, plugins): """returns pyfile ids with suited plugins""" cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2, 3, 14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins @@ -841,6 +955,8 @@ class FileMethods(object): return [x[0] for x in self.c] @style.queue + + def getUnfinished(self, pid): """return list of max length 3 ids with pyfiles in package not finished or processed""" @@ -848,21 +964,29 @@ class FileMethods(object): return [r[0] for r in self.c] @style.queue + + def deleteFinished(self): self.c.execute("DELETE FROM links WHERE status IN (0, 4)") self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") @style.queue + + def restartFailed(self): self.c.execute("UPDATE links SET status=3, error='' WHERE status IN (6, 8, 9)") @style.queue + + def findDuplicates(self, id, folder, filename): """ checks if filename exists with different id and same package """ self.c.execute("SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", (folder, id, filename)) return self.c.fetchone() @style.queue + + def purgeLinks(self): self.c.execute("DELETE FROM links;") self.c.execute("DELETE FROM packages;") diff --git a/pyload/database/Storage.py b/pyload/database/Storage.py index 75e166d39..bd7844d8d 100644 --- a/pyload/database/Storage.py +++ b/pyload/database/Storage.py @@ -6,6 +6,8 @@ from pyload.database import DatabaseBackend class StorageMethods(object): @style.queue + + def setStorage(db, identifier, key, value): db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) if db.c.fetchone() is not None: @@ -14,6 +16,8 @@ class StorageMethods(object): db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) @style.queue + + def getStorage(db, identifier, key=None): if key is not None: db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) @@ -28,6 +32,8 @@ class StorageMethods(object): return d @style.queue + + def delStorage(db, identifier, key): db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) diff --git a/pyload/database/User.py b/pyload/database/User.py index 54545e588..52b05749f 100644 --- a/pyload/database/User.py +++ b/pyload/database/User.py @@ -8,6 +8,8 @@ from pyload.database import DatabaseBackend, style class UserMethods(object): @style.queue + + def checkAuth(db, user, password): c = db.c c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user,)) @@ -25,6 +27,8 @@ class UserMethods(object): return {} @style.queue + + def addUser(db, user, password): salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) h = sha1(salt + password) @@ -39,6 +43,8 @@ class UserMethods(object): @style.queue + + def changePassword(db, user, oldpw, newpw): db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user,)) r = db.c.fetchone() @@ -60,15 +66,21 @@ class UserMethods(object): @style.async + + def setPermission(db, user, perms): db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) @style.async + + def setRole(db, user, role): db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) @style.queue + + def listUsers(db): db.c.execute('SELECT name FROM users') users = [] @@ -77,6 +89,8 @@ class UserMethods(object): return users @style.queue + + def getAllUserData(db): db.c.execute("SELECT name, permission, role, template, email FROM users") user = {} @@ -86,6 +100,8 @@ class UserMethods(object): return user @style.queue + + def removeUser(db, user): db.c.execute('DELETE FROM users WHERE name=?', (user,)) |