From 320f6f6270853b0209611ac6ec642994a90220b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Wed, 28 Jan 2009 12:58:53 +0100 Subject: Moved SQLDatabase --- portato/db/sql.py | 227 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 portato/db/sql.py (limited to 'portato/db/sql.py') diff --git a/portato/db/sql.py b/portato/db/sql.py new file mode 100644 index 0000000..c80fd91 --- /dev/null +++ b/portato/db/sql.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +# +# File: portato/db/sql.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann + +from __future__ import absolute_import, with_statement + +try: + import sqlite3 as sql +except ImportError: + from pysqlite2 import dbapi2 as sql + +import anydbm +import hashlib +import os + +from functools import wraps +from threading import RLock + +from ..helper import info, error, debug +from ..backend import system +from .database import Database, PkgData + +class SQLDatabase (Database): + + FORBIDDEN = (".bzr", ".svn", ".git", "CVS", ".hg", "_darcs") + + def __init__ (self): + """Constructor.""" + self._restrict = "" + self._lock = RLock() + + pkgdb = os.path.join(SESSION_DIR, "package.db") + pkgdb_existed = os.path.exists(pkgdb) + + if pkgdb_existed: + debug("package.db already existant") + else: + debug("package.db not existant") + + pkg_conn = sql.connect(os.path.join(SESSION_DIR, "package.db")) + pkg_conn.row_factory = sql.Row + pkg_conn.execute(""" + CREATE TABLE IF NOT EXISTS packages + ( + name TEXT, + cat TEXT, + inst INTEGER + )""") + + pkg_conn.commit() + + self.was_updated = self.updated() + if self.was_updated or not pkgdb_existed: + info(_("Cleaning database...")) + pkg_conn.execute("DELETE FROM packages") # empty db at beginning + info(_("Populating database...")) + self.populate(connection = pkg_conn) + + pkg_conn.close() + + descr_conn = sql.connect(os.path.join(SESSION_DIR, "descr.db")) + descr_conn.execute(""" + CREATE TABLE IF NOT EXISTS descriptions + ( + cp TEXT, + descr TEXT + )""") + descr_conn.close() + + def updated (self): + changed = False + + def walk (path): + debug("Walking %s", path) + + for root, dirs, files in os.walk(path): + for f in files: + path = os.path.join(root, f) + yield "%s %s" % (f, os.stat(path).st_mtime) + + for forbidden in self.FORBIDDEN: + if forbidden in dirs: + dirs.remove(forbidden) + + overlays = system.get_global_settings("PORTDIR_OVERLAY").split() + hashes = {} + for overlay in overlays: + hashes[overlay] = hashlib.md5("".join(walk(overlay))).hexdigest() + + timestamp = os.path.join(system.get_global_settings("PORTDIR"), "metadata/timestamp") + hashes["ROOT"] = hashlib.md5("%s %s" % (timestamp, os.stat(timestamp).st_mtime)).hexdigest() + + dbpath = os.path.join(SESSION_DIR, "portdirs.db") + db_existed = os.path.exists(dbpath) + db = anydbm.open(dbpath, "c") + try: + if db_existed: + debug("portdirs.db already existant") + for key in set(db.keys())- set(hashes.keys()): + debug("Overlay '%s' has been removed", key) + del db[key] + changed = True + + for key in hashes.iterkeys(): + + if key not in db.keys(): + debug("Overlay '%s' has been added.", key) + changed = True + + elif db[key] != hashes[key]: + debug("Overlay '%s' has been changed.", key) + changed = True + + db[key] = hashes[key] + else: + debug("portdirs.db not existant") + for key in hashes.iterkeys(): + db[key] = hashes[key] + + finally: + db.close() + + return changed + + def lock (f): + @wraps(f) + def wrapper (self, *args, **kwargs): + with self._lock: + r = f(self, *args, **kwargs) + + return r + + return wrapper + + def con (f): + @wraps(f) + def wrapper (*args, **kwargs): + if not "connection" in kwargs: + con= sql.connect(os.path.join(SESSION_DIR, "package.db")) + con.row_factory = sql.Row + kwargs["connection"] = con + + return f(*args, **kwargs) + + return wrapper + + @lock + @con + def populate (self, category = None, connection = None): + def _get(): + # get the lists + inst = system.find_packages(pkgSet = system.SET_INSTALLED, key=category, with_version = False) + for p in system.find_packages(key = category, with_version = False): + cat, pkg = p.split("/") + + yield (cat, pkg, p in inst) + + connection.executemany("INSERT INTO packages (cat, name, inst) VALUES (?, ?, ?)", _get()) + connection.commit() + + @lock + @con + def get_cat (self, category = None, byName = True, connection = None): + sort = "ORDER BY name" + if not byName: + sort = "ORDER BY inst DESC, name" + + if not category or category == self.ALL: + c = connection.execute("SELECT cat, name, inst FROM packages WHERE 1=1 %s %s" % (self.restrict, sort)) + else: + c = connection.execute("SELECT cat, name, inst FROM packages WHERE cat = ? %s %s" % (self.restrict ,sort), (category,)) + + for pkg in c: + yield PkgData(pkg["cat"], pkg["name"], pkg["inst"]) + c.close() + + @lock + @con + def get_categories (self, installed = False, connection = None): + + if installed: + where = "inst = 1" + else: + where = "1 = 1" + + c = connection.execute("SELECT cat FROM packages WHERE %s %s GROUP BY cat" % (where, self.restrict)) + + l = c.fetchall() + c.close() + + if len(l) > 1: + yield self.ALL + + for cat in l: + yield cat["cat"] + + @lock + @con + def reload (self, cat = None, connection = None): + if cat: + connection.execute("DELETE FROM packages WHERE cat = ?", (cat,)) + connection.commit() + self.populate(cat+"/", connection = connection) + else: + connection.execute("DELETE FROM packages") + connection.commit() + self.populate(connection = connection) + + def get_restrict (self): + return self._restrict + + @lock + def set_restrict (self, restrict): + if not restrict: + self._restrict = "" + else: + self._restrict = "AND name LIKE '%%%s%%'" % restrict + + restrict = property(get_restrict, set_restrict) -- cgit v1.2.3 From 816fbaf42407fbbb8466c0d08d64fc11f605e5b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Wed, 28 Jan 2009 13:13:56 +0100 Subject: Use the new database class layout --- portato/db/sql.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'portato/db/sql.py') diff --git a/portato/db/sql.py b/portato/db/sql.py index c80fd91..e7be91e 100644 --- a/portato/db/sql.py +++ b/portato/db/sql.py @@ -22,8 +22,8 @@ import hashlib import os from functools import wraps -from threading import RLock +from ..constants import SESSION_DIR from ..helper import info, error, debug from ..backend import system from .database import Database, PkgData @@ -31,11 +31,13 @@ from .database import Database, PkgData class SQLDatabase (Database): FORBIDDEN = (".bzr", ".svn", ".git", "CVS", ".hg", "_darcs") + lock = Database.lock def __init__ (self): """Constructor.""" + Database.__init__(self) + self._restrict = "" - self._lock = RLock() pkgdb = os.path.join(SESSION_DIR, "package.db") pkgdb_existed = os.path.exists(pkgdb) @@ -130,16 +132,6 @@ class SQLDatabase (Database): return changed - def lock (f): - @wraps(f) - def wrapper (self, *args, **kwargs): - with self._lock: - r = f(self, *args, **kwargs) - - return r - - return wrapper - def con (f): @wraps(f) def wrapper (*args, **kwargs): -- cgit v1.2.3 From 59792e7297d90cdead2e1c83e4537991b20dd11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Wed, 28 Jan 2009 22:27:59 +0100 Subject: Enable sql-db formats --- portato/db/sql.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) (limited to 'portato/db/sql.py') diff --git a/portato/db/sql.py b/portato/db/sql.py index e7be91e..3cffd88 100644 --- a/portato/db/sql.py +++ b/portato/db/sql.py @@ -30,15 +30,22 @@ from .database import Database, PkgData class SQLDatabase (Database): + FORMAT = "1" FORBIDDEN = (".bzr", ".svn", ".git", "CVS", ".hg", "_darcs") lock = Database.lock - def __init__ (self): + def __init__ (self, session): """Constructor.""" Database.__init__(self) self._restrict = "" + self.session = session + updateFormat = False + if "format" not in session or session["format"] != self.FORMAT: + session["format"] = self.FORMAT + updateFormat = True + pkgdb = os.path.join(SESSION_DIR, "package.db") pkgdb_existed = os.path.exists(pkgdb) @@ -49,18 +56,22 @@ class SQLDatabase (Database): pkg_conn = sql.connect(os.path.join(SESSION_DIR, "package.db")) pkg_conn.row_factory = sql.Row + if pkgdb_existed and updateFormat: + pkg_conn.execute("DROP TABLE packages") + pkg_conn.execute(""" CREATE TABLE IF NOT EXISTS packages ( name TEXT, cat TEXT, - inst INTEGER + inst INTEGER, + disabled INTEGER )""") pkg_conn.commit() self.was_updated = self.updated() - if self.was_updated or not pkgdb_existed: + if self.was_updated or not pkgdb_existed or updateFormat: info(_("Cleaning database...")) pkg_conn.execute("DELETE FROM packages") # empty db at beginning info(_("Populating database...")) @@ -153,9 +164,9 @@ class SQLDatabase (Database): for p in system.find_packages(key = category, with_version = False): cat, pkg = p.split("/") - yield (cat, pkg, p in inst) + yield (cat, pkg, p in inst, False) - connection.executemany("INSERT INTO packages (cat, name, inst) VALUES (?, ?, ?)", _get()) + connection.executemany("INSERT INTO packages (cat, name, inst, disabled) VALUES (?, ?, ?, ?)", _get()) connection.commit() @lock -- cgit v1.2.3 From d97a8bba4c3c877953bc6e800095ac4bc699ea45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Mon, 2 Feb 2009 11:29:08 +0100 Subject: Add the disabled field to PkgData --- portato/db/sql.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'portato/db/sql.py') diff --git a/portato/db/sql.py b/portato/db/sql.py index 3cffd88..b406981 100644 --- a/portato/db/sql.py +++ b/portato/db/sql.py @@ -153,9 +153,8 @@ class SQLDatabase (Database): return f(*args, **kwargs) - return wrapper + return Database.lock(wrapper) - @lock @con def populate (self, category = None, connection = None): def _get(): @@ -169,7 +168,6 @@ class SQLDatabase (Database): connection.executemany("INSERT INTO packages (cat, name, inst, disabled) VALUES (?, ?, ?, ?)", _get()) connection.commit() - @lock @con def get_cat (self, category = None, byName = True, connection = None): sort = "ORDER BY name" @@ -177,15 +175,14 @@ class SQLDatabase (Database): sort = "ORDER BY inst DESC, name" if not category or category == self.ALL: - c = connection.execute("SELECT cat, name, inst FROM packages WHERE 1=1 %s %s" % (self.restrict, sort)) + c = connection.execute("SELECT cat, name, inst, disabled FROM packages WHERE 1=1 %s %s" % (self.restrict, sort)) else: - c = connection.execute("SELECT cat, name, inst FROM packages WHERE cat = ? %s %s" % (self.restrict ,sort), (category,)) + c = connection.execute("SELECT cat, name, inst, disabled FROM packages WHERE cat = ? %s %s" % (self.restrict ,sort), (category,)) for pkg in c: - yield PkgData(pkg["cat"], pkg["name"], pkg["inst"]) + yield PkgData(pkg["cat"], pkg["name"], pkg["inst"], pkg["disabled"]) c.close() - @lock @con def get_categories (self, installed = False, connection = None): @@ -205,7 +202,6 @@ class SQLDatabase (Database): for cat in l: yield cat["cat"] - @lock @con def reload (self, cat = None, connection = None): if cat: @@ -217,6 +213,12 @@ class SQLDatabase (Database): connection.commit() self.populate(connection = connection) + @con + def disable (self, cpv, connection = None): + cat, pkg = cpv.split("/") + connection.execute("UPDATE packages SET disabled = 1 WHERE cat = ? AND name = ?", (cat, pkg)) + connection.commit() + def get_restrict (self): return self._restrict -- cgit v1.2.3 From a80ff9994c49b3aaba7d3f0bbf6317107fcf39bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Mon, 2 Feb 2009 12:47:49 +0100 Subject: Do not show categories which only hold disabled packages --- portato/db/sql.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'portato/db/sql.py') diff --git a/portato/db/sql.py b/portato/db/sql.py index b406981..cac5c97 100644 --- a/portato/db/sql.py +++ b/portato/db/sql.py @@ -169,15 +169,19 @@ class SQLDatabase (Database): connection.commit() @con - def get_cat (self, category = None, byName = True, connection = None): + def get_cat (self, category = None, byName = True, showDisabled = False, connection = None): sort = "ORDER BY name" if not byName: sort = "ORDER BY inst DESC, name" + disabled = "1=1" + if not showDisabled: + disabled = "disabled = 0" + if not category or category == self.ALL: - c = connection.execute("SELECT cat, name, inst, disabled FROM packages WHERE 1=1 %s %s" % (self.restrict, sort)) + c = connection.execute("SELECT cat, name, inst, disabled FROM packages WHERE %s %s %s" % (disabled, self.restrict, sort)) else: - c = connection.execute("SELECT cat, name, inst, disabled FROM packages WHERE cat = ? %s %s" % (self.restrict ,sort), (category,)) + c = connection.execute("SELECT cat, name, inst, disabled FROM packages WHERE cat = ? AND %s %s %s" % (disabled, self.restrict ,sort), (category,)) for pkg in c: yield PkgData(pkg["cat"], pkg["name"], pkg["inst"], pkg["disabled"]) @@ -191,7 +195,7 @@ class SQLDatabase (Database): else: where = "1 = 1" - c = connection.execute("SELECT cat FROM packages WHERE %s %s GROUP BY cat" % (where, self.restrict)) + c = connection.execute("SELECT cat FROM packages WHERE disabled = 0 AND %s %s GROUP BY cat" % (where, self.restrict)) l = c.fetchall() c.close() -- cgit v1.2.3 From eab7389888eb97ddcdf174acce717f45157ed079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Mon, 9 Feb 2009 13:26:28 +0100 Subject: SQLDatabase.populate needs a regexp --- portato/db/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'portato/db/sql.py') diff --git a/portato/db/sql.py b/portato/db/sql.py index cac5c97..b46374e 100644 --- a/portato/db/sql.py +++ b/portato/db/sql.py @@ -211,7 +211,7 @@ class SQLDatabase (Database): if cat: connection.execute("DELETE FROM packages WHERE cat = ?", (cat,)) connection.commit() - self.populate(cat+"/", connection = connection) + self.populate(cat+"/*", connection = connection) else: connection.execute("DELETE FROM packages") connection.commit() -- cgit v1.2.3