Merge in DB branch (even though it does not offer the change-db-in-prefs feature)

4 files changed, 535 insertions, 0 deletions

diff --git a/portato/db/__init__.py b/portato/db/__init__.py
new file mode 100644
index 0000000..05dbfcf
--- /dev/null
+++ b/portato/db/__init__.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+#
+# File: portato/db/__init__.py
+# This file is part of the Portato-Project, a graphical portage-frontend.
+#
+# Copyright (C) 2006-2009 René 'Necoro' Neumann
+# This is free software.  You may redistribute copies of it under the terms of
+# the GNU General Public License version 2.
+# There is NO WARRANTY, to the extent permitted by law.
+#
+# Written by René 'Necoro' Neumann <necoro@necoro.net>
+
+from __future__ import absolute_import
+
+from ..session import Session, SectionDict
+from ..helper import debug, warning
+
+_SESSION = None
+_TYPE = None
+
+def _set_type(t):
+    global _TYPE
+    _TYPE = t
+
+def Database():
+    global _SESSION, _TYPE
+
+    if _SESSION is None:
+        _SESSION = Session("db.cfg", name = "DB")
+        _SESSION.add_handler((["type"], _set_type, lambda: _TYPE), default = ["sql"])
+        _SESSION.load()
+
+    if _TYPE == "sql":
+        debug("Using SQLDatabase")
+        try:
+            from .sql import SQLDatabase
+        except ImportError:
+            warning(_("Cannot load SQLDatabase."))
+            _TYPE = "dict"
+            return Database()
+        else:
+            return SQLDatabase(SectionDict(_SESSION, "SQL"))
+
+    elif _TYPE == "dict":
+        debug("Using DictDatabase")
+        from .dict import DictDatabase
+        return DictDatabase(SectionDict(_SESSION, "dict"))
diff --git a/portato/db/database.py b/portato/db/database.py
new file mode 100644
index 0000000..7a23e5e
--- /dev/null
+++ b/portato/db/database.py
@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+#
+# File: portato/db/database.py
+# This file is part of the Portato-Project, a graphical portage-frontend.
+#
+# Copyright (C) 2006-2009 René 'Necoro' Neumann
+# This is free software.  You may redistribute copies of it under the terms of
+# the GNU General Public License version 2.
+# There is NO WARRANTY, to the extent permitted by law.
+#
+# Written by René 'Necoro' Neumann <necoro@necoro.net>
+
+from __future__ import absolute_import, with_statement
+
+from threading import RLock
+from functools import wraps
+
+class PkgData (object):
+    __slots__ = ("cat", "pkg", "inst", "disabled")
+
+    def __init__ (self, cat, pkg, inst = False, disabled = False):
+        self.cat = cat
+        self.pkg = pkg
+        self.inst = inst
+        self.disabled = disabled
+
+    def __iter__ (self):
+        return iter((self.cat, self.pkg, self.inst, self.disabled))
+
+    def __cmp__ (self, other):
+        return cmp(self.pkg.lower(), other.pkg.lower())
+
+    def __repr__ (self):
+        return "<Package (%(cat)s, %(pkg)s, %(inst)s)>" % {"cat" : self.cat, "pkg" : self.pkg, "inst" : self.inst}
+
+class Database (object):
+
+    ALL = _("ALL")
+
+    def __init__ (self):
+        self._lock = RLock()
+
+    @staticmethod
+    def lock (f):
+        @wraps(f)
+        def wrapper (self, *args, **kwargs):
+            with self._lock:
+                r = f(self, *args, **kwargs)
+                
+            return r
+        
+        return wrapper
+
+    def populate (self, category = None):
+        """Populates the database.
+        
+        @param category: An optional category - so only packages of this category are inserted.
+        @type category: string
+        """
+        raise NotImplentedError
+
+    def get_cat (self, cat = None, byName = True, showDisabled = False):
+        """Returns the packages in the category.
+        
+        @param cat: category to return the packages from; if None it defaults to C{ALL}
+        @type cat: string
+        @param byName: selects whether to return the list sorted by name or by installation
+        @type byName: boolean
+        @param showDisabled: should disabled packages be returned
+        @type showDisabled: boolean
+        @return: an iterator over the packages
+        @rtype: L{PkgData}<iterator>
+        """
+        raise NotImplentedError
+
+    def get_categories (self, installed = False):
+        """Returns all categories.
+        
+        @param installed: Only return these with at least one installed package.
+        @type installed: boolean
+        @returns: the list of categories
+        @rtype: string<iterator>
+        """
+        raise NotImplentedError
+
+    def disable (self, cpv):
+        """Marks the CPV as disabled.
+
+        @param cpv: the cpv to mark
+        """
+        raise NotImplentedError
+
+    def reload (self, cat = None):
+        """Reloads the given category.
+        
+        @param cat: category
+        @type cat: string
+        """
+        raise NotImplentedError
diff --git a/portato/db/dict.py b/portato/db/dict.py
new file mode 100644
index 0000000..f6edea6
--- /dev/null
+++ b/portato/db/dict.py
@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+#
+# File: portato/db/dict.py
+# This file is part of the Portato-Project, a graphical portage-frontend.
+#
+# Copyright (C) 2006-2009 René 'Necoro' Neumann
+# This is free software.  You may redistribute copies of it under the terms of
+# the GNU General Public License version 2.
+# There is NO WARRANTY, to the extent permitted by law.
+#
+# Written by René 'Necoro' Neumann <necoro@necoro.net>
+
+from __future__ import absolute_import, with_statement
+
+import re
+from collections import defaultdict
+from threading import RLock
+
+from ..backend import system
+from .database import Database, PkgData
+
+class DictDatabase (Database):
+    """An internal database which holds a simple dictionary cat -> [package_list]."""
+
+    lock = Database.lock
+
+    def __init__ (self, session):
+        """Constructor."""
+        Database.__init__(self)
+        self.session = session
+
+        self.__initialize()
+        self.populate()
+
+    def __initialize (self):
+        self._db = defaultdict(list)
+        self.inst_cats = set([self.ALL])
+        self._restrict = None
+
+    def __sort_key (self, x):
+        return x.pkg.lower()
+
+    @lock
+    def populate (self, category = None):
+        
+        # get the lists
+        packages = system.find_packages(category, with_version = False)
+        installed = system.find_packages(category, system.SET_INSTALLED, with_version = False)
+        
+        # cycle through packages
+        for p in packages:
+            cat, pkg = p.split("/")
+            inst = p in installed
+            t = PkgData(cat, pkg, inst, False)
+            self._db[cat].append(t)
+            self._db[self.ALL].append(t)
+
+            if inst:
+                self.inst_cats.add(cat)
+
+        for key in self._db: # sort alphabetically
+            self._db[key].sort(key = self.__sort_key)
+
+    @lock
+    def get_cat (self, cat = None, byName = True, showDisabled = False):
+        if not cat:
+            cat = self.ALL
+
+        def get_pkgs():
+            if byName:
+                for pkg in self._db[cat]:
+                    if showDisabled or not pkg.disabled:
+                        yield pkg
+            else:
+                ninst = []
+                for pkg in self._db[cat]:
+                    if not showDisabled and pkg.disabled: continue
+
+                    if pkg.inst:
+                        yield pkg
+                    else:
+                        ninst.append(pkg)
+
+                for pkg in ninst:
+                    yield pkg
+
+        try:
+            if self.restrict:
+                return (pkg for pkg in get_pkgs() if self.restrict.search(pkg.pkg))#if pkg[1].find(self.restrict) != -1)
+            else:
+                return get_pkgs()
+
+        except KeyError: # cat is in category list - but not in portage
+            info(_("Catched KeyError => %s seems not to be an available category. Have you played with rsync-excludes?"), cat)
+
+    @lock
+    def get_categories (self, installed = False):
+        if not self.restrict:
+            if installed:
+                cats = self.inst_cats
+            else:
+                cats = self._db.iterkeys()
+
+        else:
+            if installed:
+                cats = set((pkg.cat for pkg in self.get_cat(self.ALL) if pkg.inst))
+            else:
+                cats = set((pkg.cat for pkg in self.get_cat(self.ALL)))
+
+            if len(cats)>1:
+                cats.add(self.ALL)
+
+        return (cat for cat in cats)
+
+    @lock
+    def reload (self, cat = None):
+        if cat:
+            del self._db[cat]
+            try:
+                self.inst_cats.remove(cat)
+            except KeyError: # not in inst_cats - can be ignored
+                pass
+            
+            self._db[self.ALL] = filter(lambda x: x.cat != cat, self._db[self.ALL])
+            self.populate(cat+"/*")
+        else:
+            self.__initialize()
+            self.populate()
+
+    @lock
+    def disable (self, cpv):
+        cat, pkg = cpv.split("/")
+
+        c = self._db[cat]
+        p = c[c.index(PkgData(cat, pkg))]
+        p.disabled = True
+
+    def get_restrict (self):
+        return self._restrict
+
+    @lock
+    def set_restrict (self, restrict):
+        if not restrict:
+            self._restrict = None
+        else:
+            try:
+                regex = re.compile(restrict, re.I)
+            except re.error, e:
+                info(_("Error while compiling search expression: '%s'."), str(e))
+            else: # only set self._restrict if no error occurred
+                self._restrict = regex
+
+    restrict = property(get_restrict, set_restrict)
diff --git a/portato/db/sql.py b/portato/db/sql.py
new file mode 100644
index 0000000..ca01fd0
--- /dev/null
+++ b/portato/db/sql.py
@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+#
+# File: portato/db/sql.py
+# This file is part of the Portato-Project, a graphical portage-frontend.
+#
+# Copyright (C) 2006-2009 René 'Necoro' Neumann
+# This is free software.  You may redistribute copies of it under the terms of
+# the GNU General Public License version 2.
+# There is NO WARRANTY, to the extent permitted by law.
+#
+# Written by René 'Necoro' Neumann <necoro@necoro.net>
+
+from __future__ import absolute_import, with_statement
+
+try:
+    import sqlite3 as sql
+except ImportError:
+    from pysqlite2 import dbapi2 as sql
+
+import anydbm
+import hashlib
+import os
+
+from functools import wraps
+
+from ..constants import SESSION_DIR
+from ..helper import info, error, debug
+from ..backend import system
+from .database import Database, PkgData
+
+class SQLDatabase (Database):
+    
+    FORMAT = "1"
+    FORBIDDEN = (".bzr", ".svn", ".git", "CVS", ".hg", "_darcs")
+    lock = Database.lock
+
+    def __init__ (self, session):
+        """Constructor."""
+        Database.__init__(self)
+
+        self._restrict = ""
+        self.session = session
+        
+        updateFormat = False
+        if "format" not in session or session["format"] != self.FORMAT:
+            session["format"] = self.FORMAT
+            updateFormat = True
+
+        pkgdb = os.path.join(SESSION_DIR, "package.db")
+        pkgdb_existed = os.path.exists(pkgdb)
+
+        if pkgdb_existed:
+            debug("package.db already existant")
+        else:
+            debug("package.db not existant")
+
+        pkg_conn = sql.connect(os.path.join(SESSION_DIR, "package.db"))
+        pkg_conn.row_factory = sql.Row
+        if pkgdb_existed and updateFormat:
+            pkg_conn.execute("DROP TABLE packages")
+
+        pkg_conn.execute("""
+        CREATE TABLE IF NOT EXISTS packages
+        (
+            name TEXT,
+            cat TEXT,
+            inst INTEGER,
+            disabled INTEGER
+        )""")
+
+        pkg_conn.commit()
+        
+        self.was_updated = self.updated()
+        if self.was_updated or not pkgdb_existed or updateFormat:
+            info(_("Cleaning database..."))
+            pkg_conn.execute("DELETE FROM packages") # empty db at beginning
+            info(_("Populating database..."))
+            self.populate(connection = pkg_conn)
+            
+        pkg_conn.close()
+
+        descr_conn = sql.connect(os.path.join(SESSION_DIR, "descr.db"))
+        descr_conn.execute("""
+        CREATE TABLE IF NOT EXISTS descriptions
+        (
+            cp TEXT,
+            descr TEXT
+        )""")
+        descr_conn.close()
+
+    def updated (self):
+        changed = False
+
+        def walk (path):
+            debug("Walking %s", path)
+            
+            for root, dirs, files in os.walk(path):
+                for f in files:
+                    path = os.path.join(root, f)
+                    yield "%s %s" % (f, os.stat(path).st_mtime)
+                
+                for forbidden in self.FORBIDDEN:
+                    if forbidden in dirs:
+                        dirs.remove(forbidden)
+
+        overlays = system.get_global_settings("PORTDIR_OVERLAY").split()
+        hashes = {}
+        for overlay in overlays:
+            hashes[overlay] = hashlib.md5("".join(walk(overlay))).hexdigest()
+        
+        timestamp = os.path.join(system.get_global_settings("PORTDIR"), "metadata/timestamp")
+        hashes["ROOT"] = hashlib.md5("%s %s" % (timestamp, os.stat(timestamp).st_mtime)).hexdigest()
+    
+        dbpath = os.path.join(SESSION_DIR, "portdirs.db")
+        db_existed = os.path.exists(dbpath)
+        db = anydbm.open(dbpath, "c")
+        try:
+            if db_existed:
+                debug("portdirs.db already existant")
+                for key in set(db.keys())- set(hashes.keys()):
+                    debug("Overlay '%s' has been removed", key)
+                    del db[key]
+                    changed = True
+                
+                for key in hashes.iterkeys():
+
+                    if key not in db.keys():
+                        debug("Overlay '%s' has been added.", key)
+                        changed = True
+
+                    elif db[key] != hashes[key]:
+                        debug("Overlay '%s' has been changed.", key)
+                        changed = True
+                
+                    db[key] = hashes[key]
+            else:
+                debug("portdirs.db not existant")
+                for key in hashes.iterkeys():
+                    db[key] = hashes[key]
+
+        finally:
+            db.close()
+
+        return changed
+
+    def con (f):
+        @wraps(f)
+        def wrapper (*args, **kwargs):
+            if not "connection" in kwargs:
+                con= sql.connect(os.path.join(SESSION_DIR, "package.db"))
+                con.row_factory = sql.Row
+                kwargs["connection"] = con
+
+            return f(*args, **kwargs)
+        
+        return Database.lock(wrapper)
+
+    @con
+    def populate (self, category = None, connection = None):
+        def _get():
+            # get the lists
+            inst = system.find_packages(pkgSet = system.SET_INSTALLED, key=category, with_version = False)
+            for p in system.find_packages(key = category, with_version = False):
+                cat, pkg = p.split("/")
+
+                yield (cat, pkg, p in inst, False)
+
+        connection.executemany("INSERT INTO packages (cat, name, inst, disabled) VALUES (?, ?, ?, ?)", _get())
+        connection.commit()
+
+    @con
+    def get_cat (self, category = None, byName = True, showDisabled = False, connection = None):
+        sort = "ORDER BY name"
+        if not byName:
+            sort = "ORDER BY inst DESC, name"
+
+        disabled = "1=1"
+        if not showDisabled:
+            disabled = "disabled = 0"
+
+        if not category or category == self.ALL:
+            c = connection.execute("SELECT cat, name, inst, disabled FROM packages WHERE %s %s %s" % (disabled, self.restrict, sort))
+        else:
+            c = connection.execute("SELECT cat, name, inst, disabled FROM packages WHERE cat = ? AND %s %s %s" % (disabled, self.restrict ,sort), (category,))
+        
+        for pkg in c:
+            yield PkgData(pkg["cat"], pkg["name"], pkg["inst"], pkg["disabled"])
+        c.close()
+
+    @con
+    def get_categories (self, installed = False, connection = None):
+
+        if installed:
+            where = "inst = 1"
+        else:
+            where = "1 = 1"
+
+        c = connection.execute("SELECT cat FROM packages WHERE disabled = 0 AND %s %s GROUP BY cat" % (where, self.restrict))
+
+        l = c.fetchall()
+        c.close()
+
+        if len(l) > 1:
+            yield self.ALL
+        
+        for cat in l:
+            yield cat["cat"]
+
+    @con
+    def reload (self, cat = None, connection = None):
+        if cat:
+            connection.execute("DELETE FROM packages WHERE cat = ?", (cat,))
+            connection.commit()
+            self.populate(cat+"/*", connection = connection)
+        else:
+            connection.execute("DELETE FROM packages")
+            connection.commit()
+            self.populate(connection = connection)
+
+    @con
+    def disable (self, cpv, connection = None):
+        cat, pkg = cpv.split("/")
+        connection.execute("UPDATE packages SET disabled = 1 WHERE cat = ? AND name = ?", (cat, pkg))
+        connection.commit()
+
+    def get_restrict (self):
+        return self._restrict
+
+    @lock
+    def set_restrict (self, restrict):
+        if not restrict:
+            self._restrict = ""
+        else:
+            self._restrict = "AND name LIKE '%%%s%%'" % restrict
+
+    restrict = property(get_restrict, set_restrict)