From d44ce453d88624c8444f0733a56197d5291f52f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Fri, 14 Aug 2009 15:18:10 +0200 Subject: Rename DictDatabase to HashDatabase --- portato/db/__init__.py | 6 +- portato/db/dict.py | 153 ------------------------------------------------- portato/db/hash.py | 153 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 156 deletions(-) delete mode 100644 portato/db/dict.py create mode 100644 portato/db/hash.py (limited to 'portato/db') diff --git a/portato/db/__init__.py b/portato/db/__init__.py index da8a81e..e6f9759 100644 --- a/portato/db/__init__.py +++ b/portato/db/__init__.py @@ -43,9 +43,9 @@ def Database(type): return SQLDatabase(SectionDict(_SESSION, "SQL")) elif type == "dict": - debug("Using DictDatabase") - from .dict import DictDatabase - return DictDatabase(SectionDict(_SESSION, "dict")) + debug("Using HashDatabase") + from .hash import HashDatabase + return HashDatabase(SectionDict(_SESSION, "dict")) else: error(_("Unknown database type: %s"), type) diff --git a/portato/db/dict.py b/portato/db/dict.py deleted file mode 100644 index 279ab97..0000000 --- a/portato/db/dict.py +++ /dev/null @@ -1,153 +0,0 @@ -# -*- coding: utf-8 -*- -# -# File: portato/db/dict.py -# This file is part of the Portato-Project, a graphical portage-frontend. -# -# Copyright (C) 2006-2009 René 'Necoro' Neumann -# This is free software. You may redistribute copies of it under the terms of -# the GNU General Public License version 2. -# There is NO WARRANTY, to the extent permitted by law. -# -# Written by René 'Necoro' Neumann - -from __future__ import absolute_import, with_statement - -import re -from collections import defaultdict - -from ..helper import info -from ..backend import system -from .database import Database, PkgData - -class DictDatabase (Database): - """An internal database which holds a simple dictionary cat -> [package_list].""" - - lock = Database.lock - - def __init__ (self, session): - """Constructor.""" - Database.__init__(self) - self.session = session - - self.__initialize() - self.populate() - - def __initialize (self): - self._db = defaultdict(list) - self.inst_cats = set([self.ALL]) - self._restrict = None - - def __sort_key (self, x): - return x.pkg.lower() - - @lock - def populate (self, category = None): - - # get the lists - packages = system.find_packages(category, with_version = False) - installed = system.find_packages(category, system.SET_INSTALLED, with_version = False) - - # cycle through packages - for p in packages: - cat, pkg = p.split("/") - inst = p in installed - t = PkgData(cat, pkg, inst, False) - self._db[cat].append(t) - self._db[self.ALL].append(t) - - if inst: - self.inst_cats.add(cat) - - for key in self._db: # sort alphabetically - self._db[key].sort(key = self.__sort_key) - - @lock - def get_cat (self, cat = None, byName = True, showDisabled = False): - if not cat: - cat = self.ALL - - def get_pkgs(): - if byName: - for pkg in self._db[cat]: - if showDisabled or not pkg.disabled: - yield pkg - else: - ninst = [] - for pkg in self._db[cat]: - if not showDisabled and pkg.disabled: continue - - if pkg.inst: - yield pkg - else: - ninst.append(pkg) - - for pkg in ninst: - yield pkg - - try: - if self.restrict: - return (pkg for pkg in get_pkgs() if self.restrict.search(pkg.cat+"/"+pkg.pkg)) - else: - return get_pkgs() - - except KeyError: # cat is in category list - but not in portage - info(_("Catched KeyError => %s seems not to be an available category. Have you played with rsync-excludes?"), cat) - - @lock - def get_categories (self, installed = False): - if not self.restrict: - if installed: - cats = self.inst_cats - else: - cats = self._db.iterkeys() - - else: - if installed: - cats = set((pkg.cat for pkg in self.get_cat(self.ALL) if pkg.inst)) - else: - cats = set((pkg.cat for pkg in self.get_cat(self.ALL))) - - if len(cats)>1: - cats.add(self.ALL) - - return (cat for cat in cats) - - @lock - def reload (self, cat = None): - if cat: - del self._db[cat] - try: - self.inst_cats.remove(cat) - except KeyError: # not in inst_cats - can be ignored - pass - - self._db[self.ALL] = filter(lambda x: x.cat != cat, self._db[self.ALL]) - self.populate(cat+"/*") - else: - self.__initialize() - self.populate() - - @lock - def disable (self, cpv): - cat, pkg = cpv.split("/") - - c = self._db[cat] - p = c[c.index(PkgData(cat, pkg))] - p.disabled = True - - def get_restrict (self): - return self._restrict - - @lock - def set_restrict (self, restrict): - if not restrict: - self._restrict = None - else: - try: - regex = re.compile(restrict, re.I) - except re.error, e: - info(_("Error while compiling search expression: '%s'."), str(e)) - else: # only set self._restrict if no error occurred - self._restrict = regex - - restrict = property(get_restrict, set_restrict) diff --git a/portato/db/hash.py b/portato/db/hash.py new file mode 100644 index 0000000..8cea6f2 --- /dev/null +++ b/portato/db/hash.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +# +# File: portato/db/hash.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann + +from __future__ import absolute_import, with_statement + +import re +from collections import defaultdict + +from ..helper import info +from ..backend import system +from .database import Database, PkgData + +class HashDatabase (Database): + """An internal database which holds a simple dictionary cat -> [package_list].""" + + lock = Database.lock + + def __init__ (self, session): + """Constructor.""" + Database.__init__(self) + self.session = session + + self.__initialize() + self.populate() + + def __initialize (self): + self._db = defaultdict(list) + self.inst_cats = set([self.ALL]) + self._restrict = None + + def __sort_key (self, x): + return x.pkg.lower() + + @lock + def populate (self, category = None): + + # get the lists + packages = system.find_packages(category, with_version = False) + installed = system.find_packages(category, system.SET_INSTALLED, with_version = False) + + # cycle through packages + for p in packages: + cat, pkg = p.split("/") + inst = p in installed + t = PkgData(cat, pkg, inst, False) + self._db[cat].append(t) + self._db[self.ALL].append(t) + + if inst: + self.inst_cats.add(cat) + + for key in self._db: # sort alphabetically + self._db[key].sort(key = self.__sort_key) + + @lock + def get_cat (self, cat = None, byName = True, showDisabled = False): + if not cat: + cat = self.ALL + + def get_pkgs(): + if byName: + for pkg in self._db[cat]: + if showDisabled or not pkg.disabled: + yield pkg + else: + ninst = [] + for pkg in self._db[cat]: + if not showDisabled and pkg.disabled: continue + + if pkg.inst: + yield pkg + else: + ninst.append(pkg) + + for pkg in ninst: + yield pkg + + try: + if self.restrict: + return (pkg for pkg in get_pkgs() if self.restrict.search(pkg.cat+"/"+pkg.pkg)) + else: + return get_pkgs() + + except KeyError: # cat is in category list - but not in portage + info(_("Catched KeyError => %s seems not to be an available category. Have you played with rsync-excludes?"), cat) + + @lock + def get_categories (self, installed = False): + if not self.restrict: + if installed: + cats = self.inst_cats + else: + cats = self._db.iterkeys() + + else: + if installed: + cats = set((pkg.cat for pkg in self.get_cat(self.ALL) if pkg.inst)) + else: + cats = set((pkg.cat for pkg in self.get_cat(self.ALL))) + + if len(cats)>1: + cats.add(self.ALL) + + return (cat for cat in cats) + + @lock + def reload (self, cat = None): + if cat: + del self._db[cat] + try: + self.inst_cats.remove(cat) + except KeyError: # not in inst_cats - can be ignored + pass + + self._db[self.ALL] = filter(lambda x: x.cat != cat, self._db[self.ALL]) + self.populate(cat+"/*") + else: + self.__initialize() + self.populate() + + @lock + def disable (self, cpv): + cat, pkg = cpv.split("/") + + c = self._db[cat] + p = c[c.index(PkgData(cat, pkg))] + p.disabled = True + + def get_restrict (self): + return self._restrict + + @lock + def set_restrict (self, restrict): + if not restrict: + self._restrict = None + else: + try: + regex = re.compile(restrict, re.I) + except re.error, e: + info(_("Error while compiling search expression: '%s'."), str(e)) + else: # only set self._restrict if no error occurred + self._restrict = regex + + restrict = property(get_restrict, set_restrict) -- cgit v1.2.3 From 636eb3064e8ff8957bbf9fd172d7a0c6827e973c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Fri, 14 Aug 2009 19:24:50 +0200 Subject: Write eix-sql database backend --- portato/db/eix_sql.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 portato/db/eix_sql.py (limited to 'portato/db') diff --git a/portato/db/eix_sql.py b/portato/db/eix_sql.py new file mode 100644 index 0000000..0e4f569 --- /dev/null +++ b/portato/db/eix_sql.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# +# File: portato/db/eix_sql.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann + +from __future__ import absolute_import, with_statement + +try: + import sqlite3 as sql +except ImportError: + from pysqlite2 import dbapi2 as sql + +import os + +from .sql import SQLDatabase +from ..eix import EixReader +from ..helper import debug +from ..backend import system + +class EixSQLDatabase (SQLDatabase): + + CACHE_FILE = "/var/eix/cache" + + def __init__ (self, session): + SQLDatabase.__init__(self, session) + + if "cache" not in session: + self.cache = self.CACHE_FILE + session["cache"] = self.cache + else: + self.cache = session["cache"] + + debug("Using '%s' as eix cache file.", self.cache) + + def updated (self): + mtime = os.stat(self.cache).st_mtime + old = self.session.get("mtime", 0) + + self.session["mtime"] = mtime + + return old < mtime + + @SQLDatabase.con + def populate (self, category = None, connection = None): + inst = system.find_packages(pkgSet = system.SET_INSTALLED, key = category, with_version = False) + + def _get(): + with EixReader(self.cache) as eix: + for cat in eix.categories: + if category is None or cat.name() == category: + for pkg in cat.packages(): + yield (cat.name(), pkg.name(), pkg.name() in inst, False) + + connection.executemany("INSERT INTO packages (cat, name, inst, disabled) VALUES (?, ?, ?, ?)", _get()) + connection.commit() -- cgit v1.2.3 From 801316be64177d889ab21fc28c07dd4d77cb8184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Fri, 14 Aug 2009 19:42:01 +0200 Subject: Fix the EixSQLDatabase and EixReader --- portato/db/eix_sql.py | 18 ++++++++++-------- portato/db/sql.py | 1 + 2 files changed, 11 insertions(+), 8 deletions(-) (limited to 'portato/db') diff --git a/portato/db/eix_sql.py b/portato/db/eix_sql.py index 0e4f569..ac31019 100644 --- a/portato/db/eix_sql.py +++ b/portato/db/eix_sql.py @@ -21,29 +21,31 @@ import os from .sql import SQLDatabase from ..eix import EixReader -from ..helper import debug +from ..helper import debug, warning from ..backend import system class EixSQLDatabase (SQLDatabase): - CACHE_FILE = "/var/eix/cache" + CACHE_FILE = "/var/cache/eix" def __init__ (self, session): - SQLDatabase.__init__(self, session) - if "cache" not in session: + self.cache = session.get("cache", self.CACHE_FILE) + if not os.path.exists(self.cache): + warning(_("Cache file '%s' does not exist. Using default instead."), self.cache) self.cache = self.CACHE_FILE - session["cache"] = self.cache - else: - self.cache = session["cache"] debug("Using '%s' as eix cache file.", self.cache) + + session["cache"] = self.cache + + SQLDatabase.__init__(self, session) def updated (self): mtime = os.stat(self.cache).st_mtime old = self.session.get("mtime", 0) - self.session["mtime"] = mtime + self.session["mtime"] = str(mtime) return old < mtime diff --git a/portato/db/sql.py b/portato/db/sql.py index 6c95bb3..f2a736e 100644 --- a/portato/db/sql.py +++ b/portato/db/sql.py @@ -256,3 +256,4 @@ class SQLDatabase (Database): self._restrict = "AND (name LIKE '%%%(restrict)s%%' OR cat LIKE '%(restrict)s%%')" % {"restrict":restrict} restrict = property(get_restrict, set_restrict) + con = staticmethod(con) -- cgit v1.2.3 From 0d877faef762319348b164dba98c159ac1ec4ecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Fri, 14 Aug 2009 19:42:32 +0200 Subject: Add eixsql as db-choice --- portato/db/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'portato/db') diff --git a/portato/db/__init__.py b/portato/db/__init__.py index e6f9759..9869e3f 100644 --- a/portato/db/__init__.py +++ b/portato/db/__init__.py @@ -22,7 +22,8 @@ _SESSION = None types = { "sql": (_("SQLite"), _("Uses an SQLite-database to store package information.\nMay take longer to generate at the first time, but has advantages if portato is re-started with an unchanged portage tree. Additionally it allows to use fast SQL expressions for fetching the data.")), - "dict": (_("Hashmap"), _("Uses an in-memory hashmap to store package information.\nHas been used since at least version 0.3.3, but all information has to be regenerated on each startup.")) + "dict": (_("Hashmap"), _("Uses an in-memory hashmap to store package information.\nHas been used since at least version 0.3.3, but all information has to be regenerated on each startup.")), + "eixsql" : (_("eix + SQLite"), _("Similar to SQLite, but now uses the eix database to get the package information.\nThis should be much faster on startup, but requires that your eix database is always up-to-date.")) } def Database(type): @@ -40,13 +41,18 @@ def Database(type): warning(_("Cannot load SQLDatabase.")) return Database("dict") else: - return SQLDatabase(SectionDict(_SESSION, "SQL")) + return SQLDatabase(SectionDict(_SESSION, type)) elif type == "dict": debug("Using HashDatabase") from .hash import HashDatabase - return HashDatabase(SectionDict(_SESSION, "dict")) + return HashDatabase(SectionDict(_SESSION, type)) + elif type == "eixsql": + debug("Using EixSQLDatabase") + from .eix_sql import EixSQLDatabase + return EixSQLDatabase(SectionDict(_SESSION, type)) + else: error(_("Unknown database type: %s"), type) raise UnknownDatabaseTypeError, type -- cgit v1.2.3 From 2a945321cbc54c6acae73bc25d543e2af6f3e42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Fri, 14 Aug 2009 22:49:35 +0200 Subject: Use sets instead of lists --> faster results --- portato/db/eix_sql.py | 2 +- portato/db/sql.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'portato/db') diff --git a/portato/db/eix_sql.py b/portato/db/eix_sql.py index ac31019..089d3ed 100644 --- a/portato/db/eix_sql.py +++ b/portato/db/eix_sql.py @@ -51,7 +51,7 @@ class EixSQLDatabase (SQLDatabase): @SQLDatabase.con def populate (self, category = None, connection = None): - inst = system.find_packages(pkgSet = system.SET_INSTALLED, key = category, with_version = False) + inst = set(system.find_packages(pkgSet = system.SET_INSTALLED, key = category, with_version = False)) def _get(): with EixReader(self.cache) as eix: diff --git a/portato/db/sql.py b/portato/db/sql.py index f2a736e..415df92 100644 --- a/portato/db/sql.py +++ b/portato/db/sql.py @@ -175,7 +175,7 @@ class SQLDatabase (Database): def populate (self, category = None, connection = None): def _get(): # get the lists - inst = system.find_packages(pkgSet = system.SET_INSTALLED, key=category, with_version = False) + inst = set(system.find_packages(pkgSet = system.SET_INSTALLED, key=category, with_version = False)) for p in system.find_packages(key = category, with_version = False): cat, pkg = p.split("/") -- cgit v1.2.3 From d9c6fb6767c6873782847df168f8224d83ab30cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Fri, 14 Aug 2009 22:50:33 +0200 Subject: Rewrote eix-parser in Cython --> WAAAAAAAAAY faster --- portato/db/eix_sql.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'portato/db') diff --git a/portato/db/eix_sql.py b/portato/db/eix_sql.py index 089d3ed..3a0c6e9 100644 --- a/portato/db/eix_sql.py +++ b/portato/db/eix_sql.py @@ -56,9 +56,10 @@ class EixSQLDatabase (SQLDatabase): def _get(): with EixReader(self.cache) as eix: for cat in eix.categories: - if category is None or cat.name() == category: - for pkg in cat.packages(): - yield (cat.name(), pkg.name(), pkg.name() in inst, False) + if category is None or cat.name == category: + for pkg in cat.packages: + p = "%s/%s" % (cat.name, pkg.name) + yield (cat.name, pkg.name, p in inst, False) connection.executemany("INSERT INTO packages (cat, name, inst, disabled) VALUES (?, ?, ?, ?)", _get()) connection.commit() -- cgit v1.2.3