diff options
Diffstat (limited to 'portato')
| -rw-r--r-- | portato/__init__.py | 34 | ||||
| -rw-r--r-- | portato/backend/system_interface.py | 4 | ||||
| -rw-r--r-- | portato/db/__init__.py | 23 | ||||
| -rw-r--r-- | portato/db/eix_sql.py | 65 | ||||
| -rw-r--r-- | portato/db/hash.py (renamed from portato/db/dict.py) | 4 | ||||
| -rw-r--r-- | portato/db/sql.py | 3 | ||||
| -rw-r--r-- | portato/eix/__init__.py | 88 | ||||
| -rw-r--r-- | portato/eix/eix-format.html | 855 | ||||
| -rw-r--r-- | portato/eix/exceptions.py | 45 | ||||
| -rw-r--r-- | portato/eix/parser.pyx | 316 | ||||
| -rw-r--r-- | portato/eix/py_parser.py | 416 | ||||
| -rw-r--r-- | portato/ipc.pxd | 52 | ||||
| -rw-r--r-- | portato/ipc.pyx | 173 | ||||
| -rw-r--r-- | portato/listener.py | 64 | ||||
| -rw-r--r-- | portato/plugin.py | 35 | ||||
| -rw-r--r-- | portato/waiting_queue.py | 7 |
16 files changed, 2107 insertions, 77 deletions
diff --git a/portato/__init__.py b/portato/__init__.py index 653329c..7363429 100644 --- a/portato/__init__.py +++ b/portato/__init__.py @@ -18,7 +18,7 @@ import subprocess, threading import atexit from optparse import OptionParser, SUPPRESS_HELP -from . import log +from .log import start as logstart from .constants import LOCALE_DIR, APP, VERSION from .su import detect_su_command from .helper import debug, info, error @@ -46,7 +46,7 @@ def get_parser (use_ = False): parser = OptionParser(version = vers, prog = "portato", description = desc, usage = usage) - parser.add_option("--shm", action = "store", nargs = 3, type="long", dest = "shm", + parser.add_option("--mq", action = "store", nargs = 1, type="long", dest = "mq", default = None, help = SUPPRESS_HELP) parser.add_option("-F", "--no-fork", action = "store_true", dest = "nofork", default = False, @@ -54,6 +54,11 @@ def get_parser (use_ = False): return parser +def _sub_start (): + # set gettext stuff + locale.setlocale(locale.LC_ALL, '') + gettext.install(APP, LOCALE_DIR, unicode = True) + def start(): # set gettext stuff @@ -61,7 +66,7 @@ def start(): gettext.install(APP, LOCALE_DIR, unicode = True) # start logging - log.start(file=False) + logstart(file=False) # run parser (options, args) = get_parser().parse_args() @@ -70,15 +75,12 @@ def start(): atexit.register(get_listener().close) if options.nofork or os.getuid() == 0: # start GUI - log.start(file = True) # start logging to file + logstart(file = True) # start logging to file from .gui import run info("%s v. %s", _("Starting Portato"), VERSION) - if options.shm: - get_listener().set_send(*options.shm) - else: - get_listener().set_send() + get_listener().set_send(options.mq) try: run() @@ -87,14 +89,12 @@ def start(): else: # start us again in root modus and launch listener - import shm_wrapper as shm + from . import ipc - mem = shm.create_memory(1024, permissions=0600) - sig = shm.create_semaphore(InitialValue = 0, permissions = 0600) - rw = shm.create_semaphore(InitialValue = 1, permissions = 0600) + mq = ipc.MessageQueue(None, create = True, exclusive = True) # start listener - lt = threading.Thread(target=get_listener().set_recv, args = (mem, sig, rw)) + lt = threading.Thread(target=get_listener().set_recv, args = (mq,)) lt.setDaemon(False) lt.start() @@ -106,7 +106,7 @@ def start(): su = detect_su_command() if su: debug("Using '%s' as su command.", su.bin) - cmd = su.cmd("%s --no-fork --shm %ld %ld %ld" % (sys.argv[0], mem.key, sig.key, rw.key)) + cmd = su.cmd("%s --no-fork --mq %ld" % (sys.argv[0], mq.key)) sp = subprocess.Popen(cmd, env = env) @@ -124,3 +124,9 @@ def start(): if lt.isAlive(): debug("Listener is still running. Close it.") get_listener().close() + lt.join() + + try: + mq.remove() + except ipc.MessageQueueRemovedError: + debug("MessageQueue already removed. Ignore.") diff --git a/portato/backend/system_interface.py b/portato/backend/system_interface.py index 43de1ec..be79de2 100644 --- a/portato/backend/system_interface.py +++ b/portato/backend/system_interface.py @@ -108,8 +108,8 @@ class SystemInterface (object): @param key: the key to look for @type key: string - @param all: the package set to use - @type all: string + @param pkgSet: the package set to use + @type pkgSet: string @param masked: if True, also look for masked packages @type masked: boolean @param with_version: if True, return CPVs - else CP diff --git a/portato/db/__init__.py b/portato/db/__init__.py index da8a81e..60cb1c9 100644 --- a/portato/db/__init__.py +++ b/portato/db/__init__.py @@ -22,7 +22,8 @@ _SESSION = None types = { "sql": (_("SQLite"), _("Uses an SQLite-database to store package information.\nMay take longer to generate at the first time, but has advantages if portato is re-started with an unchanged portage tree. Additionally it allows to use fast SQL expressions for fetching the data.")), - "dict": (_("Hashmap"), _("Uses an in-memory hashmap to store package information.\nHas been used since at least version 0.3.3, but all information has to be regenerated on each startup.")) + "dict": (_("Hashmap"), _("Uses an in-memory hashmap to store package information.\nHas been used since at least version 0.3.3, but all information has to be regenerated on each startup.")), + "eixsql" : (_("eix + SQLite"), _("Similar to SQLite, but now uses the eix database to get the package information.\nThis should be much faster on startup, but requires that your eix database is always up-to-date.")) } def Database(type): @@ -37,16 +38,26 @@ def Database(type): try: from .sql import SQLDatabase except ImportError: - warning(_("Cannot load SQLDatabase.")) + warning(_("Cannot load %s."), "SQLDatabase") return Database("dict") else: - return SQLDatabase(SectionDict(_SESSION, "SQL")) + return SQLDatabase(SectionDict(_SESSION, type)) elif type == "dict": - debug("Using DictDatabase") - from .dict import DictDatabase - return DictDatabase(SectionDict(_SESSION, "dict")) + debug("Using HashDatabase") + from .hash import HashDatabase + return HashDatabase(SectionDict(_SESSION, type)) + elif type == "eixsql": + debug("Using EixSQLDatabase") + try: + from .eix_sql import EixSQLDatabase + except ImportError: + warning(_("Cannot load %s."), "EixSQLDatabase.") + return Database("sql") + else: + return EixSQLDatabase(SectionDict(_SESSION, type)) + else: error(_("Unknown database type: %s"), type) raise UnknownDatabaseTypeError, type diff --git a/portato/db/eix_sql.py b/portato/db/eix_sql.py new file mode 100644 index 0000000..3a0c6e9 --- /dev/null +++ b/portato/db/eix_sql.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# +# File: portato/db/eix_sql.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann <necoro@necoro.net> + +from __future__ import absolute_import, with_statement + +try: + import sqlite3 as sql +except ImportError: + from pysqlite2 import dbapi2 as sql + +import os + +from .sql import SQLDatabase +from ..eix import EixReader +from ..helper import debug, warning +from ..backend import system + +class EixSQLDatabase (SQLDatabase): + + CACHE_FILE = "/var/cache/eix" + + def __init__ (self, session): + + self.cache = session.get("cache", self.CACHE_FILE) + if not os.path.exists(self.cache): + warning(_("Cache file '%s' does not exist. Using default instead."), self.cache) + self.cache = self.CACHE_FILE + + debug("Using '%s' as eix cache file.", self.cache) + + session["cache"] = self.cache + + SQLDatabase.__init__(self, session) + + def updated (self): + mtime = os.stat(self.cache).st_mtime + old = self.session.get("mtime", 0) + + self.session["mtime"] = str(mtime) + + return old < mtime + + @SQLDatabase.con + def populate (self, category = None, connection = None): + inst = set(system.find_packages(pkgSet = system.SET_INSTALLED, key = category, with_version = False)) + + def _get(): + with EixReader(self.cache) as eix: + for cat in eix.categories: + if category is None or cat.name == category: + for pkg in cat.packages: + p = "%s/%s" % (cat.name, pkg.name) + yield (cat.name, pkg.name, p in inst, False) + + connection.executemany("INSERT INTO packages (cat, name, inst, disabled) VALUES (?, ?, ?, ?)", _get()) + connection.commit() diff --git a/portato/db/dict.py b/portato/db/hash.py index 279ab97..8cea6f2 100644 --- a/portato/db/dict.py +++ b/portato/db/hash.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# File: portato/db/dict.py +# File: portato/db/hash.py # This file is part of the Portato-Project, a graphical portage-frontend. # # Copyright (C) 2006-2009 René 'Necoro' Neumann @@ -19,7 +19,7 @@ from ..helper import info from ..backend import system from .database import Database, PkgData -class DictDatabase (Database): +class HashDatabase (Database): """An internal database which holds a simple dictionary cat -> [package_list].""" lock = Database.lock diff --git a/portato/db/sql.py b/portato/db/sql.py index db66764..e79a27d 100644 --- a/portato/db/sql.py +++ b/portato/db/sql.py @@ -166,7 +166,7 @@ class SQLDatabase (Database): def populate (self, category = None, connection = None): def _get(): # get the lists - inst = system.find_packages(pkgSet = system.SET_INSTALLED, key=category, with_version = False) + inst = set(system.find_packages(pkgSet = system.SET_INSTALLED, key=category, with_version = False)) for p in system.find_packages(key = category, with_version = False): cat, pkg = p.split("/") @@ -247,3 +247,4 @@ class SQLDatabase (Database): self._restrict = "AND (name LIKE '%%%(restrict)s%%' OR cat LIKE '%(restrict)s%%')" % {"restrict":restrict} restrict = property(get_restrict, set_restrict) + con = staticmethod(con) diff --git a/portato/eix/__init__.py b/portato/eix/__init__.py new file mode 100644 index 0000000..346fe82 --- /dev/null +++ b/portato/eix/__init__.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# +# File: portato/eix/__init__.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann <necoro@necoro.net> + +""" +A module to parse the eix-cache files. +""" + +from __future__ import absolute_import, with_statement +__docformat__ = "restructuredtext" + +from . import parser +from .exceptions import UnsupportedVersionError + +from ..helper import debug + +class EixReader(object): + """ + The main class to use to have access to the eix-cache. + + Note that the file used internally stays open during the whole operation. + So please call `close()` when you are finished. + + The ``EixReader`` supports the context manager protocol, so you can the ``with ... as ...``. + + :CVariables: + + supported_versions : int[] + The list of versions of the eix-cache, which are supported by this reader. + + :IVariables: + + file : file + The eix cache file. + + header : `parser.header` + The header of the eix cache. + + categories : `parser.category` [] + The list of categories. + """ + + supported_versions = (28,) + + def __init__ (self, filename): + """ + :param filename: Path to the cache file + :type filename: string + """ + + self.file = open(filename, "r") + + try: + version = parser.number(self.file) + + if version not in self.supported_versions: + raise UnsupportedVersionError(self.version) + + debug("Started EixReader for version %s.", version) + + self.file.seek(0) + + self.header = parser.header(self.file) + self.categories = parser.vector(self.file, parser.category, nelems = self.header.ncats) + except: + self.close() + raise + + def __enter__ (self): + return self + + def __exit__ (self, exc_type, exc_val, exc_tb): + self.close() + + def close (self): + """ + Closes the cache file. + """ + self.file.close() + debug("EixReader closed.") diff --git a/portato/eix/eix-format.html b/portato/eix/eix-format.html new file mode 100644 index 0000000..ee77458 --- /dev/null +++ b/portato/eix/eix-format.html @@ -0,0 +1,855 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> +<head> +<meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> +<meta name="generator" content="Docutils 0.5: http://docutils.sourceforge.net/" /> +<title>eix - index file format</title> +<meta name="authors" content="Martin Väth <vaeth AT mathematik DOT uni-wuerzburg DOT de> (active) Emil Beinroth <emilbeinroth AT gmx DOT net> (active) Wolfgang Frisch <xororand AT users DOT sourceforge DOT net> (inactive)" /> +<meta name="copyright" content="This file is part of the eix project and distributed under the terms of the GNU General Public License v2." /> +<style type="text/css"> + +/* +:Author: David Goodger (goodger@python.org) +:Id: $Id: html4css1.css 5196 2007-06-03 20:25:28Z wiemann $ +:Copyright: This stylesheet has been placed in the public domain. + +Default cascading style sheet for the HTML output of Docutils. + +See http://docutils.sf.net/docs/howto/html-stylesheets.html for how to +customize this style sheet. +*/ + +/* used to remove borders from tables and images */ +.borderless, table.borderless td, table.borderless th { + border: 0 } + +table.borderless td, table.borderless th { + /* Override padding for "table.docutils td" with "! important". + The right padding separates the table cells. */ + padding: 0 0.5em 0 0 ! important } + +.first { + /* Override more specific margin styles with "! important". */ + margin-top: 0 ! important } + +.last, .with-subtitle { + margin-bottom: 0 ! important } + +.hidden { + display: none } + +a.toc-backref { + text-decoration: none ; + color: black } + +blockquote.epigraph { + margin: 2em 5em ; } + +dl.docutils dd { + margin-bottom: 0.5em } + +/* Uncomment (and remove this text!) to get bold-faced definition list terms +dl.docutils dt { + font-weight: bold } +*/ + +div.abstract { + margin: 2em 5em } + +div.abstract p.topic-title { + font-weight: bold ; + text-align: center } + +div.admonition, div.attention, div.caution, div.danger, div.error, +div.hint, div.important, div.note, div.tip, div.warning { + margin: 2em ; + border: medium outset ; + padding: 1em } + +div.admonition p.admonition-title, div.hint p.admonition-title, +div.important p.admonition-title, div.note p.admonition-title, +div.tip p.admonition-title { + font-weight: bold ; + font-family: sans-serif } + +div.attention p.admonition-title, div.caution p.admonition-title, +div.danger p.admonition-title, div.error p.admonition-title, +div.warning p.admonition-title { + color: red ; + font-weight: bold ; + font-family: sans-serif } + +/* Uncomment (and remove this text!) to get reduced vertical space in + compound paragraphs. +div.compound .compound-first, div.compound .compound-middle { + margin-bottom: 0.5em } + +div.compound .compound-last, div.compound .compound-middle { + margin-top: 0.5em } +*/ + +div.dedication { + margin: 2em 5em ; + text-align: center ; + font-style: italic } + +div.dedication p.topic-title { + font-weight: bold ; + font-style: normal } + +div.figure { + margin-left: 2em ; + margin-right: 2em } + +div.footer, div.header { + clear: both; + font-size: smaller } + +div.line-block { + display: block ; + margin-top: 1em ; + margin-bottom: 1em } + +div.line-block div.line-block { + margin-top: 0 ; + margin-bottom: 0 ; + margin-left: 1.5em } + +div.sidebar { + margin: 0 0 0.5em 1em ; + border: medium outset ; + padding: 1em ; + background-color: #ffffee ; + width: 40% ; + float: right ; + clear: right } + +div.sidebar p.rubric { + font-family: sans-serif ; + font-size: medium } + +div.system-messages { + margin: 5em } + +div.system-messages h1 { + color: red } + +div.system-message { + border: medium outset ; + padding: 1em } + +div.system-message p.system-message-title { + color: red ; + font-weight: bold } + +div.topic { + margin: 2em } + +h1.section-subtitle, h2.section-subtitle, h3.section-subtitle, +h4.section-subtitle, h5.section-subtitle, h6.section-subtitle { + margin-top: 0.4em } + +h1.title { + text-align: center } + +h2.subtitle { + text-align: center } + +hr.docutils { + width: 75% } + +img.align-left { + clear: left } + +img.align-right { + clear: right } + +ol.simple, ul.simple { + margin-bottom: 1em } + +ol.arabic { + list-style: decimal } + +ol.loweralpha { + list-style: lower-alpha } + +ol.upperalpha { + list-style: upper-alpha } + +ol.lowerroman { + list-style: lower-roman } + +ol.upperroman { + list-style: upper-roman } + +p.attribution { + text-align: right ; + margin-left: 50% } + +p.caption { + font-style: italic } + +p.credits { + font-style: italic ; + font-size: smaller } + +p.label { + white-space: nowrap } + +p.rubric { + font-weight: bold ; + font-size: larger ; + color: maroon ; + text-align: center } + +p.sidebar-title { + font-family: sans-serif ; + font-weight: bold ; + font-size: larger } + +p.sidebar-subtitle { + font-family: sans-serif ; + font-weight: bold } + +p.topic-title { + font-weight: bold } + +pre.address { + margin-bottom: 0 ; + margin-top: 0 ; + font-family: serif ; + font-size: 100% } + +pre.literal-block, pre.doctest-block { + margin-left: 2em ; + margin-right: 2em } + +span.classifier { + font-family: sans-serif ; + font-style: oblique } + +span.classifier-delimiter { + font-family: sans-serif ; + font-weight: bold } + +span.interpreted { + font-family: sans-serif } + +span.option { + white-space: nowrap } + +span.pre { + white-space: pre } + +span.problematic { + color: red } + +span.section-subtitle { + /* font-size relative to parent (h1..h6 element) */ + font-size: 80% } + +table.citation { + border-left: solid 1px gray; + margin-left: 1px } + +table.docinfo { + margin: 2em 4em } + +table.docutils { + margin-top: 0.5em ; + margin-bottom: 0.5em } + +table.footnote { + border-left: solid 1px black; + margin-left: 1px } + +table.docutils td, table.docutils th, +table.docinfo td, table.docinfo th { + padding-left: 0.5em ; + padding-right: 0.5em ; + vertical-align: top } + +table.docutils th.field-name, table.docinfo th.docinfo-name { + font-weight: bold ; + text-align: left ; + white-space: nowrap ; + padding-left: 0 } + +h1 tt.docutils, h2 tt.docutils, h3 tt.docutils, +h4 tt.docutils, h5 tt.docutils, h6 tt.docutils { + font-size: 100% } + +ul.auto-toc { + list-style-type: none } + +</style> +</head> +<body> +<div class="document" id="eix-index-file-format"> +<h1 class="title">eix - index file format</h1> +<table class="docinfo" frame="void" rules="none"> +<col class="docinfo-name" /> +<col class="docinfo-content" /> +<tbody valign="top"> +<tr><th class="docinfo-name">Authors:</th> +<td>Martin Väth <vaeth AT mathematik DOT uni-wuerzburg DOT de> (active) +<br />Emil Beinroth <emilbeinroth AT gmx DOT net> (active) +<br />Wolfgang Frisch <xororand AT users DOT sourceforge DOT net> (inactive)</td></tr> +<tr><th class="docinfo-name">Copyright:</th> +<td>This file is part of the eix project and distributed under the +terms of the GNU General Public License v2.</td></tr> +</tbody> +</table> +<p>This article describes the format of the eix index file, usually located at /var/cache/eix. The +format includes a version field in the <a class="reference internal" href="#header">header</a> block. The current version is <strong>28</strong> (eix 0.17.0).</p> +<div class="contents topic" id="table-of-contents"> +<p class="topic-title first"><strong>Table of Contents</strong>:</p> +<ul class="simple"> +<li><a class="reference internal" href="#overall-layout" id="id3">Overall layout</a></li> +<li><a class="reference internal" href="#basic-datatypes" id="id4">Basic Datatypes</a><ul> +<li><a class="reference internal" href="#number" id="id5">Number</a></li> +<li><a class="reference internal" href="#vector" id="id6">Vector</a></li> +<li><a class="reference internal" href="#string" id="id7">String</a></li> +<li><a class="reference internal" href="#hash" id="id8">Hash</a></li> +<li><a class="reference internal" href="#hashedstring" id="id9">HashedString</a></li> +<li><a class="reference internal" href="#hashedwords" id="id10">HashedWords</a></li> +</ul> +</li> +<li><a class="reference internal" href="#data-blocks" id="id11">Data blocks</a><ul> +<li><a class="reference internal" href="#header" id="id12">Header</a></li> +<li><a class="reference internal" href="#overlay" id="id13">Overlay</a></li> +<li><a class="reference internal" href="#category" id="id14">Category</a></li> +<li><a class="reference internal" href="#package" id="id15">Package</a></li> +<li><a class="reference internal" href="#version" id="id16">Version</a></li> +<li><a class="reference internal" href="#versionpart" id="id17">VersionPart</a></li> +</ul> +</li> +<li><a class="reference internal" href="#historical-notes" id="id18">Historical notes</a></li> +</ul> +</div> +<div class="section" id="overall-layout"> +<h1>Overall layout</h1> +<p>The file is made up of blocks of data, +which may in turn contain other other blocks. <a class="footnote-reference" href="#vector-vs-blocks" id="id1">[1]</a> +The first block is a special header. +The remaining blocks are the categories which in turn contain the package blocks +which contain the version blocks, ...</p> +<blockquote> +<div class="layout-block header-block container"> +<a class="reference internal" href="#header">Header</a></div> +<div class="layout-block category-block container"> +<p>1st <a class="reference internal" href="#category">Category</a></p> +<div class="layout-block package-block container"> +<p>1st <a class="reference internal" href="#package">Package</a> in 1st Category</p> +<div class="layout-block version-block container"> +1st <a class="reference internal" href="#version">Version</a> of this Package</div> +<div class="layout-block version-block container"> +[..]</div> +</div> +<div class="layout-block package-block container"> +<p>2nd <a class="reference internal" href="#package">Package</a> in 1st Category</p> +<p>[..]</p> +</div> +</div> +<div class="layout-block category-block container"> +<p>2nd <a class="reference internal" href="#category">Category</a></p> +<p>[..]</p> +</div> +</blockquote> +<table class="docutils footnote" frame="void" id="vector-vs-blocks" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td>Most blocks here occur as a <cite>vector</cite> (described below), i.e. the +first entry is the number of elements, followed by the individual elements. +However, if not stated explicitly that a block is a <cite>vector</cite>, it is +indicated otherwise in the file how many elements it has. +For example, the number of category blocks is contained in the <a class="reference internal" href="#header">header</a>.</td></tr> +</tbody> +</table> +</div> +<div class="section" id="basic-datatypes"> +<h1>Basic Datatypes</h1> +<p>This section covers the datatypes <cite>number</cite> and <cite>vector</cite> (resp. <cite>string</cite>) which are used in the index file.</p> +<div class="section" id="number"> +<h2>Number</h2> +<p>The index file contains non-negative integer values only. The format we use avoids fixed length +integers by encoding the number of bytes into the integer itself. It has a bias towards numbers +smaller than 0xFF, which are encoded into a single byte.</p> +<blockquote> +<p>To determine the number of bytes used, you must first count how often the byte <strong>0xFF</strong> occurs at +the beginning of the number. +Let <cite>n</cite> be this count (<cite>n</cite> may be 0). Then, as a rule, there will follow <cite>n+1</cite> bytes that +contain the actual integer stored in big-endian byte order (highest byte first).</p> +<p><em>But</em> since it would be impossible to store any number that has a leading 0xFF with this format, +a leading 0xFF is stored as 0x00. Meaning, if a 0x00 byte follows the last 0xFF, you must +interpret this byte as 0xFF inside the number.</p> +</blockquote> +<p>Examples:</p> +<blockquote> +<table border="1" class="docutils"> +<colgroup> +<col width="23%" /> +<col width="77%" /> +</colgroup> +<thead valign="bottom"> +<tr><th class="head">Number</th> +<th class="head">Bytes stored in the file</th> +</tr> +</thead> +<tbody valign="top"> +<tr><td>0x00</td> +<td>0x00</td> +</tr> +<tr><td>0xFE</td> +<td>0xFE</td> +</tr> +<tr><td>0xFF</td> +<td>0xFF 0x00</td> +</tr> +<tr><td>0x0100</td> +<td>0xFF 0x01 0x00</td> +</tr> +<tr><td>0x01FF</td> +<td>0xFF 0x01 0xFF</td> +</tr> +<tr><td>0xFEFF</td> +<td>0xFF 0xFE 0xFF</td> +</tr> +<tr><td>0xFF00</td> +<td>0xFF 0xFF 0x00 0x00</td> +</tr> +<tr><td>0xFF01</td> +<td>0xFF 0xFF 0x00 0x01</td> +</tr> +<tr><td>0x010000</td> +<td>0xFF 0xFF 0x01 0x00 0x00</td> +</tr> +<tr><td>0xABCDEF</td> +<td>0xFF 0xFF 0xAB 0xCD 0xEF</td> +</tr> +<tr><td>0xFFABCD</td> +<td>0xFF 0xFF 0xFF 0x00 0xAB 0xCD</td> +</tr> +<tr><td>0x01ABCDEF</td> +<td>0xFF 0xFF 0xFF 0x01 0xAB 0xCD 0xEF</td> +</tr> +</tbody> +</table> +</blockquote> +</div> +<div class="section" id="vector"> +<h2>Vector</h2> +<p>Vectors (or lists) are extensively used throughout the index file. They are stored as the <a class="reference internal" href="#number">number</a> of +elements, followed by the elements themselves.</p> +<table border="1" class="docutils"> +<colgroup> +<col width="20%" /> +<col width="80%" /> +</colgroup> +<thead valign="bottom"> +<tr><th class="head">Type</th> +<th class="head">Content</th> +</tr> +</thead> +<tbody valign="top"> +<tr><td>Number</td> +<td>Number of elements (<cite>n</cite>)</td> +</tr> +<tr><td></td> +<td>1st element</td> +</tr> +<tr><td></td> +<td>...</td> +</tr> +<tr><td></td> +<td><cite>n</cite>th element</td> +</tr> +</tbody> +</table> +</div> +<div class="section" id="string"> +<h2>String</h2> +<p>Strings are stored as a <a class="reference internal" href="#vector">vector</a> of characters. A trailing '\0' is not included.</p> +</div> +<div class="section" id="hash"> +<h2>Hash</h2> +<p>A hash is a <a class="reference internal" href="#vector">vector</a> of <a class="reference internal" href="#string">string</a>s.</p> +</div> +<div class="section" id="hashedstring"> +<h2>HashedString</h2> +<p>A <a class="reference internal" href="#number">number</a> which is considered as an index in the corresponding <a class="reference internal" href="#hash">hash</a>; +0 denotes the first string of the hash, 1 the second, ...</p> +</div> +<div class="section" id="hashedwords"> +<h2>HashedWords</h2> +<p>A <a class="reference internal" href="#vector">vector</a> of <a class="reference internal" href="#hashedstring">HashedString</a>s. +The resulting strings are meant to be concatenated, with spaces as separators.</p> +</div> +</div> +<div class="section" id="data-blocks"> +<h1>Data blocks</h1> +<div class="section" id="header"> +<h2>Header</h2> +<table border="1" class="docutils"> +<colgroup> +<col width="19%" /> +<col width="81%" /> +</colgroup> +<thead valign="bottom"> +<tr><th class="head">Type</th> +<th class="head">Content</th> +</tr> +</thead> +<tbody valign="top"> +<tr><td>Number</td> +<td>File format version</td> +</tr> +<tr><td>Number</td> +<td>Number of <a class="reference internal" href="#category">Category</a> blocks</td> +</tr> +<tr><td>Vector</td> +<td><a class="reference internal" href="#overlay">Overlay</a>s</td> +</tr> +<tr><td>Hash</td> +<td>Hash for "Provide"</td> +</tr> +<tr><td>Hash</td> +<td>Hash for "Licenses"</td> +</tr> +<tr><td>Hash</td> +<td>Hash for "Keywords"</td> +</tr> +<tr><td>Hash</td> +<td>Hash for "Useflags"</td> +</tr> +<tr><td>Hash</td> +<td>Hash for "Slot"</td> +</tr> +<tr><td>Vector</td> +<td>names of world sets</td> +</tr> +</tbody> +</table> +<p>The names of world sets are the names (without leading @) of the world sets +stored in /var/lib/portage/world_sets. If SAVE_WORLD=false, the list is empty.</p> +</div> +<div class="section" id="overlay"> +<h2>Overlay</h2> +<table border="1" class="docutils"> +<colgroup> +<col width="21%" /> +<col width="79%" /> +</colgroup> +<thead valign="bottom"> +<tr><th class="head">Type</th> +<th class="head">Content</th> +</tr> +</thead> +<tbody valign="top"> +<tr><td>String</td> +<td>overlay path</td> +</tr> +<tr><td>String</td> +<td>label (repository name)</td> +</tr> +</tbody> +</table> +</div> +<div class="section" id="category"> +<h2>Category</h2> +<table border="1" class="docutils"> +<colgroup> +<col width="18%" /> +<col width="82%" /> +</colgroup> +<thead valign="bottom"> +<tr><th class="head">Type</th> +<th class="head">Content</th> +</tr> +</thead> +<tbody valign="top"> +<tr><td>String</td> +<td>Name of category</td> +</tr> +<tr><td>Vector</td> +<td><a class="reference internal" href="#package">Package</a>s in this category</td> +</tr> +</tbody> +</table> +</div> +<div class="section" id="package"> +<h2>Package</h2> +<table border="1" class="docutils"> +<colgroup> +<col width="8%" /> +<col width="92%" /> +</colgroup> +<thead valign="bottom"> +<tr><th class="head">Type</th> +<th class="head">Content</th> +</tr> +</thead> +<tbody valign="top"> +<tr><td>Number</td> +<td>Offset to the next package in the eix cache file (in bytes; counting starts after the number)</td> +</tr> +<tr><td>String</td> +<td>Package name</td> +</tr> +<tr><td>String</td> +<td>Description</td> +</tr> +<tr><td>HashedWords</td> +<td>Provide</td> +</tr> +<tr><td>String</td> +<td>Homepage</td> +</tr> +<tr><td>HashedString</td> +<td>Licenses, e.g. MPL-1.1 NPL-1.1</td> +</tr> +<tr><td>HashedWords</td> +<td>Useflags (all useflags of all versions of the package are added). This might "falsely" be the empty string if per-version IUSE flags are stored.</td> +</tr> +<tr><td>Vector</td> +<td><a class="reference internal" href="#version">Version</a>s</td> +</tr> +</tbody> +</table> +</div> +<div class="section" id="version"> +<h2>Version</h2> +<table border="1" class="docutils"> +<colgroup> +<col width="11%" /> +<col width="89%" /> +</colgroup> +<thead valign="bottom"> +<tr><th class="head">Type</th> +<th class="head">Content</th> +</tr> +</thead> +<tbody valign="top"> +<tr><td>char</td> +<td><p class="first">Mask bitset for the current $ARCH:</p> +<blockquote class="last"> +<table class="docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field"><th class="field-name">0x00:</th><td class="field-body">none of the following</td> +</tr> +<tr class="field"><th class="field-name">0x01:</th><td class="field-body">masked by package.mask</td> +</tr> +<tr class="field"><th class="field-name">0x02:</th><td class="field-body">masked by profile</td> +</tr> +<tr class="field"><th class="field-name">0x04:</th><td class="field-body">version is in @system</td> +</tr> +<tr class="field"><th class="field-name">0x08:</th><td class="field-body">version is in @world (if SAVE_WORLD=true)</td> +</tr> +</tbody> +</table> +</blockquote> +</td> +</tr> +<tr><td>char</td> +<td><dl class="first last docutils"> +<dt>Mask bitset for the PROPERTIES variable:</dt> +<dd><table class="first last docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field"><th class="field-name">0x01:</th><td class="field-body">PROPERTIES=interactive</td> +</tr> +<tr class="field"><th class="field-name">0x02:</th><td class="field-body">PROPERTIES=live</td> +</tr> +<tr class="field"><th class="field-name">0x04:</th><td class="field-body">PROPERTIES=virtual</td> +</tr> +</tbody> +</table> +</dd> +</dl> +</td> +</tr> +<tr><td>Number</td> +<td><dl class="first last docutils"> +<dt>Mask bitset for the RESTRICT variable:</dt> +<dd><table class="first last docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field"><th class="field-name">0x0001:</th><td class="field-body">RESTRICT=binchecks</td> +</tr> +<tr class="field"><th class="field-name">0x0002:</th><td class="field-body">RESTRICT=strip</td> +</tr> +<tr class="field"><th class="field-name">0x0004:</th><td class="field-body">RESTRICT=test</td> +</tr> +<tr class="field"><th class="field-name">0x0008:</th><td class="field-body">RESTRICT=userpriv</td> +</tr> +<tr class="field"><th class="field-name">0x0010:</th><td class="field-body">RESTRICT=installsources</td> +</tr> +<tr class="field"><th class="field-name">0x0020:</th><td class="field-body">RESTRICT=fetch</td> +</tr> +<tr class="field"><th class="field-name">0x0040:</th><td class="field-body">RESTRICT=mirror</td> +</tr> +<tr class="field"><th class="field-name">0x0080:</th><td class="field-body">RESTRICT=primaryuri</td> +</tr> +<tr class="field"><th class="field-name">0x0100:</th><td class="field-body">RESTRICT=bindist</td> +</tr> +</tbody> +</table> +</dd> +</dl> +</td> +</tr> +<tr><td>HashedWords</td> +<td>Full keywords string of the ebuild.</td> +</tr> +<tr><td>Vector</td> +<td><a class="reference internal" href="#versionpart">VersionPart</a>s</td> +</tr> +<tr><td>HashedString</td> +<td>Slot name. The slot name "0" is stored as "".</td> +</tr> +<tr><td>Number</td> +<td>Index of the portage overlay (in the overlays block)</td> +</tr> +<tr><td>HashedWords</td> +<td>Useflags of this version. This might "falsely" be empty if only per-package IUSE flags are stored.</td> +</tr> +</tbody> +</table> +</div> +<div class="section" id="versionpart"> +<h2>VersionPart</h2> +<p>A VersionPart consists of two data: a number (referred to as <em>type</em>) and a "<a class="reference internal" href="#string">string</a>" (referred to as <em>value</em>).</p> +<p>The number is encoded in the lower 5 bits of the length-part of the "<a class="reference internal" href="#string">string</a>"; +of course, the actual length is shifted by the same number of bits.</p> +<p>For the type, these named values are possible:</p> +<blockquote> +<table border="1" class="docutils"> +<colgroup> +<col width="7%" /> +<col width="16%" /> +<col width="77%" /> +</colgroup> +<thead valign="bottom"> +<tr><th class="head">Value</th> +<th class="head">Name</th> +<th class="head">Content of value</th> +</tr> +</thead> +<tbody valign="top"> +<tr><td>0</td> +<td><cite>garbage</cite></td> +<td>garbage that was found after any valid version string</td> +</tr> +<tr><td>1</td> +<td><cite>alpha</cite></td> +<td>number of "_alpha" (may be empty)</td> +</tr> +<tr><td>2</td> +<td><cite>beta</cite></td> +<td>number of "_beta" (may be empty)</td> +</tr> +<tr><td>3</td> +<td><cite>pre</cite></td> +<td>number of "_pre" (may be empty)</td> +</tr> +<tr><td>4</td> +<td><cite>rc</cite></td> +<td>number of "_rc" (may be empty)</td> +</tr> +<tr><td>5</td> +<td><cite>revision</cite></td> +<td>number of "-r" (may be empty)</td> +</tr> +<tr><td>6</td> +<td><cite>inter_rev</cite></td> +<td>number of inter-revision <a class="footnote-reference" href="#inter-revisions" id="id2">[2]</a></td> +</tr> +<tr><td>7</td> +<td><cite>patch</cite></td> +<td>number of "_p" (may be empty)</td> +</tr> +<tr><td>8</td> +<td><cite>char</cite></td> +<td>single character</td> +</tr> +<tr><td>9</td> +<td><cite>primary</cite></td> +<td>numeric version part</td> +</tr> +<tr><td>10</td> +<td><cite>first</cite></td> +<td>first numeric version part</td> +</tr> +</tbody> +</table> +</blockquote> +<p>As an example, we split the version string <cite>1.2c_pre12_alpha-r01.01-foo</cite> like this:</p> +<blockquote> +(first, "1") (primary, "2") (char, "c") (pre, "12") (alpha, "") (revision, "01") (inter_rev, "01") (garbage, "-foo")</blockquote> +<p>To reconstruct a version string, iterate through the vector of parts, +and for each part append a specific prefix and the value stored in the string. +The prefixes you need should be obvious, but here they are listed anyway:</p> +<blockquote> +<table border="1" class="docutils"> +<colgroup> +<col width="33%" /> +<col width="67%" /> +</colgroup> +<thead valign="bottom"> +<tr><th class="head">Prefix</th> +<th class="head">Name</th> +</tr> +</thead> +<tbody valign="top"> +<tr><td>"." (dot)</td> +<td>primary, inter_rev</td> +</tr> +<tr><td>"" (empty)</td> +<td>first, char, garbage</td> +</tr> +<tr><td>"_alpha"</td> +<td>alpha</td> +</tr> +<tr><td>"_beta"</td> +<td>beta</td> +</tr> +<tr><td>"_pre"</td> +<td>pre</td> +</tr> +<tr><td>"_rc"</td> +<td>rc</td> +</tr> +<tr><td>"-r"</td> +<td>revision</td> +</tr> +<tr><td>"_p"</td> +<td>patch</td> +</tr> +</tbody> +</table> +</blockquote> +<table class="docutils footnote" frame="void" id="inter-revisions" rules="none"> +<colgroup><col class="label" /><col /></colgroup> +<tbody valign="top"> +<tr><td class="label"><a class="fn-backref" href="#id2">[2]</a></td><td>inter-revision ares used by gentoo-alt to keep their prefixed portage tree in +sync with the main tree +<a class="reference external" href="http://www.gentoo.org/proj/en/gentoo-alt/prefix/techdocs.xml#doc_chap2_sect5">http://www.gentoo.org/proj/en/gentoo-alt/prefix/techdocs.xml#doc_chap2_sect5</a></td></tr> +</tbody> +</table> +</div> +</div> +<div class="section" id="historical-notes"> +<h1>Historical notes</h1> +<ul class="simple"> +<li>Since version 17, the format of this file is architecture-independent.</li> +</ul> +<!-- vim:set tw=100 ft=rst: --> +</div> +</div> +</body> +</html> diff --git a/portato/eix/exceptions.py b/portato/eix/exceptions.py new file mode 100644 index 0000000..8145af4 --- /dev/null +++ b/portato/eix/exceptions.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# +# File: portato/eix/exceptions.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann <necoro@necoro.net> + +""" +Different exceptions used in the eix module. +""" + +from __future__ import absolute_import, with_statement +__docformat__ = "restructuredtext" + +class EixError (Exception): + """ + The base class for all exceptions of this module. + + :ivar message: The error message + """ + message = _("Unknown error.") + + def __str__ (self): + return self.message + +class EndOfFileException (EixError): + """ + Denotes the unexpected EOF. + """ + + def __init__ (self, filename): + self.message = _("End of file reached though it was not expected: '%s'") % filename + +class UnsupportedVersionError (EixError): + """ + The version of the cache file found is not supported. + """ + + def __init__ (self, version): + self.message = _("Version '%s' is not supported.") % version diff --git a/portato/eix/parser.pyx b/portato/eix/parser.pyx new file mode 100644 index 0000000..01a673f --- /dev/null +++ b/portato/eix/parser.pyx @@ -0,0 +1,316 @@ +# -*- coding: utf-8 -*- +# +# File: portato/eix/_parser.pyx +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann <necoro@necoro.net> + +""" +The cache file supports different types of data. +In this module (nearly) all of these types have a corresponding function. + +For the exact way all the functions work, have a look at the eix format description. +""" + +__docformat__ = "restructuredtext" + +cdef extern from "stdio.h": + ctypedef struct FILE: + pass + + int fgetc(FILE* stream) + long ftell(FILE* stream) + int fseek(FILE* stream, long offset, int whence) + + int EOF + int SEEK_CUR + +cdef extern from "Python.h": + FILE* PyFile_AsFile(object) + +ctypedef unsigned char UChar +ctypedef long long LLong + +from portato.eix.exceptions import EndOfFileException + +# +# Helper +# + +cdef int _get_byte (FILE* file) except -1: + cdef int c = fgetc(file) + + if c == EOF: + raise EndOfFileException + + return c + +# +# Base Types +# + +cdef LLong _number (object pfile): + cdef UChar n + cdef LLong value + cdef int i + + cdef unsigned short count = 1 + cdef FILE* file = PyFile_AsFile(pfile) + + n = <UChar>_get_byte(file) + + if n < 0xFF: + return <LLong>n + else: + + n = <UChar>_get_byte(file) + while (n == 0xFF): + count += 1 + n = <UChar>_get_byte(file) + + if n == 0: + value = <LLong>0xFF # 0xFF is encoded as 0xFF 0x00 + count -= 1 + else: + value = <LLong>n + + for i in range(count): + value = (value << 8) | <LLong>(_get_byte(file)) + + return value + +def number (file): + """ + Returns a number. + + :param file: The file to read from + :type file: file + :rtype: int + """ + + return _number(file) + +def vector (file, get_type, nelems = None): + """ + Returns a vector of elements. + + :Parameters: + + file : file + The file to read from. + + get_type : function(file, bool) + The function determining type of the elements. + + nelems : int + Normally the eix-Vector has the number of elements as the first argument. + If for some reason this is not the case, you can pass it in here. + + :rtype: list + """ + + cdef LLong n + cdef LLong i + + if nelems is None: + n = _number(file) + else: + n = nelems + + return [get_type(file) for i in range(n)] + +def string (file): + """ + Returns a string. + + :param file: The file to read from + :type file: file + :rtype: str + """ + nelems = _number(file) + + s = file.read(nelems) + + if len(s) != nelems: + raise EndOfFileException, file.name + + return s + +# +# Complex Types +# + +cdef class overlay: + """ + Represents an overlay object. + + :IVariables: + + path : string + The path to the overlay + + label : string + The label/name of the overlay + """ + + cdef readonly object path + cdef readonly object label + + def __init__ (self, file): + """ + :param file: The file to read from + :type file: file + """ + + self.path = string(file) + self.label = string(file) + +cdef class header: + """ + Represents the header of the cache. + + :IVariables: + + version : int + The version of the cache file. + + ncats : int + The number of categories. + + overlays : `overlay` [] + The list of overlays. + + provide : string[] + A list of "PROVIDE" values. + + licenses : string[] + The list of licenses. + + keywords : string[] + The list of keywords. + + useflags : string[] + The list of useflags. + + slots : string[] + The list of slots different from "0". + + sets : string[] + The names of world sets are the names (without leading @) of the world sets stored in /var/lib/portage/world_sets. + If SAVE_WORLD=false, the list is empty. + """ + + cdef readonly object version + cdef readonly object ncats + cdef readonly object overlays + cdef readonly object provide + cdef readonly object licenses + cdef readonly object keywords + cdef readonly object useflags + cdef readonly object slots + cdef readonly object sets + + def __init__ (self, file): + """ + :param file: The file to read from + :type file: file + """ + self.version = _number(file) + self.ncats = _number(file) + self.overlays = vector(file, overlay) + self.provide = vector(file, string) + self.licenses = vector(file, string) + self.keywords = vector(file, string) + self.useflags = vector(file, string) + self.slots = vector(file, string) + self.sets = vector(file, string) + +cdef class package: + """ + The representation of one package. + + Currently, version information is not parsed and stored. + So you can gain general infos only. + + :IVariables: + + name : string + The name of the package. + + description : string + Description of the package. + + homepage : string + The homepage of the package. + + provide : int[] + The indices of `header.provide` representing the PROVIDE value of the package. + + license : int + The index of `header.licenses` representing the license of the package. + + useflags : int[] + The indices of `header.useflags` representing the IUSE value of the package. + """ + + cdef LLong _offset + cdef readonly object name + #cdef readonly object description + #cdef readonly object provide + #cdef readonly object homepage + #cdef readonly object license + #cdef readonly object useflags + + def __init__ (self, file): + """ + :param file: The file to read from + :type file: file + """ + cdef FILE* cfile = PyFile_AsFile(file) + cdef long after_offset + + self._offset = _number(file) + + after_offset = ftell(cfile) + + self.name = string(file) + + # skip the rest, as it is currently unneeded + #self.description = string(file) + #self.provide = vector(file, number) + #self.homepage = string(file) + #self.license = number(file) + #self.useflags = vector(file, number) + + # self.versions = LE(typed_vector(version)) + # for the moment just skip the versions + fseek(cfile, self._offset - (ftell(cfile) - after_offset), SEEK_CUR) + +cdef class category: + """ + Represents a whole category. + + :IVariables: + + name : string + The category name. + + packages : `package` [] + All the packages of the category. + """ + + cdef readonly object name + cdef readonly object packages + + def __init__ (self, file): + """ + :param file: The file to read from + :type file: file + """ + self.name = string(file) + self.packages = vector(file, package) diff --git a/portato/eix/py_parser.py b/portato/eix/py_parser.py new file mode 100644 index 0000000..cc42553 --- /dev/null +++ b/portato/eix/py_parser.py @@ -0,0 +1,416 @@ +# -*- coding: utf-8 -*- +# +# File: portato/eix/parser.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann <necoro@necoro.net> + +""" +The cache file supports different types of data. +In this module (nearly) all of these types have a corresponding function. + +For the exact way all the functions work, have a look at the eix format description. +""" + +from __future__ import absolute_import, with_statement +__docformat__ = "restructuredtext" + +import os +import struct +from functools import partial + +from ..helper import debug +from .exceptions import EndOfFileException + +# +# Helper +# + +def _get_bytes (file, length, expect_list = False): + """ + Return a number of bytes. + + :Parameters: + + file : file + The file to read from. + + length : int + The number of bytes to read. + + expect_list : bool + In case ``length`` is 1, only a single byte is returned. If ``expect_list`` is true, then a list is also returned in this case. + + :rtype: int or int[] + :raises EndOfFileException: if EOF is reached during execution + """ + + s = file.read(length) + + if len(s) != length: + raise EndOfFileException, file.name + + if length == 1 and not expect_list: + return ord(s) # is faster than unpack and we have a scalar + else: + return struct.unpack("%sB" % length, s) + +# +# Base Types +# + +def number (file, skip = False): + """ + Returns a number. + + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + + :rtype: int + """ + + n = _get_bytes(file, 1) + + if n < 0xFF: + value = n + else: + count = 0 + + while (n == 0xFF): + count += 1 + n = _get_bytes(file, 1) + + if n == 0: + n = 0xFF # 0xFF is encoded as 0xFF 0x00 + count -= 1 + + value = n << (count*8) + + if count > 0: + + if skip: + file.seek(count, os.SEEK_CUR) + return + + else: + rest = _get_bytes(file, count, expect_list = True) + + for i, r in enumerate(rest): + value += r << ((count - i - 1)*8) + + return value + +def vector (file, get_type, skip = False, nelems = None): + """ + Returns a vector of elements. + + :Parameters: + + file : file + The file to read from. + + get_type : function(file, bool) + The function determining type of the elements. + + skip : bool + Do not return the actual value, but just skip to the next datum. + + nelems : int + Normally the eix-Vector has the number of elements as the first argument. + If for some reason this is not the case, you can pass it in here. + + :rtype: list + """ + + if nelems is None: + nelems = number(file) + + if skip: + for i in range(nelems): + get_type(file, skip = True) + else: + return [get_type(file) for i in range(nelems)] + +def typed_vector(type, nelems = None): + """ + Shortcut to create a function for a special type of vector. + + :Parameters: + + type : function(file, bool) + The function determining type of the elements. + + nelems : int + Normally the eix-Vector has the number of elements as the first argument. + If for some reason this is not the case, you can pass it in here. + Do not return the actual value, but just skip to the next datum. + + :rtype: function(file, bool) + :see: `vector` + """ + + if nelems is None: + return partial(vector, get_type = type) + else: + return partial(vector, get_type = type, nelems = nelems) + +def string (file, skip = False): + """ + Returns a string. + + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + + :rtype: str + """ + nelems = number(file) + + if skip: + file.seek(nelems, os.SEEK_CUR) + return + else: + s = file.read(nelems) + + if len(s) != nelems: + raise EndOfFileException, file.name + + return s + +# +# Complex Types +# + +class LazyElement (object): + """ + This class models a value in the cache, which is only read on access. + + If not accessed directly, only the position inside the file is stored. + """ + __slots__ = ("file", "get_type", "_value", "pos") + + def __init__ (self, get_type, file): + """ + :Parameters: + + get_type : function(file, bool) + The function determining type of the elements. + + file : file + The file to read from. + """ + + self.file = file + self.get_type = get_type + self._value = None + + self.pos = file.tell() + get_type(file, skip=True) # skip it for the moment + + @property + def value (self): + """ + The value of the element. + """ + + if self._value is None: + old_pos = self.file.tell() + self.file.seek(self.pos, os.SEEK_SET) + self._value = self.get_type(self.file, skip = False) + self.file.seek(old_pos, os.SEEK_SET) + + return self._value + + def __call__ (self): + """ + Convenience function. Also returns the value. + """ + return self.value + +class overlay (object): + """ + Represents an overlay object. + + :IVariables: + + path : `LazyElement` <string> + The path to the overlay + + label : `LazyElement` <string> + The label/name of the overlay + """ + __slots__ = ("path", "label") + + def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ + + self.path = LazyElement(string, file) + self.label = LazyElement(string, file) + +class header (object): + """ + Represents the header of the cache. + + :IVariables: + + version : `LazyElement` <int> + The version of the cache file. + + ncats : `LazyElement` <int> + The number of categories. + + overlays : `LazyElement` <`overlay` []> + The list of overlays. + + provide : `LazyElement` <string[]> + A list of "PROVIDE" values. + + licenses : `LazyElement` <string[]> + The list of licenses. + + keywords : `LazyElement` <string[]> + The list of keywords. + + useflags : `LazyElement` <string[]> + The list of useflags. + + slots : `LazyElement` <string[]> + The list of slots different from "0". + + sets : `LazyElement` <string[]> + The names of world sets are the names (without leading @) of the world sets stored in /var/lib/portage/world_sets. + If SAVE_WORLD=false, the list is empty. + """ + __slots__ = ("version", "ncats", "overlays", "provide", + "licenses", "keywords", "useflags", "slots", "sets") + + def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ + def LE (t): + return LazyElement(t, file) + + self.version = LE(number) + self.ncats = LE(number) + self.overlays = LE(typed_vector(overlay)) + self.provide = LE(typed_vector(string)) + self.licenses = LE(typed_vector(string)) + self.keywords = LE(typed_vector(string)) + self.useflags = LE(typed_vector(string)) + self.slots = LE(typed_vector(string)) + self.sets = LE(typed_vector(string)) + +class package (object): + """ + The representation of one package. + + Currently, version information is not parsed and stored. + So you can gain general infos only. + + :IVariables: + + name : `LazyElement` <string> + The name of the package. + + description : `LazyElement` <string> + Description of the package. + + homepage : `LazyElement` <string> + The homepage of the package. + + provide : `LazyElement` <int[]> + The indices of `header.provide` representing the PROVIDE value of the package. + + license : `LazyElement` <int> + The index of `header.licenses` representing the license of the package. + + useflags : `LazyElement` <int[]> + The indices of `header.useflags` representing the IUSE value of the package. + """ + + __slots__ = ("_offset", "name", "description", "provide", + "homepage", "license", "useflags") + + def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ + def LE (t): + return LazyElement(t, file) + + self._offset = number(file) + + after_offset = file.tell() + + self.name = LE(string) + self.description = LE(string) + self.provide = LE(typed_vector(number)) + self.homepage = LE(string) + self.license = LE(number) + self.useflags = LE(typed_vector(number)) + + # self.versions = LE(typed_vector(version)) + # for the moment just skip the versions + file.seek(self._offset - (file.tell() - after_offset), os.SEEK_CUR) + +class category (object): + """ + Represents a whole category. + + :IVariables: + + name : `LazyElement` <string> + The category name. + + packages : `LazyElement` <`package` []> + All the packages of the category. + """ + __slots__ = ("name", "packages") + + def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ + self.name = LazyElement(string, file) + self.packages = LazyElement(typed_vector(package), file) diff --git a/portato/ipc.pxd b/portato/ipc.pxd new file mode 100644 index 0000000..64ca05d --- /dev/null +++ b/portato/ipc.pxd @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# +# File: portato/ipc.pxd +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann <necoro@necoro.net> + +from python_string cimport * +from python_mem cimport * + +cdef extern from "errno.h": + int errno + cdef enum: + EACCES, EEXIST, ENOENT, ENOMEM, ENOSPC, + EINVAL, EPERM, EIDRM, EINTR + +cdef extern from *: + int INT_MAX + int RAND_MAX + ctypedef size_t int + int rand() + +cdef extern from "string.h": + char* strerror(int errno) + void* memcpy (void* dst, void* src, size_t len) + +cdef extern from "sys/msg.h" nogil: + cdef enum: + IPC_CREAT, IPC_EXCL, IPC_NOWAIT, + IPC_RMID + + ctypedef int key_t + + struct msqid_ds: + pass + + int msgget(key_t key, int msgflg) + int msgctl(int msqid, int cmd, msqid_ds* buf) + int msgsnd(int msgid, void* msgp, size_t msgsz, int msgflg) + int msgrcv(int msgid, void* msgp, size_t msgsz, long msgtype, int msgflg) + +cdef struct msg_data: + long mtype + char mtext[1] + +cdef enum: + MAX_MESSAGE_SIZE = 2048 diff --git a/portato/ipc.pyx b/portato/ipc.pyx new file mode 100644 index 0000000..f3fb4af --- /dev/null +++ b/portato/ipc.pyx @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# +# File: portato/ipc.pyx +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann <necoro@necoro.net> + +class MessageQueueError(Exception): + """ + Base class for different queue errors. + """ + pass + +class MessageQueueRemovedError (MessageQueueError): + """ + This class is used iff the queue is already removed. + """ + pass + +cdef class MessageQueue (object): + """ + A simple interface to the SysV message queues. + """ + + cdef int msgid + cdef readonly key_t key + + def __init__ (self, key = None, create = False, exclusive = False): + """ + Create a new MessageQueue instance. Depending on the passed in flags, + different behavior occurs. See man msgget for the details. + + If key is None, a random key is created. + """ + + cdef int flags = 0600 # start mode + + if exclusive and not create: + raise ValueError("'exclusive' must be combined with 'create'.") + + if key is None and not exclusive: + raise ValueError("The key can only be None if 'exclusive' is set.") + + if create: + flags |= IPC_CREAT + + if exclusive: + flags |= IPC_EXCL + + if key is None: + check = True + while check: + self.key = self.random_key() + self.msgid = msgget(self.key, flags) + check = (self.msgid == -1 and errno == EEXIST) + else: + self.key = key + self.msgid = msgget(key, flags) + + if self.msgid == -1: + if errno == EACCES: + raise MessageQueueError("Permission denied.") + elif errno == EEXIST: + raise MessageQueueError("Queue already exists.") + elif errno == ENOENT: + raise MessageQueueError("Queue does not exist and 'create' is not set.") + elif errno == ENOMEM or errno == ENOSPC: + raise MemoryError("Insufficient ressources.") + else: + raise OSError(errno, strerror(errno)) + + def remove (self): + """ + Removes the message queue. + """ + cdef msqid_ds info + cdef int ret + + ret = msgctl(self.msgid, IPC_RMID, &info) + + if ret == -1: + if errno == EIDRM or errno == EINVAL: + raise MessageQueueRemovedError("Queue already removed.") + elif errno == EPERM: + raise MessageQueueError("Permission denied.") + else: + raise OSError(errno, strerror(errno)) + + def send (self, message, int type = 1): + """ + Sends a message with a specific type. + + The type must be larger zero. + Also note, that this is always blocking. + """ + cdef msg_data * msg + cdef int ret + cdef long size = len(message) + + if type <= 0: + raise ValueError("type must be > 0") + + if size >= MAX_MESSAGE_SIZE: + raise ValueError("Message must be smaller than %d" % MAX_MESSAGE_SIZE) + + msg = <msg_data*>PyMem_Malloc(sizeof(msg_data) + size) + + if msg is NULL: + raise MemoryError("Out of memory") + + memcpy(msg.mtext, <char*>message, size) + msg.mtype = type + + with nogil: + ret = msgsnd(self.msgid, msg, size, 0) + + try: + if ret == -1: + if errno == EIDRM or errno == EINVAL: + raise MessageQueueRemovedError("Queue was removed.") + elif errno == EINTR: + raise MessageQueueError("Signaled while waiting.") + elif errno == EACCES: + raise MessageQueueError("Permission denied.") + else: + raise OSError(errno, strerror(errno)) + finally: + PyMem_Free(msg) + + def receive (self): + """ + Receives a message from the queue and returns the (msg, type) pair. + + Note that this method is always blocking. + """ + cdef msg_data * msg + cdef int ret + cdef object retTuple + + msg = <msg_data*>PyMem_Malloc(sizeof(msg_data) + MAX_MESSAGE_SIZE) + + if msg is NULL: + raise MemoryError("Out of memory") + + msg.mtype = 0 + + with nogil: + ret = msgrcv(self.msgid, msg, <size_t>MAX_MESSAGE_SIZE, 0, 0) + + try: + if ret == -1: + if errno == EIDRM or errno == EINVAL: + raise MessageQueueRemovedError("Queue was removed.") + elif errno == EINTR: + raise MessageQueueError("Signaled while waiting.") + elif errno == EACCES: + raise MessageQueueError("Permission denied.") + else: + raise OSError(errno, strerror(errno)) + + retTuple = (PyString_FromStringAndSize(msg.mtext, ret), msg.mtype) + finally: + PyMem_Free(msg) + + return retTuple + + cdef inline key_t random_key (self): + return <int>(<double>rand() / (<double>RAND_MAX + 1) * INT_MAX) diff --git a/portato/listener.py b/portato/listener.py index 3d2dd53..c96b637 100644 --- a/portato/listener.py +++ b/portato/listener.py @@ -21,7 +21,8 @@ except ImportError: pynotify = None from .constants import APP -from .helper import debug, warning +from .helper import debug, warning, error +from . import ipc class Listener (object): """This class handles the communication between the "listener" and the GUI. @@ -32,22 +33,15 @@ class Listener (object): @ivar _send: sender socket @type _send: int""" - def set_recv (self, mem, sig, rw): - self._mem = mem - self._sig = sig - self._rw = rw + def set_recv (self, mq): + + self.mq = mq while True: try: - try: - self._sig.P() - self._rw.P() - len = self._mem.read(NumberOfBytes = 4) - string = self._mem.read(NumberOfBytes = int(len), offset = 4) - finally: - self._rw.V() - - data = string.split("\0") + msg, type = self.mq.receive() + + data = msg.split("\0") debug("Listener received: %s", data) if data[0] == "notify": @@ -59,15 +53,12 @@ class Listener (object): except KeyboardInterrupt: debug("Got KeyboardInterrupt. Aborting.") break + except ipc.MessageQueueRemovedError: + debug("MessageQueue removed. Aborting.") + break - self._mem.remove() - self._sig.remove() - self._rw.remove() - - self._mem = None - self._sig = None - self._rw = None - + self.mq = None + def do_cmd (self, cmdlist): """Starts a command as the user. @@ -89,30 +80,21 @@ class Listener (object): n.set_urgency(int(urgency)) n.show() - def set_send (self, mem = None, sig = None, rw = None): - if mem is None or sig is None or rw is None: + def set_send (self, mq = None): + if mq is None: warning(_("Listener has not been started.")) - self._mem = None - self._sig = None - self._rw = None + self.mq = None else: - import shm_wrapper as shm - - self._mem = shm.SharedMemoryHandle(mem) - self._sig = shm.SemaphoreHandle(sig) - self._rw = shm.SemaphoreHandle(rw) + self.mq = ipc.MessageQueue(mq) def __send (self, string): - self._rw.P() - self._sig.Z() try: - self._mem.write("%4d%s" % (len(string), string)) - self._sig.V() - finally: - self._rw.V() + self.mq.send(string) + except ipc.MessageQueueError, e: + error(_("An exception occured while accessing the message queue: %s"), e) def send_notify (self, base = "", descr = "", icon = "", urgency = None): - if self._sig is None: + if self.mq is None: self.do_notify(base, descr, icon, urgency) else: string = "\0".join(["notify", base, descr, icon]) @@ -125,11 +107,11 @@ class Listener (object): self.__send(string) def send_cmd (self, cmdlist): - if self._sig is None: + if self.mq is None: self.do_cmd(cmdlist) else: self.__send("\0".join(["cmd"] +cmdlist)) def close (self): - if self._sig is not None: + if self.mq is not None: self.__send("close") diff --git a/portato/plugin.py b/portato/plugin.py index 0da14f1..052dcd0 100644 --- a/portato/plugin.py +++ b/portato/plugin.py @@ -23,6 +23,7 @@ import traceback from collections import defaultdict from functools import wraps +from . import helper from .helper import debug, warning, info, error from .constants import PLUGIN_DIR from .backend import system @@ -300,6 +301,11 @@ class Plugin (object): """ self.__calls.append(Call(self, hook, callable, type, dep)) + def __str__ (self): + return self.name + + __repr__ = __str__ + class WidgetPlugin (Plugin): def __init__ (self, *args, **kwargs): @@ -430,25 +436,34 @@ class PluginQueue (object): plugin_module.__builtins__["Plugin"] = Plugin plugin_module.__builtins__["WidgetPlugin"] = WidgetPlugin plugin_module.__builtins__["register"] = register + plugin_module.__builtins__["helper"] = helper + plugin_module.__builtins__["PluginLoadException"] = PluginLoadException for p in plugins: # import them try: exec "from portato.plugins import %s" % p in {} except PluginLoadException, e: - error(_("Loading plugin '%(plugin)s' failed: %(error)s"), {"plugin" : p, "error" : e.message}) + error(_("Loading plugin module '%(plugin)s' failed: %(error)s"), {"plugin" : p, "error" : e}) except: tb = traceback.format_exc() - error(_("Loading plugin '%(plugin)s' failed: %(error)s"), {"plugin" : p, "error" : tb}) + error(_("Loading plugin module '%(plugin)s' failed: %(error)s"), {"plugin" : p, "error" : tb}) self._organize() def load_widgets(self, window): for p in self.plugins: if isinstance(p, WidgetPlugin): - p._widget_init(window) - for w in p.widgets: - WidgetSlot.slots[w.slot].add_widget(w) - info(_("Widgets of plugin '%s' loaded."), p.name) + try: + p._widget_init(window) + except PluginLoadException, e: + error(_("Loading widgets plugin '%(plugin)s' failed: %(error)s"), {"plugin" : p, "error" : e}) + except: + tb = traceback.format_exc() + error(_("Loading widgets of plugin '%(plugin)s' failed: %(error)s"), {"plugin" : p, "error" : tb}) + else: + for w in p.widgets: + WidgetSlot.slots[w.slot].add_widget(w) + info(_("Widgets of plugin '%s' loaded."), p.name) def add (self, plugin, disable = False): """ @@ -683,4 +698,10 @@ def register (plugin, disable = False): :see: `PluginQueue.add` """ if __plugins is not None: - __plugins.add(plugin, disable) + try: + __plugins.add(plugin, disable) + except PluginLoadException, e: + error(_("Registrating plugin '%(plugin)s' failed: %(error)s"), {"plugin" : plugin, "error" : e}) + except: + tb = traceback.format_exc() + error(_("Registrating plugin '%(plugin)s' failed: %(error)s"), {"plugin" : plugin, "error" : tb}) diff --git a/portato/waiting_queue.py b/portato/waiting_queue.py index d946abe..33791a3 100644 --- a/portato/waiting_queue.py +++ b/portato/waiting_queue.py @@ -36,10 +36,9 @@ class WaitingQueue (Queue): def put (self, method, *args, **kwargs): self.counter += 1; - if "caller" in kwargs: - name = "Waiting Thread #%d (called by:%s)" % (self.counter, kwargs["caller"]) - del kwargs["caller"] - else: + try: + name = "Waiting Thread #%d (called by:%s)" % (self.counter, kwargs.pop("caller")) + except KeyError: name = "Waiting Thread #%d" % self.counter t = self.threadClass(name = name, target = method, args = args, kwargs = kwargs) |