From b8f45f026ace3df864efac5d49a62be4a0fc9eae Mon Sep 17 00:00:00 2001 From: René 'Necoro' Neumann Date: Fri, 24 Jul 2009 21:30:40 +0200 Subject: First eix stuff --- portato/eix/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 portato/eix/__init__.py (limited to 'portato/eix/__init__.py') diff --git a/portato/eix/__init__.py b/portato/eix/__init__.py new file mode 100644 index 0000000..e69de29 -- cgit v1.2.3-70-g09d2 From 40344af86dbe9c7ad2f38b18f31f0f8a4917bda9 Mon Sep 17 00:00:00 2001 From: René 'Necoro' Neumann Date: Mon, 27 Jul 2009 00:38:41 +0200 Subject: Moved from Cython to normal Python --- portato/eix.py | 113 ++++++++++++++++++++++++++++++++++++++++++++++ portato/eix/__init__.py | 0 portato/eix/eix_utils.pxd | 17 ------- portato/eix/eix_utils.pyx | 15 ------ portato/eix/libeix.pyx | 102 ----------------------------------------- setup.py | 10 +--- 6 files changed, 115 insertions(+), 142 deletions(-) create mode 100644 portato/eix.py delete mode 100644 portato/eix/__init__.py delete mode 100644 portato/eix/eix_utils.pxd delete mode 100644 portato/eix/eix_utils.pyx delete mode 100644 portato/eix/libeix.pyx (limited to 'portato/eix/__init__.py') diff --git a/portato/eix.py b/portato/eix.py new file mode 100644 index 0000000..927870e --- /dev/null +++ b/portato/eix.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# +# File: portato/eix.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann + +from __future__ import absolute_import, with_statement + +import struct +from functools import wraps + +from .helper import debug + +class EixError (Exception): + message = "Unknown error." + + def __str__ (self): + return self.message + +class EndOfFileException (EixError): + + def __init__ (self, filename): + self.message = "End of file reached though it was not expected: '%s'" % filename + +class EixReaderClosed (EixError): + message = "EixReader is already closed." + +class UnsupportedVersionError (EixError): + + def __init__ (self, version): + self.message = "Version '%s' is not supported." % version + +class EixReader (object): + supported_versions = (28, ) + + def __init__ (self, filename): + self.filename = filename + self.file = open(filename, "r") + self.closed = 0 + + try: + self.version = self.get_number() + + if self.version not in self.supported_versions: + raise UnsupportedVersionError(self.version) + + debug("Started EixReader for version %s.", self.version) + except: + self.close() + raise + + def check_closed (f): + @wraps(f) + def wrapper (self, *args, **kwargs): + if self.closed: + raise EixReaderClosed + + return f(self, *args, **kwargs) + return wrapper + + @check_closed + def get_number (self): + n = self._get_bytes(1) + + if n < 0xFF: + value = n + else: + count = 0 + + while (n == 0xFF): + count += 1 + n = self._get_bytes(1) + + if n == 0: + n = 0xFF # 0xFF is encoded as 0xFF 0x00 + count -= 1 + + value = n << (count*8) + + if count > 0: + rest = self._get_bytes(count, expect_list = True) + + for i, r in enumerate(rest): + value += r << ((count - i - 1)*8) + + return value + + def _get_bytes (self, length, expect_list = False): + s = self.file.read(length) + + if len(s) != length: + raise EndOfFileException, self.filename + + if length == 1 and not expect_list: + return ord(s) # is faster than unpack and we have a scalar + else: + return struct.unpack("%sB" % length, s) + + @check_closed + def close (self): + if self.closed: + raise EixReaderClosed + + self.file.close() + self.closed = 1 + + debug("EixReader closed.") diff --git a/portato/eix/__init__.py b/portato/eix/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/portato/eix/eix_utils.pxd b/portato/eix/eix_utils.pxd deleted file mode 100644 index 39f60a9..0000000 --- a/portato/eix/eix_utils.pxd +++ /dev/null @@ -1,17 +0,0 @@ -cdef extern from *: - ctypedef int size_t - -cdef extern from "errno.h": - int errno - -cdef extern from "string.h": - char* strerror (int errno) - size_t strlen (char* s) - char* strcpy (char* dest, char* src) - -cdef extern from "Python.h": - void* PyMem_Malloc (size_t n) - void PyMem_Free (void* p) - -cdef: - char* strdup (char* other) except NULL diff --git a/portato/eix/eix_utils.pyx b/portato/eix/eix_utils.pyx deleted file mode 100644 index 0ea9603..0000000 --- a/portato/eix/eix_utils.pyx +++ /dev/null @@ -1,15 +0,0 @@ -cdef char* strdup (char * other) except NULL: - cdef size_t len - cdef char* new - - if other is NULL: - return NULL - - len = strlen(other) - new = PyMem_Malloc(len+1) - - if new is NULL: - raise MemoryError, "Malloc of new string copy" - return NULL - - return strcpy(new, other) diff --git a/portato/eix/libeix.pyx b/portato/eix/libeix.pyx deleted file mode 100644 index 2a1c7df..0000000 --- a/portato/eix/libeix.pyx +++ /dev/null @@ -1,102 +0,0 @@ -class EixError (Exception): - message = "Unknown error." - - def __str__ (self): - return self.message - -class EndOfFileException (EixError): - - def __init__ (self, filename): - self.message = "End of file reached though it was not expected: '%s'" % filename - -class EixReaderClosed (EixError): - message = "EixReader is already closed." - -class UnsupportedVersionError (EixError): - - def __init__ (self, version): - self.message = "Version '%s' is not supported." % version - -cdef class EixReader (object): - cdef object file - cdef char closed - - cdef readonly object filename - cdef readonly object version - - supported_versions = (28, ) - - def __init__ (self, filename): - self.filename = filename - self.file = open(filename, "r") - self.closed = 0 - - self.version = self._get_number() - - if self.version not in self.supported_versions: - raise UnsupportedVersionError(self.version) - - cdef unsigned long _get_number (self) except *: - cdef unsigned char n - cdef short count, i - cdef unsigned long value - cdef char* rest - cdef object orest - - n = self._get_one_byte() - - if n < 0xFF: - value = n - else: - count = 0 - - while (n == 0xFF): - count += 1 - n = self._get_one_byte() - - if n == 0: - n = 0xFF # 0xFF is encoded as 0xFF 0x00 - count -= 1 - - value = n << (count*8) - - if count > 0: - orest = self.get_string(count) - rest = orest # cast to char* - - for 0 <= i < count: - value += (rest[i]) << ((count - i - 1)*8) - - return value - - cdef unsigned char _get_one_byte (self) except? 0: - s = self.file.read(1) - - if len(s) != 1: - raise EndOfFileException, self.filename - - return ord(s) - - cdef object _get_string (self, length): - if self.closed: - raise EixReaderClosed - - s = self.file.read(length) - - if len(s) != length: - raise EndOfFileException, self.filename - - return s - - def get_number (self): - if self.closed: - raise EixReaderClosed - - return self._get_number() - - def close (self): - if self.closed: - raise EixReaderClosed - - self.file.close() - self.closed = 1 diff --git a/setup.py b/setup.py index 7276b92..f69d373 100644 --- a/setup.py +++ b/setup.py @@ -14,9 +14,6 @@ import os from distutils.core import setup -from Cython.Distutils.extension import Extension -from Cython.Distutils import build_ext - from portato.constants import VERSION, ICON_DIR, PLUGIN_DIR, TEMPLATE_DIR, APP from build_manpage import build_manpage @@ -25,14 +22,12 @@ def plugin_list (*args): """Creates a list of correct plugin pathes out of the arguments.""" return [("plugins/%s.py" % x) for x in args] -packages = ["portato", "portato.db", "portato.eix", "portato.gui", "portato.gui.windows", "portato.plugins", "portato.backend", "portato.backend.portage"] +packages = ["portato", "portato.db", "portato.gui", "portato.gui.windows", "portato.plugins", "portato.backend", "portato.backend.portage"] data_files = [ (TEMPLATE_DIR, [os.path.join("portato/gui/templates",x) for x in os.listdir("portato/gui/templates") if x.endswith(".ui")]), (ICON_DIR, ["icons/portato-icon.png"]), (PLUGIN_DIR, plugin_list("gpytage", "notify", "etc_proposals", "reload_portage", "package_details"))] -libeix = Extension("portato.eix.libeix", ["portato/eix/libeix.pyx"]) - # do the distutils setup setup(name=APP, version = VERSION, @@ -44,7 +39,6 @@ setup(name=APP, author = "René 'Necoro' Neumann", author_email = "necoro@necoro.net", packages = packages, - ext_modules = [libeix], data_files = data_files, - cmdclass={'build_manpage': build_manpage, 'build_ext' : build_ext} + cmdclass={'build_manpage': build_manpage} ) -- cgit v1.2.3-70-g09d2 From 3f70553a078aafbb029782ec092c8d037d9ea45f Mon Sep 17 00:00:00 2001 From: René 'Necoro' Neumann Date: Mon, 27 Jul 2009 23:25:21 +0200 Subject: Moved eix stuff to its own module. Also make all the parsing stuff being functions -- will be easier to deal with for the LazyElements --- portato/eix.py | 146 ---------------------------------------------- portato/eix/__init__.py | 40 +++++++++++++ portato/eix/exceptions.py | 29 +++++++++ portato/eix/parser.py | 86 +++++++++++++++++++++++++++ 4 files changed, 155 insertions(+), 146 deletions(-) delete mode 100644 portato/eix.py create mode 100644 portato/eix/__init__.py create mode 100644 portato/eix/exceptions.py create mode 100644 portato/eix/parser.py (limited to 'portato/eix/__init__.py') diff --git a/portato/eix.py b/portato/eix.py deleted file mode 100644 index 5262bf2..0000000 --- a/portato/eix.py +++ /dev/null @@ -1,146 +0,0 @@ -# -*- coding: utf-8 -*- -# -# File: portato/eix.py -# This file is part of the Portato-Project, a graphical portage-frontend. -# -# Copyright (C) 2006-2009 René 'Necoro' Neumann -# This is free software. You may redistribute copies of it under the terms of -# the GNU General Public License version 2. -# There is NO WARRANTY, to the extent permitted by law. -# -# Written by René 'Necoro' Neumann - -from __future__ import absolute_import, with_statement - -import os -import struct -from functools import wraps - -from .helper import debug - -class EixError (Exception): - message = "Unknown error." - - def __str__ (self): - return self.message - -class EndOfFileException (EixError): - - def __init__ (self, filename): - self.message = "End of file reached though it was not expected: '%s'" % filename - -class EixReaderClosed (EixError): - message = "EixReader is already closed." - -class UnsupportedVersionError (EixError): - - def __init__ (self, version): - self.message = "Version '%s' is not supported." % version - -class EixReader (object): - supported_versions = (28, ) - - def __init__ (self, filename): - self.filename = filename - self.file = open(filename, "r") - self.closed = 0 - - try: - self.version = self.number() - - if self.version not in self.supported_versions: - raise UnsupportedVersionError(self.version) - - debug("Started EixReader for version %s.", self.version) - except: - self.close() - raise - - def check_closed (f): - @wraps(f) - def wrapper (self, *args, **kwargs): - if self.closed: - raise EixReaderClosed - - return f(self, *args, **kwargs) - return wrapper - - @check_closed - def number (self): - n = self._get_bytes(1) - - if n < 0xFF: - value = n - else: - count = 0 - - while (n == 0xFF): - count += 1 - n = self._get_bytes(1) - - if n == 0: - n = 0xFF # 0xFF is encoded as 0xFF 0x00 - count -= 1 - - value = n << (count*8) - - if count > 0: - rest = self._get_bytes(count, expect_list = True) - - for i, r in enumerate(rest): - value += r << ((count - i - 1)*8) - - return value - - @check_closed - def vector (self, get_type, skip = False): - nelems = self.number() - - if skip: - for i in range(nelems): - get_type(skip = True) - else: - return (get_type() for i in range(nelems)) - - @check_closed - def string (self, skip = False): - nelems = self.number() - - if skip: - self.file.seek(nelems, os.SEEK_CUR) - else: - s = self.file.read(nelems) - - if len(s) != nelems: - raise EndOfFileException, self.filename - - return s - - @check_closed - def overlay (self, skip = False): - if skip: - self.file.seek(self.number(), os.SEEK_CUR) # path - self.file.seek(self.number(), os.SEEK_CUR) # label - else: - return (self.string(), self.string()) - - def _get_bytes (self, length, expect_list = False): - s = self.file.read(length) - - if len(s) != length: - raise EndOfFileException, self.filename - - if length == 1 and not expect_list: - return ord(s) # is faster than unpack and we have a scalar - else: - return struct.unpack("%sB" % length, s) - - @check_closed - def close (self): - if self.closed: - raise EixReaderClosed - - self.file.close() - self.closed = 1 - - debug("EixReader closed.") diff --git a/portato/eix/__init__.py b/portato/eix/__init__.py new file mode 100644 index 0000000..3c1958e --- /dev/null +++ b/portato/eix/__init__.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# File: portato/eix/__init__.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann + +from __future__ import absolute_import, with_statement + +from . import parser +from . import exceptions as ex + +from ..helper import debug + +class EixReader(object): + supported_version = (28,) + + def __init__ (self, filename): + self.filename = filename + self.file = open(filename, "r") + + try: + self.version = parser.number(self.file) + + if self.version not in self.supported_versions: + raise ex.UnsupportedVersionError(self.version) + + debug("Started EixReader for version %s.", self.version) + except: + self.close() + raise + + def close (self): + self.file.close() + debug("EixReader closed.") diff --git a/portato/eix/exceptions.py b/portato/eix/exceptions.py new file mode 100644 index 0000000..fd72dcf --- /dev/null +++ b/portato/eix/exceptions.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# +# File: portato/eix/exceptions.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann + +from __future__ import absolute_import, with_statement + +class EixError (Exception): + message = _("Unknown error.") + + def __str__ (self): + return self.message + +class EndOfFileException (EixError): + + def __init__ (self, filename): + self.message = _("End of file reached though it was not expected: '%s'") % filename + +class UnsupportedVersionError (EixError): + + def __init__ (self, version): + self.message = _("Version '%s' is not supported.") % version diff --git a/portato/eix/parser.py b/portato/eix/parser.py new file mode 100644 index 0000000..e89bffe --- /dev/null +++ b/portato/eix/parser.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# +# File: portato/eix/parser.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann + +from __future__ import absolute_import, with_statement + +import os +import struct + +from .helper import debug + +from . import exceptions as ex + +def _get_bytes (file, length, expect_list = False): + s = file.read(length) + + if len(s) != length: + raise EndOfFileException, file.name + + if length == 1 and not expect_list: + return ord(s) # is faster than unpack and we have a scalar + else: + return struct.unpack("%sB" % length, s) + +def number (file): + n = _get_bytes(file, 1) + + if n < 0xFF: + value = n + else: + count = 0 + + while (n == 0xFF): + count += 1 + n = _get_bytes(file, 1) + + if n == 0: + n = 0xFF # 0xFF is encoded as 0xFF 0x00 + count -= 1 + + value = n << (count*8) + + if count > 0: + rest = _get_bytes(file, count, expect_list = True) + + for i, r in enumerate(rest): + value += r << ((count - i - 1)*8) + + return value + +def vector (file, get_type, skip = False): + nelems = number(file) + + if skip: + for i in range(nelems): + get_type(file, skip = True) + else: + return (get_type(file) for i in range(nelems)) + +def string (file, skip = False): + nelems = number(file) + + if skip: + file.seek(nelems, os.SEEK_CUR) + else: + s = file.read(nelems) + + if len(s) != nelems: + raise EndOfFileException, file.name + + return s + +def overlay (file, skip = False): + if skip: + string(file, skip = True) # path + string(file, skip = True) # label + else: + return (string(file), string(file)) -- cgit v1.2.3-70-g09d2 From 0e6702e069aeabcd995b8968d0ebbc1593d59bf6 Mon Sep 17 00:00:00 2001 From: René 'Necoro' Neumann Date: Fri, 14 Aug 2009 15:54:22 +0200 Subject: Add header-object --- portato/eix/__init__.py | 7 ++++++- portato/eix/parser.py | 44 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 10 deletions(-) (limited to 'portato/eix/__init__.py') diff --git a/portato/eix/__init__.py b/portato/eix/__init__.py index 3c1958e..6768ca7 100644 --- a/portato/eix/__init__.py +++ b/portato/eix/__init__.py @@ -18,7 +18,7 @@ from . import exceptions as ex from ..helper import debug class EixReader(object): - supported_version = (28,) + supported_versions = (28,) def __init__ (self, filename): self.filename = filename @@ -31,10 +31,15 @@ class EixReader(object): raise ex.UnsupportedVersionError(self.version) debug("Started EixReader for version %s.", self.version) + + self.file.seek(0) except: self.close() raise + def header (self): + return parser.header(self.file) + def close (self): self.file.close() debug("EixReader closed.") diff --git a/portato/eix/parser.py b/portato/eix/parser.py index 676cdd5..90c569b 100644 --- a/portato/eix/parser.py +++ b/portato/eix/parser.py @@ -15,7 +15,8 @@ from __future__ import absolute_import, with_statement import os import struct -from .helper import debug +from ..helper import debug +from functools import partial from . import exceptions as ex @@ -30,7 +31,7 @@ def _get_bytes (file, length, expect_list = False): else: return struct.unpack("%sB" % length, s) -def number (file): +def number (file, skip = False): n = _get_bytes(file, 1) if n < 0xFF: @@ -49,10 +50,16 @@ def number (file): value = n << (count*8) if count > 0: - rest = _get_bytes(file, count, expect_list = True) - for i, r in enumerate(rest): - value += r << ((count - i - 1)*8) + if skip: + file.seek(count, os.SEEK_CUR) + return + + else: + rest = _get_bytes(file, count, expect_list = True) + + for i, r in enumerate(rest): + value += r << ((count - i - 1)*8) return value @@ -65,11 +72,15 @@ def vector (file, get_type, skip = False): else: return [get_type(file) for i in range(nelems)] +def typed_vector(type): + return partial(vector, get_type = type) + def string (file, skip = False): nelems = number(file) if skip: file.seek(nelems, os.SEEK_CUR) + return else: s = file.read(nelems) @@ -92,17 +103,32 @@ class LazyElement (object): self._value = None self.pos = file.tell() - get_type(skip=True) # skip it for the moment + get_type(file, skip=True) # skip it for the moment @property def value (self): if self._value is None: old_pos = self.file.tell() - self.file.seek(self.pos) - self._value = self.get_type(skip = False) - self.file.seek(old_pos) + self.file.seek(self.pos, os.SEEK_SET) + self._value = self.get_type(self.file, skip = False) + self.file.seek(old_pos, os.SEEK_SET) return self._value def __call__ (self): return self.value + +class header (object): + def __init__ (self, file): + def LE (t): + return LazyElement(t, file) + + self.version = LE(number) + self.ncats = LE(number) + self.overlays = LE(typed_vector(overlay)) + self.provide = LE(typed_vector(string)) + self.licenses = LE(typed_vector(string)) + self.keywords = LE(typed_vector(string)) + self.useflags = LE(typed_vector(string)) + self.slots = LE(typed_vector(string)) + self.sets = LE(typed_vector(string)) -- cgit v1.2.3-70-g09d2 From d2afc4afe4d16b3ecd475d62c1baa86d385667a9 Mon Sep 17 00:00:00 2001 From: René 'Necoro' Neumann Date: Fri, 14 Aug 2009 17:30:43 +0200 Subject: Finish EixReader --- portato/eix/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'portato/eix/__init__.py') diff --git a/portato/eix/__init__.py b/portato/eix/__init__.py index 6768ca7..977c9e2 100644 --- a/portato/eix/__init__.py +++ b/portato/eix/__init__.py @@ -13,7 +13,7 @@ from __future__ import absolute_import, with_statement from . import parser -from . import exceptions as ex +from .exceptions import UnsupportedVersionError from ..helper import debug @@ -25,21 +25,21 @@ class EixReader(object): self.file = open(filename, "r") try: - self.version = parser.number(self.file) + version = parser.number(self.file) - if self.version not in self.supported_versions: - raise ex.UnsupportedVersionError(self.version) + if version not in self.supported_versions: + raise UnsupportedVersionError(self.version) - debug("Started EixReader for version %s.", self.version) + debug("Started EixReader for version %s.", version) self.file.seek(0) + + self.header = parser.header(self.file) + self.categories = parser.vector(self.file, parser.category, nelems = self.header.ncats()) except: self.close() raise - def header (self): - return parser.header(self.file) - def close (self): self.file.close() debug("EixReader closed.") -- cgit v1.2.3-70-g09d2 From 82f6e37bdd112d223580abdbe788442a6bb22666 Mon Sep 17 00:00:00 2001 From: René 'Necoro' Neumann Date: Fri, 14 Aug 2009 18:53:17 +0200 Subject: Add documentation --- epydoc.conf | 1 - portato/eix/__init__.py | 37 ++++++- portato/eix/exceptions.py | 16 +++ portato/eix/parser.py | 252 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 300 insertions(+), 6 deletions(-) (limited to 'portato/eix/__init__.py') diff --git a/epydoc.conf b/epydoc.conf index 62b6c73..0e81aee 100644 --- a/epydoc.conf +++ b/epydoc.conf @@ -27,7 +27,6 @@ sourcecode: yes include-log: no frames: yes css: white -redundant-details: yes # # Project diff --git a/portato/eix/__init__.py b/portato/eix/__init__.py index 977c9e2..aed5855 100644 --- a/portato/eix/__init__.py +++ b/portato/eix/__init__.py @@ -10,7 +10,12 @@ # # Written by René 'Necoro' Neumann +""" +A module to parse the eix-cache files. +""" + from __future__ import absolute_import, with_statement +__docformat__ = "restructuredtext" from . import parser from .exceptions import UnsupportedVersionError @@ -18,10 +23,37 @@ from .exceptions import UnsupportedVersionError from ..helper import debug class EixReader(object): + """ + The main class to use to have access to the eix-cache. + + Note that the file used internally stays open during the whole operation. + So please call `close()` when you are finished. + + :CVariables: + + supported_versions : int[] + The list of versions of the eix-cache, which are supported by this reader. + + :IVariables: + + file : file + The eix cache file. + + header : `parser.header` + The header of the eix cache. + + categories : `parser.category` [] + The list of categories. + """ + supported_versions = (28,) def __init__ (self, filename): - self.filename = filename + """ + :param filename: Path to the cache file + :type filename: string + """ + self.file = open(filename, "r") try: @@ -41,5 +73,8 @@ class EixReader(object): raise def close (self): + """ + Closes the cache file. + """ self.file.close() debug("EixReader closed.") diff --git a/portato/eix/exceptions.py b/portato/eix/exceptions.py index fd72dcf..8145af4 100644 --- a/portato/eix/exceptions.py +++ b/portato/eix/exceptions.py @@ -10,20 +10,36 @@ # # Written by René 'Necoro' Neumann +""" +Different exceptions used in the eix module. +""" + from __future__ import absolute_import, with_statement +__docformat__ = "restructuredtext" class EixError (Exception): + """ + The base class for all exceptions of this module. + + :ivar message: The error message + """ message = _("Unknown error.") def __str__ (self): return self.message class EndOfFileException (EixError): + """ + Denotes the unexpected EOF. + """ def __init__ (self, filename): self.message = _("End of file reached though it was not expected: '%s'") % filename class UnsupportedVersionError (EixError): + """ + The version of the cache file found is not supported. + """ def __init__ (self, version): self.message = _("Version '%s' is not supported.") % version diff --git a/portato/eix/parser.py b/portato/eix/parser.py index 6e12973..2a6658b 100644 --- a/portato/eix/parser.py +++ b/portato/eix/parser.py @@ -10,7 +10,15 @@ # # Written by René 'Necoro' Neumann +""" +The cache file supports different types of data. +In this module (nearly) all of these types have a corresponding function. + +For the exact way all the functions work, have a look at the eix format description. +""" + from __future__ import absolute_import, with_statement +__docformat__ = "restructuredtext" import os import struct @@ -19,7 +27,29 @@ from functools import partial from ..helper import debug from .exceptions import EndOfFileException +# +# Helper +# + def _get_bytes (file, length, expect_list = False): + """ + Return a number of bytes. + + :Parameters: + + file : file + The file to read from. + + length : int + The number of bytes to read. + + expect_list : bool + In case ``length`` is 1, only a single byte is returned. If ``expect_list`` is true, then a list is also returned in this case. + + :rtype: int or int[] + :raises EndOfFileException: if EOF is reached during execution + """ + s = file.read(length) if len(s) != length: @@ -30,7 +60,25 @@ def _get_bytes (file, length, expect_list = False): else: return struct.unpack("%sB" % length, s) +# +# Base Types +# + def number (file, skip = False): + """ + Returns a number. + + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + + :rtype: int + """ + n = _get_bytes(file, 1) if n < 0xFF: @@ -63,6 +111,27 @@ def number (file, skip = False): return value def vector (file, get_type, skip = False, nelems = None): + """ + Returns a vector of elements. + + :Parameters: + + file : file + The file to read from. + + get_type : function(file, bool) + The function determining type of the elements. + + skip : bool + Do not return the actual value, but just skip to the next datum. + + nelems : int + Normally the eix-Vector has the number of elements as the first argument. + If for some reason this is not the case, you can pass it in here. + + :rtype: list + """ + if nelems is None: nelems = number(file) @@ -73,12 +142,42 @@ def vector (file, get_type, skip = False, nelems = None): return [get_type(file) for i in range(nelems)] def typed_vector(type, nelems = None): + """ + Shortcut to create a function for a special type of vector. + + :Parameters: + + type : function(file, bool) + The function determining type of the elements. + + nelems : int + Normally the eix-Vector has the number of elements as the first argument. + If for some reason this is not the case, you can pass it in here. + Do not return the actual value, but just skip to the next datum. + + :rtype: function(file, bool) + :see: `vector` + """ + if nelems is None: return partial(vector, get_type = type) else: return partial(vector, get_type = type, nelems = nelems) def string (file, skip = False): + """ + Returns a string. + + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + + :rtype: str + """ nelems = number(file) if skip: @@ -92,10 +191,29 @@ def string (file, skip = False): return s +# +# Complex Types +# + class LazyElement (object): + """ + This class models a value in the cache, which is only read on access. + + If not accessed directly, only the position inside the file is stored. + """ __slots__ = ("file", "get_type", "_value", "pos") def __init__ (self, get_type, file): + """ + :Parameters: + + get_type : function(file, bool) + The function determining type of the elements. + + file : file + The file to read from. + """ + self.file = file self.get_type = get_type self._value = None @@ -105,6 +223,10 @@ class LazyElement (object): @property def value (self): + """ + The value of the element. + """ + if self._value is None: old_pos = self.file.tell() self.file.seek(self.pos, os.SEEK_SET) @@ -114,20 +236,86 @@ class LazyElement (object): return self._value def __call__ (self): + """ + Convenience function. Also returns the value. + """ return self.value class overlay (object): + """ + Represents an overlay object. + + :IVariables: + + path : `LazyElement` + The path to the overlay + + label : `LazyElement` + The label/name of the overlay + """ __slots__ = ("path", "label") def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ + self.path = LazyElement(string, file) self.label = LazyElement(string, file) class header (object): + """ + Represents the header of the cache. + + :IVariables: + + version : `LazyElement` + The version of the cache file. + + ncats : `LazyElement` + The number of categories. + + overlays : `LazyElement` <`overlay` []> + The list of overlays. + + provide : `LazyElement` + A list of "PROVIDE" values. + + licenses : `LazyElement` + The list of licenses. + + keywords : `LazyElement` + The list of keywords. + + useflags : `LazyElement` + The list of useflags. + + slots : `LazyElement` + The list of slots different from "0". + + sets : `LazyElement` + The names of world sets are the names (without leading @) of the world sets stored in /var/lib/portage/world_sets. + If SAVE_WORLD=false, the list is empty. + """ __slots__ = ("version", "ncats", "overlays", "provide", "licenses", "keywords", "useflags", "slots", "sets") def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ def LE (t): return LazyElement(t, file) @@ -142,14 +330,50 @@ class header (object): self.sets = LE(typed_vector(string)) class package (object): - __slots__ = ("offset","name", "description", - "provide", "homepage", "license", "useflags") + """ + The representation of one package. + + Currently, version information is not parsed and stored. + So you can gain general infos only. + + :IVariables: + + name : `LazyElement` + The name of the package. + + description : `LazyElement` + Description of the package. + + homepage : `LazyElement` + The homepage of the package. + + provide : `LazyElement` + The indices of `header.provide` representing the PROVIDE value of the package. + + license : `LazyElement` + The index of `header.licenses` representing the license of the package. + + useflags : `LazyElement` + The indices of `header.useflags` representing the IUSE value of the package. + """ + + __slots__ = ("name", "description", "provide", + "homepage", "license", "useflags") def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ def LE (t): return LazyElement(t, file) - self.offset = number(file) + self._offset = number(file) after_offset = file.tell() @@ -162,11 +386,31 @@ class package (object): # self.versions = LE(typed_vector(version)) # for the moment just skip the versions - file.seek(self.offset - (file.tell() - after_offset), os.SEEK_CUR) + file.seek(self._offset - (file.tell() - after_offset), os.SEEK_CUR) class category (object): + """ + Represents a whole category. + + :IVariables: + + name : `LazyElement` + The category name. + + packages : `LazyElement` <`package` []> + All the packages of the category. + """ __slots__ = ("name", "packages") def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ self.name = LazyElement(string, file) self.packages = LazyElement(typed_vector(package), file) -- cgit v1.2.3-70-g09d2 From f1eee621cf0f4991fa04902f9b447de73cb41dd3 Mon Sep 17 00:00:00 2001 From: René 'Necoro' Neumann Date: Fri, 14 Aug 2009 19:22:17 +0200 Subject: Make the EixReader support the context manager protocol --- portato/eix/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'portato/eix/__init__.py') diff --git a/portato/eix/__init__.py b/portato/eix/__init__.py index aed5855..effac2d 100644 --- a/portato/eix/__init__.py +++ b/portato/eix/__init__.py @@ -29,6 +29,8 @@ class EixReader(object): Note that the file used internally stays open during the whole operation. So please call `close()` when you are finished. + The ``EixReader`` supports the context manager protocol, so you can the ``with ... as ...``. + :CVariables: supported_versions : int[] @@ -72,6 +74,13 @@ class EixReader(object): self.close() raise + def __enter__ (self): + return self + + def __exit__ (self, exc_type, exc_val, exc_tb): + self.close() + return True + def close (self): """ Closes the cache file. -- cgit v1.2.3-70-g09d2 From 6c9202841a73519586bab4f5fff9f97eb979888c Mon Sep 17 00:00:00 2001 From: René 'Necoro' Neumann Date: Fri, 14 Aug 2009 19:23:40 +0200 Subject: Make the EixReader support the context manager protocol --- portato/eix/__init__.py | 1 - 1 file changed, 1 deletion(-) (limited to 'portato/eix/__init__.py') diff --git a/portato/eix/__init__.py b/portato/eix/__init__.py index effac2d..e726073 100644 --- a/portato/eix/__init__.py +++ b/portato/eix/__init__.py @@ -79,7 +79,6 @@ class EixReader(object): def __exit__ (self, exc_type, exc_val, exc_tb): self.close() - return True def close (self): """ -- cgit v1.2.3-70-g09d2 From d9c6fb6767c6873782847df168f8224d83ab30cd Mon Sep 17 00:00:00 2001 From: René 'Necoro' Neumann Date: Fri, 14 Aug 2009 22:50:33 +0200 Subject: Rewrote eix-parser in Cython --> WAAAAAAAAAY faster --- portato/db/eix_sql.py | 7 +- portato/eix/__init__.py | 2 +- portato/eix/parser.py | 416 ----------------------------------------------- portato/eix/parser.pyx | 318 ++++++++++++++++++++++++++++++++++++ portato/eix/py_parser.py | 416 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 739 insertions(+), 420 deletions(-) delete mode 100644 portato/eix/parser.py create mode 100644 portato/eix/parser.pyx create mode 100644 portato/eix/py_parser.py (limited to 'portato/eix/__init__.py') diff --git a/portato/db/eix_sql.py b/portato/db/eix_sql.py index 089d3ed..3a0c6e9 100644 --- a/portato/db/eix_sql.py +++ b/portato/db/eix_sql.py @@ -56,9 +56,10 @@ class EixSQLDatabase (SQLDatabase): def _get(): with EixReader(self.cache) as eix: for cat in eix.categories: - if category is None or cat.name() == category: - for pkg in cat.packages(): - yield (cat.name(), pkg.name(), pkg.name() in inst, False) + if category is None or cat.name == category: + for pkg in cat.packages: + p = "%s/%s" % (cat.name, pkg.name) + yield (cat.name, pkg.name, p in inst, False) connection.executemany("INSERT INTO packages (cat, name, inst, disabled) VALUES (?, ?, ?, ?)", _get()) connection.commit() diff --git a/portato/eix/__init__.py b/portato/eix/__init__.py index e726073..346fe82 100644 --- a/portato/eix/__init__.py +++ b/portato/eix/__init__.py @@ -69,7 +69,7 @@ class EixReader(object): self.file.seek(0) self.header = parser.header(self.file) - self.categories = parser.vector(self.file, parser.category, nelems = self.header.ncats()) + self.categories = parser.vector(self.file, parser.category, nelems = self.header.ncats) except: self.close() raise diff --git a/portato/eix/parser.py b/portato/eix/parser.py deleted file mode 100644 index cc42553..0000000 --- a/portato/eix/parser.py +++ /dev/null @@ -1,416 +0,0 @@ -# -*- coding: utf-8 -*- -# -# File: portato/eix/parser.py -# This file is part of the Portato-Project, a graphical portage-frontend. -# -# Copyright (C) 2006-2009 René 'Necoro' Neumann -# This is free software. You may redistribute copies of it under the terms of -# the GNU General Public License version 2. -# There is NO WARRANTY, to the extent permitted by law. -# -# Written by René 'Necoro' Neumann - -""" -The cache file supports different types of data. -In this module (nearly) all of these types have a corresponding function. - -For the exact way all the functions work, have a look at the eix format description. -""" - -from __future__ import absolute_import, with_statement -__docformat__ = "restructuredtext" - -import os -import struct -from functools import partial - -from ..helper import debug -from .exceptions import EndOfFileException - -# -# Helper -# - -def _get_bytes (file, length, expect_list = False): - """ - Return a number of bytes. - - :Parameters: - - file : file - The file to read from. - - length : int - The number of bytes to read. - - expect_list : bool - In case ``length`` is 1, only a single byte is returned. If ``expect_list`` is true, then a list is also returned in this case. - - :rtype: int or int[] - :raises EndOfFileException: if EOF is reached during execution - """ - - s = file.read(length) - - if len(s) != length: - raise EndOfFileException, file.name - - if length == 1 and not expect_list: - return ord(s) # is faster than unpack and we have a scalar - else: - return struct.unpack("%sB" % length, s) - -# -# Base Types -# - -def number (file, skip = False): - """ - Returns a number. - - :Parameters: - - file : file - The file to read from. - - skip : bool - Do not return the actual value, but just skip to the next datum. - - :rtype: int - """ - - n = _get_bytes(file, 1) - - if n < 0xFF: - value = n - else: - count = 0 - - while (n == 0xFF): - count += 1 - n = _get_bytes(file, 1) - - if n == 0: - n = 0xFF # 0xFF is encoded as 0xFF 0x00 - count -= 1 - - value = n << (count*8) - - if count > 0: - - if skip: - file.seek(count, os.SEEK_CUR) - return - - else: - rest = _get_bytes(file, count, expect_list = True) - - for i, r in enumerate(rest): - value += r << ((count - i - 1)*8) - - return value - -def vector (file, get_type, skip = False, nelems = None): - """ - Returns a vector of elements. - - :Parameters: - - file : file - The file to read from. - - get_type : function(file, bool) - The function determining type of the elements. - - skip : bool - Do not return the actual value, but just skip to the next datum. - - nelems : int - Normally the eix-Vector has the number of elements as the first argument. - If for some reason this is not the case, you can pass it in here. - - :rtype: list - """ - - if nelems is None: - nelems = number(file) - - if skip: - for i in range(nelems): - get_type(file, skip = True) - else: - return [get_type(file) for i in range(nelems)] - -def typed_vector(type, nelems = None): - """ - Shortcut to create a function for a special type of vector. - - :Parameters: - - type : function(file, bool) - The function determining type of the elements. - - nelems : int - Normally the eix-Vector has the number of elements as the first argument. - If for some reason this is not the case, you can pass it in here. - Do not return the actual value, but just skip to the next datum. - - :rtype: function(file, bool) - :see: `vector` - """ - - if nelems is None: - return partial(vector, get_type = type) - else: - return partial(vector, get_type = type, nelems = nelems) - -def string (file, skip = False): - """ - Returns a string. - - :Parameters: - - file : file - The file to read from. - - skip : bool - Do not return the actual value, but just skip to the next datum. - - :rtype: str - """ - nelems = number(file) - - if skip: - file.seek(nelems, os.SEEK_CUR) - return - else: - s = file.read(nelems) - - if len(s) != nelems: - raise EndOfFileException, file.name - - return s - -# -# Complex Types -# - -class LazyElement (object): - """ - This class models a value in the cache, which is only read on access. - - If not accessed directly, only the position inside the file is stored. - """ - __slots__ = ("file", "get_type", "_value", "pos") - - def __init__ (self, get_type, file): - """ - :Parameters: - - get_type : function(file, bool) - The function determining type of the elements. - - file : file - The file to read from. - """ - - self.file = file - self.get_type = get_type - self._value = None - - self.pos = file.tell() - get_type(file, skip=True) # skip it for the moment - - @property - def value (self): - """ - The value of the element. - """ - - if self._value is None: - old_pos = self.file.tell() - self.file.seek(self.pos, os.SEEK_SET) - self._value = self.get_type(self.file, skip = False) - self.file.seek(old_pos, os.SEEK_SET) - - return self._value - - def __call__ (self): - """ - Convenience function. Also returns the value. - """ - return self.value - -class overlay (object): - """ - Represents an overlay object. - - :IVariables: - - path : `LazyElement` - The path to the overlay - - label : `LazyElement` - The label/name of the overlay - """ - __slots__ = ("path", "label") - - def __init__ (self, file, skip = False): - """ - :Parameters: - - file : file - The file to read from. - - skip : bool - Do not return the actual value, but just skip to the next datum. - """ - - self.path = LazyElement(string, file) - self.label = LazyElement(string, file) - -class header (object): - """ - Represents the header of the cache. - - :IVariables: - - version : `LazyElement` - The version of the cache file. - - ncats : `LazyElement` - The number of categories. - - overlays : `LazyElement` <`overlay` []> - The list of overlays. - - provide : `LazyElement` - A list of "PROVIDE" values. - - licenses : `LazyElement` - The list of licenses. - - keywords : `LazyElement` - The list of keywords. - - useflags : `LazyElement` - The list of useflags. - - slots : `LazyElement` - The list of slots different from "0". - - sets : `LazyElement` - The names of world sets are the names (without leading @) of the world sets stored in /var/lib/portage/world_sets. - If SAVE_WORLD=false, the list is empty. - """ - __slots__ = ("version", "ncats", "overlays", "provide", - "licenses", "keywords", "useflags", "slots", "sets") - - def __init__ (self, file, skip = False): - """ - :Parameters: - - file : file - The file to read from. - - skip : bool - Do not return the actual value, but just skip to the next datum. - """ - def LE (t): - return LazyElement(t, file) - - self.version = LE(number) - self.ncats = LE(number) - self.overlays = LE(typed_vector(overlay)) - self.provide = LE(typed_vector(string)) - self.licenses = LE(typed_vector(string)) - self.keywords = LE(typed_vector(string)) - self.useflags = LE(typed_vector(string)) - self.slots = LE(typed_vector(string)) - self.sets = LE(typed_vector(string)) - -class package (object): - """ - The representation of one package. - - Currently, version information is not parsed and stored. - So you can gain general infos only. - - :IVariables: - - name : `LazyElement` - The name of the package. - - description : `LazyElement` - Description of the package. - - homepage : `LazyElement` - The homepage of the package. - - provide : `LazyElement` - The indices of `header.provide` representing the PROVIDE value of the package. - - license : `LazyElement` - The index of `header.licenses` representing the license of the package. - - useflags : `LazyElement` - The indices of `header.useflags` representing the IUSE value of the package. - """ - - __slots__ = ("_offset", "name", "description", "provide", - "homepage", "license", "useflags") - - def __init__ (self, file, skip = False): - """ - :Parameters: - - file : file - The file to read from. - - skip : bool - Do not return the actual value, but just skip to the next datum. - """ - def LE (t): - return LazyElement(t, file) - - self._offset = number(file) - - after_offset = file.tell() - - self.name = LE(string) - self.description = LE(string) - self.provide = LE(typed_vector(number)) - self.homepage = LE(string) - self.license = LE(number) - self.useflags = LE(typed_vector(number)) - - # self.versions = LE(typed_vector(version)) - # for the moment just skip the versions - file.seek(self._offset - (file.tell() - after_offset), os.SEEK_CUR) - -class category (object): - """ - Represents a whole category. - - :IVariables: - - name : `LazyElement` - The category name. - - packages : `LazyElement` <`package` []> - All the packages of the category. - """ - __slots__ = ("name", "packages") - - def __init__ (self, file, skip = False): - """ - :Parameters: - - file : file - The file to read from. - - skip : bool - Do not return the actual value, but just skip to the next datum. - """ - self.name = LazyElement(string, file) - self.packages = LazyElement(typed_vector(package), file) diff --git a/portato/eix/parser.pyx b/portato/eix/parser.pyx new file mode 100644 index 0000000..453376e --- /dev/null +++ b/portato/eix/parser.pyx @@ -0,0 +1,318 @@ +# -*- coding: utf-8 -*- +# +# File: portato/eix/_parser.pyx +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann + +""" +The cache file supports different types of data. +In this module (nearly) all of these types have a corresponding function. + +For the exact way all the functions work, have a look at the eix format description. +""" + +__docformat__ = "restructuredtext" + +cdef extern from "stdio.h": + ctypedef struct FILE: + pass + + int fgetc(FILE* stream) + + int EOF + int SEEK_CUR + +cdef extern from "Python.h": + FILE* PyFile_AsFile(object) + + ctypedef int Py + +ctypedef unsigned char UChar +ctypedef long long LLong + +from portato.eix.exceptions import EndOfFileException + +# +# Helper +# + +cdef int _get_byte (FILE* file) except -1: + cdef int c = fgetc(file) + + if c == EOF: + raise EndOfFileException + + return c + + +# +# Base Types +# + +cdef LLong _number (object pfile): + cdef UChar n + cdef LLong value + cdef int i + + cdef unsigned short count = 1 + cdef FILE* file = PyFile_AsFile(pfile) + + n = _get_byte(file) + + if n < 0xFF: + return n + else: + + n = _get_byte(file) + while (n == 0xFF): + count += 1 + n = _get_byte(file) + + if n == 0: + value = 0xFF # 0xFF is encoded as 0xFF 0x00 + count -= 1 + else: + value = n + + for i in range(count): + value = (value << 8) | (_get_byte(file)) + + return value + +def number (file): + """ + Returns a number. + + :param file: The file to read from + :type file: file + :rtype: int + """ + + return _number(file) + +def vector (file, get_type, nelems = None): + """ + Returns a vector of elements. + + :Parameters: + + file : file + The file to read from. + + get_type : function(file, bool) + The function determining type of the elements. + + nelems : int + Normally the eix-Vector has the number of elements as the first argument. + If for some reason this is not the case, you can pass it in here. + + :rtype: list + """ + + cdef LLong n + cdef LLong i + + if nelems is None: + n = _number(file) + else: + n = nelems + + l = [] + for i in range(n): + l.append(get_type(file)) + + return l + +def string (file): + """ + Returns a string. + + :param file: The file to read from + :type file: file + :rtype: str + """ + cdef LLong nelems + + nelems = _number(file) + + s = file.read(nelems) + + if len(s) != nelems: + raise EndOfFileException, file.name + + return s + +# +# Complex Types +# + +cdef class overlay: + """ + Represents an overlay object. + + :IVariables: + + path : string + The path to the overlay + + label : string + The label/name of the overlay + """ + + cdef readonly object path + cdef readonly object label + + def __init__ (self, file): + """ + :param file: The file to read from + :type file: file + """ + + self.path = string(file) + self.label = string(file) + +cdef class header: + """ + Represents the header of the cache. + + :IVariables: + + version : int + The version of the cache file. + + ncats : int + The number of categories. + + overlays : `overlay` [] + The list of overlays. + + provide : string[] + A list of "PROVIDE" values. + + licenses : string[] + The list of licenses. + + keywords : string[] + The list of keywords. + + useflags : string[] + The list of useflags. + + slots : string[] + The list of slots different from "0". + + sets : string[] + The names of world sets are the names (without leading @) of the world sets stored in /var/lib/portage/world_sets. + If SAVE_WORLD=false, the list is empty. + """ + + cdef readonly object version + cdef readonly object ncats + cdef readonly object overlays + cdef readonly object provide + cdef readonly object licenses + cdef readonly object keywords + cdef readonly object useflags + cdef readonly object slots + cdef readonly object sets + + def __init__ (self, file): + """ + :param file: The file to read from + :type file: file + """ + self.version = number(file) + self.ncats = number(file) + self.overlays = vector(file, overlay) + self.provide = vector(file, string) + self.licenses = vector(file, string) + self.keywords = vector(file, string) + self.useflags = vector(file, string) + self.slots = vector(file, string) + self.sets = vector(file, string) + +cdef class package: + """ + The representation of one package. + + Currently, version information is not parsed and stored. + So you can gain general infos only. + + :IVariables: + + name : string + The name of the package. + + description : string + Description of the package. + + homepage : string + The homepage of the package. + + provide : int[] + The indices of `header.provide` representing the PROVIDE value of the package. + + license : int + The index of `header.licenses` representing the license of the package. + + useflags : int[] + The indices of `header.useflags` representing the IUSE value of the package. + """ + + cdef readonly object _offset + cdef readonly object name + cdef readonly object description + cdef readonly object provide + cdef readonly object homepage + cdef readonly object license + cdef readonly object useflags + + def __init__ (self, file): + """ + :param file: The file to read from + :type file: file + """ + self._offset = number(file) + + after_offset = file.tell() + + self.name = string(file) + self.description = string(file) + self.provide = vector(file, number) + self.homepage = string(file) + self.license = number(file) + self.useflags = vector(file, number) + + # self.versions = LE(typed_vector(version)) + # for the moment just skip the versions + file.seek(self._offset - (file.tell() - after_offset), SEEK_CUR) + +cdef class category: + """ + Represents a whole category. + + :IVariables: + + name : string + The category name. + + packages : `package` [] + All the packages of the category. + """ + + cdef readonly object name + cdef readonly object packages + + def __init__ (self, file): + """ + :param file: The file to read from + :type file: file + """ + self.name = string(file) + self.packages = vector(file, package) diff --git a/portato/eix/py_parser.py b/portato/eix/py_parser.py new file mode 100644 index 0000000..cc42553 --- /dev/null +++ b/portato/eix/py_parser.py @@ -0,0 +1,416 @@ +# -*- coding: utf-8 -*- +# +# File: portato/eix/parser.py +# This file is part of the Portato-Project, a graphical portage-frontend. +# +# Copyright (C) 2006-2009 René 'Necoro' Neumann +# This is free software. You may redistribute copies of it under the terms of +# the GNU General Public License version 2. +# There is NO WARRANTY, to the extent permitted by law. +# +# Written by René 'Necoro' Neumann + +""" +The cache file supports different types of data. +In this module (nearly) all of these types have a corresponding function. + +For the exact way all the functions work, have a look at the eix format description. +""" + +from __future__ import absolute_import, with_statement +__docformat__ = "restructuredtext" + +import os +import struct +from functools import partial + +from ..helper import debug +from .exceptions import EndOfFileException + +# +# Helper +# + +def _get_bytes (file, length, expect_list = False): + """ + Return a number of bytes. + + :Parameters: + + file : file + The file to read from. + + length : int + The number of bytes to read. + + expect_list : bool + In case ``length`` is 1, only a single byte is returned. If ``expect_list`` is true, then a list is also returned in this case. + + :rtype: int or int[] + :raises EndOfFileException: if EOF is reached during execution + """ + + s = file.read(length) + + if len(s) != length: + raise EndOfFileException, file.name + + if length == 1 and not expect_list: + return ord(s) # is faster than unpack and we have a scalar + else: + return struct.unpack("%sB" % length, s) + +# +# Base Types +# + +def number (file, skip = False): + """ + Returns a number. + + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + + :rtype: int + """ + + n = _get_bytes(file, 1) + + if n < 0xFF: + value = n + else: + count = 0 + + while (n == 0xFF): + count += 1 + n = _get_bytes(file, 1) + + if n == 0: + n = 0xFF # 0xFF is encoded as 0xFF 0x00 + count -= 1 + + value = n << (count*8) + + if count > 0: + + if skip: + file.seek(count, os.SEEK_CUR) + return + + else: + rest = _get_bytes(file, count, expect_list = True) + + for i, r in enumerate(rest): + value += r << ((count - i - 1)*8) + + return value + +def vector (file, get_type, skip = False, nelems = None): + """ + Returns a vector of elements. + + :Parameters: + + file : file + The file to read from. + + get_type : function(file, bool) + The function determining type of the elements. + + skip : bool + Do not return the actual value, but just skip to the next datum. + + nelems : int + Normally the eix-Vector has the number of elements as the first argument. + If for some reason this is not the case, you can pass it in here. + + :rtype: list + """ + + if nelems is None: + nelems = number(file) + + if skip: + for i in range(nelems): + get_type(file, skip = True) + else: + return [get_type(file) for i in range(nelems)] + +def typed_vector(type, nelems = None): + """ + Shortcut to create a function for a special type of vector. + + :Parameters: + + type : function(file, bool) + The function determining type of the elements. + + nelems : int + Normally the eix-Vector has the number of elements as the first argument. + If for some reason this is not the case, you can pass it in here. + Do not return the actual value, but just skip to the next datum. + + :rtype: function(file, bool) + :see: `vector` + """ + + if nelems is None: + return partial(vector, get_type = type) + else: + return partial(vector, get_type = type, nelems = nelems) + +def string (file, skip = False): + """ + Returns a string. + + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + + :rtype: str + """ + nelems = number(file) + + if skip: + file.seek(nelems, os.SEEK_CUR) + return + else: + s = file.read(nelems) + + if len(s) != nelems: + raise EndOfFileException, file.name + + return s + +# +# Complex Types +# + +class LazyElement (object): + """ + This class models a value in the cache, which is only read on access. + + If not accessed directly, only the position inside the file is stored. + """ + __slots__ = ("file", "get_type", "_value", "pos") + + def __init__ (self, get_type, file): + """ + :Parameters: + + get_type : function(file, bool) + The function determining type of the elements. + + file : file + The file to read from. + """ + + self.file = file + self.get_type = get_type + self._value = None + + self.pos = file.tell() + get_type(file, skip=True) # skip it for the moment + + @property + def value (self): + """ + The value of the element. + """ + + if self._value is None: + old_pos = self.file.tell() + self.file.seek(self.pos, os.SEEK_SET) + self._value = self.get_type(self.file, skip = False) + self.file.seek(old_pos, os.SEEK_SET) + + return self._value + + def __call__ (self): + """ + Convenience function. Also returns the value. + """ + return self.value + +class overlay (object): + """ + Represents an overlay object. + + :IVariables: + + path : `LazyElement` + The path to the overlay + + label : `LazyElement` + The label/name of the overlay + """ + __slots__ = ("path", "label") + + def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ + + self.path = LazyElement(string, file) + self.label = LazyElement(string, file) + +class header (object): + """ + Represents the header of the cache. + + :IVariables: + + version : `LazyElement` + The version of the cache file. + + ncats : `LazyElement` + The number of categories. + + overlays : `LazyElement` <`overlay` []> + The list of overlays. + + provide : `LazyElement` + A list of "PROVIDE" values. + + licenses : `LazyElement` + The list of licenses. + + keywords : `LazyElement` + The list of keywords. + + useflags : `LazyElement` + The list of useflags. + + slots : `LazyElement` + The list of slots different from "0". + + sets : `LazyElement` + The names of world sets are the names (without leading @) of the world sets stored in /var/lib/portage/world_sets. + If SAVE_WORLD=false, the list is empty. + """ + __slots__ = ("version", "ncats", "overlays", "provide", + "licenses", "keywords", "useflags", "slots", "sets") + + def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ + def LE (t): + return LazyElement(t, file) + + self.version = LE(number) + self.ncats = LE(number) + self.overlays = LE(typed_vector(overlay)) + self.provide = LE(typed_vector(string)) + self.licenses = LE(typed_vector(string)) + self.keywords = LE(typed_vector(string)) + self.useflags = LE(typed_vector(string)) + self.slots = LE(typed_vector(string)) + self.sets = LE(typed_vector(string)) + +class package (object): + """ + The representation of one package. + + Currently, version information is not parsed and stored. + So you can gain general infos only. + + :IVariables: + + name : `LazyElement` + The name of the package. + + description : `LazyElement` + Description of the package. + + homepage : `LazyElement` + The homepage of the package. + + provide : `LazyElement` + The indices of `header.provide` representing the PROVIDE value of the package. + + license : `LazyElement` + The index of `header.licenses` representing the license of the package. + + useflags : `LazyElement` + The indices of `header.useflags` representing the IUSE value of the package. + """ + + __slots__ = ("_offset", "name", "description", "provide", + "homepage", "license", "useflags") + + def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ + def LE (t): + return LazyElement(t, file) + + self._offset = number(file) + + after_offset = file.tell() + + self.name = LE(string) + self.description = LE(string) + self.provide = LE(typed_vector(number)) + self.homepage = LE(string) + self.license = LE(number) + self.useflags = LE(typed_vector(number)) + + # self.versions = LE(typed_vector(version)) + # for the moment just skip the versions + file.seek(self._offset - (file.tell() - after_offset), os.SEEK_CUR) + +class category (object): + """ + Represents a whole category. + + :IVariables: + + name : `LazyElement` + The category name. + + packages : `LazyElement` <`package` []> + All the packages of the category. + """ + __slots__ = ("name", "packages") + + def __init__ (self, file, skip = False): + """ + :Parameters: + + file : file + The file to read from. + + skip : bool + Do not return the actual value, but just skip to the next datum. + """ + self.name = LazyElement(string, file) + self.packages = LazyElement(typed_vector(package), file) -- cgit v1.2.3-70-g09d2