View previous topic :: View next topic |
Author |
Message |
tobidope n00b
Joined: 17 Aug 2003 Posts: 24 Location: Germany
|
Posted: Sun Dec 19, 2004 12:15 pm Post subject: |
|
|
Damn where is the error! In between I changed something that breaks the hole shit.
Code: | import portage_db_template
import os
import os.path
import cPickle
import cdb
class database(portage_db_template.database):
def module_init(self):
self.addList = {}
self.delList = []
self.modified = False
self.cdbName = os.path.join(self.path, self.category) + ".cdb"
self.cdbObject = None
self.__openCDB()
def has_key(self, key):
self.check_key(key)
retVal = 0
if self.cdbObject.get(key) is not None:
retVal = 1
if self.modified:
if key in self.delList:
retVal = 0
if key in self.addList:
retVal = 1
return retVal
def keys(self):
myKeys = self.cdbObject.keys()
if self.modified:
for k in self.delList:
myKeys.remove(k)
for k in self.addList.iterkeys():
if k not in myKeys:
myKeys.append(k)
return myKeys
def get_values(self, key):
values = None
if self.has_key(key):
if key in self.addList:
values = self.addList[key]
else:
values = cPickle.loads(self.cdbObject.get(key))
return values
def set_values(self, key, val):
self.check_key(key)
self.modified = True
self.addList[key] = val
def del_key(self, key):
retVal = 0
if self.has_key(key):
self.modified = True
retVal = 1
if key in self.addList:
del self.addList[key]
else:
self.delList.append(key)
return retVal
def sync(self):
if self.modified:
self.modified = False
newDB = cdb.cdbmake(self.cdbName, self.cdbName + ".tmp")
for key, value in iter(self.cdbObject.each, None):
if key in self.delList:
if key in self.addList:
newDB.add(key, cPickle.dumps(self.addList[key], cPickle.HIGHEST_PROTOCOL))
del self.addList[key]
elif key in self.addList:
newDB.add(key, cPickle.dumps(self.addList[key], cPickle.HIGHEST_PROTOCOL))
del self.addList[key]
else:
newDB.add(key, value)
self.__closeCDB()
for key, value in self.addList.iteritems():
newDB.add(key, cPickle.dumps(value, cPickle.HIGHEST_PROTOCOL))
newDB.finish()
del newDB
self.addList = {}
self.delList = []
self.__openCDB()
def close(self):
if self.modified:
self.sync()
self.__closeCDB()
def __del__(self):
self.close()
def __openCDB(self):
prevmask = os.umask(0)
if not os.path.exists(self.path):
os.makedirs(self.path, 02775)
os.chown(self.path, self.uid, self.gid)
if not os.path.isfile(self.cdbName):
maker = cdb.cdbmake(self.cdbName, self.cdbName + ".tmp")
maker.finish()
del maker
os.chown(self.cdbName, self.uid, self.gid)
os.chmod(self.cdbName, 0664)
os.umask(prevmask)
self.cdbObject = cdb.init(self.cdbName)
def __closeCDB(self):
if self.cdbObject:
del self.cdbObject
self.cdbObject = None
if __name__ == "__main__":
import portage
uid = os.getuid()
gid = os.getgid()
portage_db_template.test_database(database,"/home/tobias", "sys-apps", portage.auxdbkeys, uid, gid)
|
|
|
Back to top |
|
|
Jinidog Guru
Joined: 26 Nov 2003 Posts: 593 Location: Berlin
|
Posted: Mon Jan 03, 2005 10:29 am Post subject: |
|
|
What's up now?
I'm waiting for a version that speeds up the sync and works when I want to merge something. _________________ Just unused Microsoft-Software is good Microsoft-Software |
|
Back to top |
|
|
tobidope n00b
Joined: 17 Aug 2003 Posts: 24 Location: Germany
|
Posted: Wed Jan 05, 2005 8:47 am Post subject: |
|
|
Jinidog wrote: | What's up now?
I'm waiting for a version that speeds up the sync and works when I want to merge something. |
Look at the starting message. I've updated it to a version that works. |
|
Back to top |
|
|
rhill Retired Dev
Joined: 22 Oct 2004 Posts: 1629 Location: sk.ca
|
Posted: Sat Jan 08, 2005 3:59 am Post subject: |
|
|
cdb
Code: | root /var/cache/edb/dep/usr # rm -rf *
root /var/cache/edb/dep/usr # time emerge metadata
skipping sync
>>> Updating Portage cache: 100%
real 2m45.356s
user 1m35.255s
sys 0m23.820s |
normal portage
Code: | root /var/cache/edb/dep/usr # rm -rf *
root /var/cache/edb/dep/usr # time emerge metadata
skipping sync
>>> Updating Portage cache: 100%
real 5m49.752s
user 3m51.260s
sys 1m0.288s |
nice.
thanks, tobidope. _________________ by design, by neglect
for a fact or just for effect |
|
Back to top |
|
|
Cintra Advocate
Joined: 03 Apr 2004 Posts: 2111 Location: Norway
|
Posted: Tue Feb 01, 2005 11:12 am Post subject: |
|
|
tobidope wrote: |
Look at the starting message. I've updated it to a version that works. |
Hei
I have just followed the procedure on page one of this thread. I have no errors as such, but in /var/cache/edb/dep/usr/ I have /local/portage containing the .cdb files, ie not /portage ref your earlier comments.
I have the following results which look ok.
Code: | # time emerge metadata
skipping sync
>>> Updating Portage cache: 100%
real 0m40.862s
user 0m22.223s
sys 0m6.366s
|
Btw, I use the following in make.conf..
Code: | PORTDIR="/mnt/hdb9/portage"
DISTDIR="/mnt/hdb9/portage/distfiles"
PKGDIR="/mnt/hdb9/portage/packages"
|
regards
Edit: Its OK, I found the .cdb files in /var/cache/edb/dep/mnt/hdb9/portage!
Well, I assume its OK!? _________________ "I am not bound to please thee with my answers" W.S.
Last edited by Cintra on Tue Feb 01, 2005 8:13 pm; edited 2 times in total |
|
Back to top |
|
|
smithj Retired Dev
Joined: 24 Sep 2004 Posts: 111
|
Posted: Tue Feb 01, 2005 7:39 pm Post subject: |
|
|
upon creating the /etc/portage/modules file, i get these errors:
Quote: | smithj ~ # emerge metadata
Traceback (most recent call last):
File "/usr/bin/emerge", line 10, in ?
import portage
File "/usr/lib/portage/pym/portage.py", line 6999, in ?
portdb=portdbapi(settings["PORTDIR"])
File "/usr/lib/portage/pym/portage.py", line 5018, in __init__
self.eclassdb = eclass_cache(self.porttree_root, self.mysettings)
File "/usr/lib/portage/pym/portage.py", line 4854, in __init__
self.dbmodule = self.settings.load_best_module("eclass_cache.dbmodule")
File "/usr/lib/portage/pym/portage.py", line 1191, in load_best_module
return load_mod(best_mod)
File "/usr/lib/portage/pym/portage.py", line 128, in load_mod
mod = __import__(modname)
ImportError: No module named portage_db_cdb
smithj ~ # emerge --regen
Traceback (most recent call last):
File "/usr/bin/emerge", line 10, in ?
import portage
File "/usr/lib/portage/pym/portage.py", line 6999, in ?
portdb=portdbapi(settings["PORTDIR"])
File "/usr/lib/portage/pym/portage.py", line 5018, in __init__
self.eclassdb = eclass_cache(self.porttree_root, self.mysettings)
File "/usr/lib/portage/pym/portage.py", line 4854, in __init__
self.dbmodule = self.settings.load_best_module("eclass_cache.dbmodule")
File "/usr/lib/portage/pym/portage.py", line 1191, in load_best_module
return load_mod(best_mod)
File "/usr/lib/portage/pym/portage.py", line 128, in load_mod
mod = __import__(modname)
ImportError: No module named portage_db_cdb |
ideas? my portage has been getting quite slow, and i would love to get this to work |
|
Back to top |
|
|
Swoosh Apprentice
Joined: 19 Sep 2004 Posts: 203
|
Posted: Tue Feb 01, 2005 8:21 pm Post subject: |
|
|
What version of portage do you have? |
|
Back to top |
|
|
smithj Retired Dev
Joined: 24 Sep 2004 Posts: 111
|
Posted: Tue Feb 01, 2005 9:10 pm Post subject: |
|
|
emerge --info
Portage 2.0.51-r15 (default-linux/x86/2004.2, gcc-3.4.3, glibc-2.3.4.20041102-r0, 2.6.10 i686)
=================================================================
System uname: 2.6.10 i686 Intel(R) Celeron(R) CPU 2.00GHz
Gentoo Base System version 1.6.9
Python: dev-lang/python-2.3.4 [2.3.4 (#1, Jan 10 2005, 05:32:54)]
ccache version 2.3 [enabled]
dev-lang/python: 2.3.4
sys-devel/autoconf: 2.59-r6, 2.13
sys-devel/automake: 1.7.9-r1, 1.9.4, 1.8.5-r3, 1.5, 1.6.3, 1.4_p6
sys-devel/binutils: 2.15.92.0.2-r1, 2.15.92.0.2-r2
sys-devel/libtool: 1.5.10-r4
virtual/os-headers: 2.6.8.1-r2
ACCEPT_KEYWORDS="x86 ~x86"
AUTOCLEAN="yes"
CFLAGS="-O2 -march=i686 -fomit-frame-pointer -pipe"
CHOST="i686-pc-linux-gnu"
CONFIG_PROTECT="/etc /usr/kde/2/share/config /usr/kde/3.3/env /usr/kde/3.3/share/config /usr/kde/3.3/shutdown /usr/kde/3/share/config /usr/lib/X11/xkb /usr/lib/mozilla/defaults/pref /usr/share/config /usr/share/texmf/dvipdfm/config/ /usr/share/texmf/dvips/config/ /usr/share/texmf/tex/generic/config/ /usr/share/texmf/tex/platex/config/ /usr/share/texmf/xdvi/ /var/qmail/alias /var/qmail/control"
CONFIG_PROTECT_MASK="/etc/gconf /etc/terminfo /etc/env.d"
CXXFLAGS="-O2 -march=i686 -fomit-frame-pointer -pipe"
DISTDIR="/usr/portage/distfiles"
FEATURES="autoaddcvs autoconfig ccache distlocks sandbox sfperms"
GENTOO_MIRRORS="http://distfiles.gentoo.org http://distro.ibiblio.org/pub/Linux/distributions/gentoo"
MAKEOPTS="-j2"
PKGDIR="/usr/portage/packages"
PORTAGE_TMPDIR="/var/tmp"
PORTDIR="/usr/portage"
SYNC="rsync://rsync.us.gentoo.org/gentoo-portage"
USE="x86 X aalib acpi aim alsa apache2 apm arts avi berkdb bitmap-fonts cdr crypt cups dba doc dvd encode esd f77 fam flac fluxbox font-server foomaticdb fortran ftp gd gd-external gdbm gif gpm gtk gtk2 icq imagemagick imap imlib java jikes jpeg junit libg++ libwww mad mbox mikmod mmx mmx2 motif mozilla mpeg mysql ncurses nls nptl nptlonly oggvorbis opengl pam pcmcia pdflib perl php png posix postgres python qt quicktime readline samba scanner sdl slang snmp spell sse sse2 ssl svga tcltk tcpd tetex tiff truetype truetype-fonts type1-fonts udev usb vim xml2 xmms xprint xv xvid zlib"
Unset: ASFLAGS, CBUILD, CTARGET, LANG, LC_ALL, LDFLAGS, PORTDIR_OVERLAY |
|
Back to top |
|
|
rhill Retired Dev
Joined: 22 Oct 2004 Posts: 1629 Location: sk.ca
|
Posted: Wed Feb 02, 2005 8:24 am Post subject: |
|
|
Quote: | ImportError: No module named portage_db_cdb |
seems to imply it can't find the module at all. have you checked the path and spelling and everything? _________________ by design, by neglect
for a fact or just for effect |
|
Back to top |
|
|
smithj Retired Dev
Joined: 24 Sep 2004 Posts: 111
|
Posted: Wed Feb 02, 2005 12:10 pm Post subject: |
|
|
dirtyepic wrote: | Quote: | ImportError: No module named portage_db_cdb |
seems to imply it can't find the module at all. have you checked the path and spelling and everything? |
i am not sure what there is to check. i coppied and pasted everything verbatim from this site |
|
Back to top |
|
|
Cintra Advocate
Joined: 03 Apr 2004 Posts: 2111 Location: Norway
|
Posted: Wed Feb 02, 2005 12:19 pm Post subject: |
|
|
After creating /usr/lib/portage/pym/portage_db_cdb.py and dropping in the script, did you check if it was executable?
mvh _________________ "I am not bound to please thee with my answers" W.S. |
|
Back to top |
|
|
smithj Retired Dev
Joined: 24 Sep 2004 Posts: 111
|
Posted: Wed Feb 02, 2005 1:02 pm Post subject: |
|
|
Cintra wrote: | After creating /usr/lib/portage/pym/portage_db_cdb.py and dropping in the script, did you check if it was executable?
mvh |
indeed i did not. was that posted anywhere on the instructions? perhaps i missed it
in any case, i am happily updating my portage cache at the moment. thanks guys |
|
Back to top |
|
|
kesara n00b
Joined: 05 Aug 2004 Posts: 69 Location: Sri Lanka
|
Posted: Wed Feb 02, 2005 5:48 pm Post subject: |
|
|
NIce Work _________________ ---==>K3<==---
kesara.lk |
|
Back to top |
|
|
aminal Apprentice
Joined: 17 Mar 2004 Posts: 170 Location: Baltimore, MD
|
Posted: Thu Feb 03, 2005 2:56 am Post subject: |
|
|
time emerge metadata
No CDP
Code: |
real 7m48.467s
user 1m15.466s
sys 0m12.446s
|
With CDP
Code: |
real 1m38.906s
user 0m19.096s
sys 0m3.359s
|
Great job - thanks! |
|
Back to top |
|
|
shinewu n00b
Joined: 09 Jan 2005 Posts: 27
|
Posted: Thu Feb 03, 2005 3:10 am Post subject: |
|
|
I got an amazing speed up.
Really great work. Hope you can
maintain it though.
Thanks a lot! |
|
Back to top |
|
|
tobidope n00b
Joined: 17 Aug 2003 Posts: 24 Location: Germany
|
Posted: Fri Feb 04, 2005 7:51 am Post subject: |
|
|
smithj wrote: | Cintra wrote: | After creating /usr/lib/portage/pym/portage_db_cdb.py and dropping in the script, did you check if it was executable?
mvh |
indeed i did not. was that posted anywhere on the instructions? perhaps i missed it
in any case, i am happily updating my portage cache at the moment. thanks guys |
There should be no need to make the source file executable. But anyway it runs now ! Hope you like it all. Anybody knows to whom I have to talk to include it in portage? |
|
Back to top |
|
|
dr_strange Guru
Joined: 16 Apr 2002 Posts: 480 Location: Cambridge, UK
|
Posted: Fri Feb 04, 2005 10:48 am Post subject: |
|
|
for first step, submit a bugreport explaining what your app does and attaching an ebuild
(and let us know the result, great extension!!) _________________ shine on,
dr_strange
Set the Controls for the Heart of Gentoo
http://magenta.linuxforum.hu |
|
Back to top |
|
|
Chaosite Guru
Joined: 13 Dec 2003 Posts: 540 Location: Right over here.
|
Posted: Sat Feb 26, 2005 11:22 am Post subject: |
|
|
tobidope wrote: | There should be no need to make the source file executable. But anyway it runs now ! Hope you like it all. Anybody knows to whom I have to talk to include it in portage? |
IRC on to irc.freenode.org, and see what people in #gentoo-portage think of it. |
|
Back to top |
|
|
Chaosite Guru
Joined: 13 Dec 2003 Posts: 540 Location: Right over here.
|
Posted: Sat Feb 26, 2005 11:35 am Post subject: |
|
|
So, I went to #gentoo-portage, and carpaski was nice enough to explain to me the issues with this module:
Code: | 13:23 < chaosite> What do you guys think about this CDB-portage interface ... ?
13:23 < chaosite> http://forums.gentoo.org/viewtopic-t-261580-postdays-0-postorder-asc-start-0.html
13:24 <@carpaski> Constant DB ?
13:25 < chaosite> Dunno what it stands for...
13:25 < chaosite> It does have a webpage here: http://cr.yp.to/cdb.html
13:27 <@carpaski> cdb modules have been around a while...
13:27 <@carpaski> Someone hacked it in once...
13:27 <@carpaski> Didn't like that hack.
13:28 <@carpaski> First I've seen of an actual module for it though.
13:28 <@carpaski> Anything that relies on an external app that uses C calls is subject to Segfaults.
13:28 <@carpaski> This is why we don't use anydbm by default.
13:29 <@carpaski> flat and cpickle are python... So unless python explodes, portage works.
13:29 <@carpaski> Saves headaches.
13:29 < chaosite> So, basically, you're saying that this adds more runtime dependencies to portage, which might fail?
13:30 <@carpaski> Potentially, yep.
13:30 <@carpaski> If it works for you, great...
13:30 <@carpaski> But I wouldn't have any expectation of it being a default.
13:30 < chaosite> Alright.
13:31 <@carpaski> The major kicker is that segfaults outside of python segfault the entire process.
13:31 <@carpaski> Otherwise it wouldn't be a problem.
13:31 <@carpaski> Seeing this is really disturbing:
13:31 <@carpaski> emerge -e world
13:31 <@carpaski> Segmentation Fault
13:32 < chaosite> Yeah, that won't be any fun...
13:33 < chaosite> Mind if I post this to the forums?
13:33 <@carpaski> Have at it.
13:33 < chaosite> Thanks :)
13:33 <@carpaski> The module can circulate, no problem with that... it could even get included at some point.
13:33 <@carpaski> It just has pretty much no chance of being default. |
So there
EDIT:
https://bugs.gentoo.org/show_bug.cgi?id=83371 |
|
Back to top |
|
|
tobidope n00b
Joined: 17 Aug 2003 Posts: 24 Location: Germany
|
Posted: Sun Feb 27, 2005 9:40 am Post subject: |
|
|
Thanks Chaosite for submitting that ebuild. |
|
Back to top |
|
|
Chaosite Guru
Joined: 13 Dec 2003 Posts: 540 Location: Right over here.
|
Posted: Sun Feb 27, 2005 7:11 pm Post subject: |
|
|
Sure, no problem
It would be great if you guys tried out the ebuild and said "This works for me" or something like that. |
|
Back to top |
|
|
Morimando Guru
Joined: 14 Feb 2005 Posts: 339 Location: Germany
|
Posted: Tue Mar 08, 2005 12:09 pm Post subject: |
|
|
Question:
did Code: | time emerge --searchdesc mencode,
real: 1m16.127s
user: 0m20.659s
sys: 0m7.377s |
would that thing here give me a speedup on AMD 2400XP+, 512MB RAM, complete Reiser4-Fs, pure udev? _________________ That young girl is one of the least benightedly unintelligent organic life forms it has been my profound lack of pleasure not to be able to avoid meeting.
-- Marvin |
|
Back to top |
|
|
Chaosite Guru
Joined: 13 Dec 2003 Posts: 540 Location: Right over here.
|
Posted: Tue Mar 08, 2005 12:25 pm Post subject: |
|
|
Morimando wrote: | Question:
did Code: | time emerge --searchdesc mencode,
real: 1m16.127s
user: 0m20.659s
sys: 0m7.377s |
would that thing here give me a speedup on AMD 2400XP+, 512MB RAM, complete Reiser4-Fs, pure udev? |
Maybe.
Try it and tell us
It did give me a speedup on a Pentium 4, 512MB, /usr/portage on reiser4, and udev. |
|
Back to top |
|
|
Morimando Guru
Joined: 14 Feb 2005 Posts: 339 Location: Germany
|
Posted: Tue Mar 08, 2005 12:37 pm Post subject: |
|
|
Well emerge metadata took
real 1m40
user 0m16
sys 0m9
the emerge --searchdesc mencode:
Code: | real 0m47.228s
user 0m29.326s
sys 0m4.086s |
So well it did speed up a little although emerge metadata was a bit slow.. mightbe that is because it was the first time i did it? _________________ That young girl is one of the least benightedly unintelligent organic life forms it has been my profound lack of pleasure not to be able to avoid meeting.
-- Marvin |
|
Back to top |
|
|
_mikec_ Guru
Joined: 22 May 2004 Posts: 386
|
Posted: Wed Mar 09, 2005 6:04 pm Post subject: |
|
|
i am having this problem:
Code: | xBox xb0x # touch /usr/lib/portage/pym/portage_db_cdb.py
xBox xb0x # nano -w /usr/lib/portage/pym/portage_db_cdb.py // <---pasted the script
xBox xb0x # touch /etc/portage/modules
xBox xb0x # nano -w /etc/portage/modules // <---pasted the script
xBox xb0x # env-update
Traceback (most recent call last):
File "/usr/sbin/env-update", line 10, in ?
import portage
File "/usr/lib/portage/pym/portage.py", line 7064, in ?
portdb=portdbapi(settings["PORTDIR"])
File "/usr/lib/portage/pym/portage.py", line 5062, in __init__
self.eclassdb = eclass_cache(self.porttree_root, self.mysettings)
File "/usr/lib/portage/pym/portage.py", line 4890, in __init__
self.dbmodule = self.settings.load_best_module("eclass_cache.dbmodule")
File "/usr/lib/portage/pym/portage.py", line 1189, in load_best_module
return load_mod(best_mod)
File "/usr/lib/portage/pym/portage.py", line 138, in load_mod
mod = __import__(modname)
File "/usr/lib/portage/pym/portage_db_cdb.py", line 17
import portage_db_template
xBox xb0x # emerge metadata
Traceback (most recent call last):
File "/usr/bin/emerge", line 10, in ?
import portage
File "/usr/lib/portage/pym/portage.py", line 7064, in ?
portdb=portdbapi(settings["PORTDIR"])
File "/usr/lib/portage/pym/portage.py", line 5062, in __init__
self.eclassdb = eclass_cache(self.porttree_root, self.mysettings)
File "/usr/lib/portage/pym/portage.py", line 4890, in __init__
self.dbmodule = self.settings.load_best_module("eclass_cache.dbmodule")
File "/usr/lib/portage/pym/portage.py", line 1189, in load_best_module
return load_mod(best_mod)
File "/usr/lib/portage/pym/portage.py", line 138, in load_mod
mod = __import__(modname)
File "/usr/lib/portage/pym/portage_db_cdb.py", line 17
import portage_db_template
^
SyntaxError: invalid syntax
|
-rw-r--r-- 1 root root 5698 Mar 9 12:52 portage_db_cdb.py
Code: | Portage 2.0.51.19 (default-linux/x86/2004.3, gcc-3.4.3-20050110, glibc-2.3.4.20050125-r0, 2.6.11-nitro0 i686)
|
i delete both files to get emerge command to work again, btw i am compiling mozilla firefox right now so i dont want anything to get screwed up or something. How do i fix this? _________________ (8 Mars 2005 - 07:52:46)
(18 September 2017 - 11:25:35) |
|
Back to top |
|
|
|
|
You cannot post new topics in this forum You cannot reply to topics in this forum You cannot edit your posts in this forum You cannot delete your posts in this forum You cannot vote in polls in this forum
|
|