Hello community,
here is the log from the commit of package createrepo
checked in at Thu Aug 9 14:59:14 CEST 2007.
--------
--- createrepo/createrepo.changes 2007-05-21 12:49:49.000000000 +0200
+++ /mounts/work_src_done/NOARCH/createrepo/createrepo.changes 2007-08-07 15:19:28.375332000 +0200
@@ -1,0 +2,12 @@
+Tue Aug 7 14:06:52 CEST 2007 - cthiel@suse.de
+
+- update to version 0.4.10
+ * Adds a --update option to createrepo.
+ https://lists.dulug.duke.edu/pipermail/rpm-metadata/2007-March/000756.html
+ * dumpMetadata.py: Fix for older rpm versions
+- added createrepo-dmd.patch: introduces dmd to generate and apply deltas between
+ repository metadata
+- added createrepo-0.4.10-fixsymlinks.patch
+- reordered patches
+
+-------------------------------------------------------------------
Old:
----
createrepo-0.4.4-suse-changelogs.patch
createrepo-0.4.9-cachefix.patch
createrepo-0.4.9.tar.bz2
license-to-confirm.patch
missing-tags.patch
New:
----
createrepo-0.4.10-cachefix.patch
createrepo-0.4.10-fixsymlinks.patch
createrepo-0.4.10.tar.bz2
createrepo-dmd.patch
createrepo-license-to-confirm.patch
createrepo-missing-tags.patch
createrepo-suse-changelogs.patch
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ createrepo.spec ++++++
--- /var/tmp/diff_new_pack.Km7894/_old 2007-08-09 14:57:12.000000000 +0200
+++ /var/tmp/diff_new_pack.Km7894/_new 2007-08-09 14:57:12.000000000 +0200
@@ -1,5 +1,5 @@
#
-# spec file for package createrepo (Version 0.4.9)
+# spec file for package createrepo (Version 0.4.10)
#
# Copyright (c) 2007 SUSE LINUX Products GmbH, Nuernberg, Germany.
# This file and all modifications and additions to the pristine
@@ -12,18 +12,20 @@
Name: createrepo
BuildRequires: python
-Version: 0.4.9
-Release: 3
-License: GNU General Public License (GPL)
+Version: 0.4.10
+Release: 1
+License: GPL v2 or later
Summary: Creates a Common Metadata Repository
Group: System/Packages
Source: %{name}-%{version}.tar.bz2
-Patch: missing-tags.patch
-Patch1: %{name}-0.4.4-suse-changelogs.patch
-Patch2: license-to-confirm.patch
-Patch4: %{name}-0.4.9-cachefix.patch
-Patch5: %{name}-0.4.8-try_sqlitecachec.patch
-Patch6: %{name}-0.4.8-cache_utime.patch
+Patch: %{name}-missing-tags.patch
+Patch1: %{name}-suse-changelogs.patch
+Patch2: %{name}-0.4.8-try_sqlitecachec.patch
+Patch3: %{name}-0.4.8-cache_utime.patch
+Patch4: %{name}-dmd.patch
+Patch5: %{name}-0.4.10-fixsymlinks.patch
+Patch6: %{name}-0.4.10-cachefix.patch
+Patch7: %{name}-license-to-confirm.patch
URL: http://linux.duke.edu/metadata/
BuildRoot: %{_tmppath}/%{name}-%{version}-build
BuildArch: noarch
@@ -51,9 +53,11 @@
%patch
%patch1
%patch2
+%patch3
%patch4
-%patch5
+%patch5 -p1
%patch6
+%patch7
%install
%if %{suse_version} <= 1010
@@ -81,6 +85,15 @@
%{_datadir}/%{name}/*
%changelog
+* Tue Aug 07 2007 - cthiel@suse.de
+- update to version 0.4.10
+ * Adds a --update option to createrepo.
+ https://lists.dulug.duke.edu/pipermail/rpm-metadata/2007-March/000756.html
+ * dumpMetadata.py: Fix for older rpm versions
+- added createrepo-dmd.patch: introduces dmd to generate and apply deltas between
+ repository metadata
+- added createrepo-0.4.10-fixsymlinks.patch
+- reordered patches
* Mon May 21 2007 - cthiel@suse.de
- replace rpm.RPMTAG_HDRID with rpm.RPMTAG_SHA1HEADER in createrepo-0.4.8-cachefix.patch
* Wed May 16 2007 - cthiel@suse.de
++++++ createrepo-0.4.9-cachefix.patch -> createrepo-0.4.10-cachefix.patch ++++++
--- createrepo/createrepo-0.4.9-cachefix.patch 2007-05-21 13:11:42.000000000 +0200
+++ /mounts/work_src_done/NOARCH/createrepo/createrepo-0.4.10-cachefix.patch 2007-08-07 15:18:18.915791000 +0200
@@ -1,18 +1,12 @@
--- dumpMetadata.py
+++ dumpMetadata.py
-@@ -599,9 +599,12 @@
- return getChecksum(self.options['sumtype'], fo)
-
- t = []
-- t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))
-- t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
-- t.append("".join(self.hdr[rpm.RPMTAG_HDRID]))
-+ if type(self.hdr[rpm.RPMTAG_SIGGPG]) is not types.NoneType:
-+ t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))
-+ if type(self.hdr[rpm.RPMTAG_SIGPGP]) is not types.NoneType:
-+ t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
+@@ -603,8 +603,8 @@
+ t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))
+ if type(self.hdr[rpm.RPMTAG_SIGPGP]) is not types.NoneType:
+ t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
+- if type(self.hdr[rpm.RPMTAG_HDRID]) is not types.NoneType:
+- t.append("".join(self.hdr[rpm.RPMTAG_HDRID]))
+ if type(self.hdr[rpm.RPMTAG_SHA1HEADER]) is not types.NoneType:
+ t.append("".join(self.hdr[rpm.RPMTAG_SHA1HEADER]))
key = md5.new("".join(t)).hexdigest()
-
++++++ createrepo-0.4.10-fixsymlinks.patch ++++++
--- createrepo/genpkgmetadata.py.20070803 2007-06-07 06:33:34.000000000 -0400
+++ createrepo/genpkgmetadata.py 2007-08-03 15:36:17.000000000 -0400
@@ -83,11 +83,15 @@
def extension_visitor(filelist, dirname, names):
for fn in names:
- if os.path.isdir(fn):
+ if os.path.islink(dirname + '/' + fn):
+ if self.cmds['skip-symlinks']:
+ continue
+ elif os.path.isdir(dirname + '/' + fn):
+ os.path.walk(dirname + '/' + fn, extension_visitor, filelist)
+ continue
+ elif os.path.isdir(dirname + '/' + fn):
continue
- if self.cmds['skip-symlinks'] and os.path.islink(fn):
- continue
- elif fn[-extlen:].lower() == '%s' % (ext):
+ if fn[-extlen:].lower() == '%s' % (ext):
relativepath = dirname.replace(startdir, "", 1)
relativepath = relativepath.lstrip("/")
filelist.append(os.path.join(relativepath,fn))
@@ -325,9 +329,15 @@
def extension_visitor(arg, dirname, names):
for fn in names:
- if os.path.isdir(fn):
+ if os.path.islink(dirname + '/' + fn):
+ if self.cmds['skip-symlinks']:
+ continue
+ elif os.path.isdir(dirname + '/' + fn):
+ os.path.walk(dirname + '/' + fn, extension_visitor, arg)
+ continue
+ elif os.path.isdir(dirname + '/' + fn):
continue
- elif string.lower(fn[-extlen:]) == '%s' % (ext):
+ if string.lower(fn[-extlen:]) == '%s' % (ext):
reldir = os.path.basename(dirname)
if reldir == os.path.basename(directory):
reldir = ""
++++++ createrepo-0.4.9.tar.bz2 -> createrepo-0.4.10.tar.bz2 ++++++
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/ChangeLog new/createrepo-0.4.10/ChangeLog
--- old/createrepo-0.4.9/ChangeLog 2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/ChangeLog 2007-06-07 12:33:34.000000000 +0200
@@ -1,3 +1,20 @@
+2007-06-07 09:17 pnasrat
+
+ * Makefile, genpkgmetadata.py, readMetadata.py, docs/createrepo.8:
+ This patch adds a --update option to createrepo.
+ https://lists.dulug.duke.edu/pipermail/rpm-metadata/2007-March/000756.html
+ Patch from Mike Bonnet
+
+2007-05-18 16:01 pnasrat
+
+ * dumpMetadata.py: Fix for older rpm versions Christoph Thiel
+
+
+2007-05-16 09:05 pnasrat
+
+ * ChangeLog, Makefile, createrepo.spec, genpkgmetadata.py: Update
+ ChangeLog Bump version to 0.4.9
+
2007-05-16 09:02 pnasrat
* dumpMetadata.py: Figure out appropriate dbversion Jeremy Katz
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/createrepo.spec new/createrepo-0.4.10/createrepo.spec
--- old/createrepo-0.4.9/createrepo.spec 2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/createrepo.spec 2007-06-07 12:33:34.000000000 +0200
@@ -1,6 +1,6 @@
Summary: Creates a common metadata repository
Name: createrepo
-Version: 0.4.9
+Version: 0.4.10
Release: 1
License: GPL
Group: System Environment/Base
@@ -36,6 +36,9 @@
%{_mandir}/man8/createrepo.8*
%changelog
+* Thu Jun 07 2007 Paul Nasrat <pnasrat at redhat.com>
+- 0.4.10
+
* Wed May 16 2007 Paul Nasrat <pnasrat at redhat.com>
- 0.4.9
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/docs/createrepo.8 new/createrepo-0.4.10/docs/createrepo.8
--- old/createrepo-0.4.9/docs/createrepo.8 2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/docs/createrepo.8 2007-06-07 12:33:34.000000000 +0200
@@ -32,6 +32,13 @@
createrepo over the same repository of files that do not have a complete
change out of all packages this decreases the processing time dramatically.
.br
+.IP "\fB\--update\fP"
+If metadata already exists in the outputdir and an rpm is unchanged
+(based on file size and mtime) since the metadata was generated, reuse
+the existing metadata rather than recalculating it. In the case of a
+large repository with only a few new or modified rpms this can
+significantly reduce I/O and processing time.
+.br
.IP "\fB\-C --checkts\fP"
Don't generate repo metadata, if their timestamps are newer than its rpms.
This option decreases the processing time drastically again, if you happen
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/dumpMetadata.py new/createrepo-0.4.10/dumpMetadata.py
--- old/createrepo-0.4.9/dumpMetadata.py 2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/dumpMetadata.py 2007-06-07 12:33:34.000000000 +0200
@@ -16,7 +16,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2004 Duke University
-# $Id: dumpMetadata.py,v 1.47 2007/05/16 08:02:18 pnasrat Exp $
+# $Id: dumpMetadata.py,v 1.48 2007/05/18 15:01:16 pnasrat Exp $
import os
import rpm
@@ -587,9 +587,12 @@
return getChecksum(self.options['sumtype'], fo)
t = []
- t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))
- t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
- t.append("".join(self.hdr[rpm.RPMTAG_HDRID]))
+ if type(self.hdr[rpm.RPMTAG_SIGGPG]) is not types.NoneType:
+ t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))
+ if type(self.hdr[rpm.RPMTAG_SIGPGP]) is not types.NoneType:
+ t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
+ if type(self.hdr[rpm.RPMTAG_HDRID]) is not types.NoneType:
+ t.append("".join(self.hdr[rpm.RPMTAG_HDRID]))
key = md5.new("".join(t)).hexdigest()
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/genpkgmetadata.py new/createrepo-0.4.10/genpkgmetadata.py
--- old/createrepo-0.4.9/genpkgmetadata.py 2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/genpkgmetadata.py 2007-06-07 12:33:34.000000000 +0200
@@ -17,7 +17,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2004 Duke University
-# $Id: genpkgmetadata.py,v 1.69 2007/05/16 08:05:44 pnasrat Exp $
+# $Id: genpkgmetadata.py,v 1.70 2007/06/07 08:17:46 pnasrat Exp $
import os
@@ -30,6 +30,7 @@
import shutil
import dumpMetadata
+import readMetadata
from dumpMetadata import _gzipOpen
__version__ = '0.4.9'
@@ -61,6 +62,7 @@
-h, --help = show this help
-V, --version = output version
-p, --pretty = output xml files in pretty format.
+ --update = update existing metadata (if present)
-d, --database = generate the sqlite databases.
""")
@@ -124,6 +126,18 @@
"""all the heavy lifting for the package metadata"""
# rpms we're going to be dealing with
+ if self.cmds['update']:
+ #build the paths
+ basefile = os.path.join(self.cmds['outputdir'], self.cmds['finaldir'], self.cmds['primaryfile'])
+ flfile = os.path.join(self.cmds['outputdir'], self.cmds['finaldir'], self.cmds['filelistsfile'])
+ otherfile = os.path.join(self.cmds['outputdir'], self.cmds['finaldir'], self.cmds['otherfile'])
+ opts = {
+ 'verbose' : self.cmds['verbose'],
+ 'pkgdir' : os.path.normpath(os.path.join(self.cmds['basedir'], directory))
+ }
+ #and scan the old repo
+ self.oldData = readMetadata.MetadataIndex(self.cmds['outputdir'],
+ basefile, flfile, otherfile, opts)
files = self.getFileList(self.cmds['basedir'], directory, '.rpm')
files = self.trimRpms(files)
self.pkgcount = len(files)
@@ -174,61 +188,76 @@
self.otherfile.write('<otherdata xmlns="http://linux.duke.edu/metadata/other" packages="%s">\n' %
self.pkgcount)
+ def _getNodes(self, file, directory, current):
+ basenode = None
+ filesnode = None
+ othernode = None
+ try:
+ rpmdir= os.path.join(self.cmds['basedir'], directory)
+ mdobj = dumpMetadata.RpmMetaData(self.ts, rpmdir, file, self.cmds)
+ except dumpMetadata.MDError, e:
+ errorprint('\n%s - %s' % (e, file))
+ return None
+ try:
+ basenode = dumpMetadata.generateXML(self.basedoc, self.baseroot, self.formatns, mdobj, self.cmds['sumtype'])
+ except dumpMetadata.MDError, e:
+ errorprint(_('\nAn error occurred creating primary metadata: %s') % e)
+ return None
+ try:
+ filesnode = dumpMetadata.fileListXML(self.filesdoc, self.filesroot, mdobj)
+ except dumpMetadata.MDError, e:
+ errorprint(_('\nAn error occurred creating filelists: %s') % e)
+ return None
+ try:
+ othernode = dumpMetadata.otherXML(self.otherdoc, self.otherroot, mdobj)
+ except dumpMetadata.MDError, e:
+ errorprint(_('\nAn error occurred: %s') % e)
+ return None
+ return basenode,filesnode,othernode
+
def writeMetadataDocs(self, files, directory, current=0):
for file in files:
current+=1
- try:
- rpmdir= os.path.join(self.cmds['basedir'], directory)
- mdobj = dumpMetadata.RpmMetaData(self.ts, rpmdir, file, self.cmds)
- if not self.cmds['quiet']:
- if self.cmds['verbose']:
- print '%d/%d - %s' % (current, len(files), file)
- else:
- sys.stdout.write('\r' + ' ' * 80)
- sys.stdout.write("\r%d/%d - %s" % (current, self.pkgcount, file))
- sys.stdout.flush()
- except dumpMetadata.MDError, e:
- errorprint('\n%s - %s' % (e, file))
- continue
- else:
- try:
- node = dumpMetadata.generateXML(self.basedoc, self.baseroot, self.formatns, mdobj, self.cmds['sumtype'])
- except dumpMetadata.MDError, e:
- errorprint(_('\nAn error occurred creating primary metadata: %s') % e)
- continue
+ recycled = False
+ sep = '-'
+ if self.cmds['update']:
+ #see if we can pull the nodes from the old repo
+ nodes = self.oldData.getNodes(file)
+ if nodes is not None:
+ recycled = True
+ sep = '*'
+ if not recycled:
+ #scan rpm files
+ nodes = self._getNodes(file, directory, current)
+ if nodes is None:
+ return
+ basenode, filenode, othernode = nodes
+ del nodes
+ if not self.cmds['quiet']:
+ if self.cmds['verbose']:
+ print '%d/%d %s %s' % (current, self.pkgcount, sep, file)
else:
- output = node.serialize('UTF-8', self.cmds['pretty'])
- self.basefile.write(output)
- self.basefile.write('\n')
- node.unlinkNode()
- node.freeNode()
- del node
+ sys.stdout.write('\r' + ' ' * 80)
+ sys.stdout.write("\r%d/%d %s %s" % (current, self.pkgcount, sep, file))
+ sys.stdout.flush()
+ if basenode is None:
+ continue
- try:
- node = dumpMetadata.fileListXML(self.filesdoc, self.filesroot, mdobj)
- except dumpMetadata.MDError, e:
- errorprint(_('\nAn error occurred creating filelists: %s') % e)
- continue
- else:
- output = node.serialize('UTF-8', self.cmds['pretty'])
- self.flfile.write(output)
- self.flfile.write('\n')
+ for node, outfile in ((basenode,self.basefile),
+ (filenode,self.flfile),
+ (othernode,self.otherfile)):
+ if node is None:
+ break
+ output = node.serialize('UTF-8', self.cmds['pretty'])
+ outfile.write(output)
+ outfile.write('\n')
+ if not recycled:
+ #recycled nodes can be multiply referenced
node.unlinkNode()
node.freeNode()
- del node
+ if recycled:
+ self.oldData.freeNodes(file)
- try:
- node = dumpMetadata.otherXML(self.otherdoc, self.otherroot, mdobj)
- except dumpMetadata.MDError, e:
- errorprint(_('\nAn error occurred: %s') % e)
- continue
- else:
- output = node.serialize('UTF-8', self.cmds['pretty'])
- self.otherfile.write(output)
- self.otherfile.write('\n')
- node.unlinkNode()
- node.freeNode()
- del node
return current
@@ -379,6 +408,7 @@
cmds['checkts'] = False
cmds['mdtimestamp'] = 0
cmds['split'] = False
+ cmds['update'] = False
cmds['outputdir'] = ""
cmds['database'] = False
cmds['file-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$']
@@ -390,7 +420,7 @@
'quiet', 'verbose', 'cachedir=', 'basedir=',
'baseurl=', 'groupfile=', 'checksum=',
'version', 'pretty', 'split', 'outputdir=',
- 'noepoch', 'checkts', 'database',
+ 'noepoch', 'checkts', 'database', 'update',
'skip-symlinks'])
except getopt.error, e:
errorprint(_('Options Error: %s.') % e)
@@ -449,6 +479,8 @@
elif arg in ['-c', '--cachedir']:
cmds['cache'] = True
cmds['cachedir'] = a
+ elif arg == '--update':
+ cmds['update'] = True
elif arg in ['-C', '--checkts']:
cmds['checkts'] = True
elif arg == '--basedir':
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/Makefile new/createrepo-0.4.10/Makefile
--- old/createrepo-0.4.9/Makefile 2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/Makefile 2007-06-07 12:33:34.000000000 +0200
@@ -1,5 +1,5 @@
PACKAGE = createrepo
-VERSION = 0.4.9
+VERSION = 0.4.10
SHELL = /bin/sh
top_srcdir = .
srcdir = .
@@ -41,6 +41,7 @@
MODULES = $(srcdir)/genpkgmetadata.py \
$(srcdir)/dumpMetadata.py \
+ $(srcdir)/readMetadata.py \
$(srcdir)/modifyrepo.py
.SUFFIXES: .py .pyc
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/readMetadata.py new/createrepo-0.4.10/readMetadata.py
--- old/createrepo-0.4.9/readMetadata.py 1970-01-01 01:00:00.000000000 +0100
+++ new/createrepo-0.4.10/readMetadata.py 2007-06-07 12:33:34.000000000 +0200
@@ -0,0 +1,199 @@
+#!/usr/bin/python -t
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# Copyright 2006 Red Hat
+
+import os
+import sys
+import libxml2
+import pprint
+import stat
+
+def errorprint(stuff):
+ print >> sys.stderr, stuff
+
+def _(args):
+ """Stub function for translation"""
+ return args
+
+class MetadataIndex(object):
+
+ def __init__(self, outputdir, basefile, filelistfile, otherfile, opts=None):
+ if opts is None:
+ opts = {}
+ self.opts = opts
+ self.outputdir = outputdir
+ self.files = {'base' : basefile,
+ 'filelist' : filelistfile,
+ 'other' : otherfile}
+ self.scan()
+
+ def scan(self):
+ """Read in and index old repo data"""
+ self.basenodes = {}
+ self.filesnodes = {}
+ self.othernodes = {}
+ self.pkg_ids = {}
+ if self.opts.get('verbose'):
+ print _("Scanning old repo data")
+ for file in self.files.values():
+ if not os.path.exists(file):
+ #cannot scan
+ errorprint(_("Previous repo file missing: %s") % file)
+ return
+ root = libxml2.parseFile(self.files['base']).getRootElement()
+ self._scanPackageNodes(root, self._handleBase)
+ if self.opts.get('verbose'):
+ print _("Indexed %i base nodes" % len(self.basenodes))
+ root = libxml2.parseFile(self.files['filelist']).getRootElement()
+ self._scanPackageNodes(root, self._handleFiles)
+ if self.opts.get('verbose'):
+ print _("Indexed %i filelist nodes" % len(self.filesnodes))
+ root = libxml2.parseFile(self.files['other']).getRootElement()
+ self._scanPackageNodes(root, self._handleOther)
+ if self.opts.get('verbose'):
+ print _("Indexed %i other nodes" % len(self.othernodes))
+ #reverse index pkg ids to track references
+ self.pkgrefs = {}
+ for relpath, pkgid in self.pkg_ids.iteritems():
+ self.pkgrefs.setdefault(pkgid,[]).append(relpath)
+
+ def _scanPackageNodes(self, root, handler):
+ node = root.children
+ while node is not None:
+ if node.type != "element":
+ node = node.next
+ continue
+ if node.name == "package":
+ handler(node)
+ node = node.next
+
+ def _handleBase(self, node):
+ top = node
+ node = node.children
+ pkgid = None
+ mtime = None
+ size = None
+ relpath = None
+ while node is not None:
+ if node.type != "element":
+ node = node.next
+ continue
+ if node.name == "checksum":
+ pkgid = node.content
+ elif node.name == "time":
+ mtime = int(node.prop('file'))
+ elif node.name == "size":
+ size = int(node.prop('package'))
+ elif node.name == "location":
+ relpath = node.prop('href')
+ node = node.next
+ if relpath is None:
+ print _("Incomplete data for node")
+ return
+ if pkgid is None:
+ print _("pkgid missing for %s") % relpath
+ return
+ if mtime is None:
+ print _("mtime missing for %s") % relpath
+ return
+ if size is None:
+ print _("size missing for %s") % relpath
+ return
+ filepath = os.path.join(self.opts['pkgdir'], relpath)
+ try:
+ st = os.stat(filepath)
+ except OSError:
+ #file missing -- ignore
+ return
+ if not stat.S_ISREG(st.st_mode):
+ #ignore non files
+ return
+ #check size and mtime
+ if st.st_size != size:
+ if self.opts.get('verbose'):
+ print _("Size (%i -> %i) changed for file %s") % (size,st.st_size,filepath)
+ return
+ if st.st_mtime != mtime:
+ if self.opts.get('verbose'):
+ print _("Modification time changed for %s") % filepath
+ return
+ #otherwise we index
+ self.basenodes[relpath] = top
+ self.pkg_ids[relpath] = pkgid
+
+ def _handleFiles(self, node):
+ pkgid = node.prop('pkgid')
+ if pkgid:
+ self.filesnodes[pkgid] = node
+
+ def _handleOther(self, node):
+ pkgid = node.prop('pkgid')
+ if pkgid:
+ self.othernodes[pkgid] = node
+
+ def getNodes(self, relpath):
+ """Return base, filelist, and other nodes for file, if they exist
+
+ Returns a tuple of nodes, or None if not found
+ """
+ bnode = self.basenodes.get(relpath,None)
+ if bnode is None:
+ return None
+ pkgid = self.pkg_ids.get(relpath,None)
+ if pkgid is None:
+ print _("No pkgid found for: %s") % relpath
+ return None
+ fnode = self.filesnodes.get(pkgid,None)
+ if fnode is None:
+ return None
+ onode = self.othernodes.get(pkgid,None)
+ if onode is None:
+ return None
+ return bnode, fnode, onode
+
+ def freeNodes(self,relpath):
+ #causing problems
+ """Free up nodes corresponding to file, if possible"""
+ bnode = self.basenodes.get(relpath,None)
+ if bnode is None:
+ print "Missing node for %s" % relpath
+ return
+ bnode.unlinkNode()
+ bnode.freeNode()
+ del self.basenodes[relpath]
+ pkgid = self.pkg_ids.get(relpath,None)
+ if pkgid is None:
+ print _("No pkgid found for: %s") % relpath
+ return None
+ del self.pkg_ids[relpath]
+ dups = self.pkgrefs.get(pkgid)
+ dups.remove(relpath)
+ if len(dups):
+ #still referenced
+ return
+ del self.pkgrefs[pkgid]
+ for nodes in self.filesnodes, self.othernodes:
+ node = nodes.get(pkgid)
+ if node is not None:
+ node.unlinkNode()
+ node.freeNode()
+ del nodes[pkgid]
+
+
+if __name__ == "__main__":
+ #test code - attempts to read a repo in working directory
+ idx = MetadataIndex(".", "repodata/primary.xml.gz", "repodata/filelists.xml.gz",
+ "repodata/other.xml.gz", {'verbose':1})
++++++ createrepo-dmd.patch ++++++
diff --git a/dmd.py b/dmd.py
new file mode 100755
index 0000000..fdce45c
--- /dev/null
+++ b/dmd.py
@@ -0,0 +1,156 @@
+#!/usr/bin/python
+
+# dmd - Generate and apply deltas between repository metadata
+#
+# Copyright (C) 2007 James Bowes
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+import sys
+from lxml.etree import parse, tostring, Element
+
+
+class MdType(object):
+ def __init__(self, namespace, rootelem):
+ self.ns = "http://linux.duke.edu/metadata/%s" % namespace
+ self.sns = "{%s}" % self.ns
+ self.deltasns = "{http://linux.duke.edu/metadata/delta}"
+ self.root = rootelem
+
+ def get_pkg_id(self, pkg):
+ return pkg.findtext(self.sns + "checksum")
+
+ def make_hash(self, tree):
+ pkgshash = {}
+ for pkg in tree:
+ pkgid = self.get_pkg_id(pkg)
+ pkgshash[pkgid] = pkg
+
+ return pkgshash
+
+ def make_pkg_elem(self, pkgid, pkg):
+ pkgelem = Element("package")
+ pkgelem.set('name', pkg.findtext(self.sns + 'name'))
+ pkgelem.set('arch', pkg.findtext(self.sns + 'arch'))
+ pkgelem.set('pkgid', pkgid)
+ verelem = pkg.find(self.sns + 'version')
+ verelem.tag = "version"
+ pkgelem.append(verelem)
+
+ return pkgelem
+
+ def diff_trees(self, oldtree, newtree):
+ oldpkgs = oldtree.getroot().getchildren()
+ newpkgs = newtree.getroot().getchildren()
+
+ oldpkgshash = self.make_hash(oldpkgs)
+ newpkgshash = self.make_hash(newpkgs)
+
+ diff = Element(self.root,
+ nsmap = {None : self.ns,
+ "rpm" : "http://linux.duke.edu/metadata/rpm",
+ "delta" : "http://linux.duke.edu/metadata/delta"})
+ additions = Element("delta:additions")
+ diff.append(additions)
+ removals = Element("delta:removals")
+
+ diff.append(removals)
+
+ for pkgid, pkg in newpkgshash.iteritems():
+ if not oldpkgshash.has_key(pkgid):
+ additions.append(pkg)
+
+ for pkgid, pkg in oldpkgshash.iteritems():
+ if not newpkgshash.has_key(pkgid):
+ pkgelem = self.make_pkg_elem(pkgid, pkg)
+ removals.append(pkgelem)
+
+ diff.set("packages", str(len(removals) + len(additions)))
+
+ print tostring(diff, pretty_print=True)
+
+ def patch_tree(self, oldtree, deltatree):
+ oldroot = oldtree.getroot()
+ oldpkgs = oldroot.getchildren()
+
+ oldpkgshash = self.make_hash(oldpkgs)
+
+ additions = deltatree.find(self.deltasns + 'additions').getchildren()
+ removals = deltatree.find(self.deltasns + 'removals').getchildren()
+
+ for pkg in additions:
+ pkgid = self.get_pkg_id(pkg)
+ if oldpkgshash.has_key(pkgid):
+ print >> sys.stderr, "Package %s already exists" % pkgid
+ sys.exit(1)
+ oldroot.append(pkg)
+
+ for pkg in removals:
+ pkgid = pkg.get('pkgid')
+ if not oldpkgshash.has_key(pkgid):
+ print >> sys.stderr, "Package %s does not exist" % pkgid
+ sys.exit(1)
+ oldroot.remove(oldpkgshash[pkgid])
+
+ oldcount = int(oldroot.get('packages'))
+ newcount = oldcount + len(additions) - len(removals)
+ oldroot.set('packages', str(newcount))
+ print tostring(oldtree, pretty_print=True)
+
+
+class OtherMdType(MdType):
+ def get_pkg_id(self, pkg):
+ return pkg.get('pkgid')
+
+ def make_pkg_elem(self, pkgid, pkg):
+ pkgelem = Element("package")
+ pkgelem.set('name', pkg.get('name'))
+ pkgelem.set('arch', pkg.get('arch'))
+ pkgelem.set('pkgid', pkgid)
+ verelem = pkg.find(self.sns + 'version')
+ verelem.tag = "version"
+
+ return pkgelem
+
+
+mdtypeinfo = {
+ 'primary' : MdType('common', 'metadata'),
+ 'filelists' : OtherMdType('filelists', 'filelists'),
+ 'other' : OtherMdType('other', 'other'),
+ }
+
+
+def usage(progname):
+ print "usage: %s [diff|patch] MDTYPE FILE1 FILE2" % progname
+ sys.exit()
+
+def main(args):
+ if len(args) != 5:
+ usage(args[0])
+ if args[1] not in ('diff', 'patch'):
+ usage(args[0])
+ if args[2] not in ('primary', 'filelists', 'other'):
+ usage(args[0])
+
+ oldtree = parse(args[3])
+ newtree = parse(args[4])
+
+ if args[1] == 'diff':
+ mdtypeinfo[args[2]].diff_trees(oldtree, newtree)
+ else:
+ mdtypeinfo[args[2]].patch_tree(oldtree, newtree)
+
+if __name__ == "__main__":
+ main(sys.argv)
++++++ createrepo-license-to-confirm.patch ++++++
--- dumpMetadata.py
+++ dumpMetadata.py
@@ -18,6 +18,7 @@
# $Id: dumpMetadata.py,v 1.48 2007/05/18 15:01:16 pnasrat Exp $
+import glob
import os
import rpm
import exceptions
@@ -624,7 +625,7 @@
-def generateXML(doc, node, formatns, rpmObj, sumtype):
+def generateXML(doc, node, formatns, susens, rpmObj, sumtype):
"""takes an xml doc object and a package metadata entry node, populates a
package node with the md information"""
ns = node.ns()
@@ -658,6 +659,27 @@
if rpmObj.localurl is not None:
location.newProp('xml:base', rpmObj.localurl)
location.newProp('href', rpmObj.relativepath)
+
+ eulas = glob.glob("%s.eula.*" % os.path.join(rpmObj.options['basedir'], rpmObj.relativepath))
+ if eulas is not None:
+ for eula in eulas:
+ match = re.match("(.*eula)\.(.*)", eula)
+ lang = match.group(2)
+
+ eulaContent = None
+
+ try:
+ fd = open(eula, "r")
+ eulaContent = fd.read()
+ fd.close()
+ except IOError:
+ raise MDError, "Failed read contents of %s" % eula
+
+ if eulaContent is not None:
+ eulaNode = pkgNode.newChild(susens, 'license-to-confirm', None)
+ eulaNode.newProp('lang', lang)
+ eulaNode.addContent(utf8String(eulaContent))
+
format = pkgNode.newChild(ns, 'format', None)
for tag in ['license', 'vendor', 'group', 'buildhost', 'sourcerpm']:
value = rpmObj.tagByName(tag)
--- genpkgmetadata.py
+++ genpkgmetadata.py
@@ -161,11 +161,12 @@
self.baseroot = self.basedoc.newChild(None, "metadata", None)
basens = self.baseroot.newNs('http://linux.duke.edu/metadata/common', None)
self.formatns = self.baseroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm')
+ self.susens = self.baseroot.newNs('http://novell.com/package/metadata/suse/common', 'suse')
self.baseroot.setNs(basens)
basefilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['primaryfile'])
self.basefile = _gzipOpen(basefilepath, 'w')
self.basefile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
- self.basefile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">\n' %
+ self.basefile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" xmlns:suse="http://novell.com/package/metadata/suse/common" packages="%s">\n' %
self.pkgcount)
def _setupFilelists(self):
@@ -203,7 +204,7 @@
errorprint('\n%s - %s' % (e, file))
return None
try:
- basenode = dumpMetadata.generateXML(self.basedoc, self.baseroot, self.formatns, mdobj, self.cmds['sumtype'])
+ basenode = dumpMetadata.generateXML(self.basedoc, self.baseroot, self.formatns, self.susens, mdobj, self.cmds['sumtype'])
except dumpMetadata.MDError, e:
errorprint(_('\nAn error occurred creating primary metadata: %s') % e)
return None
++++++ createrepo-missing-tags.patch ++++++
--- dumpMetadata.py
+++ dumpMetadata.py
@@ -267,14 +267,10 @@
return returnflags
if type(flags) is not types.ListType:
- newflag = flags & 0xf
- returnflags.append(newflag)
+ returnflags.append(flags)
else:
for flag in flags:
- newflag = flag
- if flag is not None:
- newflag = flag & 0xf
- returnflags.append(newflag)
+ returnflags.append(flag)
return returnflags
def _checkPreReq(self, flags):
@@ -516,36 +512,48 @@
lst = zip(names, flags, ver, prereq)
return self._uniq(lst)
- def obsoletesList(self):
+ def tagsList(self, name, flags, version):
lst = []
- names = self.hdr[rpm.RPMTAG_OBSOLETENAME]
- tmpflags = self.hdr[rpm.RPMTAG_OBSOLETEFLAGS]
+ names = self.hdr[name]
+ tmpflags = self.hdr[flags]
flags = self._correctFlags(tmpflags)
- ver = self._correctVersion(self.hdr[rpm.RPMTAG_OBSOLETEVERSION])
+ ver = self._correctVersion(self.hdr[version])
if names is not None:
lst = zip(names, flags, ver)
return self._uniq(lst)
+ def obsoletesList(self):
+ return self.tagsList(rpm.RPMTAG_OBSOLETENAME, rpm.RPMTAG_OBSOLETEFLAGS,
+ rpm.RPMTAG_OBSOLETEVERSION)
+
def conflictsList(self):
- lst = []
- names = self.hdr[rpm.RPMTAG_CONFLICTNAME]
- tmpflags = self.hdr[rpm.RPMTAG_CONFLICTFLAGS]
- flags = self._correctFlags(tmpflags)
- ver = self._correctVersion(self.hdr[rpm.RPMTAG_CONFLICTVERSION])
- if names is not None:
- lst = zip(names, flags, ver)
- return self._uniq(lst)
+ return self.tagsList(rpm.RPMTAG_CONFLICTNAME, rpm.RPMTAG_CONFLICTFLAGS,
+ rpm.RPMTAG_CONFLICTVERSION)
def providesList(self):
- lst = []
- names = self.hdr[rpm.RPMTAG_PROVIDENAME]
- tmpflags = self.hdr[rpm.RPMTAG_PROVIDEFLAGS]
- flags = self._correctFlags(tmpflags)
- ver = self._correctVersion(self.hdr[rpm.RPMTAG_PROVIDEVERSION])
- if names is not None:
- lst = zip(names, flags, ver)
- return self._uniq(lst)
+ return self.tagsList(rpm.RPMTAG_PROVIDENAME, rpm.RPMTAG_PROVIDEFLAGS,
+ rpm.RPMTAG_PROVIDEVERSION)
+ def enhancesList(self):
+ lst = self.tagsList(rpm.RPMTAG_ENHANCESNAME, rpm.RPMTAG_ENHANCESFLAGS,
+ rpm.RPMTAG_ENHANCESVERSION)
+ return [ l for l in lst if not l[1] & rpm.RPMSENSE_STRONG ]
+
+ def supplementsList(self):
+ lst = self.tagsList(rpm.RPMTAG_ENHANCESNAME, rpm.RPMTAG_ENHANCESFLAGS,
+ rpm.RPMTAG_ENHANCESVERSION)
+ return [ l for l in lst if l[1] & rpm.RPMSENSE_STRONG ]
+
+ def suggestsList(self):
+ lst = self.tagsList(rpm.RPMTAG_SUGGESTSNAME, rpm.RPMTAG_SUGGESTSFLAGS,
+ rpm.RPMTAG_SUGGESTSVERSION)
+ return [ l for l in lst if not l[1] & rpm.RPMSENSE_STRONG ]
+
+ def recommendsList(self):
+ lst = self.tagsList(rpm.RPMTAG_SUGGESTSNAME, rpm.RPMTAG_SUGGESTSFLAGS,
+ rpm.RPMTAG_SUGGESTSVERSION)
+ return [ l for l in lst if l[1] & rpm.RPMSENSE_STRONG ]
+
def changelogLists(self):
lst = []
names = self.listTagByName('changelogname')
@@ -629,12 +637,18 @@
hr.newProp('end', str(rpmObj.rangeend))
for (lst, nodename) in [(rpmObj.providesList(), 'provides'),
(rpmObj.conflictsList(), 'conflicts'),
- (rpmObj.obsoletesList(), 'obsoletes')]:
+ (rpmObj.obsoletesList(), 'obsoletes'),
+ (rpmObj.enhancesList(), 'enhances'),
+ (rpmObj.supplementsList(), 'supplements'),
+ (rpmObj.suggestsList(), 'suggests'),
+ (rpmObj.recommendsList(), 'recommends')]:
if len(lst) > 0:
rpconode = format.newChild(formatns, nodename, None)
for (name, flags, (e,v,r)) in lst:
entry = rpconode.newChild(formatns, 'entry', None)
entry.newProp('name', name)
+ if flags is not None:
+ flags = flags & 0xf
if flags != 0:
if flags == 2: arg = 'LT'
if flags == 4: arg = 'GT'
@@ -656,6 +670,8 @@
for (name, flags, (e,v,r), prereq) in depsList:
entry = rpconode.newChild(formatns, 'entry', None)
entry.newProp('name', name)
+ if flags is not None:
+ flags = flags & 0xf
if flags != 0:
if flags == 2: arg = 'LT'
if flags == 4: arg = 'GT'
++++++ createrepo-suse-changelogs.patch ++++++
--- dumpMetadata.py
+++ dumpMetadata.py
@@ -744,7 +744,10 @@
for (name, time, text) in clogs:
clog = pkg.newChild(None, 'changelog', None)
clog.addContent(utf8String(text))
- clog.newProp('author', utf8String(name))
+ if name[0] != '-':
+ clog.newProp('author', utf8String(name))
+ else:
+ clog.newProp('author', utf8String(name[2:]))
clog.newProp('date', str(time))
return pkg
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Remember to have fun...
---------------------------------------------------------------------
To unsubscribe, e-mail: opensuse-commit+unsubscribe@opensuse.org
For additional commands, e-mail: opensuse-commit+help@opensuse.org