Mailinglist Archive: opensuse-commit (2092 mails)

< Previous Next >
commit createrepo
  • From: root@xxxxxxxxxxxxxxx (h_root)
  • Date: Thu, 09 Aug 2007 14:59:14 +0200
  • Message-id: <20070809125915.386E9678180@xxxxxxxxxxxxxxx>

Hello community,

here is the log from the commit of package createrepo
checked in at Thu Aug 9 14:59:14 CEST 2007.

--------
--- createrepo/createrepo.changes       2007-05-21 12:49:49.000000000 +0200
+++ /mounts/work_src_done/NOARCH/createrepo/createrepo.changes  2007-08-07 15:19:28.375332000 +0200
@@ -1,0 +2,12 @@
+Tue Aug  7 14:06:52 CEST 2007 - cthiel@xxxxxxx
+
+- update to version 0.4.10
+  * Adds a --update option to createrepo.
+    https://lists.dulug.duke.edu/pipermail/rpm-metadata/2007-March/000756.html
+  * dumpMetadata.py: Fix for older rpm versions
+- added createrepo-dmd.patch: introduces dmd to generate and apply deltas between
+  repository metadata
+- added createrepo-0.4.10-fixsymlinks.patch
+- reordered patches
+
+-------------------------------------------------------------------

Old:
----
  createrepo-0.4.4-suse-changelogs.patch
  createrepo-0.4.9-cachefix.patch
  createrepo-0.4.9.tar.bz2
  license-to-confirm.patch
  missing-tags.patch

New:
----
  createrepo-0.4.10-cachefix.patch
  createrepo-0.4.10-fixsymlinks.patch
  createrepo-0.4.10.tar.bz2
  createrepo-dmd.patch
  createrepo-license-to-confirm.patch
  createrepo-missing-tags.patch
  createrepo-suse-changelogs.patch

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

Other differences:
------------------
++++++ createrepo.spec ++++++
--- /var/tmp/diff_new_pack.Km7894/_old  2007-08-09 14:57:12.000000000 +0200
+++ /var/tmp/diff_new_pack.Km7894/_new  2007-08-09 14:57:12.000000000 +0200
@@ -1,5 +1,5 @@
 #
-# spec file for package createrepo (Version 0.4.9)
+# spec file for package createrepo (Version 0.4.10)
 #
 # Copyright (c) 2007 SUSE LINUX Products GmbH, Nuernberg, Germany.
 # This file and all modifications and additions to the pristine
@@ -12,18 +12,20 @@
 
 Name:           createrepo
 BuildRequires:  python
-Version:        0.4.9
-Release:        3
-License:        GNU General Public License (GPL)
+Version:        0.4.10
+Release:        1
+License:        GPL v2 or later
 Summary:        Creates a Common Metadata Repository
 Group:          System/Packages
 Source:         %{name}-%{version}.tar.bz2
-Patch:          missing-tags.patch
-Patch1:         %{name}-0.4.4-suse-changelogs.patch
-Patch2:         license-to-confirm.patch
-Patch4:         %{name}-0.4.9-cachefix.patch
-Patch5:         %{name}-0.4.8-try_sqlitecachec.patch
-Patch6:         %{name}-0.4.8-cache_utime.patch
+Patch:          %{name}-missing-tags.patch
+Patch1:         %{name}-suse-changelogs.patch
+Patch2:         %{name}-0.4.8-try_sqlitecachec.patch
+Patch3:         %{name}-0.4.8-cache_utime.patch
+Patch4:         %{name}-dmd.patch
+Patch5:         %{name}-0.4.10-fixsymlinks.patch
+Patch6:         %{name}-0.4.10-cachefix.patch
+Patch7:         %{name}-license-to-confirm.patch
 URL:            http://linux.duke.edu/metadata/
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
 BuildArch:      noarch
@@ -51,9 +53,11 @@
 %patch
 %patch1
 %patch2
+%patch3
 %patch4
-%patch5
+%patch5 -p1
 %patch6
+%patch7
 
 %install
 %if %{suse_version} <= 1010
@@ -81,6 +85,15 @@
 %{_datadir}/%{name}/*
 
 %changelog
+* Tue Aug 07 2007 - cthiel@xxxxxxx
+- update to version 0.4.10
+  * Adds a --update option to createrepo.
+  https://lists.dulug.duke.edu/pipermail/rpm-metadata/2007-March/000756.html
+  * dumpMetadata.py: Fix for older rpm versions
+- added createrepo-dmd.patch: introduces dmd to generate and apply deltas between
+  repository metadata
+- added createrepo-0.4.10-fixsymlinks.patch
+- reordered patches
 * Mon May 21 2007 - cthiel@xxxxxxx
 - replace rpm.RPMTAG_HDRID with rpm.RPMTAG_SHA1HEADER in createrepo-0.4.8-cachefix.patch
 * Wed May 16 2007 - cthiel@xxxxxxx

++++++ createrepo-0.4.9-cachefix.patch -> createrepo-0.4.10-cachefix.patch ++++++
--- createrepo/createrepo-0.4.9-cachefix.patch  2007-05-21 13:11:42.000000000 +0200
+++ /mounts/work_src_done/NOARCH/createrepo/createrepo-0.4.10-cachefix.patch    2007-08-07 15:18:18.915791000 +0200
@@ -1,18 +1,12 @@
 --- dumpMetadata.py
 +++ dumpMetadata.py
-@@ -599,9 +599,12 @@
-             return getChecksum(self.options['sumtype'], fo)

-         t = []
--        t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))   
--        t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
--        t.append("".join(self.hdr[rpm.RPMTAG_HDRID]))
-+        if type(self.hdr[rpm.RPMTAG_SIGGPG]) is not types.NoneType:
-+            t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))   
-+        if type(self.hdr[rpm.RPMTAG_SIGPGP]) is not types.NoneType:
-+            t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
+@@ -603,8 +603,8 @@
+             t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))   
+         if type(self.hdr[rpm.RPMTAG_SIGPGP]) is not types.NoneType:
+             t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
+-        if type(self.hdr[rpm.RPMTAG_HDRID]) is not types.NoneType:
+-            t.append("".join(self.hdr[rpm.RPMTAG_HDRID]))
 +        if type(self.hdr[rpm.RPMTAG_SHA1HEADER]) is not types.NoneType:
 +            t.append("".join(self.hdr[rpm.RPMTAG_SHA1HEADER]))
  
          key = md5.new("".join(t)).hexdigest()
-                                         

++++++ createrepo-0.4.10-fixsymlinks.patch ++++++
--- createrepo/genpkgmetadata.py.20070803       2007-06-07 06:33:34.000000000 -0400
+++ createrepo/genpkgmetadata.py        2007-08-03 15:36:17.000000000 -0400
@@ -83,11 +83,15 @@
 
         def extension_visitor(filelist, dirname, names):
             for fn in names:
-                if os.path.isdir(fn):
+                if os.path.islink(dirname + '/' + fn):
+                    if self.cmds['skip-symlinks']:
+                        continue
+                    elif os.path.isdir(dirname + '/' + fn):
+                        os.path.walk(dirname + '/' + fn, extension_visitor, filelist)
+                        continue
+                elif os.path.isdir(dirname + '/' + fn):
                     continue
-                if self.cmds['skip-symlinks'] and os.path.islink(fn):
-                    continue
-                elif fn[-extlen:].lower() == '%s' % (ext):
+                if fn[-extlen:].lower() == '%s' % (ext):
                     relativepath = dirname.replace(startdir, "", 1)
                     relativepath = relativepath.lstrip("/")
                     filelist.append(os.path.join(relativepath,fn))
@@ -325,9 +329,15 @@
 
         def extension_visitor(arg, dirname, names):
             for fn in names:
-                if os.path.isdir(fn):
+                if os.path.islink(dirname + '/' + fn):
+                    if self.cmds['skip-symlinks']:
+                        continue
+                    elif os.path.isdir(dirname + '/' + fn):
+                        os.path.walk(dirname + '/' + fn, extension_visitor, arg)
+                        continue
+                elif os.path.isdir(dirname + '/' + fn):
                     continue
-                elif string.lower(fn[-extlen:]) == '%s' % (ext):
+                if string.lower(fn[-extlen:]) == '%s' % (ext):
                     reldir = os.path.basename(dirname)
                     if reldir == os.path.basename(directory):
                         reldir = ""
++++++ createrepo-0.4.9.tar.bz2 -> createrepo-0.4.10.tar.bz2 ++++++
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/ChangeLog new/createrepo-0.4.10/ChangeLog
--- old/createrepo-0.4.9/ChangeLog      2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/ChangeLog     2007-06-07 12:33:34.000000000 +0200
@@ -1,3 +1,20 @@
+2007-06-07 09:17  pnasrat
+
+       * Makefile, genpkgmetadata.py, readMetadata.py, docs/createrepo.8:
+         This patch adds a --update option to createrepo.
+         https://lists.dulug.duke.edu/pipermail/rpm-metadata/2007-March/000756.html
+         Patch from Mike Bonnet <mikeb@xxxxxxxxxx>
+
+2007-05-18 16:01  pnasrat
+
+       * dumpMetadata.py: Fix for older rpm versions Christoph Thiel
+         <cthiel@xxxxxxx>
+
+2007-05-16 09:05  pnasrat
+
+       * ChangeLog, Makefile, createrepo.spec, genpkgmetadata.py: Update
+         ChangeLog Bump version to 0.4.9
+
 2007-05-16 09:02  pnasrat
 
        * dumpMetadata.py: Figure out appropriate dbversion Jeremy Katz
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/createrepo.spec new/createrepo-0.4.10/createrepo.spec
--- old/createrepo-0.4.9/createrepo.spec        2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/createrepo.spec       2007-06-07 12:33:34.000000000 +0200
@@ -1,6 +1,6 @@
 Summary: Creates a common metadata repository
 Name: createrepo
-Version: 0.4.9
+Version: 0.4.10
 Release: 1
 License: GPL
 Group: System Environment/Base
@@ -36,6 +36,9 @@
 %{_mandir}/man8/createrepo.8*
 
 %changelog
+* Thu Jun 07 2007 Paul Nasrat <pnasrat at redhat.com>
+- 0.4.10
+
 * Wed May 16 2007 Paul Nasrat <pnasrat at redhat.com>
 - 0.4.9 
 
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/docs/createrepo.8 new/createrepo-0.4.10/docs/createrepo.8
--- old/createrepo-0.4.9/docs/createrepo.8      2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/docs/createrepo.8     2007-06-07 12:33:34.000000000 +0200
@@ -32,6 +32,13 @@
 createrepo over the same repository of files that do not have a complete
 change out of all packages this decreases the processing time dramatically.
 .br
+.IP "\fB\--update\fP"
+If metadata already exists in the outputdir and an rpm is unchanged
+(based on file size and mtime) since the metadata was generated, reuse
+the existing metadata rather than recalculating it. In the case of a
+large repository with only a few new or modified rpms this can
+significantly reduce I/O and processing time.
+.br
 .IP "\fB\-C --checkts\fP"
 Don't generate repo metadata, if their timestamps are newer than its rpms.
 This option decreases the processing time drastically again, if you happen
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/dumpMetadata.py new/createrepo-0.4.10/dumpMetadata.py
--- old/createrepo-0.4.9/dumpMetadata.py        2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/dumpMetadata.py       2007-06-07 12:33:34.000000000 +0200
@@ -16,7 +16,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 # Copyright 2004 Duke University
 
-# $Id: dumpMetadata.py,v 1.47 2007/05/16 08:02:18 pnasrat Exp $
+# $Id: dumpMetadata.py,v 1.48 2007/05/18 15:01:16 pnasrat Exp $
 
 import os
 import rpm
@@ -587,9 +587,12 @@
             return getChecksum(self.options['sumtype'], fo)
 
         t = []
-        t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))   
-        t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
-        t.append("".join(self.hdr[rpm.RPMTAG_HDRID]))
+        if type(self.hdr[rpm.RPMTAG_SIGGPG]) is not types.NoneType:
+            t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG]))   
+        if type(self.hdr[rpm.RPMTAG_SIGPGP]) is not types.NoneType:
+            t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP]))
+        if type(self.hdr[rpm.RPMTAG_HDRID]) is not types.NoneType:
+            t.append("".join(self.hdr[rpm.RPMTAG_HDRID]))
 
         key = md5.new("".join(t)).hexdigest()
                                         
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/genpkgmetadata.py new/createrepo-0.4.10/genpkgmetadata.py
--- old/createrepo-0.4.9/genpkgmetadata.py      2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/genpkgmetadata.py     2007-06-07 12:33:34.000000000 +0200
@@ -17,7 +17,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 # Copyright 2004 Duke University
 
-# $Id: genpkgmetadata.py,v 1.69 2007/05/16 08:05:44 pnasrat Exp $
+# $Id: genpkgmetadata.py,v 1.70 2007/06/07 08:17:46 pnasrat Exp $
 
 
 import os
@@ -30,6 +30,7 @@
 import shutil
 
 import dumpMetadata
+import readMetadata
 from dumpMetadata import _gzipOpen
 __version__ = '0.4.9'
 
@@ -61,6 +62,7 @@
      -h, --help = show this help
      -V, --version = output version
      -p, --pretty = output xml files in pretty format.
+     --update = update existing metadata (if present)
      -d, --database = generate the sqlite databases.
     """)
 
@@ -124,6 +126,18 @@
         """all the heavy lifting for the package metadata"""
 
         # rpms we're going to be dealing with
+        if self.cmds['update']:
+            #build the paths
+            basefile = os.path.join(self.cmds['outputdir'], self.cmds['finaldir'], self.cmds['primaryfile'])
+            flfile = os.path.join(self.cmds['outputdir'], self.cmds['finaldir'], self.cmds['filelistsfile'])
+            otherfile = os.path.join(self.cmds['outputdir'], self.cmds['finaldir'], self.cmds['otherfile'])
+            opts = {
+                'verbose' : self.cmds['verbose'],
+                'pkgdir' : os.path.normpath(os.path.join(self.cmds['basedir'], directory))
+            }
+            #and scan the old repo
+            self.oldData = readMetadata.MetadataIndex(self.cmds['outputdir'],
+                                                      basefile, flfile, otherfile, opts)
         files = self.getFileList(self.cmds['basedir'], directory, '.rpm')
         files = self.trimRpms(files)
         self.pkgcount = len(files)
@@ -174,61 +188,76 @@
         self.otherfile.write('<otherdata xmlns="http://linux.duke.edu/metadata/other" packages="%s";>\n' %
                        self.pkgcount)
 
+    def _getNodes(self, file, directory, current):
+        basenode = None
+        filesnode = None
+        othernode = None
+        try:
+            rpmdir= os.path.join(self.cmds['basedir'], directory)
+            mdobj = dumpMetadata.RpmMetaData(self.ts, rpmdir, file, self.cmds)
+        except dumpMetadata.MDError, e:
+            errorprint('\n%s - %s' % (e, file))
+            return None
+        try:
+            basenode = dumpMetadata.generateXML(self.basedoc, self.baseroot, self.formatns, mdobj, self.cmds['sumtype'])
+        except dumpMetadata.MDError, e:
+            errorprint(_('\nAn error occurred creating primary metadata: %s') % e)
+            return None
+        try:
+            filesnode = dumpMetadata.fileListXML(self.filesdoc, self.filesroot, mdobj)
+        except dumpMetadata.MDError, e:
+            errorprint(_('\nAn error occurred creating filelists: %s') % e)
+            return None
+        try:
+            othernode = dumpMetadata.otherXML(self.otherdoc, self.otherroot, mdobj)
+        except dumpMetadata.MDError, e:
+            errorprint(_('\nAn error occurred: %s') % e)
+            return None
+        return basenode,filesnode,othernode
+
     def writeMetadataDocs(self, files, directory, current=0):
         for file in files:
             current+=1
-            try:
-                rpmdir= os.path.join(self.cmds['basedir'], directory)
-                mdobj = dumpMetadata.RpmMetaData(self.ts, rpmdir, file, self.cmds)
-                if not self.cmds['quiet']:
-                    if self.cmds['verbose']:
-                        print '%d/%d - %s' % (current, len(files), file)
-                    else:
-                        sys.stdout.write('\r' + ' ' * 80)
-                        sys.stdout.write("\r%d/%d - %s" % (current, self.pkgcount, file))
-                        sys.stdout.flush()
-            except dumpMetadata.MDError, e:
-                errorprint('\n%s - %s' % (e, file))
-                continue
-            else:
-                try:
-                    node = dumpMetadata.generateXML(self.basedoc, self.baseroot, self.formatns, mdobj, self.cmds['sumtype'])
-                except dumpMetadata.MDError, e:
-                    errorprint(_('\nAn error occurred creating primary metadata: %s') % e)
-                    continue
+            recycled = False
+            sep = '-'
+            if self.cmds['update']:
+                #see if we can pull the nodes from the old repo
+                nodes = self.oldData.getNodes(file)
+                if nodes is not None:
+                    recycled = True
+                    sep = '*'
+            if not recycled:
+                #scan rpm files
+                nodes = self._getNodes(file, directory, current)
+            if nodes is None:
+                return
+            basenode, filenode, othernode = nodes
+            del nodes
+            if not self.cmds['quiet']:
+                if self.cmds['verbose']:
+                    print '%d/%d %s %s' % (current, self.pkgcount, sep, file)
                 else:
-                    output = node.serialize('UTF-8', self.cmds['pretty'])
-                    self.basefile.write(output)
-                    self.basefile.write('\n')
-                    node.unlinkNode()
-                    node.freeNode()
-                    del node
+                    sys.stdout.write('\r' + ' ' * 80)
+                    sys.stdout.write("\r%d/%d %s %s" % (current, self.pkgcount, sep, file))
+                    sys.stdout.flush()
+            if basenode is None:
+                continue
 
-                try:
-                    node = dumpMetadata.fileListXML(self.filesdoc, self.filesroot, mdobj)
-                except dumpMetadata.MDError, e:
-                    errorprint(_('\nAn error occurred creating filelists: %s') % e)
-                    continue
-                else:
-                    output = node.serialize('UTF-8', self.cmds['pretty'])
-                    self.flfile.write(output)
-                    self.flfile.write('\n')
+            for node, outfile in ((basenode,self.basefile),
+                                  (filenode,self.flfile),
+                                  (othernode,self.otherfile)):
+                if node is None:
+                    break
+                output = node.serialize('UTF-8', self.cmds['pretty'])
+                outfile.write(output)
+                outfile.write('\n')
+                if not recycled:
+                    #recycled nodes can be multiply referenced
                     node.unlinkNode()
                     node.freeNode()
-                    del node
+            if recycled:
+                self.oldData.freeNodes(file)
 
-                try:
-                    node = dumpMetadata.otherXML(self.otherdoc, self.otherroot, mdobj)
-                except dumpMetadata.MDError, e:
-                    errorprint(_('\nAn error occurred: %s') % e)
-                    continue
-                else:
-                    output = node.serialize('UTF-8', self.cmds['pretty'])
-                    self.otherfile.write(output)
-                    self.otherfile.write('\n')
-                    node.unlinkNode()
-                    node.freeNode()
-                    del node
         return current
 
 
@@ -379,6 +408,7 @@
     cmds['checkts'] = False
     cmds['mdtimestamp'] = 0
     cmds['split'] = False
+    cmds['update'] = False
     cmds['outputdir'] = ""
     cmds['database'] = False
     cmds['file-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$']
@@ -390,7 +420,7 @@
                                                                   'quiet', 'verbose', 'cachedir=', 'basedir=',
                                                                   'baseurl=', 'groupfile=', 'checksum=',
                                                                   'version', 'pretty', 'split', 'outputdir=',
-                                                                  'noepoch', 'checkts', 'database', 
+                                                                  'noepoch', 'checkts', 'database', 'update',
                                                                   'skip-symlinks'])
     except getopt.error, e:
         errorprint(_('Options Error: %s.') % e)
@@ -449,6 +479,8 @@
             elif arg in ['-c', '--cachedir']:
                 cmds['cache'] = True
                 cmds['cachedir'] = a
+            elif arg == '--update':
+                cmds['update'] = True
             elif arg in ['-C', '--checkts']:
                 cmds['checkts'] = True
             elif arg == '--basedir':
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/Makefile new/createrepo-0.4.10/Makefile
--- old/createrepo-0.4.9/Makefile       2007-05-16 10:05:49.000000000 +0200
+++ new/createrepo-0.4.10/Makefile      2007-06-07 12:33:34.000000000 +0200
@@ -1,5 +1,5 @@
 PACKAGE = createrepo
-VERSION = 0.4.9
+VERSION = 0.4.10
 SHELL = /bin/sh
 top_srcdir = .
 srcdir = .
@@ -41,6 +41,7 @@
 
 MODULES = $(srcdir)/genpkgmetadata.py \
                  $(srcdir)/dumpMetadata.py \
+                 $(srcdir)/readMetadata.py \
                  $(srcdir)/modifyrepo.py
 
 .SUFFIXES: .py .pyc
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/createrepo-0.4.9/readMetadata.py new/createrepo-0.4.10/readMetadata.py
--- old/createrepo-0.4.9/readMetadata.py        1970-01-01 01:00:00.000000000 +0100
+++ new/createrepo-0.4.10/readMetadata.py       2007-06-07 12:33:34.000000000 +0200
@@ -0,0 +1,199 @@
+#!/usr/bin/python -t
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# Copyright 2006 Red Hat
+
+import os
+import sys
+import libxml2
+import pprint
+import stat
+
+def errorprint(stuff):
+    print >> sys.stderr, stuff
+
+def _(args):
+    """Stub function for translation"""
+    return args
+
+class MetadataIndex(object):
+
+    def __init__(self, outputdir, basefile, filelistfile, otherfile, opts=None):
+        if opts is None:
+            opts = {}
+        self.opts = opts
+        self.outputdir = outputdir
+        self.files = {'base' : basefile,
+                      'filelist' : filelistfile,
+                      'other' : otherfile}
+        self.scan()
+
+    def scan(self):
+        """Read in and index old repo data"""
+        self.basenodes = {}
+        self.filesnodes = {}
+        self.othernodes = {}
+        self.pkg_ids = {}
+        if self.opts.get('verbose'):
+            print _("Scanning old repo data")
+        for file in self.files.values():
+            if not os.path.exists(file):
+                #cannot scan
+                errorprint(_("Previous repo file missing: %s") % file)
+                return
+        root = libxml2.parseFile(self.files['base']).getRootElement()
+        self._scanPackageNodes(root, self._handleBase)
+        if self.opts.get('verbose'):
+            print _("Indexed %i base nodes" % len(self.basenodes))
+        root = libxml2.parseFile(self.files['filelist']).getRootElement()
+        self._scanPackageNodes(root, self._handleFiles)
+        if self.opts.get('verbose'):
+            print _("Indexed %i filelist nodes" % len(self.filesnodes))
+        root = libxml2.parseFile(self.files['other']).getRootElement()
+        self._scanPackageNodes(root, self._handleOther)
+        if self.opts.get('verbose'):
+            print _("Indexed %i other nodes" % len(self.othernodes))
+        #reverse index pkg ids to track references
+        self.pkgrefs = {}
+        for relpath, pkgid in self.pkg_ids.iteritems():
+            self.pkgrefs.setdefault(pkgid,[]).append(relpath)
+
+    def _scanPackageNodes(self, root, handler):
+        node = root.children
+        while node is not None:
+            if node.type != "element":
+                node = node.next
+                continue
+            if node.name == "package":
+                handler(node)
+            node = node.next
+
+    def _handleBase(self, node):
+        top = node
+        node = node.children
+        pkgid = None
+        mtime = None
+        size = None
+        relpath = None
+        while node is not None:
+            if node.type != "element":
+                node = node.next
+                continue
+            if node.name == "checksum":
+                pkgid = node.content
+            elif node.name == "time":
+                mtime = int(node.prop('file'))
+            elif node.name == "size":
+                size = int(node.prop('package'))
+            elif node.name == "location":
+                relpath = node.prop('href')
+            node = node.next
+        if relpath is None:
+            print _("Incomplete data for node")
+            return
+        if pkgid is None:
+            print _("pkgid missing for %s") % relpath
+            return
+        if mtime is None:
+            print _("mtime missing for %s") % relpath
+            return
+        if size is None:
+            print _("size missing for %s") % relpath
+            return
+        filepath = os.path.join(self.opts['pkgdir'], relpath)
+        try:
+            st = os.stat(filepath)
+        except OSError:
+            #file missing -- ignore
+            return
+        if not stat.S_ISREG(st.st_mode):
+            #ignore non files
+            return
+        #check size and mtime
+        if st.st_size != size:
+            if self.opts.get('verbose'):
+                print _("Size (%i -> %i) changed for file %s") % (size,st.st_size,filepath)
+            return
+        if st.st_mtime != mtime:
+            if self.opts.get('verbose'):
+                print _("Modification time changed for %s") % filepath
+            return
+        #otherwise we index
+        self.basenodes[relpath] = top
+        self.pkg_ids[relpath] = pkgid
+
+    def _handleFiles(self, node):
+        pkgid = node.prop('pkgid')
+        if pkgid:
+            self.filesnodes[pkgid] = node
+
+    def _handleOther(self, node):
+        pkgid = node.prop('pkgid')
+        if pkgid:
+            self.othernodes[pkgid] = node
+
+    def getNodes(self, relpath):
+        """Return base, filelist, and other nodes for file, if they exist
+
+        Returns a tuple of nodes, or None if not found
+        """
+        bnode = self.basenodes.get(relpath,None)
+        if bnode is None:
+            return None
+        pkgid = self.pkg_ids.get(relpath,None)
+        if pkgid is None:
+            print _("No pkgid found for: %s") % relpath
+            return None
+        fnode = self.filesnodes.get(pkgid,None)
+        if fnode is None:
+            return None
+        onode = self.othernodes.get(pkgid,None)
+        if onode is None:
+            return None
+        return bnode, fnode, onode
+
+    def freeNodes(self,relpath):
+        #causing problems
+        """Free up nodes corresponding to file, if possible"""
+        bnode = self.basenodes.get(relpath,None)
+        if bnode is None:
+            print "Missing node for %s" % relpath
+            return
+        bnode.unlinkNode()
+        bnode.freeNode()
+        del self.basenodes[relpath]
+        pkgid = self.pkg_ids.get(relpath,None)
+        if pkgid is None:
+            print _("No pkgid found for: %s") % relpath
+            return None
+        del self.pkg_ids[relpath]
+        dups = self.pkgrefs.get(pkgid)
+        dups.remove(relpath)
+        if len(dups):
+            #still referenced
+            return
+        del self.pkgrefs[pkgid]
+        for nodes in self.filesnodes, self.othernodes:
+            node = nodes.get(pkgid)
+            if node is not None:
+                node.unlinkNode()
+                node.freeNode()
+                del nodes[pkgid]
+
+
+if __name__ == "__main__":
+    #test code - attempts to read a repo in working directory
+    idx = MetadataIndex(".", "repodata/primary.xml.gz", "repodata/filelists.xml.gz",
+                        "repodata/other.xml.gz", {'verbose':1})

++++++ createrepo-dmd.patch ++++++
diff --git a/dmd.py b/dmd.py
new file mode 100755
index 0000000..fdce45c
--- /dev/null
+++ b/dmd.py
@@ -0,0 +1,156 @@
+#!/usr/bin/python
+
+# dmd - Generate and apply deltas between repository metadata
+#
+# Copyright (C) 2007 James Bowes <jbowes@xxxxxxxxxx>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+import sys
+from lxml.etree import parse, tostring, Element
+
+
+class MdType(object):
+    def __init__(self, namespace, rootelem):
+        self.ns = "http://linux.duke.edu/metadata/%s" % namespace
+        self.sns = "{%s}" % self.ns
+        self.deltasns = "{http://linux.duke.edu/metadata/delta}";
+        self.root = rootelem
+
+    def get_pkg_id(self, pkg):
+        return pkg.findtext(self.sns + "checksum")
+
+    def make_hash(self, tree):
+        pkgshash = {}
+        for pkg in tree:
+            pkgid = self.get_pkg_id(pkg)
+            pkgshash[pkgid] = pkg
+
+        return pkgshash
+
+    def make_pkg_elem(self, pkgid, pkg):
+        pkgelem = Element("package")
+        pkgelem.set('name', pkg.findtext(self.sns + 'name'))
+        pkgelem.set('arch', pkg.findtext(self.sns + 'arch'))
+        pkgelem.set('pkgid', pkgid)
+        verelem = pkg.find(self.sns + 'version')
+        verelem.tag = "version"
+        pkgelem.append(verelem)
+
+        return pkgelem
+
+    def diff_trees(self, oldtree, newtree):
+        oldpkgs = oldtree.getroot().getchildren()
+        newpkgs = newtree.getroot().getchildren()
+
+        oldpkgshash = self.make_hash(oldpkgs)
+        newpkgshash = self.make_hash(newpkgs)
+
+        diff =  Element(self.root,
+                nsmap = {None : self.ns,
+                         "rpm" : "http://linux.duke.edu/metadata/rpm";,
+                         "delta" : "http://linux.duke.edu/metadata/delta"})
+        additions = Element("delta:additions")
+        diff.append(additions)
+        removals = Element("delta:removals")
+
+        diff.append(removals)
+
+        for pkgid, pkg in newpkgshash.iteritems():
+            if not oldpkgshash.has_key(pkgid):
+                additions.append(pkg)
+
+        for pkgid, pkg in oldpkgshash.iteritems():
+            if not newpkgshash.has_key(pkgid):
+                pkgelem = self.make_pkg_elem(pkgid, pkg)
+                removals.append(pkgelem)
+
+        diff.set("packages", str(len(removals) + len(additions)))
+
+        print tostring(diff, pretty_print=True)
+
+    def patch_tree(self, oldtree, deltatree):
+        oldroot = oldtree.getroot()
+        oldpkgs = oldroot.getchildren()
+
+        oldpkgshash = self.make_hash(oldpkgs)
+
+        additions = deltatree.find(self.deltasns + 'additions').getchildren()
+        removals = deltatree.find(self.deltasns + 'removals').getchildren()
+        
+        for pkg in additions:
+            pkgid = self.get_pkg_id(pkg)
+            if oldpkgshash.has_key(pkgid):
+                print >> sys.stderr, "Package %s already exists" % pkgid
+                sys.exit(1)
+            oldroot.append(pkg)
+
+        for pkg in removals:
+            pkgid = pkg.get('pkgid')
+            if not oldpkgshash.has_key(pkgid):
+                print >> sys.stderr, "Package %s does not exist" % pkgid
+                sys.exit(1)
+            oldroot.remove(oldpkgshash[pkgid])
+
+        oldcount = int(oldroot.get('packages'))
+        newcount = oldcount + len(additions) - len(removals)
+        oldroot.set('packages', str(newcount)) 
+        print tostring(oldtree, pretty_print=True)
+
+
+class OtherMdType(MdType):
+    def get_pkg_id(self, pkg):
+        return pkg.get('pkgid')
+
+    def make_pkg_elem(self, pkgid, pkg):
+        pkgelem = Element("package")
+        pkgelem.set('name', pkg.get('name'))
+        pkgelem.set('arch', pkg.get('arch'))
+        pkgelem.set('pkgid', pkgid)
+        verelem = pkg.find(self.sns + 'version')
+        verelem.tag = "version"
+
+        return pkgelem
+
+
+mdtypeinfo = {
+        'primary' : MdType('common', 'metadata'),
+        'filelists' : OtherMdType('filelists', 'filelists'),
+        'other' : OtherMdType('other', 'other'),
+        }
+
+
+def usage(progname):
+    print "usage: %s [diff|patch] MDTYPE FILE1 FILE2" % progname
+    sys.exit()
+
+def main(args):
+    if len(args) != 5:
+        usage(args[0])
+    if args[1] not in ('diff', 'patch'):
+        usage(args[0])
+    if args[2] not in ('primary', 'filelists', 'other'):
+        usage(args[0])
+
+    oldtree = parse(args[3])
+    newtree = parse(args[4])
+
+    if args[1] == 'diff':
+        mdtypeinfo[args[2]].diff_trees(oldtree, newtree)
+    else:
+        mdtypeinfo[args[2]].patch_tree(oldtree, newtree)
+
+if __name__ == "__main__":
+    main(sys.argv)
++++++ createrepo-license-to-confirm.patch ++++++
--- dumpMetadata.py
+++ dumpMetadata.py
@@ -18,6 +18,7 @@
 
 # $Id: dumpMetadata.py,v 1.48 2007/05/18 15:01:16 pnasrat Exp $
 
+import glob
 import os
 import rpm
 import exceptions
@@ -624,7 +625,7 @@
 
 
     
-def generateXML(doc, node, formatns, rpmObj, sumtype):
+def generateXML(doc, node, formatns, susens, rpmObj, sumtype):
     """takes an xml doc object and a package metadata entry node, populates a 
        package node with the md information"""
     ns = node.ns()
@@ -658,6 +659,27 @@
     if rpmObj.localurl is not None:
         location.newProp('xml:base', rpmObj.localurl)
     location.newProp('href', rpmObj.relativepath)
+
+    eulas = glob.glob("%s.eula.*" % os.path.join(rpmObj.options['basedir'], rpmObj.relativepath))
+    if eulas is not None:
+        for eula in eulas:
+            match = re.match("(.*eula)\.(.*)", eula)
+            lang = match.group(2)
+
+            eulaContent = None
+
+            try:
+                fd = open(eula, "r")
+                eulaContent = fd.read()
+                fd.close()
+            except IOError:
+                raise MDError, "Failed read contents of %s" % eula
+
+            if eulaContent is not None:
+                eulaNode = pkgNode.newChild(susens, 'license-to-confirm', None)
+                eulaNode.newProp('lang', lang)
+                eulaNode.addContent(utf8String(eulaContent))
+
     format = pkgNode.newChild(ns, 'format', None)
     for tag in ['license', 'vendor', 'group', 'buildhost', 'sourcerpm']:
         value = rpmObj.tagByName(tag)
--- genpkgmetadata.py
+++ genpkgmetadata.py
@@ -161,11 +161,12 @@
         self.baseroot =  self.basedoc.newChild(None, "metadata", None)
         basens = self.baseroot.newNs('http://linux.duke.edu/metadata/common', None)
         self.formatns = self.baseroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm')
+        self.susens = self.baseroot.newNs('http://novell.com/package/metadata/suse/common', 'suse')
         self.baseroot.setNs(basens)
         basefilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['primaryfile'])
         self.basefile = _gzipOpen(basefilepath, 'w')
         self.basefile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
-        self.basefile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s";>\n' %
+        self.basefile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" xmlns:suse="http://novell.com/package/metadata/suse/common" packages="%s";>\n' %
                        self.pkgcount)
 
     def _setupFilelists(self):
@@ -203,7 +204,7 @@
             errorprint('\n%s - %s' % (e, file))
             return None
         try:
-            basenode = dumpMetadata.generateXML(self.basedoc, self.baseroot, self.formatns, mdobj, self.cmds['sumtype'])
+            basenode = dumpMetadata.generateXML(self.basedoc, self.baseroot, self.formatns, self.susens, mdobj, self.cmds['sumtype'])
         except dumpMetadata.MDError, e:
             errorprint(_('\nAn error occurred creating primary metadata: %s') % e)
             return None
++++++ createrepo-missing-tags.patch ++++++
--- dumpMetadata.py
+++ dumpMetadata.py
@@ -267,14 +267,10 @@
             return returnflags
 
         if type(flags) is not types.ListType:
-            newflag = flags & 0xf
-            returnflags.append(newflag)
+            returnflags.append(flags)
         else:
             for flag in flags:
-                newflag = flag
-                if flag is not None:
-                    newflag = flag & 0xf
-                returnflags.append(newflag)
+                returnflags.append(flag)
         return returnflags
 
     def _checkPreReq(self, flags):
@@ -516,36 +512,48 @@
             lst = zip(names, flags, ver, prereq)
         return self._uniq(lst)
         
-    def obsoletesList(self):
+    def tagsList(self, name, flags, version):
         lst = []
-        names = self.hdr[rpm.RPMTAG_OBSOLETENAME]
-        tmpflags = self.hdr[rpm.RPMTAG_OBSOLETEFLAGS]
+        names = self.hdr[name]
+        tmpflags = self.hdr[flags]
         flags = self._correctFlags(tmpflags)
-        ver = self._correctVersion(self.hdr[rpm.RPMTAG_OBSOLETEVERSION])
+        ver = self._correctVersion(self.hdr[version])
         if names is not None:
             lst = zip(names, flags, ver)
         return self._uniq(lst)
 
+    def obsoletesList(self):
+        return self.tagsList(rpm.RPMTAG_OBSOLETENAME, rpm.RPMTAG_OBSOLETEFLAGS,
+                             rpm.RPMTAG_OBSOLETEVERSION)
+
     def conflictsList(self):
-        lst = []
-        names = self.hdr[rpm.RPMTAG_CONFLICTNAME]
-        tmpflags = self.hdr[rpm.RPMTAG_CONFLICTFLAGS]
-        flags = self._correctFlags(tmpflags)
-        ver = self._correctVersion(self.hdr[rpm.RPMTAG_CONFLICTVERSION])
-        if names is not None:
-            lst = zip(names, flags, ver)
-        return self._uniq(lst)
+        return self.tagsList(rpm.RPMTAG_CONFLICTNAME, rpm.RPMTAG_CONFLICTFLAGS,
+                             rpm.RPMTAG_CONFLICTVERSION)
 
     def providesList(self):
-        lst = []
-        names = self.hdr[rpm.RPMTAG_PROVIDENAME]
-        tmpflags = self.hdr[rpm.RPMTAG_PROVIDEFLAGS]
-        flags = self._correctFlags(tmpflags)
-        ver = self._correctVersion(self.hdr[rpm.RPMTAG_PROVIDEVERSION])
-        if names is not None:
-            lst = zip(names, flags, ver)
-        return self._uniq(lst)
+        return self.tagsList(rpm.RPMTAG_PROVIDENAME, rpm.RPMTAG_PROVIDEFLAGS,
+                             rpm.RPMTAG_PROVIDEVERSION)
         
+    def enhancesList(self):
+        lst = self.tagsList(rpm.RPMTAG_ENHANCESNAME, rpm.RPMTAG_ENHANCESFLAGS,
+                            rpm.RPMTAG_ENHANCESVERSION)
+        return [ l for l in lst if not l[1] & rpm.RPMSENSE_STRONG ]
+
+    def supplementsList(self):
+        lst = self.tagsList(rpm.RPMTAG_ENHANCESNAME, rpm.RPMTAG_ENHANCESFLAGS,
+                            rpm.RPMTAG_ENHANCESVERSION)
+        return [ l for l in lst if l[1] & rpm.RPMSENSE_STRONG ]
+
+    def suggestsList(self):
+        lst = self.tagsList(rpm.RPMTAG_SUGGESTSNAME, rpm.RPMTAG_SUGGESTSFLAGS,
+                            rpm.RPMTAG_SUGGESTSVERSION)
+        return [ l for l in lst if not l[1] & rpm.RPMSENSE_STRONG ]
+
+    def recommendsList(self):
+        lst = self.tagsList(rpm.RPMTAG_SUGGESTSNAME, rpm.RPMTAG_SUGGESTSFLAGS,
+                            rpm.RPMTAG_SUGGESTSVERSION)
+        return [ l for l in lst if l[1] & rpm.RPMSENSE_STRONG ]
+
     def changelogLists(self):
         lst = []
         names = self.listTagByName('changelogname')
@@ -629,12 +637,18 @@
     hr.newProp('end', str(rpmObj.rangeend))
     for (lst, nodename) in [(rpmObj.providesList(), 'provides'),
                             (rpmObj.conflictsList(), 'conflicts'),
-                            (rpmObj.obsoletesList(), 'obsoletes')]:
+                            (rpmObj.obsoletesList(), 'obsoletes'),
+                            (rpmObj.enhancesList(), 'enhances'),
+                            (rpmObj.supplementsList(), 'supplements'),
+                            (rpmObj.suggestsList(), 'suggests'),
+                            (rpmObj.recommendsList(), 'recommends')]:
         if len(lst) > 0:               
             rpconode = format.newChild(formatns, nodename, None)
             for (name, flags, (e,v,r)) in lst:
                 entry = rpconode.newChild(formatns, 'entry', None)
                 entry.newProp('name', name)
+                if flags is not None:
+                    flags = flags & 0xf
                 if flags != 0:
                     if flags == 2: arg = 'LT'
                     if flags == 4: arg = 'GT'
@@ -656,6 +670,8 @@
         for (name, flags, (e,v,r), prereq) in depsList:
             entry = rpconode.newChild(formatns, 'entry', None)
             entry.newProp('name', name)
+            if flags is not None:
+                flags = flags & 0xf
             if flags != 0:
                 if flags == 2: arg = 'LT'
                 if flags == 4: arg = 'GT'
++++++ createrepo-suse-changelogs.patch ++++++
--- dumpMetadata.py
+++ dumpMetadata.py
@@ -744,7 +744,10 @@
     for (name, time, text) in clogs:
         clog = pkg.newChild(None, 'changelog', None)
         clog.addContent(utf8String(text))
-        clog.newProp('author', utf8String(name))
+        if name[0] != '-':
+          clog.newProp('author', utf8String(name))
+        else:
+          clog.newProp('author', utf8String(name[2:]))
         clog.newProp('date', str(time))
     return pkg
     

++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++



Remember to have fun...

---------------------------------------------------------------------
To unsubscribe, e-mail: opensuse-commit+unsubscribe@xxxxxxxxxxxx
For additional commands, e-mail: opensuse-commit+help@xxxxxxxxxxxx

< Previous Next >
This Thread
  • No further messages