Mailinglist Archive: zypp-commit (266 mails)

< Previous Next >
[zypp-commit] r11648 - in /trunk/libzypp: VERSION.cmake package/libzypp.changes tests/zypp/Fetcher_test.cc zypp/Fetcher.cc
  • From: mlandres@xxxxxxxxxxxxxxxx
  • Date: Tue, 11 Nov 2008 13:50:25 -0000
  • Message-id: <20081111135025.7E51727B31@xxxxxxxxxxxxxxxx>
Author: mlandres
Date: Tue Nov 11 14:50:25 2008
New Revision: 11648

URL: http://svn.opensuse.org/viewcvs/zypp?rev=11648&view=rev
Log:
- Prevent fetcher from processing the same index file twice. (bnc #443644)

Modified:
trunk/libzypp/VERSION.cmake
trunk/libzypp/package/libzypp.changes
trunk/libzypp/tests/zypp/Fetcher_test.cc
trunk/libzypp/zypp/Fetcher.cc

Modified: trunk/libzypp/VERSION.cmake
URL:
http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/VERSION.cmake?rev=11648&r1=11647&r2=11648&view=diff
==============================================================================
--- trunk/libzypp/VERSION.cmake (original)
+++ trunk/libzypp/VERSION.cmake Tue Nov 11 14:50:25 2008
@@ -60,9 +60,9 @@
#
SET(LIBZYPP_MAJOR "5")
SET(LIBZYPP_COMPATMINOR "21")
-SET(LIBZYPP_MINOR "21")
+SET(LIBZYPP_MINOR "22")
SET(LIBZYPP_PATCH "0")
#
-# LAST RELEASED: 5.21.0 (21)
+# LAST RELEASED: 5.22.0 (21)
# (The number in parenthesis is LIBZYPP_COMPATMINOR)
#=======

Modified: trunk/libzypp/package/libzypp.changes
URL:
http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/package/libzypp.changes?rev=11648&r1=11647&r2=11648&view=diff
==============================================================================
--- trunk/libzypp/package/libzypp.changes (original)
+++ trunk/libzypp/package/libzypp.changes Tue Nov 11 14:50:25 2008
@@ -1,4 +1,11 @@
-------------------------------------------------------------------
+Tue Nov 11 14:48:52 CET 2008 - ma@xxxxxxx
+
+- Prevent fetcher from processing the same index file twice. (bnc #443644)
+- revision 11648
+- version 5.22.0 (21)
+
+-------------------------------------------------------------------
Fri Nov 7 23:55:47 CET 2008 - ma@xxxxxxx

- Fix retrieval of patch contents and references attributes. (bnc #442200)

Modified: trunk/libzypp/tests/zypp/Fetcher_test.cc
URL:
http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/tests/zypp/Fetcher_test.cc?rev=11648&r1=11647&r2=11648&view=diff
==============================================================================
--- trunk/libzypp/tests/zypp/Fetcher_test.cc (original)
+++ trunk/libzypp/tests/zypp/Fetcher_test.cc Tue Nov 11 14:50:25 2008
@@ -1,26 +1,11 @@
-#include <stdio.h>
-#include <iostream>
-#define BOOST_TEST_MODULE fetcher_test
-#include <boost/test/auto_unit_test.hpp>
-#include <boost/test/parameterized_test.hpp>
-#include <boost/test/unit_test_log.hpp>
+#include "TestSetup.h"

-#include "zypp/base/Logger.h"
-#include "zypp/ZYppFactory.h"
#include "zypp/MediaSetAccess.h"
#include "zypp/Fetcher.h"
-#include "zypp/Url.h"
-#include "zypp/TmpPath.h"

#include "WebServer.h"
-#include <boost/thread.hpp>

-using std::cout;
-using std::endl;
-using std::string;
-using namespace zypp;
-using namespace zypp::media;
-using namespace boost::unit_test;
+#define BOOST_TEST_MODULE fetcher_test

#define DATADIR (Pathname(TESTS_SRC_DIR) + "/zypp/data/Fetcher/remote-site")

@@ -47,6 +32,8 @@

BOOST_AUTO_TEST_CASE(fetcher_enqueuedir_autoindex)
{
+ base::LogControl::TmpLineWriter shutUp( new zypp::log::FileLineWriter(
"/tmp/YLOG" ) );
+ MIL << "GO" << endl;
MediaSetAccess media( ( DATADIR).asUrl(), "/" );
// do the test by trusting the SHA1SUMS file signature key
{
@@ -192,7 +179,7 @@
{
MediaSetAccess media( (DATADIR).asUrl(), "/" );
Fetcher fetcher;
-
+
{
filesystem::TmpDir dest;
OnMediaLocation loc("/complexdir/subdir1/subdir1-file1.txt");
@@ -220,7 +207,7 @@
OnMediaLocation loc("/contentindex/subdir1/subdir1-file1.txt");
// trust the key manually
getZYpp()->keyRing()->importKey(PublicKey(DATADIR +
"/contentindex/content.key"), true);
- fetcher.addIndex(OnMediaLocation("/contentindex/content", 1));
+ fetcher.addIndex(OnMediaLocation("/contentindex/content", 1));
fetcher.enqueue(loc);
fetcher.start(dest.path(), media);
fetcher.reset();

Modified: trunk/libzypp/zypp/Fetcher.cc
URL:
http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/zypp/Fetcher.cc?rev=11648&r1=11647&r2=11648&view=diff
==============================================================================
--- trunk/libzypp/zypp/Fetcher.cc (original)
+++ trunk/libzypp/zypp/Fetcher.cc Tue Nov 11 14:50:25 2008
@@ -15,7 +15,8 @@
#include <map>

#include "zypp/base/Easy.h"
-#include "zypp/base/Logger.h"
+#include "zypp/base/LogControl.h"
+#include "zypp/base/LogTools.h"
#include "zypp/base/PtrTypes.h"
#include "zypp/base/DefaultIntegral.h"
#include "zypp/base/String.h"
@@ -27,6 +28,9 @@

using namespace std;

+#undef ZYPP_BASE_LOGGER_LOGGROUP
+#define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher"
+
///////////////////////////////////////////////////////////////////
namespace zypp
{ /////////////////////////////////////////////////////////////////
@@ -40,13 +44,34 @@
{
FetcherIndex( const OnMediaLocation &loc )
: location(loc)
+ {}
+ /** Index localtion. */
+ OnMediaLocation location;
+ /** Whether we read this index. */
+ DefaultIntegral<bool,false> read;
+ };
+
+ typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
+
+ /** std::set ordering (less semantic) */
+ struct SameFetcherIndex
+ {
+ bool operator()( const FetcherIndex_Ptr & lhs, const FetcherIndex_Ptr &
rhs )
{
+ if ( lhs == rhs )
+ return false; // incl. NULL == NULL
+ if ( ! lhs )
+ return true; // NULL < nonNULL
+ if ( ! rhs )
+ return false; // nonNULL > NULL
+ // both nonNULL ==> compare medianr and path
+ if ( lhs->location.medianr() == rhs->location.medianr() )
+ return lhs->location.filename() < rhs->location.filename();
+ //else
+ return lhs->location.medianr() < rhs->location.medianr();
}
-
- OnMediaLocation location;
};
- typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr;
-
+
/**
* Class to encapsulate the \ref OnMediaLocation object
* and the \ref FileChecker together
@@ -55,7 +80,7 @@
{
enum Flag
{
- None = 0x0000,
+ None = 0x0000,
Directory = 0x0001,
Recursive = 0x0002,
RecursiveDirectory = Directory | Recursive,
@@ -64,7 +89,7 @@
AlwaysVerifyChecksum = 0x0004,
};
ZYPP_DECLARE_FLAGS(Flags, Flag);
-
+

FetcherJob( const OnMediaLocation &loc )
: location(loc)
@@ -72,7 +97,7 @@
{
//MIL << location << endl;
}
-
+
~FetcherJob()
{
//MIL << location << " | * " << checkers.size() << endl;
@@ -83,7 +108,8 @@
list<FileChecker> checkers;
Flags flags;
};
- ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
+
+ ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
typedef shared_ptr<FetcherJob> FetcherJob_Ptr;

std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
@@ -91,7 +117,6 @@
return str << obj->location;
}

-
///////////////////////////////////////////////////////////////////
//
// CLASS NAME : Fetcher::Impl
@@ -100,23 +125,24 @@
class Fetcher::Impl
{
friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl
& obj );
+
public:
- Impl();
-
+ Impl();
+
~Impl() {
MIL << endl;
- }
-
- void setOptions( Fetcher::Options options );
- Fetcher::Options options() const;
+ }
+
+ void setOptions( Fetcher::Options options );
+ Fetcher::Options options() const;

- void addIndex( const OnMediaLocation &resource );
+ void addIndex( const OnMediaLocation &resource );

- void enqueueDir( const OnMediaLocation &resource, bool recursive, const
FileChecker &checker = FileChecker() );
- void enqueueDigestedDir( const OnMediaLocation &resource, bool
recursive, const FileChecker &checker = FileChecker() );
+ void enqueueDir( const OnMediaLocation &resource, bool recursive, const
FileChecker &checker = FileChecker() );
+ void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive,
const FileChecker &checker = FileChecker() );

- void enqueue( const OnMediaLocation &resource, const FileChecker
&checker = FileChecker() );
- void enqueueDigested( const OnMediaLocation &resource, const FileChecker
&checker = FileChecker() );
+ void enqueue( const OnMediaLocation &resource, const FileChecker &checker
= FileChecker() );
+ void enqueueDigested( const OnMediaLocation &resource, const FileChecker
&checker = FileChecker() );
void addCachePath( const Pathname &cache_dir );
void reset();
void start( const Pathname &dest_dir,
@@ -151,13 +177,13 @@

/** specific version of \ref readIndex for SHA1SUMS file */
void readSha1sumsIndex( const Pathname &index, const Pathname &basedir );
-
+
/** specific version of \ref readIndex for SHA1SUMS file */
void readContentFileIndex( const Pathname &index, const Pathname
&basedir );
-
+
/** reads the content of a directory but keeps a cache **/
void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation
&resource, filesystem::DirContent &content );
-
+
/**
* tries to provide the file represented by job into dest_dir by
* looking at the cache. If success, returns true, and the desired
@@ -195,15 +221,15 @@
Impl * clone() const
{ return new Impl( *this ); }

- list<FetcherJob_Ptr> _resources;
- list<FetcherIndex_Ptr> _indexes;
- list<Pathname> _caches;
+ list<FetcherJob_Ptr> _resources;
+ std::set<FetcherIndex_Ptr,SameFetcherIndex> _indexes;
+ std::set<Pathname> _caches;
// checksums read from the indexes
map<string, CheckSum> _checksums;
// cache of dir contents
map<string, filesystem::DirContent> _dircontent;
-
- Fetcher::Options _options;
+
+ Fetcher::Options _options;
};
///////////////////////////////////////////////////////////////////

@@ -214,18 +240,18 @@
job->flags |= FetcherJob:: AlwaysVerifyChecksum;
_resources.push_back(job);
}
-
+
Fetcher::Impl::Impl()
: _options(0)
{
}
-
+
void Fetcher::Impl::setOptions( Fetcher::Options options )
{ _options = options; }
-
+
Fetcher::Options Fetcher::Impl::options() const
{ return _options; }
-
+
void Fetcher::Impl::enqueueDir( const OnMediaLocation &resource,
bool recursive,
const FileChecker &checker )
@@ -269,10 +295,8 @@

void Fetcher::Impl::addIndex( const OnMediaLocation &resource )
{
- MIL << "adding index " << resource << endl;
- FetcherIndex_Ptr index;
- index.reset(new FetcherIndex(resource));
- _indexes.push_back(index);
+ MIL << "adding index " << resource << endl;
+ _indexes.insert(FetcherIndex_Ptr(new FetcherIndex(resource)));
}


@@ -292,7 +316,7 @@
if ( info.isDir() )
{
DBG << "Adding fetcher cache: '" << cache_dir << "'." << endl;
- _caches.push_back(cache_dir);
+ _caches.insert(cache_dir);
}
else
{
@@ -361,7 +385,7 @@
} // iterate over caches
return false;
}
-
+
void Fetcher::Impl::validate( const OnMediaLocation &resource, const
Pathname &dest_dir, const list<FileChecker> &checkers )
{
// no matter where did we got the file, try to validate it:
@@ -411,7 +435,7 @@
filesystem::DirEntry shafile;
shafile.name = "SHA1SUMS"; shafile.type = filesystem::FT_FILE;
if ( find( content.begin(), content.end(), shafile ) !=
content.end() )
- {
+ {
// add the index of this directory
OnMediaLocation indexloc(resource);
indexloc.changeFilename(resource.filename() + "SHA1SUMS");
@@ -426,7 +450,7 @@
filesystem::DirEntry contentfile;
contentfile.name = "content"; contentfile.type = filesystem::FT_FILE;
if ( find( content.begin(), content.end(), contentfile ) !=
content.end() )
- {
+ {
// add the index of this directory
OnMediaLocation indexloc(resource);
indexloc.changeFilename(resource.filename() + "content");
@@ -441,11 +465,11 @@
const OnMediaLocation &resource,
filesystem::DirContent &content )
{
- if ( _dircontent.find(resource.filename().asString())
+ if ( _dircontent.find(resource.filename().asString())
!= _dircontent.end() )
{
filesystem::DirContent
filled(_dircontent[resource.filename().asString()]);
-
+
std::copy(filled.begin(), filled.end(), std::back_inserter(content));
}
else
@@ -459,7 +483,7 @@
_dircontent[resource.filename().asString()] = tofill;
}
}
-
+
void Fetcher::Impl::addDirJobs( MediaSetAccess &media,
const OnMediaLocation &resource,
const Pathname &dest_dir, FetcherJob::Flags
flags )
@@ -469,11 +493,11 @@
MIL << "Adding directory " << resource.filename() << endl;
filesystem::DirContent content;
getDirectoryContent(media, resource, content);
-
+
// this method test for the option flags so indexes are added
// only if the options are enabled
autoaddIndexes(content, media, resource, dest_dir);
-
+
for ( filesystem::DirContent::const_iterator it = content.begin();
it != content.end();
++it )
@@ -489,7 +513,7 @@
case filesystem::FT_NOT_AVAIL: // old directory.yast contains no
typeinfo at all
case filesystem::FT_FILE:
{
- CheckSum chksm(resource.checksum());
+ CheckSum chksm(resource.checksum());
if ( _checksums.find(filename.asString()) != _checksums.end() )
{
// the checksum can be replaced with the one in the index.
@@ -558,7 +582,7 @@
return;
}
else
- {
+ {
ZYPP_RETHROW(excpt_r);
}
}
@@ -578,7 +602,7 @@
{
setRepoIndexConsumer( bind( &ContentReaderHelper::consumeIndex, this, _1
) );
}
-
+
void consumeIndex( const parser::susetags::RepoIndex_Ptr & data_r )
{ _repoindex = data_r; }

@@ -634,14 +658,13 @@
void Fetcher::Impl::downloadIndex( MediaSetAccess &media, const
OnMediaLocation &resource, const Pathname &dest_dir)
{
MIL << "downloading index " << resource << endl;
-
// create a new fetcher with a different state to transfer the
// file containing checksums and its signature
Fetcher fetcher;
// signature checker for index. We havent got the signature from
// the nextwork yet.
SignatureFileChecker sigchecker;
-
+
// build the name of the index and the signature
OnMediaLocation idxloc(resource);
OnMediaLocation sigloc(resource);
@@ -651,27 +674,27 @@
// the checking will warn later
sigloc.setOptional(true);
keyloc.setOptional(true);
-
+
// calculate signature and key name
sigloc.changeFilename( sigloc.filename().extend(".asc") );
keyloc.changeFilename( keyloc.filename().extend(".key") );
-
+
//assert_dir(dest_dir + idxloc.filename().dirname());
-
+
// transfer the signature
fetcher.enqueue(sigloc);
fetcher.start( dest_dir, media );
// if we get the signature, update the checker
if ( PathInfo(dest_dir + sigloc.filename()).isExist() )
sigchecker = SignatureFileChecker(dest_dir + sigloc.filename());
-
+
fetcher.reset();
-
+
// now the key
fetcher.enqueue(keyloc);
fetcher.start( dest_dir, media );
fetcher.reset();
-
+
// now the index itself
fetcher.enqueue( idxloc, FileChecker(sigchecker) );
fetcher.start( dest_dir, media );
@@ -691,16 +714,26 @@
return;
}

- for ( list<FetcherIndex_Ptr>::const_iterator it_idx = _indexes.begin();
- it_idx != _indexes.end(); ++it_idx )
+ for_( it_idx, _indexes.begin(), _indexes.end() )
{
+ if ( (*it_idx)->read )
+ {
+ DBG << "Already read index " << PathInfo(dest_dir +
(*it_idx)->location.filename()) << endl;
+ }
+ else
+ {
+ // base::LogControl::TmpLineWriter shutUp;
downloadIndex( media, (*it_idx)->location, dest_dir );
// now we have the indexes in dest_dir
readIndex( dest_dir + (*it_idx)->location.filename(),
(*it_idx)->location.filename().dirname() );
+ // Take care we don't process it again
+ MIL << "Remember read index " << PathInfo(dest_dir +
(*it_idx)->location.filename()) << endl;
+ (*it_idx)->read = true;
+ }
}
MIL << "done reading indexes" << endl;
}
-
+
// start processing all fetcher jobs.
// it processes any user pointed index first
void Fetcher::Impl::start( const Pathname &dest_dir,
@@ -738,7 +771,7 @@
MIL << "Autodiscovering signed indexes on '"
<< (*it_res)->location.filename().dirname() << "' for '"
<< (*it_res)->location.filename() << "'" << endl;
-
+
autoaddIndexes(content, media,
(*it_res)->location.filename().dirname(), dest_dir);

// also look in the root of the media
@@ -749,9 +782,9 @@
MIL << "Autodiscovering signed indexes on '"
<< "/" << "' for '"
<< (*it_res)->location.filename() << "'" << endl;
-
+
autoaddIndexes(content, media, Pathname("/"), dest_dir);
- }
+ }

provideToDest(media, (*it_res)->location, dest_dir);

@@ -764,21 +797,21 @@
// indexes checksum, then add a checker
if ( (*it_res)->location.checksum().empty() )
{
- if ( _checksums.find((*it_res)->location.filename().asString())
+ if ( _checksums.find((*it_res)->location.filename().asString())
!= _checksums.end() )
{
CheckSum chksm =
_checksums[(*it_res)->location.filename().asString()];
- ChecksumFileChecker digest_check(chksm);
+ ChecksumFileChecker digest_check(chksm);
(*it_res)->checkers.push_back(digest_check);
}
else
{
// if the index checksum is empty too, we only add the checker
// if the AlwaysVerifyChecksum option is set on
- if ( (*it_res)->flags & FetcherJob::AlwaysVerifyChecksum )
+ if ( (*it_res)->flags & FetcherJob::AlwaysVerifyChecksum )
{
// add the checker with the empty checksum
- ChecksumFileChecker
digest_check((*it_res)->location.checksum());
+ ChecksumFileChecker
digest_check((*it_res)->location.checksum());
(*it_res)->checkers.push_back(digest_check);
}
}
@@ -786,10 +819,10 @@
else
{
// checksum is not empty, so add a checksum checker
- ChecksumFileChecker digest_check((*it_res)->location.checksum());
+ ChecksumFileChecker digest_check((*it_res)->location.checksum());
(*it_res)->checkers.push_back(digest_check);
}
-
+
// validate job, this throws if not valid
validate((*it_res)->location, dest_dir, (*it_res)->checkers);

@@ -819,7 +852,7 @@
{
_pimpl->setOptions(options);
}
-
+
Fetcher::Options Fetcher::options() const
{
return _pimpl->options();

--
To unsubscribe, e-mail: zypp-commit+unsubscribe@xxxxxxxxxxxx
For additional commands, e-mail: zypp-commit+help@xxxxxxxxxxxx

< Previous Next >
This Thread
  • No further messages