Author: dmacvicar Date: Tue Nov 4 18:04:32 2008 New Revision: 11577 URL: http://svn.opensuse.org/viewcvs/zypp?rev=11577&view=rev Log: Do autoindexing for simple files to if enabled. Warning, this is expensive, and I already wait for someone to use it in the wrong way and then complain that it is not efficient. If you want to transfer lot of files with one index use enqueueDir, not enqueue Modified: trunk/libzypp/tests/zypp/Fetcher_test.cc trunk/libzypp/zypp/Fetcher.cc Modified: trunk/libzypp/tests/zypp/Fetcher_test.cc URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/tests/zypp/Fetcher_test.cc?rev=11577&r1=11576&r2=11577&view=diff ============================================================================== --- trunk/libzypp/tests/zypp/Fetcher_test.cc (original) +++ trunk/libzypp/tests/zypp/Fetcher_test.cc Tue Nov 4 18:04:32 2008 @@ -1,5 +1,6 @@ #include #include <iostream> +#define BOOST_TEST_MODULE fetcher_test #include #include #include @@ -23,6 +24,8 @@ #define DATADIR (Pathname(TESTS_SRC_DIR) + "/zypp/data/Fetcher/remote-site") +BOOST_AUTO_TEST_SUITE( fetcher_test ); + BOOST_AUTO_TEST_CASE(fetcher_enqueuedir_noindex) { MediaSetAccess media( ( DATADIR).asUrl(), "/" ); @@ -121,7 +124,7 @@ } -BOOST_AUTO_TEST_CASE(fetcher_enqueue_digested_brokendir_with_index) +BOOST_AUTO_TEST_CASE(fetcher_enqueue_digesteddir_brokendir_with_index) { MediaSetAccess media( ( DATADIR).asUrl(), "/" ); // do the test by trusting the SHA1SUMS file signature key but with a broken file @@ -137,6 +140,39 @@ } } +BOOST_AUTO_TEST_CASE(fetcher_enqueue_digested_broken_with_autoindex) +{ + MediaSetAccess media( ( DATADIR).asUrl(), "/" ); + // do the test by trusting the SHA1SUMS file signature key but with a broken file + { + filesystem::TmpDir dest; + Fetcher fetcher; + // add the key as trusted + getZYpp()->keyRing()->importKey(PublicKey(DATADIR + "/complexdir-broken/subdir1/SHA1SUMS.key"), true); + fetcher.setOptions( Fetcher::AutoAddIndexes ); + fetcher.enqueueDigested(OnMediaLocation("/complexdir-broken/subdir1/subdir1-file1.txt")); + BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), Exception); + fetcher.reset(); + } +} + +BOOST_AUTO_TEST_CASE(fetcher_enqueue_digested_with_autoindex) +{ + MediaSetAccess media( ( DATADIR).asUrl(), "/" ); + // do the test by trusting the SHA1SUMS file signature key with a good file + // checksum in auto discovered index + { + filesystem::TmpDir dest; + Fetcher fetcher; + // add the key as trusted + getZYpp()->keyRing()->importKey(PublicKey(DATADIR + "/complexdir/subdir1/SHA1SUMS.key"), true); + fetcher.setOptions( Fetcher::AutoAddIndexes ); + fetcher.enqueueDigested(OnMediaLocation("/complexdir/subdir1/subdir1-file1.txt")); + fetcher.start( dest.path(), media ); + fetcher.reset(); + } +} + BOOST_AUTO_TEST_CASE(fetcher_enqueuefile_noindex) { @@ -197,9 +233,8 @@ { MediaSetAccess media( ( DATADIR).asUrl(), "/" ); Fetcher fetcher; - + filesystem::TmpDir dest; { - filesystem::TmpDir dest; OnMediaLocation loc("/contentindex-broken-digest/subdir1/subdir1-file1.txt",1); // key was already imported as trusted fetcher.addIndex(OnMediaLocation("/contentindex-broken-digest/content", 1)); @@ -210,28 +245,8 @@ // now retrieve a file that is modified, so the checksum has to fail loc = OnMediaLocation("/contentindex-broken-digest/subdir1/subdir1-file2.txt",1); - fetcher.enqueue(loc); - BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), Exception); - fetcher.reset(); - } -} - -BOOST_AUTO_TEST_CASE(enqueue_digested_broken_content_index) -{ - MediaSetAccess media( ( DATADIR).asUrl(), "/" ); - Fetcher fetcher; - - { - filesystem::TmpDir dest; - OnMediaLocation loc("/contentindex-broken-digest/subdir1/subdir1-file1.txt",1); - // key was already imported as trusted fetcher.addIndex(OnMediaLocation("/contentindex-broken-digest/content", 1)); - fetcher.enqueueDigested(loc); - fetcher.start(dest.path(), media); - fetcher.reset(); - // now retrieve a file that is modified, so the checksum has to fail - loc = OnMediaLocation("/contentindex-broken-digest/subdir1/subdir1-file2.txt",1); - fetcher.enqueueDigested(loc); + fetcher.enqueue(loc); BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), Exception); fetcher.reset(); } @@ -296,5 +311,6 @@ } } +BOOST_AUTO_TEST_SUITE_END(); // vim: set ts=2 sts=2 sw=2 ai et: Modified: trunk/libzypp/zypp/Fetcher.cc URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/zypp/Fetcher.cc?rev=11577&r1=11576&r2=11577&view=diff ============================================================================== --- trunk/libzypp/zypp/Fetcher.cc (original) +++ trunk/libzypp/zypp/Fetcher.cc Tue Nov 4 18:04:32 2008 @@ -133,7 +133,7 @@ /** * download the indexes and reads them */ - void readIndexes( MediaSetAccess &media, const Pathname &dest_dir); + void downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir); /** * reads a downloaded index file and updates internal @@ -150,6 +150,9 @@ /** specific version of \ref readIndex for SHA1SUMS file */ void readContentFileIndex( const Pathname &index, const Pathname &basedir ); + /** reads the content of a directory but keeps a cache **/ + void getDirectoryContent( MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content ); + /** * tries to provide the file represented by job into dest_dir by * looking at the cache. If success, returns true, and the desired @@ -192,6 +195,9 @@ list<Pathname> _caches; // checksums read from the indexes map _checksums; + // cache of dir contents + map _dircontent; + Fetcher::Options _options; }; /////////////////////////////////////////////////////////////////// @@ -269,6 +275,8 @@ { _resources.clear(); _indexes.clear(); + _checksums.clear(); + _dircontent.clear(); } void Fetcher::Impl::addCachePath( const Pathname &cache_dir ) @@ -404,7 +412,7 @@ indexloc.changeFilename(resource.filename() + "SHA1SUMS"); addIndex(indexloc); // we need to read it now - readIndexes(media, dest_dir); + downloadAndReadIndexList(media, dest_dir); } } if ( _options & AutoAddContentFileIndexes ) @@ -419,10 +427,33 @@ indexloc.changeFilename(resource.filename() + "content"); addIndex(indexloc); // we need to read it now - readIndexes(media, dest_dir); + downloadAndReadIndexList(media, dest_dir); } } } + + void Fetcher::Impl::getDirectoryContent( MediaSetAccess &media, + const OnMediaLocation &resource, + filesystem::DirContent &content ) + { + if ( _dircontent.find(resource.filename().asString()) + != _dircontent.end() ) + { + filesystem::DirContent filled(_dircontent[resource.filename().asString()]); + + std::copy(filled.begin(), filled.end(), std::back_inserter(content)); + } + else + { + filesystem::DirContent tofill; + media.dirInfo( tofill, + resource.filename(), + false /* dots */, + resource.medianr()); + std::copy(tofill.begin(), tofill.end(), std::back_inserter(content)); + _dircontent[resource.filename().asString()] = tofill; + } + } void Fetcher::Impl::addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource, @@ -432,7 +463,7 @@ // individual transfer jobs MIL << "Adding directory " << resource.filename() << endl; filesystem::DirContent content; - media.dirInfo( content, resource.filename(), false /* dots */, resource.medianr()); + getDirectoryContent(media, resource, content); // this method test for the option flags so indexes are added // only if the options are enabled @@ -462,6 +493,7 @@ } else WAR << "Resource " << filename << " has no checksum in the index either." << endl; + if ( flags & FetcherJob::AlwaysVerifyChecksum ) enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm)); else @@ -499,6 +531,12 @@ ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString())); if ( filesystem::copy(tmp_file, dest_full_path ) != 0 ) { + if ( ! PathInfo(tmp_file).isExist() ) + ERR << tmp_file << " does not exist" << endl; + if ( ! PathInfo(dest_full_path.dirname()).isExist() ) + ERR << dest_full_path.dirname() << " does not exist" << endl; + + media.releaseFile(resource); //not needed anymore, only eat space ZYPP_THROW( Exception("Can't copy " + tmp_file.asString() + " to " + dest_dir.asString())); } @@ -582,7 +620,7 @@ // this method takes all the user pointed indexes, gets them and also tries to // download their signature, and verify them. After that, its parses each one // to fill the checksum cache. - void Fetcher::Impl::readIndexes( MediaSetAccess &media, const Pathname &dest_dir) + void Fetcher::Impl::downloadAndReadIndexList( MediaSetAccess &media, const Pathname &dest_dir) { // if there is no indexes, then just return to avoid // the directory listing @@ -646,7 +684,7 @@ ProgressData progress(_resources.size()); progress.sendTo(progress_receiver); - readIndexes(media, dest_dir); + downloadAndReadIndexList(media, dest_dir); for ( list::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res ) { @@ -658,6 +696,26 @@ continue; } + // may be this code can be factored out + // together with the autodiscovery of indexes + // of addDirJobs + if ( ( _options & AutoAddSha1sumsIndexes ) || + ( _options & AutoAddContentFileIndexes ) ) + { + // if auto indexing is enabled, then we need to read the + // index for each file. We look only in the directory + // where the file is. this is expensive of course. + filesystem::DirContent content; + getDirectoryContent(media, (*it_res)->location.filename().dirname(), content); + // this method test for the option flags so indexes are added + // only if the options are enabled + MIL << "Autodiscovering signed indexes on '" + << (*it_res)->location.filename().dirname() << "' for '" + << (*it_res)->location.filename() << "'" << endl; + + autoaddIndexes(content, media, (*it_res)->location.filename().dirname(), dest_dir); + } + provideToDest(media, (*it_res)->location, dest_dir); // if the checksum is empty, but the checksum is in one of the -- To unsubscribe, e-mail: zypp-commit+unsubscribe@opensuse.org For additional commands, e-mail: zypp-commit+help@opensuse.org