Author: dmacvicar Date: Mon Oct 27 18:51:36 2008 New Revision: 11513 URL: http://svn.opensuse.org/viewcvs/zypp?rev=11513&view=rev Log: - first refactoring towards real fixing of slideshow bug - refactor the index so any index can be specified - the index supports now relative paths, so the usage of one SHA1SUMS file per tree is also possble Modified: trunk/libzypp/tests/zypp/Fetcher_test.cc trunk/libzypp/zypp/Fetcher.cc trunk/libzypp/zypp/Fetcher.h Modified: trunk/libzypp/tests/zypp/Fetcher_test.cc URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/tests/zypp/Fetcher_test.cc?rev=11513&r1=11512&r2=11513&view=diff ============================================================================== --- trunk/libzypp/tests/zypp/Fetcher_test.cc (original) +++ trunk/libzypp/tests/zypp/Fetcher_test.cc Mon Oct 27 18:51:36 2008 @@ -23,6 +23,28 @@ #define DATADIR (Pathname(TESTS_SRC_DIR) + "/zypp/data/Fetcher/remote-site") +BOOST_AUTO_TEST_CASE(fetcher_simple) +{ + MediaSetAccess media( (DATADIR).asUrl(), "/" ); + Fetcher fetcher; + + { + filesystem::TmpDir dest; + OnMediaLocation loc("/complexdir/subdir1/subdir1-file1.txt"); + loc.setChecksum(CheckSum::sha1("f1d2d2f924e986ac86fdf7b36c94bcdf32beec15")); + fetcher.enqueueDigested(loc); + fetcher.start(dest.path(), media); + fetcher.reset(); + // now we break the checksum and it should fail + loc.setChecksum(CheckSum::sha1("f1d2d2f924e986ac86fdf7b36c94bcdf32beec16")); + fetcher.enqueueDigested(loc); + BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), Exception); + fetcher.reset(); + + } + +} + BOOST_AUTO_TEST_CASE(fetcher) { MediaSetAccess media( ( DATADIR).asUrl(), "/" ); Modified: trunk/libzypp/zypp/Fetcher.cc URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/zypp/Fetcher.cc?rev=11513&r1=11512&r2=11513&view=diff ============================================================================== --- trunk/libzypp/zypp/Fetcher.cc (original) +++ trunk/libzypp/zypp/Fetcher.cc Mon Oct 27 18:51:36 2008 @@ -14,6 +14,7 @@ #include <list> #include <map> +#include "zypp/base/Flags.h" #include "zypp/base/Easy.h" #include "zypp/base/Logger.h" #include "zypp/base/PtrTypes.h" @@ -30,12 +31,27 @@ { ///////////////////////////////////////////////////////////////// /** + * class that represents indexes which add metadata + * to fetcher jobs and therefore need to be retrieved + * in advance. + */ + struct FetcherIndex + { + FetcherIndex( const OnMediaLocation &loc ) + : location(loc) + { + } + + OnMediaLocation location; + }; + typedef shared_ptr<FetcherIndex> FetcherIndex_Ptr; + + /** * Class to encapsulate the \ref OnMediaLocation object * and the \ref FileChecker together */ struct FetcherJob { - FetcherJob( const OnMediaLocation &loc ) : location(loc) , directory(false) @@ -43,7 +59,7 @@ { //MIL << location << endl; } - + ~FetcherJob() { //MIL << location << " | * " << checkers.size() << endl; @@ -77,7 +93,8 @@ ~Impl() { MIL << endl; } - + + void addIndex( const OnMediaLocation &resource ); void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() ); void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() ); void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() ); @@ -95,6 +112,20 @@ } private: /** + * download the indexes and reads them + */ + void readIndexes( MediaSetAccess &media, const Pathname &dest_dir); + + /** + * reads a downloaded index file and updates internal + * attributes table + * + * The index lists files relative to a directory, which is + * normally the same as the index file is located. + */ + void readIndex( const Pathname &index, const Pathname &basedir ); + + /** * tries to provide the file represented by job into dest_dir by * looking at the cache. If success, returns true, and the desired * file should be available on dest_dir @@ -123,8 +154,11 @@ Impl * clone() const { return new Impl( *this ); } - std::list _resources; - std::list<Pathname> _caches; + list _resources; + list _indexes; + list<Pathname> _caches; + // checksums read from the indexes + map _checksums; }; /////////////////////////////////////////////////////////////////// @@ -161,9 +195,19 @@ _resources.push_back(job); } + void Fetcher::Impl::addIndex( const OnMediaLocation &resource ) + { + MIL << "adding index " << resource << endl; + FetcherIndex_Ptr index; + index.reset(new FetcherIndex(resource)); + _indexes.push_back(index); + } + + void Fetcher::Impl::reset() { _resources.clear(); + _indexes.clear(); } void Fetcher::Impl::addCachePath( const Pathname &cache_dir ) @@ -240,14 +284,16 @@ } // iterate over caches return false; } - + void Fetcher::Impl::validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers ) { // no matter where did we got the file, try to validate it: Pathname localfile = dest_dir + resource.filename(); // call the checker function - try { + try + { MIL << "Checking job [" << localfile << "] (" << checkers.size() << " checkers )" << endl; + for ( list<FileChecker>::const_iterator it = checkers.begin(); it != checkers.end(); ++it ) @@ -287,93 +333,17 @@ filesystem::DirContent content; media.dirInfo( content, resource.filename(), false /* dots */, resource.medianr()); - filesystem::DirEntry shafile, shasig, shakey; + // only try to add an index if it exists + filesystem::DirEntry shafile; shafile.name = "SHA1SUMS"; shafile.type = filesystem::FT_FILE; - shasig.name = "SHA1SUMS.asc"; shasig.type = filesystem::FT_FILE; - shakey.name = "SHA1SUMS.key"; shakey.type = filesystem::FT_FILE; - - // create a new fetcher with a different state to transfer the - // file containing checksums and its signature - Fetcher fetcher; - // signature checker for SHA1SUMS. We havent got the signature from - // the nextwork yet. - SignatureFileChecker sigchecker; - - // now try to find the SHA1SUMS signature - if ( find(content.begin(), content.end(), shasig) - != content.end() ) - { - MIL << "found checksums signature file: " << shasig.name << endl; - // TODO refactor with OnMediaLocation::extend or something - fetcher.enqueue( OnMediaLocation(resource.filename() + shasig.name, resource.medianr()).setOptional(true) ); - assert_dir(dest_dir + resource.filename()); - fetcher.start( dest_dir, media ); - - // if we get the signature, update the checker - sigchecker = SignatureFileChecker(dest_dir + resource.filename() + shasig.name); - fetcher.reset(); - } - else - MIL << "no signature for " << shafile.name << endl; - - // look for the SHA1SUMS.key file - if ( find(content.begin(), content.end(), shakey) - != content.end() ) - { - MIL << "found public key file: " << shakey.name << endl; - fetcher.enqueue( OnMediaLocation(resource.filename() + shakey.name, resource.medianr()).setOptional(true) ); - assert_dir(dest_dir + resource.filename()); - fetcher.start( dest_dir, media ); - fetcher.reset(); - KeyContext context; - sigchecker.addPublicKey(dest_dir + resource.filename() + shakey.name, context); - } - - // look for the SHA1SUMS public key file - if ( find(content.begin(), content.end(), shafile) - != content.end() ) - { - MIL << "found checksums file: " << shafile.name << endl; - fetcher.enqueue( OnMediaLocation(resource.filename() + shafile.name, resource.medianr()).setOptional(true) ); - assert_dir(dest_dir + resource.filename()); - fetcher.start( dest_dir, media ); - fetcher.reset(); - } - - // hash table to store checksums - map checksums; - - // look for the SHA1SUMS file - if ( find(content.begin(), content.end(), shafile) != content.end() ) - { - MIL << "found checksums file: " << shafile.name << endl; - fetcher.enqueue( OnMediaLocation( - resource.filename() + shafile.name, - resource.medianr()).setOptional(true), - FileChecker(sigchecker) ); - assert_dir(dest_dir + resource.filename()); - fetcher.start( dest_dir + resource.filename(), media ); - fetcher.reset(); - - // now read the SHA1SUMS file - Pathname pShafile = dest_dir + resource.filename() + shafile.name; - std::ifstream in( pShafile.c_str() ); - string buffer; - if ( ! in.fail() ) - { - while ( getline(in, buffer) ) - { - vector<string> words; - str::split( buffer, back_inserter(words) ); - if ( words.size() != 2 ) - ZYPP_THROW(Exception("Wrong format for SHA1SUMS file")); - //MIL << "check: '" << words[0] << "' | '" << words[1] << "'" << endl; - if ( ! words[1].empty() ) - checksums[words[1]] = CheckSum::sha1(words[0]); - } - } - else - ZYPP_THROW(Exception("Can't open SHA1SUMS file: " + pShafile.asString())); + if ( find( content.begin(), content.end(), shafile ) != content.end() ) + { + // add the index of this directory + OnMediaLocation indexloc(resource); + indexloc.changeFilename(resource.filename() + "SHA1SUMS"); + addIndex(indexloc); + // we need to read it now + readIndexes(media, dest_dir); } for ( filesystem::DirContent::const_iterator it = content.begin(); @@ -391,17 +361,17 @@ case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all case filesystem::FT_FILE: { - CheckSum checksum; - if ( checksums.find(it->name) != checksums.end() ) - checksum = checksums[it->name]; - - // create a resource from the file - // if checksum was not available we will have a empty - // checksum, which will end in a validation anyway - // warning the user that there is no checksum - enqueueDigested(OnMediaLocation(filename, resource.medianr()) - .setChecksum(checksum)); + CheckSum chksm(resource.checksum()); + if ( _checksums.find(filename.asString()) != _checksums.end() ) + { + // the checksum can be replaced with the one in the index. + chksm = _checksums[filename.asString()]; + //MIL << "resource " << filename << " has checksum in the index file." << endl; + } + else + WAR << "Resource " << filename << " has no checksum in the index either." << endl; + enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm)); break; } case filesystem::FT_DIR: // newer directory.yast contain at least directory info @@ -455,6 +425,82 @@ } } + void Fetcher::Impl::readIndex( const Pathname &index, const Pathname &basedir ) + { + std::ifstream in( index.c_str() ); + string buffer; + if ( ! in.fail() ) + { + while ( getline(in, buffer) ) + { + vector<string> words; + str::split( buffer, back_inserter(words) ); + if ( words.size() != 2 ) + ZYPP_THROW(Exception("Wrong format for SHA1SUMS file")); + //MIL << "check: '" << words[0] << "' | '" << words[1] << "'" << endl; + if ( ! words[1].empty() ) + _checksums[(basedir + words[1]).asString()] = CheckSum::sha1(words[0]); + } + } + else + ZYPP_THROW(Exception("Can't open SHA1SUMS file: " + index.asString())); + } + + void Fetcher::Impl::readIndexes( MediaSetAccess &media, const Pathname &dest_dir) + { + // if there is no indexes, then just return to avoid + // the directory listing + if ( _indexes.empty() ) + { + MIL << "No indexes to read." << endl; + return; + } + + // create a new fetcher with a different state to transfer the + // file containing checksums and its signature + Fetcher fetcher; + // signature checker for index. We havent got the signature from + // the nextwork yet. + SignatureFileChecker sigchecker; + + for ( list::const_iterator it_idx = _indexes.begin(); + it_idx != _indexes.end(); ++it_idx ) + { + MIL << "reading index " << (*it_idx)->location << endl; + // build the name of the index and the signature + OnMediaLocation idxloc((*it_idx)->location); + OnMediaLocation sigloc((*it_idx)->location.setOptional(true)); + OnMediaLocation keyloc((*it_idx)->location.setOptional(true)); + + // calculate signature and key name + sigloc.changeFilename( sigloc.filename().extend(".asc") ); + keyloc.changeFilename( keyloc.filename().extend(".key") ); + + //assert_dir(dest_dir + idxloc.filename().dirname()); + + // transfer the signature + fetcher.enqueue(sigloc); + fetcher.start( dest_dir, media ); + // if we get the signature, update the checker + sigchecker = SignatureFileChecker(dest_dir + sigloc.filename()); + fetcher.reset(); + + // now the key + fetcher.enqueue(keyloc); + fetcher.start( dest_dir, media ); + fetcher.reset(); + + // now the index itself + fetcher.enqueue( idxloc, FileChecker(sigchecker) ); + fetcher.start( dest_dir, media ); + fetcher.reset(); + + // now we have the indexes in dest_dir + readIndex( dest_dir + idxloc.filename(), idxloc.filename().dirname() ); + } + MIL << "done reading indexes" << endl; + } + void Fetcher::Impl::start( const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc & progress_receiver ) @@ -462,6 +508,8 @@ ProgressData progress(_resources.size()); progress.sendTo(progress_receiver); + readIndexes(media, dest_dir); + for ( list::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res ) { @@ -527,6 +575,12 @@ _pimpl->enqueueDir(resource, recursive, checker); } + void Fetcher::addIndex( const OnMediaLocation &resource ) + { + _pimpl->addIndex(resource); + } + + void Fetcher::enqueue( const OnMediaLocation &resource, const FileChecker &checker ) { _pimpl->enqueue(resource, checker); Modified: trunk/libzypp/zypp/Fetcher.h URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/zypp/Fetcher.h?rev=11513&r1=11512&r2=11513&view=diff ============================================================================== --- trunk/libzypp/zypp/Fetcher.h (original) +++ trunk/libzypp/zypp/Fetcher.h Mon Oct 27 18:51:36 2008 @@ -76,6 +76,23 @@ virtual ~Fetcher(); public: + + + /** + * Adds an index containing metadata (for example + * checksums ) that will be retrieved and read + * before the job processing starts. + * + * Nothing will be transfered or checked + * until \ref start() is called. + * + * The index is relative to the media path, and + * the listed files too. + * + */ + void addIndex( const OnMediaLocation &resource ); + + /** * Enqueue a object for transferal, they will not * be transfered until \ref start() is called -- To unsubscribe, e-mail: zypp-commit+unsubscribe@opensuse.org For additional commands, e-mail: zypp-commit+help@opensuse.org