[zypp-commit] r11555 - in /trunk/libzypp: tests/zypp/CMakeLists.txt tests/zypp/Fetcher_test.cc zypp/Fetcher.cc zypp/Fetcher.h zypp/FileChecker.cc
Author: dmacvicar
Date: Fri Oct 31 13:54:40 2008
New Revision: 11555
URL: http://svn.opensuse.org/viewcvs/zypp?rev=11555&view=rev
Log:
- add ability to autodiscover indexes.
- enqueueDir is not enqueueDigestedDir
- enqueueDir stays, but does not check automatically
- lot of new testcases and docs on those classes
Modified:
trunk/libzypp/tests/zypp/CMakeLists.txt
trunk/libzypp/tests/zypp/Fetcher_test.cc
trunk/libzypp/zypp/Fetcher.cc
trunk/libzypp/zypp/Fetcher.h
trunk/libzypp/zypp/FileChecker.cc
Modified: trunk/libzypp/tests/zypp/CMakeLists.txt
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/tests/zypp/CMakeLists.txt?rev=11555&r1=11554&r2=11555&view=diff
==============================================================================
--- trunk/libzypp/tests/zypp/CMakeLists.txt (original)
+++ trunk/libzypp/tests/zypp/CMakeLists.txt Fri Oct 31 13:54:40 2008
@@ -24,6 +24,7 @@
RWPtr
RepoInfo
RepoManager
+ Reports
RepoStatus
ResKind
ResStatus
Modified: trunk/libzypp/tests/zypp/Fetcher_test.cc
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/tests/zypp/Fetcher_test.cc?rev=11555&r1=11554&r2=11555&view=diff
==============================================================================
--- trunk/libzypp/tests/zypp/Fetcher_test.cc (original)
+++ trunk/libzypp/tests/zypp/Fetcher_test.cc Fri Oct 31 13:54:40 2008
@@ -23,27 +23,31 @@
#define DATADIR (Pathname(TESTS_SRC_DIR) + "/zypp/data/Fetcher/remote-site")
-BOOST_AUTO_TEST_CASE(fetcher)
+BOOST_AUTO_TEST_CASE(fetcher_enqueuedir_noindex)
{
MediaSetAccess media( ( DATADIR).asUrl(), "/" );
- Fetcher fetcher;
-
// Now test that without trusting it, it should throw
// do the test by trusting the SHA1SUMS file signature key
{
filesystem::TmpDir dest;
-
+ Fetcher fetcher;
fetcher.enqueueDir(OnMediaLocation("/complexdir"), true);
- BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), Exception);
+ fetcher.start( dest.path(), media );
fetcher.reset();
}
+}
+BOOST_AUTO_TEST_CASE(fetcher_enqueuedir_autoindex)
+{
+ MediaSetAccess media( ( DATADIR).asUrl(), "/" );
// do the test by trusting the SHA1SUMS file signature key
{
filesystem::TmpDir dest;
// add the key as trusted
getZYpp()->keyRing()->importKey(PublicKey(DATADIR + "/complexdir/subdir1/SHA1SUMS.key"), true);
+ Fetcher fetcher;
+ fetcher.setOptions( Fetcher::AutoAddIndexes );
fetcher.enqueueDir(OnMediaLocation("/complexdir"), true);
fetcher.start( dest.path(), media );
@@ -54,22 +58,81 @@
fetcher.reset();
}
+}
+
+BOOST_AUTO_TEST_CASE(fetcher_enqueue_digested_dir_autoindex)
+{
+ MediaSetAccess media( ( DATADIR).asUrl(), "/" );
+ // do the test by trusting the SHA1SUMS file signature key but with a broken file
+ {
+ filesystem::TmpDir dest;
+ Fetcher fetcher;
+ // add the key as trusted
+ getZYpp()->keyRing()->importKey(PublicKey(DATADIR + "/complexdir-broken/subdir1/SHA1SUMS.key"), true);
+ fetcher.setOptions( Fetcher::AutoAddIndexes );
+ fetcher.enqueueDigestedDir(OnMediaLocation("/complexdir-broken"), true);
+ BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), FileCheckException);
+ fetcher.reset();
+ }
+}
+BOOST_AUTO_TEST_CASE(fetcher_enqueuebrokendir_noindex)
+{
+ MediaSetAccess media( ( DATADIR).asUrl(), "/" );
// do the test by trusting the SHA1SUMS file signature key but with a broken file
{
filesystem::TmpDir dest;
+ Fetcher fetcher;
+ // add the key as trusted
+ getZYpp()->keyRing()->importKey(PublicKey(DATADIR + "/complexdir-broken/subdir1/SHA1SUMS.key"), true);
+ fetcher.enqueueDir(OnMediaLocation("/complexdir-broken"), true);
+ // this should not throw as we provided no indexes and the
+ // enqueue is not digested
+ fetcher.start( dest.path(), media );
+ fetcher.reset();
+ }
+}
+BOOST_AUTO_TEST_CASE(fetcher_enqueuebrokendir_index)
+{
+ MediaSetAccess media( ( DATADIR).asUrl(), "/" );
+ // do the test by trusting the SHA1SUMS file signature key but with a broken file
+ {
+ filesystem::TmpDir dest;
+ Fetcher fetcher;
// add the key as trusted
getZYpp()->keyRing()->importKey(PublicKey(DATADIR + "/complexdir-broken/subdir1/SHA1SUMS.key"), true);
+ fetcher.setOptions( Fetcher::AutoAddIndexes );
fetcher.enqueueDir(OnMediaLocation("/complexdir-broken"), true);
BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), Exception);
fetcher.reset();
}
+}
+BOOST_AUTO_TEST_CASE(fetcher_enqueue_digested_brokendir_with_index)
+{
+ MediaSetAccess media( ( DATADIR).asUrl(), "/" );
+ // do the test by trusting the SHA1SUMS file signature key but with a broken file
{
filesystem::TmpDir dest;
+ Fetcher fetcher;
+ // add the key as trusted
+ getZYpp()->keyRing()->importKey(PublicKey(DATADIR + "/complexdir-broken/subdir1/SHA1SUMS.key"), true);
+ fetcher.setOptions( Fetcher::AutoAddIndexes );
+ fetcher.enqueueDigestedDir(OnMediaLocation("/complexdir-broken"), true);
+ BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), Exception);
+ fetcher.reset();
+ }
+}
+
+BOOST_AUTO_TEST_CASE(fetcher_enqueuefile_noindex)
+{
+ MediaSetAccess media( ( DATADIR).asUrl(), "/" );
+ {
+ filesystem::TmpDir dest;
+ Fetcher fetcher;
fetcher.enqueue(OnMediaLocation("/file-1.txt"));
fetcher.start( dest.path(), media );
BOOST_CHECK( PathInfo(dest.path() + "/file-1.txt").isExist() );
@@ -97,7 +160,6 @@
fetcher.reset();
}
-
}
BOOST_AUTO_TEST_CASE(content_index)
@@ -119,7 +181,7 @@
}
-BOOST_AUTO_TEST_CASE(content_index_broken)
+BOOST_AUTO_TEST_CASE(enqueue_broken_content_index)
{
MediaSetAccess media( ( DATADIR).asUrl(), "/" );
Fetcher fetcher;
@@ -140,6 +202,48 @@
}
}
+BOOST_AUTO_TEST_CASE(enqueue_digested_broken_content_index)
+{
+ MediaSetAccess media( ( DATADIR).asUrl(), "/" );
+ Fetcher fetcher;
+
+ {
+ filesystem::TmpDir dest;
+ OnMediaLocation loc("/contentindex-broken-digest/subdir1/subdir1-file1.txt",1);
+ // key was already imported as trusted
+ fetcher.addIndex(OnMediaLocation("/contentindex-broken-digest/content", 1));
+ fetcher.enqueueDigested(loc);
+ fetcher.start(dest.path(), media);
+ fetcher.reset();
+ // now retrieve a file that is modified, so the checksum has to fail
+ loc = OnMediaLocation("/contentindex-broken-digest/subdir1/subdir1-file2.txt",1);
+ fetcher.enqueueDigested(loc);
+ BOOST_CHECK_THROW( fetcher.start( dest.path(), media ), Exception);
+ fetcher.reset();
+ }
+}
+
+BOOST_AUTO_TEST_CASE(enqueue_broken_content_noindex)
+{
+ MediaSetAccess media( ( DATADIR).asUrl(), "/" );
+ Fetcher fetcher;
+
+ {
+ filesystem::TmpDir dest;
+ OnMediaLocation loc("/contentindex-broken-digest/subdir1/subdir1-file1.txt",1);
+ // key was already imported as trusted
+ fetcher.enqueue(loc);
+ fetcher.start(dest.path(), media);
+ fetcher.reset();
+ // now retrieve a file that is modified, so the checksum has to fail
+ loc = OnMediaLocation("/contentindex-broken-digest/subdir1/subdir1-file2.txt",1);
+ fetcher.enqueue(loc);
+ fetcher.start( dest.path(), media );
+ fetcher.reset();
+ }
+}
+
+
BOOST_AUTO_TEST_CASE(fetcher_remove)
{
// at this point the key is already trusted
@@ -154,6 +258,8 @@
Fetcher fetcher;
filesystem::TmpDir dest;
+ // auto add the SHA1SUMS
+ fetcher.setOptions( Fetcher::AutoAddIndexes );
fetcher.enqueueDir(OnMediaLocation("/complexdir"), true);
fetcher.start( dest.path(), media );
Modified: trunk/libzypp/zypp/Fetcher.cc
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/zypp/Fetcher.cc?rev=11555&r1=11554&r2=11555&view=diff
==============================================================================
--- trunk/libzypp/zypp/Fetcher.cc (original)
+++ trunk/libzypp/zypp/Fetcher.cc Fri Oct 31 13:54:40 2008
@@ -53,10 +53,22 @@
*/
struct FetcherJob
{
+ enum Flag
+ {
+ None = 0x0000,
+ Directory = 0x0001,
+ Recursive = 0x0002,
+ RecursiveDirectory = Directory | Recursive,
+ // check checksums even if there is no such
+ // checksum (warns of no checksum)
+ AlwaysVerifyChecksum = 0x0004,
+ };
+ ZYPP_DECLARE_FLAGS(Flags, Flag);
+
+
FetcherJob( const OnMediaLocation &loc )
: location(loc)
- , directory(false)
- , recursive(false)
+ , flags(None)
{
//MIL << location << endl;
}
@@ -69,10 +81,9 @@
OnMediaLocation location;
//CompositeFileChecker checkers;
list<FileChecker> checkers;
- bool directory;
- bool recursive;
+ Flags flags;
};
-
+ ZYPP_DECLARE_OPERATORS_FOR_FLAGS(FetcherJob::Flags);
typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
@@ -100,8 +111,11 @@
Fetcher::Options options() const;
void addIndex( const OnMediaLocation &resource );
- void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
+
void enqueueDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
+ void enqueueDigestedDir( const OnMediaLocation &resource, bool recursive, const FileChecker &checker = FileChecker() );
+
+ void enqueue( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
void enqueueDigested( const OnMediaLocation &resource, const FileChecker &checker = FileChecker() );
void addCachePath( const Pathname &cache_dir );
void reset();
@@ -152,8 +166,16 @@
/**
* scan the directory and adds the individual jobs
*/
- void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
- const Pathname &dest_dir, bool recursive );
+ void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
+ const Pathname &dest_dir, FetcherJob::Flags flags );
+
+ /**
+ * auto discovery and reading of indexes
+ */
+ void autoaddIndexes( const filesystem::DirContent &content,
+ MediaSetAccess &media,
+ const OnMediaLocation &resource,
+ const Pathname &dest_dir );
/**
* Provide the resource to \ref dest_dir
*/
@@ -178,10 +200,7 @@
{
FetcherJob_Ptr job;
job.reset(new FetcherJob(resource));
- ChecksumFileChecker digest_check(resource.checksum());
- job->checkers.push_back(digest_check);
- if ( checker )
- job->checkers.push_back(checker);
+ job->flags |= FetcherJob:: AlwaysVerifyChecksum;
_resources.push_back(job);
}
@@ -204,11 +223,30 @@
job.reset(new FetcherJob(resource));
if ( checker )
job->checkers.push_back(checker);
- job->directory = true;
- job->recursive = recursive;
+ if ( recursive )
+ job->flags |= FetcherJob::Recursive;
+ job->flags |= FetcherJob::Directory;
+
_resources.push_back(job);
}
+ void Fetcher::Impl::enqueueDigestedDir( const OnMediaLocation &resource,
+ bool recursive,
+ const FileChecker &checker )
+ {
+ FetcherJob_Ptr job;
+ job.reset(new FetcherJob(resource));
+ if ( checker )
+ job->checkers.push_back(checker);
+ if ( recursive )
+ job->flags |= FetcherJob::Recursive;
+ job->flags |= FetcherJob::Directory;
+ job->flags |= FetcherJob::AlwaysVerifyChecksum;
+
+ _resources.push_back(job);
+
+ }
+
void Fetcher::Impl::enqueue( const OnMediaLocation &resource, const FileChecker &checker )
{
FetcherJob_Ptr job;
@@ -256,6 +294,9 @@
}
+ // tries to provide resource to dest_dir from any of the configured additional
+ // cache paths where the file may already be present. returns true if the
+ // file was provided from the cache.
bool Fetcher::Impl::provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir )
{
Pathname dest_full_path = dest_dir + resource.filename();
@@ -346,29 +387,57 @@
}
}
+ void Fetcher::Impl::autoaddIndexes( const filesystem::DirContent &content,
+ MediaSetAccess &media,
+ const OnMediaLocation &resource,
+ const Pathname &dest_dir )
+ {
+ if ( _options & AutoAddSha1sumsIndexes )
+ {
+ // only try to add an index if it exists
+ filesystem::DirEntry shafile;
+ shafile.name = "SHA1SUMS"; shafile.type = filesystem::FT_FILE;
+ if ( find( content.begin(), content.end(), shafile ) != content.end() )
+ {
+ // add the index of this directory
+ OnMediaLocation indexloc(resource);
+ indexloc.changeFilename(resource.filename() + "SHA1SUMS");
+ addIndex(indexloc);
+ // we need to read it now
+ readIndexes(media, dest_dir);
+ }
+ }
+ if ( _options & AutoAddContentFileIndexes )
+ {
+ // only try to add an index if it exists
+ filesystem::DirEntry contentfile;
+ contentfile.name = "content"; contentfile.type = filesystem::FT_FILE;
+ if ( find( content.begin(), content.end(), contentfile ) != content.end() )
+ {
+ // add the index of this directory
+ OnMediaLocation indexloc(resource);
+ indexloc.changeFilename(resource.filename() + "content");
+ addIndex(indexloc);
+ // we need to read it now
+ readIndexes(media, dest_dir);
+ }
+ }
+ }
+
void Fetcher::Impl::addDirJobs( MediaSetAccess &media,
const OnMediaLocation &resource,
- const Pathname &dest_dir, bool recursive )
+ const Pathname &dest_dir, FetcherJob::Flags flags )
{
// first get the content of the directory so we can add
// individual transfer jobs
MIL << "Adding directory " << resource.filename() << endl;
filesystem::DirContent content;
media.dirInfo( content, resource.filename(), false /* dots */, resource.medianr());
-
- // only try to add an index if it exists
- filesystem::DirEntry shafile;
- shafile.name = "SHA1SUMS"; shafile.type = filesystem::FT_FILE;
- if ( find( content.begin(), content.end(), shafile ) != content.end() )
- {
- // add the index of this directory
- OnMediaLocation indexloc(resource);
- indexloc.changeFilename(resource.filename() + "SHA1SUMS");
- addIndex(indexloc);
- // we need to read it now
- readIndexes(media, dest_dir);
- }
-
+
+ // this method test for the option flags so indexes are added
+ // only if the options are enabled
+ autoaddIndexes(content, media, resource, dest_dir);
+
for ( filesystem::DirContent::const_iterator it = content.begin();
it != content.end();
++it )
@@ -393,13 +462,15 @@
}
else
WAR << "Resource " << filename << " has no checksum in the index either." << endl;
-
- enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
+ if ( flags & FetcherJob::AlwaysVerifyChecksum )
+ enqueueDigested(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
+ else
+ enqueue(OnMediaLocation(filename, resource.medianr()).setChecksum(chksm));
break;
}
case filesystem::FT_DIR: // newer directory.yast contain at least directory info
- if ( recursive )
- addDirJobs(media, filename, dest_dir, recursive);
+ if ( flags & FetcherJob::Recursive )
+ addDirJobs(media, filename, dest_dir, flags);
break;
default:
// don't provide devices, sockets, etc.
@@ -508,6 +579,9 @@
ZYPP_THROW(Exception("Can't open SHA1SUMS file: " + index.asString()));
}
+ // this method takes all the user pointed indexes, gets them and also tries to
+ // download their signature, and verify them. After that, its parses each one
+ // to fill the checksum cache.
void Fetcher::Impl::readIndexes( MediaSetAccess &media, const Pathname &dest_dir)
{
// if there is no indexes, then just return to avoid
@@ -563,6 +637,8 @@
MIL << "done reading indexes" << endl;
}
+ // start processing all fetcher jobs.
+ // it processes any user pointed index first
void Fetcher::Impl::start( const Pathname &dest_dir,
MediaSetAccess &media,
const ProgressData::ReceiverFnc & progress_receiver )
@@ -575,10 +651,10 @@
for ( list
participants (1)
-
dmacvicar@svn.opensuse.org