Author: dmacvicar
Date: Tue Sep 23 18:29:53 2008
New Revision: 11138
URL: http://svn.opensuse.org/viewcvs/zypp?rev=11138&view=rev
Log:
- refactoring of Fetcher into smaller methods
- start to implement enqueueDir, to allow crypto checking of slideshow
- create a testcase to do changes without making anyone nervous :-)
Added:
trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/
trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/SHA1SUMS
trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir1/
trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir1/subdir1-file1.txt
trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir1/subdir1-file2.txt
trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir2/
trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir2/subdir2-file1.txt
Modified:
trunk/libzypp/tests/zypp/Fetcher_test.cc
trunk/libzypp/zypp/Fetcher.cc
trunk/libzypp/zypp/MediaSetAccess.cc
trunk/libzypp/zypp/MediaSetAccess.h
trunk/libzypp/zypp/PathInfo.h
Modified: trunk/libzypp/tests/zypp/Fetcher_test.cc
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/tests/zypp/Fetcher_test.cc?rev=11138&r1=11137&r2=11138&view=diff
==============================================================================
--- trunk/libzypp/tests/zypp/Fetcher_test.cc (original)
+++ trunk/libzypp/tests/zypp/Fetcher_test.cc Tue Sep 23 18:29:53 2008
@@ -4,9 +4,11 @@
#include
#include
+#include "zypp/base/Logger.h"
#include "zypp/MediaSetAccess.h"
#include "zypp/Fetcher.h"
#include "zypp/Url.h"
+#include "zypp/TmpPath.h"
using std::cout;
using std::endl;
@@ -19,8 +21,29 @@
BOOST_AUTO_TEST_CASE(fetcher)
{
- MediaSetAccess media( ("dir:" + DATADIR).asUrl(), "/" );
+ MediaSetAccess media( ( DATADIR).asUrl(), "/" );
Fetcher fetcher;
+
+ filesystem::TmpDir dest;
+
+ fetcher.enqueueDir(OnMediaLocation().setFilename("/complexdir"), true);
+ fetcher.start( dest.path(), media );
+
+ BOOST_CHECK( PathInfo(dest.path() + "/complexdir/subdir2").isExist() );
+ BOOST_CHECK( PathInfo(dest.path() + "/complexdir/subdir2/subdir2-file1.txt").isExist() );
+ BOOST_CHECK( PathInfo(dest.path() + "/complexdir/subdir1/subdir1-file1.txt").isExist() );
+ BOOST_CHECK( PathInfo(dest.path() + "/complexdir/subdir1/subdir1-file2.txt").isExist() );
+
+ fetcher.reset();
+
+ fetcher.enqueue(OnMediaLocation().setFilename("/file-1.txt"));
+ fetcher.start( dest.path(), media );
+
+ BOOST_CHECK( PathInfo(dest.path() + "/file-1.txt").isExist() );
+
+ MIL << fetcher;
}
+
+
// vim: set ts=2 sts=2 sw=2 ai et:
Added: trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/SHA1SUMS
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/SHA1SUMS?rev=11138&view=auto
==============================================================================
--- trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/SHA1SUMS (added)
+++ trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/SHA1SUMS Tue Sep 23 18:29:53 2008
@@ -0,0 +1 @@
+ffoo
Added: trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir1/subdir1-file1.txt
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir1/subdir1-file1.txt?rev=11138&view=auto
==============================================================================
--- trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir1/subdir1-file1.txt (added)
+++ trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir1/subdir1-file1.txt Tue Sep 23 18:29:53 2008
@@ -0,0 +1 @@
+foo
Added: trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir1/subdir1-file2.txt
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir1/subdir1-file2.txt?rev=11138&view=auto
==============================================================================
--- trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir1/subdir1-file2.txt (added)
+++ trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir1/subdir1-file2.txt Tue Sep 23 18:29:53 2008
@@ -0,0 +1 @@
+bar
Added: trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir2/subdir2-file1.txt
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir2/subdir2-file1.txt?rev=11138&view=auto
==============================================================================
--- trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir2/subdir2-file1.txt (added)
+++ trunk/libzypp/tests/zypp/data/Fetcher/remote-site/complexdir/subdir2/subdir2-file1.txt Tue Sep 23 18:29:53 2008
@@ -0,0 +1 @@
+hello
Modified: trunk/libzypp/zypp/Fetcher.cc
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/zypp/Fetcher.cc?rev=11138&r1=11137&r2=11138&view=diff
==============================================================================
--- trunk/libzypp/zypp/Fetcher.cc (original)
+++ trunk/libzypp/zypp/Fetcher.cc Tue Sep 23 18:29:53 2008
@@ -31,6 +31,7 @@
*/
struct FetcherJob
{
+
FetcherJob( const OnMediaLocation &loc )
: location(loc)
, directory(false)
@@ -53,6 +54,12 @@
typedef shared_ptr<FetcherJob> FetcherJob_Ptr;
+ std::ostream & operator<<( std::ostream & str, const FetcherJob_Ptr & obj )
+ {
+ return str << obj->location;
+ }
+
+
///////////////////////////////////////////////////////////////////
//
// CLASS NAME : Fetcher::Impl
@@ -60,7 +67,7 @@
/** Fetcher implementation. */
class Fetcher::Impl
{
-
+ friend std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj );
public:
Impl() {}
~Impl() {
@@ -88,13 +95,24 @@
* looking at the cache. If success, returns true, and the desired
* file should be available on dest_dir
*/
- bool provideFromCache( FetcherJob_Ptr job, const Pathname &dest_dir );
+ bool provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir );
/**
* Validates the job against is checkers, by using the file instance
* on dest_dir
* \throws Exception
*/
- void validate( FetcherJob_Ptr job, const Pathname &dest_dir );
+ void validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers );
+
+ /**
+ * scan the directory and adds the individual jobs
+ */
+ void addDirJobs( MediaSetAccess &media, const OnMediaLocation &resource,
+ const Pathname &dest_dir, bool recursive );
+ /**
+ * Provide the resource to \ref dest_dir
+ */
+ void provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir );
+
private:
friend Impl * rwcowClone<Impl>( const Impl * rhs );
/** clone for RWCOW_pointer */
@@ -167,25 +185,34 @@
}
- bool Fetcher::Impl::provideFromCache( FetcherJob_Ptr job, const Pathname &dest_dir )
+ bool Fetcher::Impl::provideFromCache( const OnMediaLocation &resource, const Pathname &dest_dir )
{
+ Pathname dest_full_path = dest_dir + resource.filename();
+
+ // first check in the destination directory
+ if ( PathInfo(dest_full_path).isExist() )
+ {
+ if ( is_checksum( dest_full_path, resource.checksum() )
+ && (! resource.checksum().empty() ) )
+ return true;
+ }
+
MIL << "start fetcher with " << _caches.size() << " cache directories." << endl;
for_ ( it_cache, _caches.begin(), _caches.end() )
{
// does the current file exists in the current cache?
- Pathname cached_file = *it_cache + job->location.filename();
+ Pathname cached_file = *it_cache + resource.filename();
if ( PathInfo( cached_file ).isExist() )
{
- DBG << "File '" << cached_file << "' exist, testing checksum " << job->location.checksum() << endl;
+ DBG << "File '" << cached_file << "' exist, testing checksum " << resource.checksum() << endl;
// check the checksum
- if ( is_checksum( cached_file, job->location.checksum() ) && (! job->location.checksum().empty() ) )
+ if ( is_checksum( cached_file, resource.checksum() ) && (! resource.checksum().empty() ) )
{
// cached
- MIL << "file " << job->location.filename() << " found in previous cache. Using cached copy." << endl;
+ MIL << "file " << resource.filename() << " found in previous cache. Using cached copy." << endl;
// checksum is already checked.
// we could later implement double failover and try to download if file copy fails.
// replicate the complete path in the target directory
- Pathname dest_full_path = dest_dir + job->location.filename();
if( dest_full_path != cached_file )
{
if ( assert_dir( dest_full_path.dirname() ) != 0 )
@@ -210,15 +237,15 @@
return false;
}
- void Fetcher::Impl::validate( FetcherJob_Ptr job, const Pathname &dest_dir )
+ void Fetcher::Impl::validate( const OnMediaLocation &resource, const Pathname &dest_dir, const list<FileChecker> &checkers )
{
// no matter where did we got the file, try to validate it:
- Pathname localfile = dest_dir + job->location.filename();
+ Pathname localfile = dest_dir + resource.filename();
// call the checker function
try {
- MIL << "Checking job [" << localfile << "] (" << job->checkers.size() << " checkers )" << endl;
- for ( list<FileChecker>::const_iterator it = job->checkers.begin();
- it != job->checkers.end();
+ MIL << "Checking job [" << localfile << "] (" << checkers.size() << " checkers )" << endl;
+ for ( list<FileChecker>::const_iterator it = checkers.begin();
+ it != checkers.end();
++it )
{
if (*it)
@@ -242,58 +269,132 @@
}
catch (...)
{
- ZYPP_THROW(Exception("Unknown error while validating " + job->location.filename().asString()));
+ ZYPP_THROW(Exception("Unknown error while validating " + resource.filename().asString()));
}
}
- void Fetcher::Impl::start( const Pathname &dest_dir,
- MediaSetAccess &media,
- const ProgressData::ReceiverFnc & progress_receiver )
+ void Fetcher::Impl::addDirJobs( MediaSetAccess &media,
+ const OnMediaLocation &resource,
+ const Pathname &dest_dir, bool recursive )
{
- ProgressData progress(_resources.size());
- progress.sendTo(progress_receiver);
+ filesystem::DirContent content;
+
+ media.dirInfo( content, resource.filename(), false /* dots */, resource.medianr());
- for ( list::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res )
- {
- bool got_from_cache = false;
+ for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it )
+ {
+ MIL << (*it).name << endl;
+ }
- // start look in cache
- got_from_cache = provideFromCache((*it_res), dest_dir);
+ filesystem::DirEntry shafile, shasig;
+ shafile.name = "SHA1SUMS";
+ shafile.name = "SHA1SUMS.asc";
+ shasig.type = filesystem::FT_FILE;
+ shasig.type = filesystem::FT_FILE;
+
+ // look for the SHA1SUMS file
+ if ( find(content.begin(), content.end(), shafile) != content.end() )
+ {
+ MIL << "found checksums file: " << shafile.name << endl;
+
+ provideToDest(media, resource.filename() + shafile.name, dest_dir);
+ // look for the SHA1SUMS.asc signature
+ if ( find(content.begin(), content.end(), shasig) != content.end() )
+ {
+ provideToDest(media, resource.filename() + shasig.name, dest_dir);
+ }
+
+ }
- if ( ! got_from_cache )
+ for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it )
{
- MIL << "Not found in cache, downloading" << endl;
-
- // try to get the file from the net
- try
- {
- Pathname tmp_file = media.provideFile((*it_res)->location);
- Pathname dest_full_path = dest_dir + (*it_res)->location.filename();
- if ( assert_dir( dest_full_path.dirname() ) != 0 )
- ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
- if ( filesystem::copy(tmp_file, dest_full_path ) != 0 )
+ Pathname filename = resource.filename() + it->name;
+
+ switch ( it->type )
{
- ZYPP_THROW( Exception("Can't copy " + tmp_file.asString() + " to " + dest_dir.asString()));
+ case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
+ case filesystem::FT_FILE:
+ enqueue(OnMediaLocation().setFilename(filename), FileChecker());
+
+ MIL << filename << endl;
+ break;
+ case filesystem::FT_DIR: // newer directory.yast contain at least directory info
+ if ( recursive )
+ {
+ addDirJobs(media, filename, dest_dir, recursive);
+ }
+ else
+ {
+ }
+ break;
+ default:
+ // don't provide devices, sockets, etc.
+ break;
}
+ }
+ }
- media.releaseFile((*it_res)->location); //not needed anymore, only eat space
- }
- catch (Exception & excpt_r)
+ void Fetcher::Impl::provideToDest( MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir )
+ {
+ bool got_from_cache = false;
+
+ // start look in cache
+ got_from_cache = provideFromCache(resource, dest_dir);
+
+ if ( ! got_from_cache )
+ {
+ MIL << "Not found in cache, downloading" << endl;
+
+ // try to get the file from the net
+ try
+ {
+ Pathname tmp_file = media.provideFile(resource);
+ Pathname dest_full_path = dest_dir + resource.filename();
+ if ( assert_dir( dest_full_path.dirname() ) != 0 )
+ ZYPP_THROW( Exception("Can't create " + dest_full_path.dirname().asString()));
+ if ( filesystem::copy(tmp_file, dest_full_path ) != 0 )
{
- ZYPP_CAUGHT(excpt_r);
- excpt_r.remember("Can't provide " + (*it_res)->location.filename().asString() + " : " + excpt_r.msg());
- ZYPP_RETHROW(excpt_r);
+ ZYPP_THROW( Exception("Can't copy " + tmp_file.asString() + " to " + dest_dir.asString()));
}
+
+ media.releaseFile(resource); //not needed anymore, only eat space
}
- else
+ catch (Exception & excpt_r)
+ {
+ ZYPP_CAUGHT(excpt_r);
+ excpt_r.remember("Can't provide " + resource.filename().asString() + " : " + excpt_r.msg());
+ ZYPP_RETHROW(excpt_r);
+ }
+ }
+ else
+ {
+ // We got the file from cache
+ // continue with next file
+ return;
+ }
+ }
+
+ void Fetcher::Impl::start( const Pathname &dest_dir,
+ MediaSetAccess &media,
+ const ProgressData::ReceiverFnc & progress_receiver )
+ {
+ ProgressData progress(_resources.size());
+ progress.sendTo(progress_receiver);
+
+ for ( list::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res )
+ {
+
+ if ( (*it_res)->directory )
{
- // We got the file from cache
- // continue with next file
- continue;
+ const OnMediaLocation location((*it_res)->location);
+ addDirJobs(media, location, dest_dir, true);
+ continue;
}
+ provideToDest(media, (*it_res)->location, dest_dir);
+
// validate job, this throws if not valid
- validate(*it_res, dest_dir);
+ validate((*it_res)->location, dest_dir, (*it_res)->checkers);
if ( ! progress.incr() )
ZYPP_THROW(AbortRequestException());
@@ -303,7 +404,11 @@
/** \relates Fetcher::Impl Stream output */
inline std::ostream & operator<<( std::ostream & str, const Fetcher::Impl & obj )
{
- return str << "Fetcher::Impl";
+ for ( list::const_iterator it_res = obj._resources.begin(); it_res != obj._resources.end(); ++it_res )
+ {
+ str << *it_res;
+ }
+ return str;
}
///////////////////////////////////////////////////////////////////
Modified: trunk/libzypp/zypp/MediaSetAccess.cc
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/zypp/MediaSetAccess.cc?rev=11138&r1=11137&r2=11138&view=diff
==============================================================================
--- trunk/libzypp/zypp/MediaSetAccess.cc (original)
+++ trunk/libzypp/zypp/MediaSetAccess.cc Tue Sep 23 18:29:53 2008
@@ -103,6 +103,20 @@
media_mgr.releaseFile (media, file);
}
+ void MediaSetAccess::dirInfo( filesystem::DirContent &retlist, const Pathname &dirname,
+ bool dots, unsigned media_nr )
+ {
+ media::MediaManager media_mgr;
+ media::MediaAccessId media;
+ media = getMediaAccessId(media_nr);
+
+ // try to attach the media
+ if ( ! media_mgr.isAttached(media) )
+ media_mgr.attachDesiredMedia(media);
+
+ media_mgr.dirInfo(media, retlist, dirname, dots);
+ }
+
Pathname MediaSetAccess::provideFile( const OnMediaLocation & on_media_file )
{
return provideFile( on_media_file.filename(), on_media_file.medianr() );
Modified: trunk/libzypp/zypp/MediaSetAccess.h
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/zypp/MediaSetAccess.h?rev=11138&r1=11137&r2=11138&view=diff
==============================================================================
--- trunk/libzypp/zypp/MediaSetAccess.h (original)
+++ trunk/libzypp/zypp/MediaSetAccess.h Tue Sep 23 18:29:53 2008
@@ -171,6 +171,12 @@
bool doesFileExist(const Pathname & file, unsigned media_nr = 1 );
/**
+ * Fills \ref retlist with directory information
+ */
+ void dirInfo( filesystem::DirContent &retlist, const Pathname &dirname,
+ bool dots = true, unsigned media_nr = 1 );
+
+ /**
* Release all attached media of this set.
*
* \throws MediaNotOpenException for invalid access IDs.
Modified: trunk/libzypp/zypp/PathInfo.h
URL: http://svn.opensuse.org/viewcvs/zypp/trunk/libzypp/zypp/PathInfo.h?rev=11138&r1=11137&r2=11138&view=diff
==============================================================================
--- trunk/libzypp/zypp/PathInfo.h (original)
+++ trunk/libzypp/zypp/PathInfo.h Tue Sep 23 18:29:53 2008
@@ -489,6 +489,9 @@
: name( name_r )
, type( type_r )
{}
+
+ bool operator==( const DirEntry &rhs ) const
+ { return ((name == rhs.name ) && (type == rhs.type)); }
};
/** Returned by readdir. */
--
To unsubscribe, e-mail: zypp-commit+unsubscribe@opensuse.org
For additional commands, e-mail: zypp-commit+help@opensuse.org
