[zypp-commit] r5932 - in /branches/REFACTORING-10_3/libzypp: tests/zypp/ zypp/ zypp/parser/yum/ zypp/repo/ zypp/repo/cached/

Author: dmacvicar Date: Mon Jul 9 14:24:33 2007 New Revision: 5932 URL: http://svn.opensuse.org/viewcvs/zypp?rev=5932&view=rev Log: - Add CombinedProgressData, see docs - Add test for ProgressData - make RepoImpl::resolvables() load lazy - Fix MediaProducts inlining - Start fixing YUM progress report Added: branches/REFACTORING-10_3/libzypp/tests/zypp/ProgressData_test.cc Modified: branches/REFACTORING-10_3/libzypp/tests/zypp/CMakeLists.txt branches/REFACTORING-10_3/libzypp/zypp/MediaProducts.cc branches/REFACTORING-10_3/libzypp/zypp/MediaProducts.h branches/REFACTORING-10_3/libzypp/zypp/ProgressData.cc branches/REFACTORING-10_3/libzypp/zypp/ProgressData.h branches/REFACTORING-10_3/libzypp/zypp/RepoManager.cc branches/REFACTORING-10_3/libzypp/zypp/RepoManager.h branches/REFACTORING-10_3/libzypp/zypp/parser/yum/RepoParser.cc branches/REFACTORING-10_3/libzypp/zypp/repo/RepositoryImpl.cc branches/REFACTORING-10_3/libzypp/zypp/repo/RepositoryImpl.h branches/REFACTORING-10_3/libzypp/zypp/repo/cached/RepoImpl.cc branches/REFACTORING-10_3/libzypp/zypp/repo/cached/RepoImpl.h Modified: branches/REFACTORING-10_3/libzypp/tests/zypp/CMakeLists.txt URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/tests/zypp/CMakeLists.txt?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/tests/zypp/CMakeLists.txt (original) +++ branches/REFACTORING-10_3/libzypp/tests/zypp/CMakeLists.txt Mon Jul 9 14:24:33 2007 @@ -52,6 +52,9 @@ ADD_EXECUTABLE( RepoInfo RepoInfo_test.cc ) TARGET_LINK_LIBRARIES( RepoInfo zypp boost_unit_test_framework ) +ADD_EXECUTABLE( ProgressData ProgressData_test.cc ) +TARGET_LINK_LIBRARIES( ProgressData zypp boost_unit_test_framework ) + ADD_TEST(ArchTest ${CMAKE_CURRENT_BINARY_DIR}/Arch) ADD_TEST(UrlTest ${CMAKE_CURRENT_BINARY_DIR}/Url) ADD_TEST(Edition ${CMAKE_CURRENT_BINARY_DIR}/Edition) @@ -69,3 +72,4 @@ ADD_TEST(RepoManagerTest ${CMAKE_CURRENT_BINARY_DIR}/RepoManager ${CMAKE_CURRENT_SOURCE_DIR}/data/RepoManager) ADD_TEST(RepoInfoTest ${CMAKE_CURRENT_BINARY_DIR}/RepoInfo ${CMAKE_CURRENT_SOURCE_DIR}/data/RepoInfo) ADD_TEST(FetcherTest ${CMAKE_CURRENT_BINARY_DIR}/Fetcher) +ADD_TEST(ProgressData_test ${CMAKE_CURRENT_BINARY_DIR}/ProgressData) Added: branches/REFACTORING-10_3/libzypp/tests/zypp/ProgressData_test.cc URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/tests/zypp/ProgressData_test.cc?rev=5932&view=auto ============================================================================== --- branches/REFACTORING-10_3/libzypp/tests/zypp/ProgressData_test.cc (added) +++ branches/REFACTORING-10_3/libzypp/tests/zypp/ProgressData_test.cc Mon Jul 9 14:24:33 2007 @@ -0,0 +1,59 @@ + +#include <iostream> +#include <fstream> +#include <list> +#include <string> + +#include "zypp/base/Logger.h" +#include "zypp/base/Exception.h" + +#include +#include +#include + +#include "zypp/ProgressData.h" + +using boost::unit_test::test_suite; +using boost::unit_test::test_case; +using namespace boost::unit_test::log; + +using namespace std; +using namespace zypp; + + +struct ProgressCollector +{ + bool operator()( const ProgressData &progress ) + { + + } + + ProgressData::value_type lastReported; +}; + +void progressdata_test() +{ + ProgressData progress; + CombinedProgressData sub1rcv(progress, 80); + CombinedProgressData sub2rcv(progress, 20); + + ProgressData sub1progress; + sub1progress.sendTo(sub1rcv); + + ProgressData sub2progress; + sub2progress.sendTo(sub2rcv); + + // task 1 goes to 50% + sub1progress.set(50); + // which is 50% of 80% in task 1 + BOOST_CHECK_EQUAL( progress.val(), 40 ); +} + +test_suite* +init_unit_test_suite( int argc, char* argv[] ) +{ + test_suite* test= BOOST_TEST_SUITE( "ProgressData_test" ); + test->add( BOOST_TEST_CASE( &progressdata_test ), 0 /* expected zero error */ ); + return test; +} + Modified: branches/REFACTORING-10_3/libzypp/zypp/MediaProducts.cc URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/zypp/MediaProducts.cc?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/zypp/MediaProducts.cc (original) +++ branches/REFACTORING-10_3/libzypp/zypp/MediaProducts.cc Mon Jul 9 14:24:33 2007 @@ -18,28 +18,6 @@ namespace zypp { -template <class _OutputIterator> -static void scanProductsFile( const Pathname & file_r, _OutputIterator result ) -{ - std::ifstream pfile( file_r.asString().c_str() ); - while ( pfile.good() ) { - - std::string value = str::getline( pfile, str::TRIM ); - if ( pfile.bad() ) { - ERR << "Error parsing " << file_r << endl; - ZYPP_THROW(Exception("Error parsing " + file_r.asString())); - } - if ( pfile.fail() ) { - break; // no data on last line - } - std::string tag = str::stripFirstWord( value, true ); - - if ( tag.size() ) { - *result = MediaProductEntry( tag, value ); - } - } -} - // template<class _OutputIterator> // void productsInMedia( const Url & url_r, _OutputIterator result ) Modified: branches/REFACTORING-10_3/libzypp/zypp/MediaProducts.h URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/zypp/MediaProducts.h?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/zypp/MediaProducts.h (original) +++ branches/REFACTORING-10_3/libzypp/zypp/MediaProducts.h Mon Jul 9 14:24:33 2007 @@ -55,7 +55,26 @@ * FIXME: add a comment here... */ template <class _OutputIterator> - static void scanProductsFile( const Pathname & file_r, _OutputIterator result ); + static void scanProductsFile( const Pathname & file_r, _OutputIterator result ) + { + std::ifstream pfile( file_r.asString().c_str() ); + while ( pfile.good() ) { + + std::string value = str::getline( pfile, str::TRIM ); + if ( pfile.bad() ) { + ERR << "Error parsing " << file_r << std::endl; + ZYPP_THROW(Exception("Error parsing " + file_r.asString())); + } + if ( pfile.fail() ) { + break; // no data on last line + } + std::string tag = str::stripFirstWord( value, true ); + + if ( tag.size() ) { + *result = MediaProductEntry( tag, value ); + } + } + } /** * \short Available products in a url location Modified: branches/REFACTORING-10_3/libzypp/zypp/ProgressData.cc URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/zypp/ProgressData.cc?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/zypp/ProgressData.cc (original) +++ branches/REFACTORING-10_3/libzypp/zypp/ProgressData.cc Mon Jul 9 14:24:33 2007 @@ -128,6 +128,26 @@ return ret; } + CombinedProgressData::CombinedProgressData( ProgressData &pd, + ProgressData::value_type weight ) + : _weight(weight), + _last_value(0), + _pd(pd) + { + + } + + bool CombinedProgressData::operator()( const ProgressData &progress ) + { + MIL << endl; + ProgressData::value_type increment = progress.val() - _last_value; + + int real_increment = (int)((float)(_weight*increment)/100); + //MIL << "Value: " << v << ". Increment of " << increment << ". Real of " << real_increment << endl; + _last_value = progress.val(); + return _pd.incr(real_increment); + } + ///////////////////////////////////////////////////////////////// } // namespace zypp /////////////////////////////////////////////////////////////////// Modified: branches/REFACTORING-10_3/libzypp/zypp/ProgressData.h URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/zypp/ProgressData.h?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/zypp/ProgressData.h (original) +++ branches/REFACTORING-10_3/libzypp/zypp/ProgressData.h Mon Jul 9 14:24:33 2007 @@ -319,6 +319,69 @@ /** \relates ProgressData Setup from \ref InputStream. */ ProgressData makeProgressData( const InputStream & input_r ); + /////////////////////////////////////////////////////////////////// + + /** + * \short Progress callback from another progress + * + * This class allows you to pass a progress callback to a + * subtask based on a current progress data, plus a weight + * value. Every progress reported by the subtask via + * this callback will be forwarded to the main progress + * data, with the corresponding weight. + * + * Example: + * + * \code + * + * // receiver for main task + * void task_receiver( ProgressData &progress ); + * + * // subtask prototypes + * void do_subtask_one( ProgressData::ReceiverFnc &fnc ); + * void do_subtask_two( ProgressData::ReceiverFnc &fnc ); + * + * // main task + * ProgressData progress; + * //progress for subtask 1 + * // which is 80% + * CombinedProgressData sub1(pd, 80); + * // the second is only 20% + * CombinedProgressData sub2(pd, 20); + * do_subtask_one( sub1 ); + * do_subtask_two( sub2 ); + * + * \endcode + */ + class CombinedProgressData + { + public: + /** + * \short Ctor + * + * Creates a \ref ProgressData::ReceiverFnc + * from a \ref ProgressData object + * + * \param pd \ref ProgressData object + * \param weight Weight of the subtask + * relative to the main task range. + * + */ + CombinedProgressData( ProgressData &pd, + ProgressData::value_type weight ); + + /** + * Implements the \ref ProgressData::ReceiverFnc + * callback interface + */ + bool operator()( const ProgressData &progress ); + + private: + ProgressData::value_type _weight; + ProgressData::value_type _last_value; + ProgressData &_pd; + }; + ///////////////////////////////////////////////////////////////// } // namespace zypp /////////////////////////////////////////////////////////////////// Modified: branches/REFACTORING-10_3/libzypp/zypp/RepoManager.cc URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/zypp/RepoManager.cc?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/zypp/RepoManager.cc (original) +++ branches/REFACTORING-10_3/libzypp/zypp/RepoManager.cc Mon Jul 9 14:24:33 2007 @@ -610,7 +610,6 @@ repo::cached::RepoImpl::Ptr repoimpl = new repo::cached::RepoImpl( info, _pimpl->options.repoCachePath, id ); // read the resolvables from cache - repoimpl->createResolvables(); return Repository(repoimpl); } Modified: branches/REFACTORING-10_3/libzypp/zypp/RepoManager.h URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/zypp/RepoManager.h?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/zypp/RepoManager.h (original) +++ branches/REFACTORING-10_3/libzypp/zypp/RepoManager.h Mon Jul 9 14:24:33 2007 @@ -172,12 +172,25 @@ * Creating from cache requires that the repository is * refreshed (metadata downloaded) and cached * + * \throws repo::RepoNoAliasException if can't figure an alias to look in cache * \throw RepoNotCachedException When the source is not cached. */ Repository createFromCache( const RepoInfo &info, const ProgressData::ReceiverFnc & progressrcv = ProgressData::ReceiverFnc() ); /** + * \short Create a repository object from raw metadata + * + * Creating from cache requires that the repository is + * refreshed (metadata downloaded) + * + * \throw Exception If there are errors parsing the + * raw metadata + */ + Repository createFromMetadata( const RepoInfo &info, + const ProgressData::ReceiverFnc & progressrcv = ProgressData::ReceiverFnc() ); + + /** * \short Probe repo metadata type. * * \todo FIXME Should this be private? Modified: branches/REFACTORING-10_3/libzypp/zypp/parser/yum/RepoParser.cc URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/zypp/parser/yum/RepoParser.cc?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/zypp/parser/yum/RepoParser.cc (original) +++ branches/REFACTORING-10_3/libzypp/zypp/parser/yum/RepoParser.cc Mon Jul 9 14:24:33 2007 @@ -13,7 +13,7 @@ #include "zypp/ZConfig.h" #include "zypp/base/Logger.h" - +#include "zypp/PathInfo.h" #include "zypp/base/UserRequestException.h" #include "zypp/repo/yum/ResourceType.h" @@ -316,15 +316,15 @@ cache_dir + "/repodata/repomd.xml", bind(&RepoParser::Impl::repomd_CB, this, _1, _2)); -// long long totalsize = 0 -// for(list<RepoParserJob>::const_iterator it = _jobs.begin(); -// it != _jobs.end(); ++it) -// { -// RepoParserJob job = *it; -// totalsize += PathInfo(cache_dir + job.filename()).size(); -// } + long long totalsize = 0; + for(list<RepoParserJob>::const_iterator it = _jobs.begin(); + it != _jobs.end(); ++it) + { + RepoParserJob job = *it; + totalsize += PathInfo(cache_dir + job.filename()).size(); + } - //_ticks.range(totalsize); + _ticks.range(totalsize); _ticks.toMin(); doJobs(cache_dir); @@ -340,9 +340,12 @@ it != _jobs.end(); ++it) { RepoParserJob job = *it; - - MIL << "going to parse " << job.type() << " file " << job.filename() << endl; - + // FIXME better way to do progress here? + int jobsize = PathInfo(cache_dir + job.filename()).size(); + + MIL << "going to parse " << job.type() << " file " + << job.filename() << " (" << jobsize << " bytes)" << endl; + switch(job.type().toEnum()) { // parse primary.xml.gz @@ -362,7 +365,8 @@ bind(&RepoParser::Impl::patches_CB, this, _1, _2)); // reset progress reporter max value (number of jobs changed if // there are patches to parse) - _ticks.range(_jobs.size()); + _ticks.range( _ticks.max() + jobsize ); + break; } @@ -427,7 +431,8 @@ } } - if (!_ticks.incr()) + // increase in the total bytes of the file + if (!_ticks.incr( jobsize )) ZYPP_THROW(AbortRequestException()); } } Modified: branches/REFACTORING-10_3/libzypp/zypp/repo/RepositoryImpl.cc URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/zypp/repo/RepositoryImpl.cc?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/zypp/repo/RepositoryImpl.cc (original) +++ branches/REFACTORING-10_3/libzypp/zypp/repo/RepositoryImpl.cc Mon Jul 9 14:24:33 2007 @@ -1,12 +1,17 @@ +#include "zypp/base/Logger.h" #include "zypp/repo/RepositoryImpl.h" +using namespace zypp; +using namespace std; + namespace zypp { namespace repo { IMPL_PTR_TYPE(RepositoryImpl) RepositoryImpl::RepositoryImpl( const RepoInfo &info ) - : _info(info) + : _restore_lazy_initialized(false), + _info(info) { } @@ -28,9 +33,24 @@ const ResStore & RepositoryImpl::resolvables() const { + if ( ! _restore_lazy_initialized ) + { + const_cast(this)->createResolvables(); + const_cast(this)->_restore_lazy_initialized = true; + } return _store; } +void RepositoryImpl::createResolvables() +{ + WAR << "createResolvables() not implemented" << endl; +} + +void RepositoryImpl::createPatchAndDeltas() +{ + WAR << "createPatchAndDeltas() not implemented" << endl; +} + const std::listpackagedelta::PatchRpm & RepositoryImpl::patchRpms() const { Modified: branches/REFACTORING-10_3/libzypp/zypp/repo/RepositoryImpl.h URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/zypp/repo/RepositoryImpl.h?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/zypp/repo/RepositoryImpl.h (original) +++ branches/REFACTORING-10_3/libzypp/zypp/repo/RepositoryImpl.h Mon Jul 9 14:24:33 2007 @@ -54,6 +54,8 @@ const std::listpackagedelta::PatchRpm & patchRpms() const; const std::listpackagedelta::DeltaRpm & deltaRpms() const; + virtual void createResolvables(); + virtual void createPatchAndDeltas(); public: struct null {}; @@ -73,6 +75,7 @@ ResStore _store; std::listpackagedelta::PatchRpm _patchRpms; std::listpackagedelta::DeltaRpm _deltaRpms; + bool _restore_lazy_initialized; private: RepoInfo _info; }; Modified: branches/REFACTORING-10_3/libzypp/zypp/repo/cached/RepoImpl.cc URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/zypp/repo/cached/RepoImpl.cc?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/zypp/repo/cached/RepoImpl.cc (original) +++ branches/REFACTORING-10_3/libzypp/zypp/repo/cached/RepoImpl.cc Mon Jul 9 14:24:33 2007 @@ -73,11 +73,13 @@ MIL << "Plaindir repository initialized." << std::endl; } -void read_capabilities( sqlite3_connection &con, map &nvras ); +void read_capabilities( sqlite3_connection &con, map &nvras, ProgressData &progress ); void RepoImpl::createResolvables() { + ProgressData progress; + debug::Measure m("create resolvables"); CapFactory capfactory; try @@ -86,6 +88,11 @@ con.executenonquery("PRAGMA cache_size=8000;"); con.executenonquery("BEGIN;"); +// We could use this to calculate total resolvables +// sqlite3_command total_cmd("select count(id) from resolvables where repository_id=:repository_id;"); +// total_cmd.bind(":repository_id", _repository_id); +// int total = total_cmd.executeint(); + sqlite3_command cmd( con, "select id,name,version,release,epoch,arch,kind from resolvables where repository_id=:repository_id;"); cmd.bind(":repository_id", _repository_id); map > nvras; @@ -104,8 +111,9 @@ } MIL << "Done reading resolvables nvra" << endl; - - read_capabilities( con, nvras); + progress.set(15); + + read_capabilities( con, nvras, progress); for ( map >::const_iterator it = nvras.begin(); it != nvras.end(); ++it ) { @@ -164,7 +172,7 @@ cerr << "Exception Occured: " << ex.what() << endl; } //extract_packages_from_directory( _store, thePath, selfRepositoryRef(), true ); - + progress.toMax(); } static CheckSum encoded_string_to_checksum( const std::string &encoded ) @@ -298,7 +306,9 @@ return _rquery; } -void RepoImpl::read_capabilities( sqlite3_connection &con, map > &nvras ) +void RepoImpl::read_capabilities( sqlite3_connection &con, + map > &nvras, + ProgressData &progress ) { CapFactory capfactory; @@ -346,6 +356,8 @@ } } + progress.incr(49); + { debug::Measure mnf("read file capabilities"); sqlite3_reader reader = select_file_cmd.executereader(); @@ -359,6 +371,8 @@ } } + progress.incr(10); + MIL << nvras.size() << " capabilities" << endl; } Modified: branches/REFACTORING-10_3/libzypp/zypp/repo/cached/RepoImpl.h URL: http://svn.opensuse.org/viewcvs/zypp/branches/REFACTORING-10_3/libzypp/zypp/repo/cached/RepoImpl.h?rev=5932&r1=5931&r2=5932&view=diff ============================================================================== --- branches/REFACTORING-10_3/libzypp/zypp/repo/cached/RepoImpl.h (original) +++ branches/REFACTORING-10_3/libzypp/zypp/repo/cached/RepoImpl.h Mon Jul 9 14:24:33 2007 @@ -16,6 +16,7 @@ #include "zypp/Arch.h" #include "zypp/Rel.h" #include "zypp/Pathname.h" +#include "zypp/ProgressData.h" #include "zypp/data/RecordId.h" #include "zypp/repo/RepositoryImpl.h" #include "zypp/ResStore.h" @@ -52,17 +53,21 @@ ~RepoImpl(); void factoryInit(); public: + virtual void createResolvables(); + virtual void createPatchAndDeltas(); cache::ResolvableQuery resolvableQuery(); - void createResolvables(); - void createPatchAndDeltas(); private: - void read_capabilities( sqlite3x::sqlite3_connection &con, std::map > &nvras ); + void read_capabilities( sqlite3x::sqlite3_connection &con, + std::map > &nvras, + zypp::ProgressData &progress ); Pathname _dbdir; cache::CacheTypes _type_cache; data::RecordId _repository_id; cache::ResolvableQuery _rquery; + }; /////////////////////////////////////////////////////////////////// -- To unsubscribe, e-mail: zypp-commit+unsubscribe@opensuse.org For additional commands, e-mail: zypp-commit+help@opensuse.org