ref: refs/heads/ma-tmp-pqt commit adecb3b0a2a73ad08ee73bcaa3d21631a4cfa42f Author: Michael Andres Date: Thu Apr 9 16:16:22 2009 +0200 Refactor PoolQuery, separate iterator and matching --- devel/devel.ma/NewPool.cc | 96 +++++++++++--- zypp/PoolQuery.cc | 318 ++++++++++++++++++++++++++++++++++++++++++++- zypp/PoolQuery.h | 46 +++++++ 3 files changed, 438 insertions(+), 22 deletions(-) diff --git a/devel/devel.ma/NewPool.cc b/devel/devel.ma/NewPool.cc index 029bfc4..546cdc5 100644 --- a/devel/devel.ma/NewPool.cc +++ b/devel/devel.ma/NewPool.cc @@ -44,6 +44,7 @@ #include "zypp/sat/WhatObsoletes.h" #include "zypp/PoolQuery.h" #include "zypp/ServiceInfo.h" +#include "zypp/media/MediaPriority.h" #include @@ -428,6 +429,59 @@ void testCMP( const L & lhs, const R & rhs ) #undef OUTS } +template +void diffquery( const Lcont & l, const Rcont & r ) +{ + MIL << "DIFF " << l.size() << " <-> " << r.size() << endl; + sat::SolvableSet ls; + sat::SolvableSet bb; + sat::SolvableSet rs; + + for_( it, l.begin(), l.end() ) + { + ( r.contains( *it ) ? bb : ls ).insert( *it ); + } + for_( it, r.begin(), r.end() ) + { + if ( ! l.contains( *it ) ) + rs.insert( *it ); + } + + MIL << "(" << ls.size() << ") (" << bb.size() << ") (" << rs.size() << ")" << endl; + INT << ls << endl; + INT << rs << endl; +} + +bool querycompare( const PoolQuery & q ) +{ + SEC << q << endl; + unsigned oc = 0; + unsigned nc = 0; + if ( 1 ) + { + Measure x( "new query" ); + for_( it, q.nbegin(), q.nend() ) + { + ++nc; + } + MIL << nc << endl; + } + if ( 1 ) + { + Measure x( "old query" ); + oc = q.size(); + MIL << oc << endl; + } + + bool ret = true; + if ( ret && oc != nc ) + { + ERR << "Diff in result count: " << oc << " <-> " << nc << endl; + ret = false; + } + return ret; +} + /****************************************************************** ** ** FUNCTION NAME : main @@ -444,7 +498,7 @@ try { ResPool pool( ResPool::instance() ); sat::Pool satpool( sat::Pool::instance() ); - if ( 0 ) + if ( 1 ) { Measure x( "INIT TARGET" ); { @@ -477,7 +531,7 @@ try { } } - if ( 0 ) + if ( 1 ) { RepoManager repoManager( makeRepoManager( sysRoot ) ); RepoInfoList repos = repoManager.knownRepositories(); @@ -530,7 +584,7 @@ try { repoManager.loadFromCache( nrepo ); } - USR << "pool: " << pool << endl; + //USR << "pool: " << pool << endl; } } } @@ -548,22 +602,32 @@ try { } /////////////////////////////////////////////////////////////////// + + + /////////////////////////////////////////////////////////////////// - RepoManager repoManager( makeRepoManager( sysRoot ) ); - RepoInfoList repos = repoManager.knownRepositories(); - // launch repos - for ( RepoInfoList::iterator it = repos.begin(); it != repos.end(); ++it ) - { - RepoInfo & nrepo( *it ); - Url url_r( nrepo.url() ); + std::string search("devel"); + //search = ""; + + PoolQuery q; + //querycompare( q ); + + q.addString(search); + q.addAttribute(sat::SolvAttr::name); + //q.addAttribute(sat::SolvAttr::summary); + q.setMatchSubstring(); + q.setCaseSensitive( true ); + + //q.addRepo( "11.1-update" ); + //q.addRepo( "@System" ); + + //q.addKind( ResKind::package ); + + //q.setEdition( Edition("1.0"), Rel::GE ); + + querycompare( q ); - SEC << url_r << endl; - MIL << RepoManager::makeStupidAlias( url_r ) << endl; - } - MIL << RepoManager::makeStupidAlias() << endl; - MIL << RepoManager::makeStupidAlias() << endl; - MIL << RepoManager::makeStupidAlias() << endl; #if 0 getZYpp()->resolver()->addRequire( Capability("amarok") ); diff --git a/zypp/PoolQuery.cc b/zypp/PoolQuery.cc index 5722d6d..a7f5028 100644 --- a/zypp/PoolQuery.cc +++ b/zypp/PoolQuery.cc @@ -21,6 +21,7 @@ #include "zypp/sat/Pool.h" #include "zypp/sat/Solvable.h" +#include "zypp/sat/AttrMatcher.h" #include "zypp/PoolQuery.h" @@ -270,7 +271,7 @@ attremptycheckend: } _compiled = true; - DBG << asString() << endl; + //DBG << asString() << endl; } @@ -397,11 +398,9 @@ attremptycheckend: } } - DBG << "_cflags:" << _cflags << endl; + //DBG << "_cflags:" << _cflags << endl; scoped_ptr< ::_Dataiterator> _rdit( new ::Dataiterator ); - // needed while LookupAttr::iterator::dip_equal does ::memcmp: - ::memset( _rdit.get(), 0, sizeof(::_Dataiterator) ); // initialize the Dataiterator for different cases if (_rcattrs.empty()) @@ -413,6 +412,7 @@ attremptycheckend: 0, // attribute id - only if 1 attr key specified _rcstrings.empty() ? 0 : _rcstrings.c_str(), // compiled search string _cflags); + INT << "rcstrings >" << _rcstrings << "<" << endl; } else if (_rcattrs.size() == 1) { @@ -712,7 +712,7 @@ attremptycheckend: /* After calling dataiterator_match (with any string matcher set) the kv.str member will be filled with something sensible. */ /*INT << "value: " << base().get()->kv.str << endl - << " str: " << _str << endl;*/ + << " str: " << _str << endl; */ } } } @@ -750,7 +750,7 @@ attremptycheckend: return matches; } - /////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////// } //namespace detail /////////////////////////////////////////////////////////////////// @@ -1347,6 +1347,312 @@ attremptycheckend: return true; } + /////////////////////////////////////////////////////////////////// + namespace detail + { ///////////////////////////////////////////////////////////////// + +//#define PQM_LOG_MATCHES + +#ifdef PQM_LOG_MATCHES +#warning PQM_LOG_MATCHES debug loggin still enabled (slows down queries) +#endif + + /////////////////////////////////////////////////////////////////// + // + // CLASS NAME : PoolQueryMatcher + // + /** Store \ref PoolQuery settings and assist \ref PoolQueryIterator. + * + * Basically the matcher performs a base query, which should preselect + * candidates for a match. And has some filter conditions on top of it. + * Query and fileter depend on the \ref PoolQuery settings. + * + * Matcher must be stateless, as it is shared between multiple + * \ref PoolQueryIterator instances. + * + * If \ref base_iterator is at the \ref end, \ref advance moves it + * to the first match. Otherwise advance moves to the next match, or + * to the \ref end, if there is no more match. + * + * \todo I guess the attribute content matching deserves some polishing, + * it's more or less taken from the old iterator. + */ + class PoolQueryMatcher + { + public: + typedef sat::LookupAttr::iterator base_iterator; + typedef std::pair StringMatchData; + + public: + const base_iterator & end() const + { + static base_iterator _end; + return _end; + } + + bool advance( base_iterator & base_r ) const + { + if ( base_r == end() ) + base_r = startNewQyery(); // first candidate + else + ++base_r; // advance to next candidate + + while ( base_r != end() ) + { + if ( isAMatch( base_r ) ) + { + base_r.nextSkipSolvable(); // assert we don't visit this Solvable again + return true; + } + // No match: try next + ++base_r; + } + return false; + } + + public: + /** Ctor stores the \ref PoolQuery settings. */ + PoolQueryMatcher( const shared_ptr<const PoolQuery::Impl> & query_r ) + { + query_r->compile(); + + // Repo restriction: + sat::Pool satpool( sat::Pool::instance() ); + for_( it, query_r->_repos.begin(), query_r->_repos.end() ) + { + Repository r( satpool.reposFind( *it ) ); + if ( r ) + _repos.insert( r ); + } + // Kind restriction: + _kinds = query_r->_kinds; + // Edition restriction: + _op = query_r->_op; + _edition = query_r->_edition; + // Status restriction: + _status_flags = query_r->_status_flags; + + // Attribute content matching: (beautify it) + _flags = query_r->_cflags; + _str = query_r->_rcstrings; + _regex = query_r->_regex; + _attrs_str = query_r->_rcattrs; + _attrs_regex = query_r->_rattrs; + + // this setup depends on how PoolQuery::compile + // sets up the quert strings. + if ( _attrs_str.empty() ) + { + _matchList.push_back( StringMatchData( SolvAttr::allAttr, (_str.empty() ? 0 : _str.c_str() ) ) ); + _m.setSearchstring( _str, _flags ); + USR << _m << endl; + } + else if ( _attrs_str.size() == 1 ) + { + _matchList.push_back( StringMatchData( _attrs_str.begin()->first, (_str.empty() ? 0 : _str.c_str() ) ) ); + _m.setSearchstring( _str, _flags ); + USR << _m << endl; + } + else + { + _m.setSearchstring( _str, _flags ); + bool doRegex = ( (_flags & SEARCH_STRINGMASK) == SEARCH_REGEX ); + for_( ai, _attrs_str.begin(), _attrs_str.end() ) + { + if ( doRegex ) + { + const regex_t * regex_p; + if ( _str.empty() ) + { + PoolQuery::AttrRegexMap::iterator rai = _attrs_regex.find( ai->first ); + if ( rai != _attrs_regex.end() ) + regex_p = rai->second.get(); + else + { + INT << "No compiled regex found for " << ai->first << endl; + continue; + } + } + else + regex_p = _regex.get(); + + _matchList.push_back( StringMatchData( ai->first, regex_p ) ); + } + else + { + _matchList.push_back( StringMatchData( ai->first, (_str.empty() ? ai->second : _str).c_str() ) ); + } + } + } + } + + ~PoolQueryMatcher() + { + SEC << "Match? " << _counter << endl; + // + } + + private: + /** Initialize a new base query. */ + base_iterator startNewQyery() const + { + sat::LookupAttr q( sat::SolvAttr::allAttr ); + + // Repo restriction: + if ( _repos.size() == 1 ) + q.setRepo( *_repos.begin() ); + // else: handled in isAMatch. + + // Attribute restriction: + if( !_attrs_str.empty() ) + { + // Jump to the 1st one, the rest is done in isAMatch. + q.setAttr( _attrs_str.begin()->first ); + //q.setAttrMatcher( _m ); + } + + return q.begin(); + } + + /** Check whether we are on a match. + * + * The check covers the whole Solvable, not just the current + * attribute \c base_r points to. If there's no match, also + * prepare \c base_r to advance appropriately. If there is + * a match, simply return \c true. \ref advance always moves + * to the next Solvable if there was a match. + * + * \note: Caller asserts we're not at \ref end. + */ + bool isAMatch( base_iterator & base_r ) const + { + ++_counter; +#if 0 + ///////////////////////////////////////////////////////////////////// + Repository inRepo( base_r.inRepo() ); + // Status restriction: + if ( _status_flags + && ( (_status_flags == PoolQuery::INSTALLED_ONLY) != inRepo.isSystemRepo() ) ) + { + base_r.nextSkipRepo(); + return false; + } + // Repo restriction: + if ( _repos.size() > 1 && _repos.find( inRepo ) == _repos.end() ) + { + base_r.nextSkipRepo(); + return false; + } + ///////////////////////////////////////////////////////////////////// + sat::Solvable inSolvable( base_r.inSolvable() ); + // Kind restriction: + if ( ! _kinds.empty() && _kinds.find( inSolvable.kind() ) == _kinds.end() ) + { + base_r.nextSkipSolvable(); + return false; + } + // Edition restriction: + if ( _op != Rel::ANY && !compareByRel( _op, inSolvable.edition(), _edition, Edition::Match() ) ) + { + base_r.nextSkipSolvable(); + return false; + } + ///////////////////////////////////////////////////////////////////// +#else + sat::Solvable inSolvable( base_r.inSolvable() ); +#endif + // string matching: + if( _attrs_str.empty() ) + { + // search all attributes ;( + const StringMatchData & data( *_matchList.begin() ); + sat::LookupAttr q( sat::SolvAttr::allAttr, inSolvable ); + for_( it, q.begin(), q.end() ) + if ( attrMatchString( it, data ) ) + return true; + } + else + { + // base iterator always visits the current (first) attribute: + std::vector<StringMatchData>::const_iterator mi( _matchList.begin() ); + if ( attrMatchString( base_r, *_matchList.begin() ) ) + return true; + // then check the remaining ones: + for_( mi, ++_matchList.begin(), _matchList.end() ) + { + sat::LookupAttr q( mi->first, inSolvable ); + for_( it, q.begin(), q.end() ) + if ( attrMatchString( it, *mi ) ) + return true; + } + } + base_r.nextSkipSolvable(); + return false; + } + + private: + /** */ + bool attrMatchString( const base_iterator & base_r, const StringMatchData & data_r ) const + { + //return true; + return _m( base_r.c_str() ); + if ( ! data_r.second ) // empty string matches always + return true; +#ifdef PQM_LOG_MATCHES + bool ret = ::dataiterator_match_obsolete( base_r.get(), _flags, data_r.second ); + if ( ret ) + DBG << "Match in " << base_r << endl; + return ret; +#else + return ::dataiterator_match_obsolete( base_r.get(), _flags, data_r.second ); +#endif + } + + private: + mutable DefaultIntegral _counter; + /** Repositories include in the search. */ + std::set<Repository> _repos; + /** Resolvable kinds to include. */ + std::set<ResKind> _kinds; + /** Edition filter. */ + Rel _op; + Edition _edition; + /** Installed status filter flags. \see PoolQuery::StatusFilter */ + int _status_flags; + + /** string matching option flags */ + int _flags; + /** global query string compiled */ + std::string _str; + /** global query compiled regex */ + str::regex _regex; + /** Attribute to string map holding per-attribute query strings (compiled) */ + PoolQuery::AttrCompiledStrMap _attrs_str; + /** Attribute to regex map holding per-attribute compiled regex */ + PoolQuery::AttrRegexMap _attrs_regex; + /** Prepared dataiterator_match_obsolete args */ + std::vector<StringMatchData> _matchList; + sat::AttrMatcher _m; + }; + /////////////////////////////////////////////////////////////////// + + void NewPoolQueryIterator::increment() + { + // matcher restarts if at end! It is called from the ctor + // to get the 1st match. But if the end is reached, it should + // be deleted, otherwise we'd start over again. + if ( _matcher && ! _matcher->advance( base_reference() ) ) + _matcher.reset(); + } + + /////////////////////////////////////////////////////////////////// + } //namespace detail + /////////////////////////////////////////////////////////////////// + + detail::NewPoolQueryIterator PoolQuery::nbegin() const + { + return shared_ptrdetail::PoolQueryMatcher( new detail::PoolQueryMatcher( _pimpl.getPtr() ) ); + } ///////////////////////////////////////////////////////////////// } // namespace zypp diff --git a/zypp/PoolQuery.h b/zypp/PoolQuery.h index 5f4a9e3..c3c6045 100644 --- a/zypp/PoolQuery.h +++ b/zypp/PoolQuery.h @@ -31,6 +31,7 @@ namespace zypp namespace detail { class PoolQueryIterator; + class NewPoolQueryIterator; } /////////////////////////////////////////////////////////////////// @@ -100,6 +101,9 @@ namespace zypp /** Query result accessers. */ //@{ + detail::NewPoolQueryIterator nbegin() const; + detail::NewPoolQueryIterator nend() const; + /** * Compile the query and return an iterator to the result. * @@ -392,6 +396,45 @@ namespace zypp namespace detail { ///////////////////////////////////////////////////////////////// + class PoolQueryMatcher; + + /////////////////////////////////////////////////////////////////// + // + // CLASS NAME : PoolQuery::PoolQueryIterator + // + /** \ref PoolQuery iterator as returned by \ref PoolQuery::begin. + */ + class NewPoolQueryIterator : public boost::iterator_adaptor< + NewPoolQueryIterator // Derived + , sat::LookupAttr::iterator // Base + , const sat::Solvable // Value + , boost::forward_traversal_tag // CategoryOrTraversal + , const sat::Solvable // Reference + > + { + public: + /** Default ctor is also \c end.*/ + NewPoolQueryIterator() + {} + + /** \Ref PoolQuery ctor. */ + NewPoolQueryIterator( const shared_ptr<PoolQueryMatcher> & matcher_r ) + : _matcher( matcher_r ) + { increment(); } + + private: + friend class boost::iterator_core_access; + void increment(); + + private: + shared_ptr<PoolQueryMatcher> _matcher; + }; + /////////////////////////////////////////////////////////////////// + + /** \relates PoolQueryIterator Stream output. */ + inline std::ostream & operator<<( std::ostream & str, const NewPoolQueryIterator & obj ) + { return str << obj.base(); } + /////////////////////////////////////////////////////////////////// // @@ -482,6 +525,9 @@ namespace zypp } //namespace detail /////////////////////////////////////////////////////////////////// + inline detail::NewPoolQueryIterator PoolQuery::nend() const + { return detail::NewPoolQueryIterator(); } + ///////////////////////////////////////////////////////////////// } // namespace zypp /////////////////////////////////////////////////////////////////// -- To unsubscribe, e-mail: zypp-commit+unsubscribe@opensuse.org For additional commands, e-mail: zypp-commit+help@opensuse.org