[zypp-commit] r8389 - in /trunk/sat-solver: doc/ src/ tools/

Author: mlschroe Date: Tue Jan 29 17:14:03 2008 New Revision: 8389 URL: http://svn.opensuse.org/viewcvs/zypp?rev=8389&view=rev Log: - add key filtering to repo_write - change repo_write so that it combines all available data - integrate attr_store into repodata - write storage parameter with every key, bump solv revision to 5 - don't create system rules for atoms - change repo_susetags to use repodata interface Added: trunk/sat-solver/src/repodata.c trunk/sat-solver/src/repodata.h Modified: trunk/sat-solver/doc/README.format trunk/sat-solver/src/CMakeLists.txt trunk/sat-solver/src/dirpool.c trunk/sat-solver/src/dirpool.h trunk/sat-solver/src/pool.c trunk/sat-solver/src/pool.h trunk/sat-solver/src/poolid.c trunk/sat-solver/src/poolid.h trunk/sat-solver/src/pooltypes.h trunk/sat-solver/src/repo.c trunk/sat-solver/src/repo.h trunk/sat-solver/src/repo_solv.c trunk/sat-solver/src/solver.c trunk/sat-solver/src/strpool.c trunk/sat-solver/src/strpool.h trunk/sat-solver/src/util.h trunk/sat-solver/tools/CMakeLists.txt trunk/sat-solver/tools/content2solv.c trunk/sat-solver/tools/dumpsolv.c trunk/sat-solver/tools/helix2solv.c trunk/sat-solver/tools/mergesolv.c trunk/sat-solver/tools/patchxml2solv.c trunk/sat-solver/tools/repo_susetags.c trunk/sat-solver/tools/repo_write.c trunk/sat-solver/tools/repo_write.h trunk/sat-solver/tools/rpmdb2solv.c trunk/sat-solver/tools/rpmmd2solv.c trunk/sat-solver/tools/susetags2solv.c Modified: trunk/sat-solver/doc/README.format URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/doc/README.format?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/doc/README.format (original) +++ trunk/sat-solver/doc/README.format Tue Jan 29 17:14:03 2008 @@ -96,3 +96,15 @@ NUMINFO * ID schema U8* data + + + + +--- + +key sizes for storage types: + + +KEY_STORAGE_VERTICAL_OFFSET: packed size +KEY_STORAGE_INCORE: packed size +KEY_STORAGE_SOLVABLE: unpacked size Modified: trunk/sat-solver/src/CMakeLists.txt URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/CMakeLists.txt?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/CMakeLists.txt (original) +++ trunk/sat-solver/src/CMakeLists.txt Tue Jan 29 17:14:03 2008 @@ -1,10 +1,10 @@ -SET(libsatsolver_SRCS attr_store.c bitmap.c poolarch.c poolvendor.c poolid.c strpool.c -solver.c repo_solv.c evr.c pool.c queue.c repo.c util.c policy.c) +SET(libsatsolver_SRCS bitmap.c poolarch.c poolvendor.c poolid.c strpool.c dirpool.c +solver.c repo_solv.c evr.c pool.c queue.c repo.c repodata.c util.c policy.c) ADD_LIBRARY(satsolver STATIC ${libsatsolver_SRCS}) -SET(libsatsolver_HEADERS attr_store.h bitmap.h evr.h hash.h policy.h poolarch.h poolvendor.h pool.h poolid.h pooltypes.h queue.h solvable.h solver.h repo.h repo_solv.h util.h strpool.h) +SET(libsatsolver_HEADERS bitmap.h evr.h hash.h policy.h poolarch.h poolvendor.h pool.h poolid.h pooltypes.h queue.h solvable.h solver.h repo.h repodata.h repo_solv.h util.h strpool.h dirpool.h) SET( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -Wall -fPIC" ) Modified: trunk/sat-solver/src/dirpool.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/dirpool.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/dirpool.c (original) +++ trunk/sat-solver/src/dirpool.c Tue Jan 29 17:14:03 2008 @@ -31,8 +31,9 @@ Id parent, i, *dirtraverse; if (!dp->ndirs) return; - dp->dirs = sat_realloc2(dp->dirs, (dp->ndirs + DIR_BLOCK) &~ DIR_BLOCK, sizeof(Id)); - dirtraverse = sat_calloc((dp->ndirs + DIR_BLOCK) &~ DIR_BLOCK, sizeof(Id)); + dp->dirs = sat_extend_resize(dp->dirs, dp->ndirs, sizeof(Id), DIR_BLOCK); + dirtraverse = sat_extend_resize(0, dp->ndirs, sizeof(Id), DIR_BLOCK); + memset(dirtraverse, 0, dp->ndirs * sizeof(Id)); for (parent = 0, i = 0; i < dp->ndirs; i++) { if (dp->dirs[i] > 0) @@ -45,14 +46,14 @@ } Id -dirpool_add_dir(Dirpool *dp, Id parent, Id comp) +dirpool_add_dir(Dirpool *dp, Id parent, Id comp, int create) { Id did, d, ds, *dirtraverse; if (!dp->ndirs) { - dp->dirs = sat_malloc2(DIR_BLOCK, sizeof(Id)); dp->ndirs = 2; + dp->dirs = sat_extend_resize(dp->dirs, dp->ndirs, sizeof(Id), DIR_BLOCK); dp->dirs[0] = 0; dp->dirs[1] = 1; /* "" */ } @@ -76,27 +77,25 @@ if (ds) ds = dp->dirtraverse[ds]; } + if (!create) + return 0; /* a new one, find last parent */ for (did = dp->ndirs - 1; did > 0; did--) if (dp->dirs[did] <= 0) break; if (dp->dirs[did] != -parent) { - if ((dp->ndirs & DIR_BLOCK) == 0) - { - dp->dirs = sat_realloc2(dp->dirs, dp->ndirs + DIR_BLOCK, sizeof(Id)); - dp->dirtraverse = sat_realloc2(dp->dirtraverse, dp->ndirs + DIR_BLOCK, sizeof(Id)); - } + /* make room for parent entry */ + dp->dirs = sat_extend(dp->dirs, dp->ndirs, 1, sizeof(Id), DIR_BLOCK); + dp->dirtraverse = sat_extend(dp->dirtraverse, dp->ndirs, 1, sizeof(Id), DIR_BLOCK); /* new parent block, link in */ dp->dirs[dp->ndirs] = -parent; dp->dirtraverse[dp->ndirs] = dp->dirtraverse[parent]; dp->dirtraverse[parent] = ++dp->ndirs; } - if ((dp->ndirs & DIR_BLOCK) == 0) - { - dp->dirs = sat_realloc2(dp->dirs, dp->ndirs + DIR_BLOCK, sizeof(Id)); - dp->dirtraverse = sat_realloc2(dp->dirtraverse, dp->ndirs + DIR_BLOCK, sizeof(Id)); - } + /* make room for new entry */ + dp->dirs = sat_extend(dp->dirs, dp->ndirs, 1, sizeof(Id), DIR_BLOCK); + dp->dirtraverse = sat_extend(dp->dirtraverse, dp->ndirs, 1, sizeof(Id), DIR_BLOCK); dp->dirs[dp->ndirs] = comp; dp->dirtraverse[dp->ndirs] = 0; return dp->ndirs++; Modified: trunk/sat-solver/src/dirpool.h URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/dirpool.h?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/dirpool.h (original) +++ trunk/sat-solver/src/dirpool.h Tue Jan 29 17:14:03 2008 @@ -21,7 +21,7 @@ void dirpool_create(Dirpool *dp); void dirpool_make_dirtraverse(Dirpool *dp); -Id dirpool_add_dir(Dirpool *dp, Id parent, Id comp); +Id dirpool_add_dir(Dirpool *dp, Id parent, Id comp, int create); static inline Id dirpool_parent(Dirpool *dp, Id did) { Modified: trunk/sat-solver/src/pool.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/pool.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/pool.c (original) +++ trunk/sat-solver/src/pool.c Tue Jan 29 17:14:03 2008 @@ -76,19 +76,22 @@ stringpool_init (&pool->ss, initpool_data); - // pre-alloc space for a RelDep - pool->rels = (Reldep *)sat_calloc(1 + REL_BLOCK, sizeof(Reldep)); + /* alloc space for ReDep 0 */ + pool->rels = sat_extend_resize(0, 1, sizeof(Reldep), REL_BLOCK); pool->nrels = 1; + memset(pool->rels, 0, sizeof(Reldep)); - // pre-alloc space for a Solvable - pool->solvables = (Solvable *)sat_calloc(SOLVABLE_BLOCK + 1, sizeof(Solvable)); + /* alloc space for Solvable 0 and system solvable */ + pool->solvables = sat_extend_resize(0, 2, sizeof(Solvable), SOLVABLE_BLOCK); pool->nsolvables = 2; - queue_init(&pool->vendormap); + memset(pool->solvables, 0, 2 * sizeof(Solvable)); s = pool->solvables + SYSTEMSOLVABLE; s->name = SYSTEM_SYSTEM; s->arch = ARCH_NOARCH; s->evr = ID_EMPTY; + queue_init(&pool->vendormap); + pool->debugmask = SAT_DEBUG_RESULT; /* FIXME */ return pool; } @@ -117,8 +120,7 @@ Id pool_add_solvable(Pool *pool) { - if ((pool->nsolvables & SOLVABLE_BLOCK) == 0) - pool->solvables = sat_realloc2(pool->solvables, pool->nsolvables + (SOLVABLE_BLOCK + 1), sizeof(Solvable)); + pool->solvables = sat_extend(pool->solvables, pool->nsolvables, 1, sizeof(Solvable), SOLVABLE_BLOCK); memset(pool->solvables + pool->nsolvables, 0, sizeof(Solvable)); return pool->nsolvables++; } @@ -129,8 +131,7 @@ Id nsolvables = pool->nsolvables; if (!count) return nsolvables; - if (((nsolvables - 1) | SOLVABLE_BLOCK) != ((nsolvables + count - 1) | SOLVABLE_BLOCK)) - pool->solvables = sat_realloc2(pool->solvables, (nsolvables + count + SOLVABLE_BLOCK) & ~SOLVABLE_BLOCK, sizeof(Solvable)); + pool->solvables = sat_extend(pool->solvables, pool->nsolvables, count, sizeof(Solvable), SOLVABLE_BLOCK); memset(pool->solvables + nsolvables, 0, sizeof(Solvable) * count); pool->nsolvables += count; return nsolvables; @@ -305,8 +306,10 @@ pool_freeidhashes(pool); /* XXX: should not be here! */ pool_freewhatprovides(pool); num = pool->ss.nstrings; - pool->whatprovides = whatprovides = sat_calloc((num + WHATPROVIDES_BLOCK) & ~WHATPROVIDES_BLOCK, sizeof(Offset)); - pool->whatprovides_rel = sat_calloc((pool->nrels + WHATPROVIDES_BLOCK) & ~WHATPROVIDES_BLOCK, sizeof(Offset)); + pool->whatprovides = whatprovides = sat_extend_resize(0, num, sizeof(Offset), WHATPROVIDES_BLOCK); + memset(whatprovides, 0, num * sizeof(Offset)); + pool->whatprovides_rel = sat_extend_resize(0, pool->nrels, sizeof(Offset), WHATPROVIDES_BLOCK); + memset(pool->whatprovides_rel, 0, pool->nrels * sizeof(Offset)); /* count providers for each name */ for (i = 1; i < pool->nsolvables; i++) @@ -593,7 +596,7 @@ va_list args; char buf[1024]; - if ((type & SAT_FATAL) == 0) + if ((type & (SAT_FATAL|SAT_ERROR)) == 0) { if ((pool->debugmask & type) == 0) return; @@ -691,8 +694,7 @@ s = id2str(pool, dep); if (*s != '/') continue; - if ((sf->nfiles & SEARCHFILES_BLOCK) == 0) - sf->files = sat_realloc2(sf->files, sf->nfiles + (SEARCHFILES_BLOCK + 1), sizeof(const char *)); + sf->files = sat_extend(sf->files, sf->nfiles, 1, sizeof(const char *), SEARCHFILES_BLOCK); sf->files[sf->nfiles++] = strdup(s); } } @@ -756,8 +758,7 @@ for (i = 0; i < sf.nfiles; i++) POOL_DEBUG(SAT_DEBUG_STATS, "looking up %s in filelist\n", sf.files[i]); #endif - if ((sf.nfiles & SEARCHFILES_BLOCK) == 0) - sf.files = sat_realloc2(sf.files, sf.nfiles + (SEARCHFILES_BLOCK + 1), sizeof(const char *)); + sf.files = sat_extend(sf.files, sf.nfiles, 1, sizeof(const char *), SEARCHFILES_BLOCK); sf.files[sf.nfiles++] = 0; #if 0 pool_search(0, SOLVABLE_FILELIST, (const char *)sf.files, SEARCH_STRING|SEARCH_MULTIPLE, addfileprovides_cb, 0); @@ -770,8 +771,7 @@ for (i = 0; i < isf.nfiles; i++) POOL_DEBUG(SAT_DEBUG_STATS, "looking up %s in installed filelist\n", isf.files[i]); #endif - if ((isf.nfiles & SEARCHFILES_BLOCK) == 0) - isf.files = sat_realloc2(isf.files, isf.nfiles + (SEARCHFILES_BLOCK + 1), sizeof(const char *)); + isf.files = sat_extend(isf.files, isf.nfiles, 1, sizeof(const char *), SEARCHFILES_BLOCK); isf.files[isf.nfiles++] = 0; #if 0 repo_search(installed, 0, SOLVABLE_FILELIST, (const char *)isf.files, SEARCH_STRING|SEARCH_MULTIPLE, addfileprovides_cb, 0); Modified: trunk/sat-solver/src/pool.h URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/pool.h?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/pool.h (original) +++ trunk/sat-solver/src/pool.h Tue Jan 29 17:14:03 2008 @@ -153,9 +153,13 @@ #define TYPE_IDVALUEARRAY 13 #define TYPE_DIR 14 -#define TYPE_DIRVALUEVALUEARRAY 15 +#define TYPE_DIRNUMNUMARRAY 15 +#define TYPE_DIRSTRARRAY 16 -#define TYPE_ATTR_TYPE_MAX TYPE_DIRVALUEVALUEARRAY +#define TYPE_CONSTANT 17 +#define TYPE_NUM 18 + +#define TYPE_ATTR_TYPE_MAX 18 //----------------------------------------------- Modified: trunk/sat-solver/src/poolid.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/poolid.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/poolid.c (original) +++ trunk/sat-solver/src/poolid.c Tue Jan 29 17:14:03 2008 @@ -38,6 +38,20 @@ } Id +strn2id(Pool *pool, const char *str, unsigned int len, int create) +{ + int oldnstrings = pool->ss.nstrings; + Id id = stringpool_strn2id (&pool->ss, str, len, create); + if (create && oldnstrings != pool->ss.nstrings && (id & WHATPROVIDES_BLOCK) == 0) + { + /* grow whatprovides array */ + pool->whatprovides = sat_realloc(pool->whatprovides, (id + (WHATPROVIDES_BLOCK + 1)) * sizeof(Offset)); + memset(pool->whatprovides + id, 0, (WHATPROVIDES_BLOCK + 1) * sizeof(Offset)); + } + return id; +} + +Id rel2id(Pool *pool, Id name, Id evr, int flags, int create) { Hashval h; @@ -86,8 +100,7 @@ id = pool->nrels++; /* extend rel space if needed */ - if ((id & REL_BLOCK) == 0) - pool->rels = sat_realloc(pool->rels, ((pool->nrels + REL_BLOCK) & ~REL_BLOCK) * sizeof(Reldep)); + pool->rels = sat_extend(pool->rels, id, 1, sizeof(Reldep), REL_BLOCK); hashtbl[h] = id; ran = pool->rels + id; ran->name = name; @@ -247,7 +260,7 @@ void pool_shrink_rels(Pool *pool) { - pool->rels = (Reldep *)sat_realloc(pool->rels, ((pool->nrels + REL_BLOCK) & ~REL_BLOCK) * sizeof(Reldep)); + pool->rels = sat_extend_resize(pool->rels, pool->nrels, sizeof(Reldep), REL_BLOCK); } // reset all hash tables Modified: trunk/sat-solver/src/poolid.h URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/poolid.h?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/poolid.h (original) +++ trunk/sat-solver/src/poolid.h Tue Jan 29 17:14:03 2008 @@ -26,6 +26,7 @@ } Reldep; extern Id str2id(Pool *pool, const char *, int); +extern Id strn2id(Pool *pool, const char *, unsigned int, int); extern Id rel2id(Pool *pool, Id, Id, int, int); extern const char *id2str(Pool *pool, Id); extern const char *dep2str(Pool *pool, Id); Modified: trunk/sat-solver/src/pooltypes.h URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/pooltypes.h?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/pooltypes.h (original) +++ trunk/sat-solver/src/pooltypes.h Tue Jan 29 17:14:03 2008 @@ -19,6 +19,7 @@ #define SOLV_VERSION_2 2 #define SOLV_VERSION_3 3 #define SOLV_VERSION_4 4 +#define SOLV_VERSION_5 5 #define SOLV_FLAG_PACKEDSIZES 1 #define SOLV_FLAG_VERTICAL 2 Modified: trunk/sat-solver/src/repo.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/repo.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/repo.c (original) +++ trunk/sat-solver/src/repo.c Tue Jan 29 17:14:03 2008 @@ -12,15 +12,22 @@ * */ +#define _GNU_SOURCE +#include +#include + #include #include -#include + + #include "repo.h" #include "pool.h" #include "poolid_private.h" #include "util.h" +#if 0 #include "attr_store_p.h" +#endif #define IDARRAY_BLOCK 4095 @@ -74,17 +81,16 @@ if (!idarray) /* alloc idarray if not done yet */ { - idarray = sat_malloc2(1 + IDARRAY_BLOCK, sizeof(Id)); - idarray[0] = 0; idarraysize = 1; + idarray = sat_extend_resize(0, 1, sizeof(Id), IDARRAY_BLOCK); + idarray[0] = 0; repo->lastoff = 0; } if (!olddeps) /* no deps yet */ { olddeps = idarraysize; - if ((idarraysize & IDARRAY_BLOCK) == 0) - idarray = sat_realloc2(idarray, idarraysize + 1 + IDARRAY_BLOCK, sizeof(Id)); + idarray = sat_extend(idarray, idarraysize, 1, sizeof(Id), IDARRAY_BLOCK); } else if (olddeps == repo->lastoff) /* extend at end */ idarraysize--; @@ -94,19 +100,14 @@ olddeps = idarraysize; for (; idarray[i]; i++) { - if ((idarraysize & IDARRAY_BLOCK) == 0) - idarray = sat_realloc2(idarray, idarraysize + 1 + IDARRAY_BLOCK, sizeof(Id)); + idarray = sat_extend(idarray, idarraysize, 1, sizeof(Id), IDARRAY_BLOCK); idarray[idarraysize++] = idarray[i]; } - if ((idarraysize & IDARRAY_BLOCK) == 0) - idarray = sat_realloc2(idarray, idarraysize + 1 + IDARRAY_BLOCK, sizeof(Id)); + idarray = sat_extend(idarray, idarraysize, 1, sizeof(Id), IDARRAY_BLOCK); } idarray[idarraysize++] = id; /* insert Id into array */ - - if ((idarraysize & IDARRAY_BLOCK) == 0) /* realloc if at block boundary */ - idarray = sat_realloc2(idarray, idarraysize + 1 + IDARRAY_BLOCK, sizeof(Id)); - + idarray = sat_extend(idarray, idarraysize, 1, sizeof(Id), IDARRAY_BLOCK); idarray[idarraysize++] = 0; /* ensure NULL termination */ repo->idarraydata = idarray; @@ -215,7 +216,7 @@ if (!repo->idarraysize) /* ensure buffer space */ { repo->idarraysize = 1; - repo->idarraydata = sat_malloc2((1 + num + IDARRAY_BLOCK) & ~IDARRAY_BLOCK, sizeof(Id)); + repo->idarraydata = sat_extend_resize(0, 1 + num, sizeof(Id), IDARRAY_BLOCK); repo->idarraydata[0] = 0; repo->lastoff = 1; return 1; @@ -235,10 +236,7 @@ ; count = idend - idstart - 1 + num; /* new size */ - /* realloc if crossing block boundary */ - if (((repo->idarraysize - 1) | IDARRAY_BLOCK) != ((repo->idarraysize + count - 1) | IDARRAY_BLOCK)) - repo->idarraydata = sat_realloc2(repo->idarraydata, (repo->idarraysize + count + IDARRAY_BLOCK) & ~IDARRAY_BLOCK, sizeof(Id)); - + repo->idarraydata = sat_extend(repo->idarraydata, repo->idarraysize, count, sizeof(Id), IDARRAY_BLOCK); /* move old deps to end */ olddeps = repo->lastoff = repo->idarraysize; memcpy(repo->idarraydata + olddeps, idstart, count - num); @@ -250,9 +248,8 @@ if (olddeps) /* appending */ repo->idarraysize--; - /* realloc if crossing block boundary */ - if (((repo->idarraysize - 1) | IDARRAY_BLOCK) != ((repo->idarraysize + num - 1) | IDARRAY_BLOCK)) - repo->idarraydata = sat_realloc2(repo->idarraydata, (repo->idarraysize + num + IDARRAY_BLOCK) & ~IDARRAY_BLOCK, sizeof(Id)); + /* make room*/ + repo->idarraydata = sat_extend(repo->idarraydata, repo->idarraysize, num, sizeof(Id), IDARRAY_BLOCK); /* appending or new */ repo->lastoff = olddeps ? olddeps : repo->idarraysize; @@ -457,413 +454,272 @@ return supplements; } -static unsigned char * -data_read_id(unsigned char *dp, Id *idp) -{ - Id x = 0; - unsigned char c; - for (;;) - { - c = *dp++; - if (!(c & 0x80)) - { - *idp = (x << 7) ^ c; - return dp; - } - x = (x << 7) ^ c ^ 128; - } -} - -static unsigned char * -data_skip(unsigned char *dp, int type) -{ - switch (type) - { - case TYPE_VOID: - return dp; - case TYPE_ID: - case TYPE_DIR: - while ((*dp & 0x80) != 0) - dp++; - return dp; - case TYPE_IDARRAY: - case TYPE_REL_IDARRAY: - case TYPE_IDVALUEARRAY: - case TYPE_DIRVALUEVALUEARRAY: - while ((*dp & 0xc0) != 0) - dp++; - return dp; - default: - fprintf(stderr, "unknown type in data_skip\n"); - exit(1); - } -} - -static unsigned char * -forward_to_key(Repodata *data, Id key, Id schema, unsigned char *dp) -{ - Id k, *keyp; - - keyp = data->schemadata + schema; - while ((k = *keyp++) != 0) - { - if (k == key) - return dp; - if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET) - { - /* skip that offset */ - dp = data_skip(dp, TYPE_ID); - continue; - } - if (data->keys[k].storage != KEY_STORAGE_INCORE) - continue; - dp = data_skip(dp, data->keys[k].type); - } - return 0; -} - -static unsigned char * -load_page_range(Repodata *data, unsigned int pstart, unsigned int pend) -{ - /* add smart paging here */ - return 0; -} - -static unsigned char * -get_data(Repodata *data, Repokey *key, unsigned char **dpp) -{ - Id off; - unsigned char *dp = *dpp; - int i, max; - unsigned int pstart, pend, poff, plen; - - if (!dp) - return 0; - if (key->storage == KEY_STORAGE_INCORE) - { - *dpp = data_skip(dp, key->type); - return dp; - } - if (key->storage != KEY_STORAGE_VERTICAL_OFFSET) - return 0; - if (!data->fp) - return 0; - dp = data_read_id(dp, &off); - *dpp = dp; - if (key->type == TYPE_VOID) - return 0; - max = key->size - off; - if (max <= 0) - return 0; - /* we now have the offset, go into vertical */ - for (i = key - data->keys - 1; i > 0; i--) - if (data->keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET) - off += data->keys[i].size; - pstart = off / BLOB_PAGESIZE; - pend = pstart; - poff = off % BLOB_PAGESIZE; - plen = BLOB_PAGESIZE - poff; - for (;;) - { - if (plen > max) - plen = max; - dp = load_page_range(data, pstart, pend) + poff; - if (!dp) - return 0; - switch (key->type) - { - case TYPE_STR: - if (memchr(dp, 0, plen)) - return dp; - break; - case TYPE_ID: - for (i = 0; i < plen; i++) - if ((dp[i] & 0x80) == 0) - return dp; - break; - case TYPE_IDARRAY: - case TYPE_REL_IDARRAY: - case TYPE_IDVALUEARRAY: - case TYPE_DIRVALUEVALUEARRAY: - for (i = 0; i < plen; i++) - if ((dp[i] & 0xc0) == 0) - return dp; - break; - } - if (plen == max) - return 0; - pend++; - plen += BLOB_PAGESIZE; - } -} - - -const char * -repodata_lookup_str(Repodata *data, Id entry, Id key) -{ - Id schema; - Id id, k, *kp, *keyp; - unsigned char *dp; - - if (data->entryschemau8) - schema = data->entryschemau8[entry]; - else - schema = data->entryschema[entry]; - keyp = data->schemadata + schema; - /* make sure the schema of this solvable contains the key */ - for (kp = keyp; (k = *kp++) != 0; ) - if (k == key) - break; - if (k == 0) - return 0; - dp = forward_to_key(data, key, schema, data->incoredata + data->incoreoffset[entry]); - dp = get_data(data, data->keys + key, &dp); - if (!dp) - return 0; - if (data->keys[key].type == TYPE_STR) - return (const char *)dp; - /* id type, must either use global or local string strore*/ - dp = data_read_id(dp, &id); -#if 0 - /* not yet working */ - return data->ss.stringspace + data->ss.strings[id]; -#else - return id2str(data->repo->pool, id); -#endif -} - -#define SEARCH_NEXT_KEY 1 -#define SEARCH_NEXT_SOLVABLE 2 -#define SEACH_STOP 3 - struct matchdata { Pool *pool; - const char *matchstr; + const char *match; int flags; #if 0 regex_t regex; #endif int stop; - int (*callback)(void *data, Solvable *s, Id key, const char *str); + int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv); void *callback_data; }; -static void -domatch(Id p, Id key, struct matchdata *md, const char *str) +int +repo_matchvalue(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv) { - /* fill match code here */ - md->stop = md->callback(md->callback_data, md->pool->solvables + p, key, str); -} + struct matchdata *md = cbdata; + int flags = md->flags; -static void -repodata_search(Repodata *data, Id entry, Id key, struct matchdata *md) -{ - Id schema; - Id id, k, *kp, *keyp; - unsigned char *dp, *ddp; - int onekey = 0; - - if (data->entryschemau8) - schema = data->entryschemau8[entry]; - else - schema = data->entryschema[entry]; - keyp = data->schemadata + schema; - dp = data->incoredata + data->incoreoffset[entry]; - if (key) - { - /* search in a specific key */ - for (kp = keyp; (k = *kp++) != 0; ) - if (k == key) - break; - if (k == 0) - return; - dp = forward_to_key(data, key, schema, dp); - if (!dp) - return; - keyp = kp - 1; - onekey = 1; - } - md->stop = 0; - while ((key = *keyp++) != 0) + if ((flags & SEARCH_STRINGMASK) != 0) { - ddp = get_data(data, data->keys + key, &dp); - while (ddp) + switch (key->type) { - switch (data->keys[key].type) - { - case TYPE_STR: - domatch(data->start + entry, data->keys[key].name, md, (const char *)ddp); - ddp = 0; - break; - case TYPE_ID: - data_read_id(ddp, &id); - /* domatch */ - ddp = 0; - break; - case TYPE_IDARRAY: - ddp = data_read_id(ddp, &id); - if ((id & 0x40) == 0) - ddp = 0; - id = (id & 0x3f) | ((id >> 1) & ~0x3f); - /* domatch */ - while (md->stop == SEARCH_NEXT_KEY && ddp) - ddp = data_read_id(ddp, &id); - break; - default: - ddp = 0; - } + case TYPE_ID: + case TYPE_IDARRAY: + if (data->localpool) + kv->str = stringpool_id2str(&data->spool, kv->id); + else + kv->str = id2str(data->repo->pool, kv->id); + break; + case TYPE_STR: + break; + default: + return 0; + } + switch ((flags & SEARCH_STRINGMASK)) + { + case SEARCH_SUBSTRING: + if (flags & SEARCH_NOCASE) + { + if (!strcasestr(kv->str, md->match)) + return 0; + } + else + { + if (!strstr(kv->str, md->match)) + return 0; + } + break; + case SEARCH_STRING: + if (flags & SEARCH_NOCASE) + { + if (strcasecmp(md->match, kv->str)) + return 0; + } + else + { + if (strcmp(md->match, kv->str)) + return 0; + } + break; + case SEARCH_GLOB: + if (fnmatch(md->match, kv->str, (flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0)) + return 0; + break; +#if 0 + case SEARCH_REGEX: + if (regexec(&md->regexp, kv->str, 0, NULL, 0)) + return 0; +#endif + default: + return 0; } - if (onekey || md->stop > SEARCH_NEXT_KEY) - return; } + md->stop = md->callback(md->callback_data, s, data, key, kv); + return md->stop; } + +static Repokey solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1] = { + { SOLVABLE_NAME, TYPE_ID, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_ARCH, TYPE_ID, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_EVR, TYPE_ID, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_VENDOR, TYPE_ID, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_PROVIDES, TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_OBSOLETES, TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_CONFLICTS, TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_REQUIRES, TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_RECOMMENDS, TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_SUGGESTS, TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_SUPPLEMENTS, TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_ENHANCES, TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, + { SOLVABLE_FRESHENS, TYPE_IDARRAY, 0, KEY_STORAGE_SOLVABLE }, + { RPM_RPMDBID, TYPE_U32, 0, KEY_STORAGE_SOLVABLE }, +}; + static void -domatch_idarray(Id p, Id key, struct matchdata *md, Id *ida) +domatch_idarray(Solvable *s, Id keyname, struct matchdata *md, Id *ida) { + KeyValue kv; for (; *ida && !md->stop; ida++) - domatch(p, key, md, id2str(md->pool, *ida)); + { + kv.id = *ida; + kv.eof = ida[1] ? 0 : 1; + repo_matchvalue(md, s, 0, solvablekeys + (keyname - SOLVABLE_NAME), &kv); + } } static void -repo_search_md(Repo *repo, Id p, Id key, struct matchdata *md) +repo_search_md(Repo *repo, Id p, Id keyname, struct matchdata *md) { + KeyValue kv; Pool *pool = repo->pool; Repodata *data; Solvable *s; - int i; + int i, j, flags; md->stop = 0; if (!p) { -#if 0 - switch(key) - { - case 0: - case SOLVABLE_NAME: - case SOLVABLE_ARCH: - case SOLVABLE_EVR: - case SOLVABLE_VENDOR: - case SOLVABLE_PROVIDES: - case SOLVABLE_OBSOLETES: - case SOLVABLE_CONFLICTS: - case SOLVABLE_REQUIRES: - case SOLVABLE_RECOMMENDS: - case SOLVABLE_SUPPLEMENTS: - case SOLVABLE_SUGGESTS: - case SOLVABLE_ENHANCES: - case SOLVABLE_FRESHENS: - break; - default: - for (i = 0, data = repo->repodata; i < repo->nrepodata; i++, data++) - repodata_search(data, -1, key, md); - return; - } -#endif for (p = repo->start, s = repo->pool->solvables + p; p < repo->end; p++, s++) { if (s->repo == repo) - repo_search_md(repo, p, key, md); + repo_search_md(repo, p, keyname, md); if (md->stop > SEARCH_NEXT_SOLVABLE) break; } return; } s = pool->solvables + p; - switch(key) + flags = md->flags; + if (!(flags & SEARCH_NO_STORAGE_SOLVABLE)) { - case 0: - case SOLVABLE_NAME: - if (s->name) - domatch(p, SOLVABLE_NAME, md, id2str(pool, s->name)); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_ARCH: - if (s->arch) - domatch(p, SOLVABLE_ARCH, md, id2str(pool, s->arch)); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_EVR: - if (s->evr) - domatch(p, SOLVABLE_EVR, md, id2str(pool, s->evr)); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_VENDOR: - if (s->vendor) - domatch(p, SOLVABLE_VENDOR, md, id2str(pool, s->vendor)); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_PROVIDES: - if (s->provides) - domatch_idarray(p, SOLVABLE_PROVIDES, md, repo->idarraydata + s->provides); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_OBSOLETES: - if (s->obsoletes) - domatch_idarray(p, SOLVABLE_OBSOLETES, md, repo->idarraydata + s->obsoletes); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_CONFLICTS: - if (s->obsoletes) - domatch_idarray(p, SOLVABLE_CONFLICTS, md, repo->idarraydata + s->conflicts); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_REQUIRES: - if (s->requires) - domatch_idarray(p, SOLVABLE_REQUIRES, md, repo->idarraydata + s->requires); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_RECOMMENDS: - if (s->recommends) - domatch_idarray(p, SOLVABLE_RECOMMENDS, md, repo->idarraydata + s->recommends); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_SUPPLEMENTS: - if (s->supplements) - domatch_idarray(p, SOLVABLE_SUPPLEMENTS, md, repo->idarraydata + s->supplements); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_SUGGESTS: - if (s->suggests) - domatch_idarray(p, SOLVABLE_SUGGESTS, md, repo->idarraydata + s->suggests); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_ENHANCES: - if (s->enhances) - domatch_idarray(p, SOLVABLE_ENHANCES, md, repo->idarraydata + s->enhances); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - case SOLVABLE_FRESHENS: - if (s->freshens) - domatch_idarray(p, SOLVABLE_FRESHENS, md, repo->idarraydata + s->freshens); - if (key || md->stop > SEARCH_NEXT_KEY) - return; - default: - break; + switch(keyname) + { + case 0: + case SOLVABLE_NAME: + if (s->name) + { + kv.id = s->name; + repo_matchvalue(md, s, 0, solvablekeys + 0, &kv); + } + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_ARCH: + if (s->arch) + { + kv.id = s->arch; + repo_matchvalue(md, s, 0, solvablekeys + 1, &kv); + } + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_EVR: + if (s->evr) + { + kv.id = s->evr; + repo_matchvalue(md, s, 0, solvablekeys + 2, &kv); + } + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_VENDOR: + if (s->vendor) + { + kv.id = s->vendor; + repo_matchvalue(md, s, 0, solvablekeys + 3, &kv); + } + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_PROVIDES: + if (s->provides) + domatch_idarray(s, SOLVABLE_PROVIDES, md, repo->idarraydata + s->provides); + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_OBSOLETES: + if (s->obsoletes) + domatch_idarray(s, SOLVABLE_OBSOLETES, md, repo->idarraydata + s->obsoletes); + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_CONFLICTS: + if (s->conflicts) + domatch_idarray(s, SOLVABLE_CONFLICTS, md, repo->idarraydata + s->conflicts); + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_REQUIRES: + if (s->requires) + domatch_idarray(s, SOLVABLE_REQUIRES, md, repo->idarraydata + s->requires); + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_RECOMMENDS: + if (s->recommends) + domatch_idarray(s, SOLVABLE_RECOMMENDS, md, repo->idarraydata + s->recommends); + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_SUPPLEMENTS: + if (s->supplements) + domatch_idarray(s, SOLVABLE_SUPPLEMENTS, md, repo->idarraydata + s->supplements); + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_SUGGESTS: + if (s->suggests) + domatch_idarray(s, SOLVABLE_SUGGESTS, md, repo->idarraydata + s->suggests); + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_ENHANCES: + if (s->enhances) + domatch_idarray(s, SOLVABLE_ENHANCES, md, repo->idarraydata + s->enhances); + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case SOLVABLE_FRESHENS: + if (s->freshens) + domatch_idarray(s, SOLVABLE_FRESHENS, md, repo->idarraydata + s->freshens); + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + case RPM_RPMDBID: + if (repo->rpmdbid) + { + kv.num = repo->rpmdbid[p - repo->start]; + repo_matchvalue(md, s, 0, solvablekeys + (RPM_RPMDBID - SOLVABLE_NAME), &kv); + } + if (keyname || md->stop > SEARCH_NEXT_KEY) + return; + break; + default: + break; + } } for (i = 0, data = repo->repodata; i < repo->nrepodata; i++, data++) { if (p < data->start || p >= data->end) continue; - repodata_search(data, p - data->start, key, md); + if (data->state == REPODATA_STUB) + { + if (keyname) + { + for (j = 1; j < data->nkeys; j++) + if (keyname == data->keys[j].name) + break; + if (j == data->nkeys) + continue; + } + /* load it */ + if (data->loadcallback) + data->loadcallback(data); + else + data->state = REPODATA_ERROR; + } + if (data->state == REPODATA_ERROR) + continue; + repodata_search(data, p - data->start, keyname, repo_matchvalue, md); if (md->stop > SEARCH_NEXT_KEY) break; } } void -repo_search(Repo *repo, Id p, Id key, const char *match, int flags, int (*callback)(void *data, Solvable *s, Id key, const char *str), void *callback_data) +repo_search(Repo *repo, Id p, Id key, const char *match, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata) { struct matchdata md; memset(&md, 0, sizeof(md)); md.pool = repo->pool; - md.matchstr = match; + md.match = match; md.flags = flags; md.callback = callback; - md.callback_data = callback_data; + md.callback_data = cbdata; repo_search_md(repo, p, key, &md); } @@ -900,7 +756,91 @@ return 0; } +Repodata * +repo_add_repodata(Repo *repo) +{ + Repodata *data; + repo->nrepodata++; + repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata, sizeof(*data)); + data = repo->repodata + repo->nrepodata - 1; + memset(data, 0, sizeof (*data)); + data->repo = repo; + data->start = repo->start; + data->end = repo->end; + data->localpool = 0; + data->keys = sat_calloc(1, sizeof(Repokey)); + data->nkeys = 1; + data->schemata = sat_calloc(1, sizeof(Id)); + data->schemadata = sat_calloc(1, sizeof(Id)); + data->nschemata = 1; + data->schemadatalen = 1; + data->entryschemau8 = sat_calloc(data->end - data->start, 1); + data->incoreoffset = sat_calloc(data->end - data->start, sizeof(Id)); + return data; +} + +static Repodata *findrepodata(Repo *repo, Id p, Id keyname) +{ + int i; + Repodata *data; + + /* FIXME: enter nice code here */ + for (i = 0, data = repo->repodata; i < repo->nrepodata; i++, data++) + if (p >= data->start && p < data->end) + return data; + for (i = 0, data = repo->repodata; i < repo->nrepodata; i++, data++) + if (p == data->end) + break; + if (i < repo->nrepodata) + { + repodata_extend(data, p); + return data; + } + return repo_add_repodata(repo); +} + +void +repo_set_id(Repo *repo, Id p, Id keyname, Id id) +{ + Repodata *data = findrepodata(repo, p, keyname); + repodata_set_id(data, p - data->start, keyname, id); +} + +void +repo_set_num(Repo *repo, Id p, Id keyname, Id num) +{ + Repodata *data = findrepodata(repo, p, keyname); + repodata_set_num(data, p - data->start, keyname, num); +} + +void +repo_set_str(Repo *repo, Id p, Id keyname, const char *str) +{ + Repodata *data = findrepodata(repo, p, keyname); + repodata_set_str(data, p - data->start, keyname, str); +} + +void +repo_set_poolstr(Repo *repo, Id p, Id keyname, const char *str) +{ + Repodata *data = findrepodata(repo, p, keyname); + repodata_set_poolstr(data, p - data->start, keyname, str); +} + +void +repo_internalize(Repo *repo) +{ + int i; + Repodata *data; + + for (i = 0, data = repo->repodata; i < repo->nrepodata; i++, data++) + if (data->attrs) + repodata_internalize(data); +} + + +#if 0 static int key_cmp (const void *pa, const void *pb) @@ -948,5 +888,6 @@ if (location) data->location = strdup(location); } +#endif // EOF Modified: trunk/sat-solver/src/repo.h URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/repo.h?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/repo.h (original) +++ trunk/sat-solver/src/repo.h Tue Jan 29 17:14:03 2008 @@ -15,7 +15,10 @@ #include "pooltypes.h" #include "pool.h" +#if 0 #include "attr_store.h" +#endif +#include "repodata.h" typedef struct _Repokey { Id name; @@ -29,57 +32,6 @@ #define KEY_STORAGE_INCORE 2 #define KEY_STORAGE_VERTICAL_OFFSET 3 -struct _Repo; - -typedef struct _Repodata { - struct _Repo *repo; /* back pointer to repo */ - - int start; /* start of solvables this repodata is valid for */ - int end; /* last solvable + 1 of this repodata */ - - FILE *fp; /* file pointer of solv file */ - int error; /* corrupt solv file */ - - /* Keys provided by this attribute store, sorted by name value. - The same keys may be provided by multiple attribute stores, but - then only for different solvables. I.e. the relation - (solvable,name) -> store - has to be injective. */ - - Repokey *keys; /* keys, first entry is always zero */ - unsigned int nkeys; /* length of keys array */ - - Id *schemata; /* schema -> offset into schemadata */ - unsigned int nschemata; /* number of schemata */ - - Id *schemadata; /* schema storage */ - - unsigned char *entryschemau8; /* schema for entry */ - Id *entryschema; /* schema for entry */ - - unsigned char *incoredata; /* in-core data (flat_attrs) */ - unsigned int incoredatalen; /* data len (attr_next_free) */ - unsigned int incoredatafree; /* free data len */ - - Id *incoreoffset; /* offset for all entries (ent2attr) */ - - Id verticaloffset; /* file offset of verticals */ - - Id *dirs; /* directory list */ - int ndirs; /* its size */ - - /* The attribute store itself. */ - Attrstore *s; - /* A filename where to find this attribute store, or where to store - it. May be "", in which case we can't load it on demand or store - into it. It may also be NULL for at most one of the repodata per - repo, in which case these are the embedded attributes. */ - - const char *location; - /* The SHA1 checksum of the file. */ - unsigned char checksum[20]; -} Repodata; - typedef struct _Repo { const char *name; @@ -95,12 +47,10 @@ int idarraysize; Offset lastoff; - Id *rpmdbid; + Id *rpmdbid; /* hmm, go to repodata? */ - /* The attribute stores we know about. */ - Repodata *repodata; - /* Number of attribute stores.. */ - unsigned nrepodata; + Repodata *repodata; /* our stores for non-solvable related data */ + unsigned nrepodata; /* number of our stores.. */ } Repo; extern Repo *repo_create(Pool *pool, const char *name); @@ -112,7 +62,9 @@ extern Offset repo_reserve_ids(Repo *repo, Offset olddeps, int num); extern Offset repo_fix_legacy(Repo *repo, Offset provides, Offset supplements); +#if 0 extern void repo_add_attrstore (Repo *repo, Attrstore *s, const char *location); +#endif static inline const char *repo_name(const Repo *repo) { @@ -183,4 +135,38 @@ for (p = (r)->start, s = (r)->pool->solvables + p; p < (r)->end; p++, s++) \ if (s->repo == (r)) + +/* search callback values */ + +#define SEARCH_NEXT_KEY 1 +#define SEARCH_NEXT_SOLVABLE 2 +#define SEACH_STOP 3 + +typedef struct _KeyValue { + Id id; + const char *str; + int num; + int num2; + int eof; +} KeyValue; + +/* search flags */ +#define SEARCH_STRINGMASK 15 +#define SEARCH_STRING 1 +#define SEARCH_SUBSTRING 2 +#define SEARCH_GLOB 3 +#define SEARCH_REGEX 4 + +#define SEARCH_NOCASE (1<<8) +#define SEARCH_NO_STORAGE_SOLVABLE (1<<9) + +Repodata *repo_add_repodata(Repo *repo); +void repo_search(Repo *repo, Id p, Id key, const char *match, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata); + +void repo_set_id(Repo *repo, Id p, Id keyname, Id id); +void repo_set_num(Repo *repo, Id p, Id keyname, Id num); +void repo_set_str(Repo *repo, Id p, Id keyname, const char *str); +void repo_set_poolstr(Repo *repo, Id p, Id keyname, const char *str); +void repo_internalize(Repo *repo); + #endif /* SATSOLVER_REPO_H */ Modified: trunk/sat-solver/src/repo_solv.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/repo_solv.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/repo_solv.c (original) +++ trunk/sat-solver/src/repo_solv.c Tue Jan 29 17:14:03 2008 @@ -25,7 +25,9 @@ #include "repo_solv.h" #include "util.h" +#if 0 #include "attr_store_p.h" +#endif #define INTERESTED_START SOLVABLE_NAME #define INTERESTED_END SOLVABLE_FRESHENS @@ -253,29 +255,50 @@ switch (type) { case TYPE_VOID: + case TYPE_CONSTANT: break; case TYPE_ID: read_id(data, numid + numrel); /* just check Id */ break; case TYPE_DIR: - read_id(data, numid + data->ndirs); /* just check Id */ + read_id(data, numid + data->dirpool.ndirs); /* just check Id */ + break; + case TYPE_NUM: + read_id(data, 0); break; case TYPE_U32: read_u32(data); break; case TYPE_ATTR_STRING: case TYPE_STR: - while(read_u8(data) != 0) + while (read_u8(data) != 0) ; break; case TYPE_IDARRAY: - case TYPE_IDVALUEARRAY: - case TYPE_DIRVALUEVALUEARRAY: case TYPE_REL_IDARRAY: case TYPE_ATTR_INTLIST: while ((read_u8(data) & 0xc0) != 0) ; break; + case TYPE_DIRNUMNUMARRAY: + for (;;) + { + read_id(data, numid + data->dirpool.ndirs); /* just check Id */ + read_id(data, 0); + if (!(read_id(data, 0) & 0x40)) + break; + } + break; + case TYPE_DIRSTRARRAY: + for (;;) + { + Id id = read_id(data, 0); + while (read_u8(data) != 0) + ; + if (!(id & 0x40)) + break; + } + break; case TYPE_COUNT_NAMED: { unsigned count = read_id(data, 0); @@ -320,6 +343,8 @@ return a->name - b->name; } +static void repodata_load_solv(Repodata *data); + static void parse_repodata(Repodata *maindata, Id *keyp, Repokey *keys, Id *idmap, unsigned numid, unsigned numrel, Repo *repo) { @@ -330,8 +355,10 @@ repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata + 1, sizeof (*data)); data = repo->repodata + repo->nrepodata++; - data->repo = repo; memset(data, 0, sizeof(*data)); + data->repo = repo; + data->state = REPODATA_STUB; + data->loadcallback = repodata_load_solv; while ((key = *keyp++) != 0) { @@ -349,6 +376,7 @@ n = ide - ida - 1; if (n & 1) { + pool_debug (mypool, SAT_ERROR, "invalid attribute data\n"); data->error = SOLV_ERROR_CORRUPT; return; } @@ -359,8 +387,8 @@ { if (*ide >= numid) { - data->error = SOLV_ERROR_CORRUPT; pool_debug (mypool, SAT_ERROR, "invalid attribute data\n"); + data->error = SOLV_ERROR_CORRUPT; return; } data->keys[i].name = idmap ? idmap[*ide++] : *ide++; @@ -381,7 +409,9 @@ unsigned len = sizeof (buf); char *filename = buf; read_str(maindata, &filename, &len); +#if 0 data->location = strdup(filename); +#endif if (filename != buf) free(filename); } @@ -469,15 +499,18 @@ data->incoredatalen++; } + + // ---------------------------------------------- + /* * read repo from .solv file * and add it to pool */ -int -repo_add_solv(Repo *repo, FILE *fp) +static int +repo_add_solv_parent(Repo *repo, FILE *fp, Repodata *parent) { Pool *pool = repo->pool; int i, l; @@ -508,6 +541,9 @@ Repokey *keys; Id *schemadata, *schemadatap, *schemadataend; Id *schemata, key; + int have_xdata; + + struct _Stringpool *spool; Repodata data; @@ -529,6 +565,7 @@ case SOLV_VERSION_2: case SOLV_VERSION_3: case SOLV_VERSION_4: + case SOLV_VERSION_5: break; default: pool_debug(pool, SAT_ERROR, "unsupported SOLV version\n"); @@ -549,6 +586,9 @@ numinfo = read_u32(&data); solvflags = read_u32(&data); + if (solvversion < SOLV_VERSION_5) + numschemata++; + if (numdir && numdir < 2) { pool_debug(pool, SAT_ERROR, "bad number of dirs\n"); @@ -560,6 +600,25 @@ return SOLV_ERROR_UNSUPPORTED; } + if (parent) + { + if (numrel) + { + pool_debug(pool, SAT_ERROR, "relations are forbidden in a store\n"); + return SOLV_ERROR_CORRUPT; + } + if (numsolv) + { + pool_debug(pool, SAT_ERROR, "solvables are forbidden in a store\n"); + return SOLV_ERROR_CORRUPT; + } + if (numinfo) + { + pool_debug(pool, SAT_ERROR, "info blocks are forbidden in a store\n"); + return SOLV_ERROR_CORRUPT; + } + } + /******* Part 1: string IDs *****************************************/ sizeid = read_u32(&data); /* size of string+Id space */ @@ -574,16 +633,27 @@ * alloc buffers */ + if (!parent) + spool = &pool->ss; + else + { + spool = &data.spool; + spool->stringspace = sat_malloc(7); + strcpy(spool->stringspace, "<NULL>"); + spool->sstrings = 7; + spool->nstrings = 0; + } + /* alloc string buffer */ - pool->ss.stringspace = sat_realloc(pool->ss.stringspace, pool->ss.sstrings + sizeid + 1); + spool->stringspace = sat_realloc(spool->stringspace, spool->sstrings + sizeid + 1); /* alloc string offsets (Id -> Offset into string space) */ - pool->ss.strings = sat_realloc2(pool->ss.strings, pool->ss.nstrings + numid, sizeof(Offset)); + spool->strings = sat_realloc2(spool->strings, spool->nstrings + numid, sizeof(Offset)); - strsp = pool->ss.stringspace; - str = pool->ss.strings; /* array of offsets into strsp, indexed by Id */ + strsp = spool->stringspace; + str = spool->strings; /* array of offsets into strsp, indexed by Id */ /* point to _BEHIND_ already allocated string/Id space */ - strsp += pool->ss.sstrings; + strsp += spool->sstrings; /* @@ -603,7 +673,7 @@ unsigned int pfsize = read_u32(&data); char *prefix = sat_malloc(pfsize); char *pp = prefix; - char *old_str = ""; + char *old_str = 0; char *dest = strsp; if (fread(prefix, pfsize, 1, fp) != 1) { @@ -627,87 +697,113 @@ strsp[sizeid] = 0; /* make string space \0 terminated */ sp = strsp; + /* make sure first entry is "" for a store */ + if (parent) + { + /* no shared pool, thus no idmap and no unification */ + idmap = 0; + sp += 7; + if (*sp) + { + pool_debug(pool, SAT_ERROR, "store strings don't start with ''\n"); + return SOLV_ERROR_CORRUPT; + } + str[0] = 0; + for (i = 1; i < spool->nstrings; i++) + { + if (sp >= strsp + sizeid) + { + pool_debug(pool, SAT_ERROR, "not enough strings\n"); + return SOLV_ERROR_OVERFLOW; + } + str[i] = sp - strsp; + sp += strlen(sp) + 1; + } + } + else + { - /* alloc id map for name and rel Ids. this maps ids in the solv files - * to the ids in our pool */ - idmap = sat_calloc(numid + numrel, sizeof(Id)); + /* alloc id map for name and rel Ids. this maps ids in the solv files + * to the ids in our pool */ + idmap = sat_calloc(numid + numrel, sizeof(Id)); - /* - * build hashes for all read strings - * - */ - - hashmask = mkmask(pool->ss.nstrings + numid); + /* + * build hashes for all read strings + * + */ + + hashmask = mkmask(spool->nstrings + numid); #if 0 - POOL_DEBUG(SAT_DEBUG_STATS, "read %d strings\n", numid); - POOL_DEBUG(SAT_DEBUG_STATS, "string hash buckets: %d\n", hashmask + 1); + POOL_DEBUG(SAT_DEBUG_STATS, "read %d strings\n", numid); + POOL_DEBUG(SAT_DEBUG_STATS, "string hash buckets: %d\n", hashmask + 1); #endif - /* - * create hashtable with strings already in pool - */ - - hashtbl = sat_calloc(hashmask + 1, sizeof(Id)); - for (i = 1; i < pool->ss.nstrings; i++) /* leave out our dummy zero id */ - { - h = strhash(pool->ss.stringspace + pool->ss.strings[i]) & hashmask; - hh = HASHCHAIN_START; - while (hashtbl[h]) - h = HASHCHAIN_NEXT(h, hh, hashmask); - hashtbl[h] = i; - } + /* + * create hashtable with strings already in pool + */ - /* - * run over string space, calculate offsets - * - * build id map (maps solv Id -> pool Id) - */ - - for (i = 1; i < numid; i++) - { - if (sp >= strsp + sizeid) + hashtbl = sat_calloc(hashmask + 1, sizeof(Id)); + for (i = 1; i < spool->nstrings; i++) /* leave out our dummy zero id */ { - sat_free(hashtbl); - sat_free(idmap); - pool_debug(pool, SAT_ERROR, "not enough strings\n"); - return SOLV_ERROR_OVERFLOW; + h = strhash(spool->stringspace + spool->strings[i]) & hashmask; + hh = HASHCHAIN_START; + while (hashtbl[h]) + h = HASHCHAIN_NEXT(h, hh, hashmask); + hashtbl[h] = i; } - if (!*sp) /* empty string */ + + /* + * run over string space, calculate offsets + * + * build id map (maps solv Id -> pool Id) + */ + + for (i = 1; i < numid; i++) { - idmap[i] = ID_EMPTY; - sp++; - continue; - } + if (sp >= strsp + sizeid) + { + sat_free(hashtbl); + sat_free(idmap); + pool_debug(pool, SAT_ERROR, "not enough strings\n"); + return SOLV_ERROR_OVERFLOW; + } + if (!*sp) /* empty string */ + { + idmap[i] = ID_EMPTY; + sp++; + continue; + } + + /* find hash slot */ + h = strhash(sp) & hashmask; + hh = HASHCHAIN_START; + for (;;) + { + id = hashtbl[h]; + if (id == 0) + break; + if (!strcmp(spool->stringspace + spool->strings[id], sp)) + break; /* existing string */ + h = HASHCHAIN_NEXT(h, hh, hashmask); + } - /* find hash slot */ - h = strhash(sp) & hashmask; - hh = HASHCHAIN_START; - for (;;) - { - id = hashtbl[h]; - if (id == 0) - break; - if (!strcmp(pool->ss.stringspace + pool->ss.strings[id], sp)) - break; /* existing string */ - h = HASHCHAIN_NEXT(h, hh, hashmask); - } - - /* length == offset to next string */ - l = strlen(sp) + 1; - if (id == ID_NULL) /* end of hash chain -> new string */ - { - id = pool->ss.nstrings++; - hashtbl[h] = id; - str[id] = pool->ss.sstrings; /* save Offset */ - if (sp != pool->ss.stringspace + pool->ss.sstrings) /* not at end-of-buffer */ - memmove(pool->ss.stringspace + pool->ss.sstrings, sp, l); /* append to pool buffer */ - pool->ss.sstrings += l; + /* length == offset to next string */ + l = strlen(sp) + 1; + if (id == ID_NULL) /* end of hash chain -> new string */ + { + id = spool->nstrings++; + hashtbl[h] = id; + str[id] = spool->sstrings; /* save Offset */ + if (sp != spool->stringspace + spool->sstrings) /* not at end-of-buffer */ + memmove(spool->stringspace + spool->sstrings, sp, l); /* append to pool buffer */ + spool->sstrings += l; + } + idmap[i] = id; /* repo relative -> pool relative */ + sp += l; /* next string */ } - idmap[i] = id; /* repo relative -> pool relative */ - sp += l; /* next string */ + sat_free(hashtbl); } - sat_free(hashtbl); pool_shrink_strings(pool); /* vacuum */ @@ -783,18 +879,19 @@ /******* Part 3: Dirs ***********************************************/ if (numdir) { - data.dirs = sat_calloc(numdir, sizeof(Id)); - data.ndirs = numdir; - /* dir 0: no directory - * dir 1: / - */ + data.dirpool.dirs = sat_malloc2(numdir, sizeof(Id)); + data.dirpool.ndirs = numdir; + data.dirpool.dirs[0] = 0; /* dir 0: virtual root */ + data.dirpool.dirs[1] = 1; /* dir 1: / */ for (i = 2; i < numdir; i++) { id = read_id(&data, i + numid); - if (i > numid) - data.dirs[i] = -(id - numid); + if (id >= numid) + data.dirpool.dirs[i] = -(id - numid); + else if (idmap) + data.dirpool.dirs[i] = idmap[id]; else - data.dirs[i] = idmap[id]; + data.dirpool.dirs[i] = id; } } @@ -810,9 +907,26 @@ keys[i].name = id; keys[i].type = read_id(&data, 0); keys[i].size = read_id(&data, 0); + if (solvversion >= SOLV_VERSION_5) + { + keys[i].storage = read_id(&data, 0); + continue; + } keys[i].storage = KEY_STORAGE_DROPPED; + if (parent) + { + keys[i].storage = KEY_STORAGE_INCORE; + continue; + } switch (keys[i].type) { + case TYPE_VOID: + case TYPE_CONSTANT: + case TYPE_STR: + case TYPE_NUM: + case TYPE_DIRNUMNUMARRAY: + keys[i].storage = KEY_STORAGE_INCORE; + break; case TYPE_ID: switch(id) { @@ -834,9 +948,6 @@ else keys[i].storage = KEY_STORAGE_INCORE; break; - case TYPE_STR: - keys[i].storage = KEY_STORAGE_INCORE; - break; case TYPE_U32: if (id == RPM_RPMDBID) keys[i].storage = KEY_STORAGE_SOLVABLE; @@ -848,17 +959,22 @@ } } + have_xdata = parent ? 1 : 0; + for (i = 1; i < numkeys; i++) + if (keys[i].storage == KEY_STORAGE_INCORE || keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET) + have_xdata = 1; + data.keys = keys; data.nkeys = numkeys; /******* Part 5: Schemata ********************************************/ id = read_id(&data, 0); - schemadata = sat_calloc(id, sizeof(Id)); - schemadatap = schemadata; - schemadataend = schemadata + id; + schemadata = sat_calloc(id + 1, sizeof(Id)); + schemadatap = schemadata + 1; + schemadataend = schemadatap + id; schemata = sat_calloc(numschemata, sizeof(Id)); - for (i = 0; i < numschemata; i++) + for (i = 1; i < numschemata; i++) { schemata[i] = schemadatap - schemadata; schemadatap = read_idarray(&data, numid, 0, schemadatap, schemadataend, 0); @@ -866,6 +982,8 @@ data.schemata = schemata; data.nschemata = numschemata; data.schemadata = schemadata; + data.schemadatalen = schemadataend - data.schemadata; + /******* Part 6: Info ***********************************************/ for (i = 0; i < numinfo; i++) @@ -873,7 +991,11 @@ /* for now we're just interested in data that starts with * the repodata_external id */ - Id *keyp = schemadata + schemata[read_id(&data, numschemata)]; + Id *keyp; + id = read_id(&data, numschemata); + if (solvversion < SOLV_VERSION_5) + id++; + keyp = schemadata + schemata[id]; key = *keyp; if (keys[key].name == REPODATA_EXTERNAL && keys[key].type == TYPE_VOID) { @@ -884,6 +1006,7 @@ skip_schema(&data, keyp, keys, numid, numrel); } + /******* Part 7: packed sizes (optional) ****************************/ char *exists = 0; if ((solvflags & SOLV_FLAG_PACKEDSIZES) != 0) @@ -893,6 +1016,7 @@ exists[i] = read_id(&data, 0) != 0; } + /******* Part 8: item data *******************************************/ /* calculate idarray size */ @@ -921,24 +1045,63 @@ } /* read solvables */ - s = pool_id2solvable(pool, repo_add_solvable_block(repo, numsolv)); - - /* store start and end of our id block */ - data.start = s - pool->solvables; - data.end = data.start + numsolv; + if (numsolv) + { + s = pool_id2solvable(pool, repo_add_solvable_block(repo, numsolv)); + /* store start and end of our id block */ + data.start = s - pool->solvables; + data.end = data.start + numsolv; + } + else + s = 0; + if (have_xdata) + { + if (numschemata < 255) + data.entryschemau8 = sat_calloc(numsolv, 1); + else + data.entryschema = sat_calloc(numsolv, sizeof(Id)); + data.incoreoffset = sat_calloc(numsolv, sizeof(Id)); + } for (i = 0; i < numsolv; i++, s++) { + Id *keyp; if (data.error) break; if (exists && !exists[i]) continue; - Id *keyp = schemadata + schemata[read_id(&data, numschemata)]; + id = read_id(&data, numschemata); + if (solvversion < SOLV_VERSION_5) + id++; + if (have_xdata) + { + if (data.entryschemau8) + data.entryschemau8[i] = id; + else + data.entryschema[i] = id; + data.incoreoffset[i] = data.incoredatalen; + } + keyp = schemadata + schemata[id]; while ((key = *keyp++) != 0) { id = keys[key].name; +#if 0 +fprintf(stderr, "solv %d name %d type %d class %d\n", i, id, keys[key].type, keys[key].storage); +#endif + if (keys[key].storage == KEY_STORAGE_VERTICAL_OFFSET) + { + /* copy offset/length into incore */ + did = read_id(&data, 0); + incore_add_id(&data, did); + did = read_id(&data, 0); + incore_add_id(&data, did); + continue; + } switch (keys[key].type) { + case TYPE_VOID: + case TYPE_CONSTANT: + break; case TYPE_ID: did = read_id(&data, numid + numrel); if (idmap) @@ -957,6 +1120,14 @@ POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %s\n", id2str(pool, id), id2str(pool, did)); #endif break; + case TYPE_NUM: + did = read_id(&data, 0); + if (keys[key].storage == KEY_STORAGE_INCORE) + incore_add_id(&data, did); +#if 0 + POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %d\n", id2str(pool, id), did); +#endif + break; case TYPE_U32: h = read_u32(&data); #if 0 @@ -1073,6 +1244,27 @@ add_attr_from_file (embedded_store, i, id, keys[key].type, idmap, numid, &data, keys[key].size); break; #endif + case TYPE_DIRNUMNUMARRAY: + for (;;) + { + Id num, num2; + did = read_id(&data, numdir); + num = read_id(&data, 0); + num2 = read_id(&data, 0); + if (keys[key].storage == KEY_STORAGE_INCORE) + { +#if 0 + POOL_DEBUG(SAT_DEBUG_STATS, "%s -> %d %d %d\n", id2str(pool, id), did, num, num2); +#endif + incore_add_id(&data, did); + incore_add_id(&data, num); + incore_add_id(&data, num2); + } + if (!(num2 & 0x40)) + break; + } + break; + default: skip_item(&data, keys[key].type, numid, numrel); } @@ -1102,8 +1294,20 @@ break; if (i < numkeys && !data.error) { + Id fileoffset = 0; + unsigned int pagesize; + /* we have vertical data, make it available */ - data.verticaloffset = ftell(fp); + data.verticaloffset = sat_calloc(numkeys, sizeof(Id)); + for (i = 1; i < numkeys; i++) + if (keys[i].storage == KEY_STORAGE_VERTICAL_OFFSET) + { + data.verticaloffset[i] = fileoffset; + fileoffset += keys[i].size; + } + data.lastverticaloffset = fileoffset; + pagesize = read_u32(&data); + repodata_read_or_setup_pages(&data, pagesize, fileoffset); } else { @@ -1111,7 +1315,15 @@ data.fp = 0; } - if (data.incoredatalen || data.fp) + if (parent) + { + /* we're a store */ + sat_free(parent->schemata); + sat_free(parent->schemadata); + sat_free(parent->keys); + *parent = data; + } + else if (data.incoredatalen || data.fp) { /* we got some data, make it available */ repo->repodata = sat_realloc2(repo->repodata, repo->nrepodata + 1, sizeof(data)); @@ -1120,7 +1332,10 @@ else { /* discard data */ - sat_free(data.dirs); + sat_free(data.dirpool.dirs); + sat_free(data.entryschemau8); + sat_free(data.entryschema); + sat_free(data.incoreoffset); sat_free(schemata); sat_free(schemadata); sat_free(keys); @@ -1142,4 +1357,38 @@ return data.error; } +int +repo_add_solv(Repo *repo, FILE *fp) +{ + return repo_add_solv_parent(repo, fp, 0); +} + +static void +repodata_load_solv(Repodata *data) +{ + FILE *fp; +#if 0 + Pool *pool = data->repo->pool; + if (!pool->loadcallback) + { + data->state = REPODATA_ERROR; + return; + } + fp = pool->loadcallback(pool->loadcallback_data, pool, data); +#else + fp = 0; +#endif + + if (!fp) + { + data->state = REPODATA_ERROR; + return; + } + if (repo_add_solv_parent(data->repo, fp, data)) + data->state = REPODATA_ERROR; + else + data->state = REPODATA_AVAILABLE; +} + + // EOF Added: trunk/sat-solver/src/repodata.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/repodata.c?rev=8389&view=auto ============================================================================== --- trunk/sat-solver/src/repodata.c (added) +++ trunk/sat-solver/src/repodata.c Tue Jan 29 17:14:03 2008 @@ -0,0 +1,1157 @@ +/* + * Copyright (c) 2007, Novell Inc. + * + * This program is licensed under the BSD license, read LICENSE.BSD + * for further information + */ + +/* + * repodata.c + * + * Manage data coming from one repository + * + */ + +#define _GNU_SOURCE +#include + +#include +#include +#include +#include + +#include "repo.h" +#include "pool.h" +#include "poolid_private.h" +#include "util.h" + +#include "fastlz.c" + +unsigned char * +data_read_id(unsigned char *dp, Id *idp) +{ + Id x = 0; + unsigned char c; + for (;;) + { + c = *dp++; + if (!(c & 0x80)) + { + *idp = (x << 7) ^ c; + return dp; + } + x = (x << 7) ^ c ^ 128; + } +} + +static unsigned char * +data_read_ideof(unsigned char *dp, Id *idp, int *eof) +{ + Id x = 0; + unsigned char c; + for (;;) + { + c = *dp++; + if (!(c & 0x80)) + { + if (c & 0x40) + { + c ^= 0x40; + *eof = 0; + } + else + *eof = 1; + *idp = (x << 6) ^ c; + return dp; + } + x = (x << 7) ^ c ^ 128; + } +} + +static unsigned char * +data_skip(unsigned char *dp, int type) +{ + unsigned char x; + switch (type) + { + case TYPE_VOID: + case TYPE_CONSTANT: + return dp; + case TYPE_ID: + case TYPE_NUM: + case TYPE_DIR: + while ((*dp & 0x80) != 0) + dp++; + return dp + 1; + case TYPE_IDARRAY: + while ((*dp & 0xc0) != 0) + dp++; + return dp + 1; + case TYPE_STR: + while ((*dp) != 0) + dp++; + return dp + 1; + case TYPE_DIRSTRARRAY: + for (;;) + { + while ((*dp & 0x80) != 0) + dp++; + x = *dp++; + while ((*dp) != 0) + dp++; + dp++; + if (!(x & 0x40)) + return dp; + } + case TYPE_DIRNUMNUMARRAY: + for (;;) + { + while ((*dp & 0x80) != 0) + dp++; + dp++; + while ((*dp & 0x80) != 0) + dp++; + dp++; + while ((*dp & 0x80) != 0) + dp++; + if (!(*dp & 0x40)) + return dp + 1; + dp++; + } + default: + fprintf(stderr, "unknown type in data_skip\n"); + exit(1); + } +} + +static unsigned char * +data_fetch(unsigned char *dp, KeyValue *kv, Repokey *key) +{ + if (!dp) + return 0; + kv->eof = 1; + switch (key->type) + { + case TYPE_VOID: + return dp; + case TYPE_CONSTANT: + kv->num = key->size; + return dp; + case TYPE_STR: + kv->str = (const char *)dp; + return dp + strlen(kv->str) + 1; + case TYPE_ID: + return data_read_id(dp, &kv->id); + case TYPE_NUM: + return data_read_id(dp, &kv->num); + case TYPE_IDARRAY: + return data_read_ideof(dp, &kv->id, &kv->eof); + case TYPE_DIR: + return data_read_id(dp, &kv->id); + case TYPE_DIRSTRARRAY: + dp = data_read_ideof(dp, &kv->id, &kv->eof); + kv->str = (const char *)dp; + return dp + strlen(kv->str) + 1; + case TYPE_DIRNUMNUMARRAY: + dp = data_read_id(dp, &kv->id); + dp = data_read_id(dp, &kv->num); + return data_read_ideof(dp, &kv->num2, &kv->eof); + default: + return 0; + } +} + +static unsigned char * +forward_to_key(Repodata *data, Id key, Id schemaid, unsigned char *dp) +{ + Id k, *keyp; + + keyp = data->schemadata + data->schemata[schemaid]; + while ((k = *keyp++) != 0) + { + if (k == key) + return dp; + if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET) + { + dp = data_skip(dp, TYPE_ID); /* skip that offset */ + dp = data_skip(dp, TYPE_ID); /* skip that length */ + continue; + } + if (data->keys[k].storage != KEY_STORAGE_INCORE) + continue; + dp = data_skip(dp, data->keys[k].type); + } + return 0; +} + +#define BLOB_PAGEBITS 15 +#define BLOB_PAGESIZE (1 << BLOB_PAGEBITS) + +static unsigned char * +load_page_range(Repodata *data, unsigned int pstart, unsigned int pend) +{ +/* Make sure all pages from PSTART to PEND (inclusive) are loaded, + and are consecutive. Return a pointer to the mapping of PSTART. */ + unsigned char buf[BLOB_PAGESIZE]; + unsigned int i; + + /* Quick check in case all pages are there already and consecutive. */ + for (i = pstart; i <= pend; i++) + if (data->pages[i].mapped_at == -1 + || (i > pstart + && data->pages[i].mapped_at + != data->pages[i-1].mapped_at + BLOB_PAGESIZE)) + break; + if (i > pend) + return data->blob_store + data->pages[pstart].mapped_at; + + /* Ensure that we can map the numbers of pages we need at all. */ + if (pend - pstart + 1 > data->ncanmap) + { + unsigned int oldcan = data->ncanmap; + data->ncanmap = pend - pstart + 1; + if (data->ncanmap < 4) + data->ncanmap = 4; + data->mapped = sat_realloc2(data->mapped, data->ncanmap, sizeof(data->mapped[0])); + memset (data->mapped + oldcan, 0, (data->ncanmap - oldcan) * sizeof (data->mapped[0])); + data->blob_store = sat_realloc2(data->blob_store, data->ncanmap, BLOB_PAGESIZE); +#ifdef DEBUG_PAGING + fprintf (stderr, "PAGE: can map %d pages\n", data->ncanmap); +#endif + } + + /* Now search for "cheap" space in our store. Space is cheap if it's either + free (very cheap) or contains pages we search for anyway. */ + + /* Setup cost array. */ + unsigned int cost[data->ncanmap]; + for (i = 0; i < data->ncanmap; i++) + { + unsigned int pnum = data->mapped[i]; + if (pnum == 0) + cost[i] = 0; + else + { + pnum--; + Attrblobpage *p = data->pages + pnum; + assert (p->mapped_at != -1); + if (pnum >= pstart && pnum <= pend) + cost[i] = 1; + else + cost[i] = 3; + } + } + + /* And search for cheapest space. */ + unsigned int best_cost = -1; + unsigned int best = 0; + unsigned int same_cost = 0; + for (i = 0; i + pend - pstart < data->ncanmap; i++) + { + unsigned int c = cost[i]; + unsigned int j; + for (j = 0; j < pend - pstart + 1; j++) + c += cost[i+j]; + if (c < best_cost) + best_cost = c, best = i; + else if (c == best_cost) + same_cost++; + /* A null cost won't become better. */ + if (c == 0) + break; + } + /* If all places have the same cost we would thrash on slot 0. Avoid + this by doing a round-robin strategy in this case. */ + if (same_cost == data->ncanmap - pend + pstart - 1) + best = data->rr_counter++ % (data->ncanmap - pend + pstart); + + /* So we want to map our pages from [best] to [best+pend-pstart]. + Use a very simple strategy, which doesn't make the best use of + our resources, but works. Throw away all pages in that range + (even ours) then copy around ours (in case they were outside the + range) or read them in. */ + for (i = best; i < best + pend - pstart + 1; i++) + { + unsigned int pnum = data->mapped[i]; + if (pnum-- + /* If this page is exactly at the right place already, + no need to evict it. */ + && pnum != pstart + i - best) + { + /* Evict this page. */ +#ifdef DEBUG_PAGING + fprintf (stderr, "PAGE: evict page %d from %d\n", pnum, i); +#endif + cost[i] = 0; + data->mapped[i] = 0; + data->pages[pnum].mapped_at = -1; + } + } + + /* Everything is free now. Read in the pages we want. */ + for (i = pstart; i <= pend; i++) + { + Attrblobpage *p = data->pages + i; + unsigned int pnum = i - pstart + best; + void *dest = data->blob_store + pnum * BLOB_PAGESIZE; + if (p->mapped_at != -1) + { + if (p->mapped_at != pnum * BLOB_PAGESIZE) + { +#ifdef DEBUG_PAGING + fprintf (stderr, "PAGECOPY: %d to %d\n", i, pnum); +#endif + /* Still mapped somewhere else, so just copy it from there. */ + memcpy (dest, data->blob_store + p->mapped_at, BLOB_PAGESIZE); + data->mapped[p->mapped_at / BLOB_PAGESIZE] = 0; + } + } + else + { + unsigned int in_len = p->file_size; + unsigned int compressed = in_len & 1; + in_len >>= 1; +#ifdef DEBUG_PAGING + fprintf (stderr, "PAGEIN: %d to %d", i, pnum); +#endif + /* Not mapped, so read in this page. */ + if (fseek(data->fp, p->file_offset, SEEK_SET) < 0) + { + perror ("mapping fseek"); + exit (1); + } + if (fread(compressed ? buf : dest, in_len, 1, data->fp) != 1) + { + perror ("mapping fread"); + exit (1); + } + if (compressed) + { + unsigned int out_len; + out_len = unchecked_decompress_buf(buf, in_len, + dest, BLOB_PAGESIZE); + if (out_len != BLOB_PAGESIZE + && i < data->num_pages - 1) + { + fprintf (stderr, "can't decompress\n"); + exit (1); + } +#ifdef DEBUG_PAGING + fprintf (stderr, " (expand %d to %d)", in_len, out_len); +#endif + } +#ifdef DEBUG_PAGING + fprintf (stderr, "\n"); +#endif + } + p->mapped_at = pnum * BLOB_PAGESIZE; + data->mapped[pnum] = i + 1; + } + return data->blob_store + best * BLOB_PAGESIZE; +} + +static unsigned char * +make_vertical_available(Repodata *data, Repokey *key, Id off, Id len) +{ + unsigned char *dp; + if (key->type == TYPE_VOID) + return 0; + if (off >= data->lastverticaloffset) + { + off -= data->lastverticaloffset; + if (off + len > data->vincorelen) + return 0; + return data->vincore + off; + } + if (!data->fp) + return 0; + if (off + len > key->size) + return 0; + /* we now have the offset, go into vertical */ + off += data->verticaloffset[key - data->keys]; + dp = load_page_range(data, off / BLOB_PAGESIZE, (off + len - 1) / BLOB_PAGESIZE); + if (dp) + dp += off % BLOB_PAGESIZE; + return dp; +} + +static inline unsigned char * +get_data(Repodata *data, Repokey *key, unsigned char **dpp) +{ + unsigned char *dp = *dpp; + + if (!dp) + return 0; + if (key->storage == KEY_STORAGE_INCORE) + { + /* hmm, this is a bit expensive */ + *dpp = data_skip(dp, key->type); + return dp; + } + else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET) + { + Id off, len; + dp = data_read_id(dp, &off); + dp = data_read_id(dp, &len); + *dpp = dp; + return make_vertical_available(data, key, off, len); + } + return 0; +} + + +const char * +repodata_lookup_str(Repodata *data, Id entry, Id keyid) +{ + Id schema; + Repokey *key; + Id id, *keyp; + unsigned char *dp; + + if (data->entryschemau8) + schema = data->entryschemau8[entry]; + else + schema = data->entryschema[entry]; + /* make sure the schema of this solvable contains the key */ + for (keyp = data->schemadata + data->schemata[schema]; *keyp != keyid; keyp++) + if (!*keyp) + return 0; + dp = forward_to_key(data, keyid, schema, data->incoredata + data->incoreoffset[entry]); + key = data->keys + keyid; + dp = get_data(data, key, &dp); + if (!dp) + return 0; + if (key->type == TYPE_STR) + return (const char *)dp; + if (key->type != TYPE_ID) + return 0; + /* id type, must either use global or local string strore*/ + dp = data_read_id(dp, &id); + if (data->localpool) + return data->spool.stringspace + data->spool.strings[id]; + return id2str(data->repo->pool, id); +} + +void +repodata_search(Repodata *data, Id entry, Id keyname, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata) +{ + Id schema; + Repokey *key; + Id k, keyid, *kp, *keyp; + unsigned char *dp, *ddp; + int onekey = 0; + int stop; + KeyValue kv; + + if (data->entryschemau8) + schema = data->entryschemau8[entry]; + else + schema = data->entryschema[entry]; + keyp = data->schemadata + data->schemata[schema]; + dp = data->incoredata + data->incoreoffset[entry]; + if (keyname) + { + /* search in a specific key */ + for (kp = keyp; (k = *kp++) != 0; ) + if (data->keys[k].name == keyname) + break; + if (k == 0) + return; + dp = forward_to_key(data, k, schema, dp); + if (!dp) + return; + keyp = kp - 1; + onekey = 1; + } + while ((keyid = *keyp++) != 0) + { + stop = 0; + key = data->keys + keyid; + ddp = get_data(data, key, &dp); + do + { + ddp = data_fetch(ddp, &kv, key); + if (ddp) + stop = callback(cbdata, data->repo->pool->solvables + data->start + entry, data, key, &kv); + } + while (!kv.eof && !stop); + if (onekey || stop > SEARCH_NEXT_KEY) + return; + } +} + + +/* extend repodata so that it includes solvables p */ +void +repodata_extend(Repodata *data, Id p) +{ + if (data->start == data->end) + data->start = data->end = p; + if (p >= data->end) + { + int old = data->end - data->start; + int new = p - data->end + 1; + if (data->entryschemau8) + { + data->entryschemau8 = sat_realloc(data->entryschemau8, old + new); + memset(data->entryschemau8 + old, 0, new); + } + if (data->entryschema) + { + data->entryschema = sat_realloc2(data->entryschema, old + new, sizeof(Id)); + memset(data->entryschema + old, 0, new * sizeof(Id)); + } + if (data->attrs) + { + data->attrs = sat_realloc2(data->attrs, old + new, sizeof(Id *)); + memset(data->attrs + old, 0, new * sizeof(Id *)); + } + data->incoreoffset = sat_realloc2(data->incoreoffset, old + new, sizeof(Id)); + memset(data->incoreoffset + old, 0, new * sizeof(Id)); + data->end = p + 1; + } + if (p < data->start) + { + int old = data->end - data->start; + int new = data->start - p; + if (data->entryschemau8) + { + data->entryschemau8 = sat_realloc(data->entryschemau8, old + new); + memmove(data->entryschemau8 + new, data->entryschemau8, old); + memset(data->entryschemau8, 0, new); + } + if (data->entryschema) + { + data->entryschema = sat_realloc2(data->entryschema, old + new, sizeof(Id)); + memmove(data->entryschema + new, data->entryschema, old * sizeof(Id)); + memset(data->entryschema, 0, new * sizeof(Id)); + } + if (data->attrs) + { + data->attrs = sat_realloc2(data->attrs, old + new, sizeof(Id *)); + memmove(data->attrs + new, data->attrs, old * sizeof(Id *)); + memset(data->attrs, 0, new * sizeof(Id *)); + } + data->incoreoffset = sat_realloc2(data->incoreoffset, old + new, sizeof(Id)); + memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id)); + memset(data->incoreoffset, 0, new * sizeof(Id)); + data->start = p; + } +} + +void +repodata_set(Repodata *data, Id entry, Repokey *key, Id val) +{ + Id keyid, *pp; + int i; + + /* find key in keys */ + for (keyid = 1; keyid < data->nkeys; keyid++) + if (data->keys[keyid].name == key->name && data->keys[keyid].type == key->type) + { + if (key->type == TYPE_CONSTANT && key->size != data->keys[keyid].size) + continue; + break; + } + if (keyid == data->nkeys) + { + /* allocate new key */ + data->keys = sat_realloc2(data->keys, data->nkeys + 1, sizeof(Repokey)); + data->keys[data->nkeys++] = *key; + if (data->verticaloffset) + { + data->verticaloffset = sat_realloc2(data->verticaloffset, data->nkeys, sizeof(Id)); + data->verticaloffset[data->nkeys - 1] = 0; + } + } + key = data->keys + keyid; + if (!data->attrs) + data->attrs = sat_calloc(data->end - data->start + 1, sizeof(Id *)); + i = 0; + if (data->attrs[entry]) + { + for (pp = data->attrs[entry]; *pp; pp += 2) + if (*pp == keyid) + break; + if (*pp) + { + pp[1] = val; + return; + } + i = pp - data->attrs[entry]; + } + data->attrs[entry] = sat_realloc2(data->attrs[entry], i + 3, sizeof(Id)); + pp = data->attrs[entry] + i; + *pp++ = keyid; + *pp++ = val; + *pp = 0; +} + +void +repodata_set_id(Repodata *data, Id entry, Id keyname, Id id) +{ + Repokey key; + key.name = keyname; + key.type = TYPE_ID; + key.size = 0; + key.storage = KEY_STORAGE_INCORE; + repodata_set(data, entry, &key, id); +} + +void +repodata_set_num(Repodata *data, Id entry, Id keyname, Id num) +{ + Repokey key; + key.name = keyname; + key.type = TYPE_NUM; + key.size = 0; + key.storage = KEY_STORAGE_INCORE; + repodata_set(data, entry, &key, num); +} + +void +repodata_set_poolstr(Repodata *data, Id entry, Id keyname, const char *str) +{ + Repokey key; + Id id; + if (data->localpool) + id = stringpool_str2id(&data->spool, str, 1); + else + id = str2id(data->repo->pool, str, 1); + key.name = keyname; + key.type = TYPE_ID; + key.size = 0; + key.storage = KEY_STORAGE_INCORE; + repodata_set(data, entry, &key, id); +} + +void +repodata_set_constant(Repodata *data, Id entry, Id keyname, Id constant) +{ + Repokey key; + key.name = keyname; + key.type = TYPE_CONSTANT; + key.size = constant; + key.storage = KEY_STORAGE_INCORE; + repodata_set(data, entry, &key, 0); +} + +void +repodata_set_void(Repodata *data, Id entry, Id keyname) +{ + Repokey key; + key.name = keyname; + key.type = TYPE_VOID; + key.size = 0; + key.storage = KEY_STORAGE_INCORE; + repodata_set(data, entry, &key, 0); +} + +void +repodata_set_str(Repodata *data, Id entry, Id keyname, const char *str) +{ + Repokey key; + int l; + + l = strlen(str) + 1; + key.name = keyname; + key.type = TYPE_STR; + key.size = 0; + key.storage = KEY_STORAGE_INCORE; + data->attrdata = sat_realloc(data->attrdata, data->attrdatalen + l); + memcpy(data->attrdata + data->attrdatalen, str, l); + repodata_set(data, entry, &key, data->attrdatalen); + data->attrdatalen += l; +} + +void +repodata_add_dirnumnum(Repodata *data, Id entry, Id keyname, Id dir, Id num, Id num2) +{ + Id *ida, *pp; + Repokey key; + +#if 0 +fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", entry, dir, num, num2, data->attriddatalen); +#endif + if (data->attrs[entry]) + { + for (pp = data->attrs[entry]; *pp; pp += 2) + if (data->keys[*pp].name == keyname && data->keys[*pp].type == TYPE_DIRNUMNUMARRAY) + break; + if (*pp) + { + int oldsize = 0; + for (ida = data->attriddata + pp[1]; *ida; ida += 3) + oldsize += 3; + if (ida + 1 == data->attriddata + data->attriddatalen) + { + /* this was the last entry, just append it */ + data->attriddata = sat_realloc2(data->attriddata, data->attriddatalen + 3, sizeof(Id)); + data->attriddatalen--; /* overwrite terminating 0 */ + } + else + { + /* too bad. move to back. */ + data->attriddata = sat_realloc2(data->attriddata, data->attriddatalen + oldsize + 4, sizeof(Id)); + memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id)); + pp[1] = data->attriddatalen; + data->attriddatalen += oldsize; + } + data->attriddata[data->attriddatalen++] = dir; + data->attriddata[data->attriddatalen++] = num; + data->attriddata[data->attriddatalen++] = num2; + data->attriddata[data->attriddatalen++] = 0; + return; + } + } + key.name = keyname; + key.type = TYPE_DIRNUMNUMARRAY; + key.size = 0; + key.storage = KEY_STORAGE_INCORE; + data->attriddata = sat_realloc2(data->attriddata, data->attriddatalen + 4, sizeof(Id)); + repodata_set(data, entry, &key, data->attriddatalen); + data->attriddata[data->attriddatalen++] = dir; + data->attriddata[data->attriddatalen++] = num; + data->attriddata[data->attriddatalen++] = num2; + data->attriddata[data->attriddatalen++] = 0; +} + +/*********************************/ + +/* unify with repo_write! */ + +#define EXTDATA_BLOCK 1023 +#define SCHEMATA_BLOCK 31 +#define SCHEMATADATA_BLOCK 255 + +struct extdata { + unsigned char *buf; + int len; +}; + +static void +data_addid(struct extdata *xd, Id x) +{ + unsigned char *dp; + xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK); + dp = xd->buf + xd->len; + + if (x >= (1 << 14)) + { + if (x >= (1 << 28)) + *dp++ = (x >> 28) | 128; + if (x >= (1 << 21)) + *dp++ = (x >> 21) | 128; + *dp++ = (x >> 14) | 128; + } + if (x >= (1 << 7)) + *dp++ = (x >> 7) | 128; + *dp++ = x & 127; + xd->len = dp - xd->buf; +} + +static void +data_addideof(struct extdata *xd, Id x, int eof) +{ + if (x >= 64) + x = (x & 63) | ((x & ~63) << 1); + data_addid(xd, (eof ? x: x | 64)); +} + +static void +data_addblob(struct extdata *xd, unsigned char *blob, int len) +{ + xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK); + memcpy(xd->buf + xd->len, blob, len); + xd->len += len; +} + +/*********************************/ + +static void +addschema_prepare(Repodata *data, Id *schematacache) +{ + int h, len, i; + Id *sp; + + memset(schematacache, 0, 256 * sizeof(Id)); + for (i = 0; i < data->nschemata; i++) + { + for (sp = data->schemadata + data->schemata[i], h = 0; *sp; len++) + h = h * 7 + *sp++; + h &= 255; + schematacache[h] = i + 1; + } + data->schemadata = sat_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK); + data->schemata = sat_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK); +} + +static Id +addschema(Repodata *data, Id *schema, Id *schematacache) +{ + int h, len; + Id *sp, cid; + + for (sp = schema, len = 0, h = 0; *sp; len++) + h = h * 7 + *sp++; + h &= 255; + len++; + + cid = schematacache[h]; + if (cid) + { + cid--; + if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id))) + return cid; + /* cache conflict */ + for (cid = 0; cid < data->nschemata; cid++) + if (!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id))) + return cid; + } + /* a new one. make room. */ + data->schemadata = sat_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK); + data->schemata = sat_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK); + /* add schema */ + memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id)); + data->schemata[data->nschemata] = data->schemadatalen; + data->schemadatalen += len; + schematacache[h] = data->nschemata + 1; +#if 0 +fprintf(stderr, "addschema: new schema\n"); +#endif + return data->nschemata++; +} + + +void +repodata_internalize(Repodata *data) +{ + int i; + Repokey *key; + Id id, entry, nentry, *ida; + Id schematacache[256]; + Id schemaid, *schema, *sp, oldschema, *keyp, *seen; + unsigned char *dp, *ndp; + int newschema, oldcount; + struct extdata newincore; + struct extdata newvincore; + + if (!data->attrs) + return; + + newvincore.buf = data->vincore; + newvincore.len = data->vincorelen; + + schema = sat_malloc2(data->nkeys, sizeof(Id)); + seen = sat_malloc2(data->nkeys, sizeof(Id)); + + nentry = data->end - data->start; + addschema_prepare(data, schematacache); + memset(&newincore, 0, sizeof(newincore)); + for (entry = 0; entry < nentry; entry++) + { + memset(seen, 0, data->nkeys * sizeof(Id)); + sp = schema; + if (data->entryschemau8) + oldschema = data->entryschemau8[entry]; + else + oldschema = data->entryschema[entry]; +#if 0 +fprintf(stderr, "oldschema %d\n", oldschema); +fprintf(stderr, "schemata %d\n", data->schemata[oldschema]); +fprintf(stderr, "schemadata %p\n", data->schemadata); +#endif + /* seen: -1: old data 0: skipped >0: id + 1 */ + newschema = 0; + oldcount = 0; + for (keyp = data->schemadata + data->schemata[oldschema]; *keyp; keyp++) + { + if (seen[*keyp]) + { + newschema = 1; + continue; + } + seen[*keyp] = -1; + *sp++ = *keyp; + oldcount++; + } + for (keyp = data->attrs[entry]; *keyp; keyp += 2) + { + if (!seen[*keyp]) + { + newschema = 1; + *sp++ = *keyp; + } + seen[*keyp] = keyp[1] + 1; + } + *sp++ = 0; + if (newschema) + { + schemaid = addschema(data, schema, schematacache); + if (schemaid > 255 && data->entryschemau8) + { + data->entryschema = sat_malloc2(nentry, sizeof(Id)); + for (i = 0; i < nentry; i++) + data->entryschema[i] = data->entryschemau8[i]; + data->entryschemau8 = sat_free(data->entryschemau8); + } + if (data->entryschemau8) + data->entryschemau8[entry] = schemaid; + else + data->entryschema[entry] = schemaid; + } + else + schemaid = oldschema; + + + /* now create data blob */ + dp = data->incoredata + data->incoreoffset[entry]; + data->incoreoffset[entry] = newincore.len; + for (keyp = data->schemadata + data->schemata[schemaid]; *keyp; keyp++) + { + key = data->keys + *keyp; + ndp = dp; + if (oldcount) + { + if (key->storage == KEY_STORAGE_VERTICAL_OFFSET) + { + ndp = data_skip(dp, TYPE_ID); + ndp = data_skip(dp, TYPE_ID); + } + else if (key->storage == KEY_STORAGE_INCORE) + ndp = data_skip(dp, key->type); + oldcount--; + } + if (seen[*keyp] == -1) + { + if (dp != ndp) + data_addblob(&newincore, dp, ndp - dp); + seen[*keyp] = 0; + } + else if (seen[*keyp]) + { + struct extdata *xd; + unsigned int oldvincorelen = 0; + + xd = &newincore; + if (key->storage == KEY_STORAGE_VERTICAL_OFFSET) + { + xd = &newvincore; + oldvincorelen = xd->len; + } + id = seen[*keyp] - 1; + switch (key->type) + { + case TYPE_VOID: + case TYPE_CONSTANT: + break; + case TYPE_STR: + data_addblob(xd, data->attrdata + id, strlen((char *)(data->attrdata + id)) + 1); + break; + case TYPE_ID: + case TYPE_NUM: + case TYPE_DIR: + data_addid(xd, id); + break; + case TYPE_DIRNUMNUMARRAY: + ida = data->attriddata + id; + for (ida = data->attriddata + id; *ida; ida += 3) + { + data_addid(xd, ida[0]); + data_addid(xd, ida[1]); + data_addideof(xd, ida[2], ida[3] ? 0 : 1); + } + break; + default: + fprintf(stderr, "don't know how to handle type %d\n", key->type); + exit(1); + } + if (key->storage == KEY_STORAGE_VERTICAL_OFFSET) + { + /* put offset/len in incore */ + data_addid(&newincore, data->lastverticaloffset + oldvincorelen); + oldvincorelen = xd->len - oldvincorelen; + data_addid(&newincore, oldvincorelen); + } + } + dp = ndp; + } + } + data->incoredata = newincore.buf; + data->incoredatalen = newincore.len; + data->incoredatafree = 0; + + data->vincore = newvincore.buf; + data->vincorelen = newvincore.len; + + data->attrs = sat_free(data->attrs); + data->attrdata = sat_free(data->attrdata); + data->attrdatalen = 0; +} + +Id +repodata_str2dir(Repodata *data, const char *dir, int create) +{ + Id id, parent; + const char *dire; + + parent = 0; + while (*dir == '/' && dir[1] == '/') + dir++; + while (*dir) + { + dire = strchrnul(dir, '/'); + if (data->localpool) + id = stringpool_strn2id(&data->spool, dir, dire - dir, create); + else + id = strn2id(data->repo->pool, dir, dire - dir, create); + if (!id) + return 0; + parent = dirpool_add_dir(&data->dirpool, parent, id, create); + if (!parent) + return 0; + if (!*dire) + break; + dir = dire + 1; + while (*dir == '/') + dir++; + } + return parent; +} + +unsigned int +repodata_compress_page(unsigned char *page, unsigned int len, unsigned char *cpage, unsigned int max) +{ + return compress_buf(page, len, cpage, max); +} + +/* Try to either setup on-demand paging (using FP as backing + file), or in case that doesn't work (FP not seekable) slurps in + all pages and deactivates paging. */ + +#define SOLV_ERROR_EOF 3 + +static inline unsigned int +read_u32(FILE *fp) +{ + int c, i; + unsigned int x = 0; + + for (i = 0; i < 4; i++) + { + c = getc(fp); + if (c == EOF) + return 0; + x = (x << 8) | c; + } + return x; +} + +void +repodata_read_or_setup_pages(Repodata *data, unsigned int pagesz, unsigned int blobsz) +{ + FILE *fp = data->fp; + unsigned int npages; + unsigned int i; + unsigned int can_seek; + long cur_file_ofs; + unsigned char buf[BLOB_PAGESIZE]; + if (pagesz != BLOB_PAGESIZE) + { + /* We could handle this by slurping in everything. */ + fprintf (stderr, "non matching page size\n"); + exit (1); + } + can_seek = 1; + if ((cur_file_ofs = ftell(fp)) < 0) + can_seek = 0; + clearerr (fp); +#ifdef DEBUG_PAGING + fprintf (stderr, "can %sseek\n", can_seek ? "" : "NOT "); +#endif + npages = (blobsz + BLOB_PAGESIZE - 1) / BLOB_PAGESIZE; + + data->num_pages = npages; + data->pages = sat_malloc2(npages, sizeof(data->pages[0])); + + /* If we can't seek on our input we have to slurp in everything. */ + if (!can_seek) + data->blob_store = sat_malloc(npages * BLOB_PAGESIZE); + for (i = 0; i < npages; i++) + { + unsigned int in_len = read_u32(fp); + unsigned int compressed = in_len & 1; + Attrblobpage *p = data->pages + i; + in_len >>= 1; +#ifdef DEBUG_PAGING + fprintf (stderr, "page %d: len %d (%scompressed)\n", + i, in_len, compressed ? "" : "not "); +#endif + if (can_seek) + { + cur_file_ofs += 4; + p->mapped_at = -1; + p->file_offset = cur_file_ofs; + p->file_size = in_len * 2 + compressed; + if (fseek(fp, in_len, SEEK_CUR) < 0) + { + perror ("fseek"); + fprintf (stderr, "can't seek after we thought we can\n"); + /* We can't fall back to non-seeking behaviour as we already + read over some data pages without storing them away. */ + exit (1); + } + cur_file_ofs += in_len; + } + else + { + unsigned int out_len; + void *dest = data->blob_store + i * BLOB_PAGESIZE; + p->mapped_at = i * BLOB_PAGESIZE; + p->file_offset = 0; + p->file_size = 0; + /* We can't seek, so suck everything in. */ + if (fread(compressed ? buf : dest, in_len, 1, fp) != 1) + { + perror ("fread"); + exit (1); + } + if (compressed) + { + out_len = unchecked_decompress_buf(buf, in_len, dest, BLOB_PAGESIZE); + if (out_len != BLOB_PAGESIZE + && i < npages - 1) + { + fprintf (stderr, "can't decompress\n"); + exit (1); + } + } + } + } + + if (can_seek) + { + /* If we are here we were able to seek to all page + positions, so activate paging by copying FP into our structure. + We dup() the file, so that our callers can fclose() it and we + still have it open. But this means that we share file positions + with the input filedesc. So in case our caller reads it after us, + and calls back into us we might change the file position unexpectedly + to him. */ + int fd = dup (fileno (fp)); + if (fd < 0) + { + /* Jeez! What a bloody system, we can't dup() anymore. */ + perror ("dup"); + exit (1); + } + /* XXX we don't close this yet anywhere. */ + data->fp = fdopen (fd, "r"); + if (!data->fp) + { + /* My God! What happened now? */ + perror ("fdopen"); + exit (1); + } + } +} Added: trunk/sat-solver/src/repodata.h URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/repodata.h?rev=8389&view=auto ============================================================================== --- trunk/sat-solver/src/repodata.h (added) +++ trunk/sat-solver/src/repodata.h Tue Jan 29 17:14:03 2008 @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2007, Novell Inc. + * + * This program is licensed under the BSD license, read LICENSE.BSD + * for further information + */ + +/* + * repodata.h + * + */ + +#ifndef SATSOLVER_REPODATA_H +#define SATSOLVER_REPODATA_H + +#include "pooltypes.h" +#include "pool.h" +#include "dirpool.h" + +struct _Repo; +struct _Repokey; +struct _KeyValue; + +typedef struct _Attrblobpage +{ + /* mapped_at == -1 --> not loaded, otherwise offset into + store->blob_store. The size of the mapping is BLOB_PAGESIZE + except for the last page. */ + unsigned int mapped_at; + long file_offset; + /* file_size == 0 means the page is not backed by some file storage. + Otherwise it is L*2+(compressed ? 1 : 0), with L being the data + length. */ + long file_size; +} Attrblobpage; + +typedef struct _Repodata { + struct _Repo *repo; /* back pointer to repo */ + + int state; /* available, stub or error */ + + void (*loadcallback)(struct _Repodata *); + + int start; /* start of solvables this repodata is valid for */ + int end; /* last solvable + 1 of this repodata */ + + FILE *fp; /* file pointer of solv file */ + int error; /* corrupt solv file */ + + struct _Repokey *keys; /* keys, first entry is always zero */ + unsigned int nkeys; /* length of keys array */ + + Id *schemata; /* schema -> offset into schemadata */ + unsigned int nschemata; /* number of schemata */ + + Id *schemadata; /* schema storage */ + unsigned int schemadatalen; /* schema storage size */ + + unsigned char *entryschemau8; /* schema for entry */ + Id *entryschema; /* schema for entry */ + + Stringpool spool; /* local string pool */ + int localpool; /* is local string pool used */ + + Dirpool dirpool; /* local dir pool */ + + unsigned char *incoredata; /* in-core data (flat_attrs) */ + unsigned int incoredatalen; /* data len (attr_next_free) */ + unsigned int incoredatafree; /* free data len */ + + Id *incoreoffset; /* offset for all entries (ent2attr) */ + + Id *verticaloffset; /* offset for all verticals, nkeys elements */ + Id lastverticaloffset; /* end of verticals */ + + unsigned char *blob_store; + + Attrblobpage *pages; + unsigned int num_pages; + + /* mapped[i] is zero if nothing is mapped at logical page I, + otherwise it contains the pagenumber plus one (of the mapped page). */ + unsigned int *mapped; + unsigned int nmapped, ncanmap; + unsigned int rr_counter; + + unsigned char *vincore; + unsigned int vincorelen; + + Id **attrs; /* un-internalized attributes */ + unsigned char *attrdata; /* their string data space */ + unsigned int attrdatalen; + Id *attriddata; /* their id space */ + unsigned int attriddatalen; + +} Repodata; + +#define REPODATA_AVAILABLE 0 +#define REPODATA_STUB 1 +#define REPODATA_ERROR 2 +#define REPODATA_STORE 3 + +void repodata_search(Repodata *data, Id entry, Id keyname, int (*callback)(void *cbdata, Solvable *s, Repodata *data, struct _Repokey *key, struct _KeyValue *kv), void *cbdata); +const char *repodata_lookup_str(Repodata *data, Id entry, Id keyid); + +void repodata_extend(Repodata *data, Id p); + +void repodata_set_id(Repodata *data, Id entry, Id keyname, Id id); +void repodata_set_num(Repodata *data, Id entry, Id keyname, Id num); +void repodata_set_poolstr(Repodata *data, Id entry, Id keyname, const char *str); +void repodata_set_constant(Repodata *data, Id entry, Id keyname, Id constant); +void repodata_set_void(Repodata *data, Id entry, Id keyname); +void repodata_set_str(Repodata *data, Id entry, Id keyname, const char *str); +void repodata_add_dirnumnum(Repodata *data, Id entry, Id keyname, Id dir, Id num, Id num2); + +void repodata_internalize(Repodata *data); + +Id repodata_str2dir(Repodata *data, const char *dir, int create); + +unsigned int repodata_compress_page(unsigned char *, unsigned int, unsigned char *, unsigned int); +void repodata_read_or_setup_pages(Repodata *data, unsigned int pagesz, unsigned int blobsz); + +#endif /* SATSOLVER_REPODATA_H */ Modified: trunk/sat-solver/src/solver.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/solver.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/solver.c (original) +++ trunk/sat-solver/src/solver.c Tue Jan 29 17:14:03 2008 @@ -294,7 +294,7 @@ /* adapt rule buffer */ solv->nrules = j; - solv->rules = (Rule *)sat_realloc(solv->rules, ((solv->nrules + RULES_BLOCK) & ~RULES_BLOCK) * sizeof(Rule)); + solv->rules = sat_extend_resize(solv->rules, solv->nrules, sizeof(Rule), RULES_BLOCK); IF_POOLDEBUG (SAT_DEBUG_STATS) { int binr = 0; @@ -461,12 +461,8 @@ * allocate new rule */ - /* check and extend rule buffer */ - if ((solv->nrules & RULES_BLOCK) == 0) - { - solv->rules = (Rule *)sat_realloc(solv->rules, (solv->nrules + (RULES_BLOCK + 1)) * sizeof(Rule)); - } - + /* extend rule buffer */ + solv->rules = sat_extend(solv->rules, solv->nrules, 1, sizeof(Rule), RULES_BLOCK); r = solv->rules + solv->nrules++; /* point to rule space */ r->p = p; @@ -921,6 +917,46 @@ } } +#if 0 +static void +addpatchatomrequires(Solver *solv, Solvable *s, Id *dp, Queue *q, Map *m) +{ + Pool *pool = solv->pool; + Id fre, *frep, p, *pp, ndp; + Solvable *ps; + Queue fq; + Id qbuf[64]; + int i, used = 0; + + queue_init_buffer(&fq, qbuf, sizeof(qbuf)/sizeof(*qbuf)); + queue_push(&fq, -(s - pool->solvables)); + for (; *dp; dp++) + queue_push(&fq, *dp); + ndp = pool_queuetowhatprovides(pool, &fq); + frep = s->repo->idarraydata + s->freshens; + while ((fre = *frep++) != 0) + { + FOR_PROVIDES(p, pp, fre) + { + ps = pool->solvables + p; + addrule(solv, -p, ndp); + used = 1; + if (!MAPTST(m, p)) + queue_push(q, p); + } + } + if (used) + { + for (i = 1; i < fq.count; i++) + { + p = fq.elements[i]; + if (!MAPTST(m, p)) + queue_push(q, p); + } + } + queue_free(&fq); +} +#endif /* * add (install) rules for solvable @@ -937,6 +973,7 @@ Id qbuf[64]; int i; int dontfix; + int patchatom; Id req, *reqp; Id con, *conp; Id obs, *obsp; @@ -979,6 +1016,14 @@ addrule(solv, -n, 0); /* uninstallable */ } + patchatom = 0; + if (s->freshens && !s->supplements) + { + const char *name = id2str(pool, s->name); + if (name[0] == 'a' && !strncmp(name, "atom:", 5)) + patchatom = 1; + } + /*----------------------------------------- * check requires of s */ @@ -996,6 +1041,13 @@ if (*dp == SYSTEMSOLVABLE) /* always installed */ continue; +#if 0 + if (patchatom) + { + addpatchatomrequires(solv, s, dp, &q, m); + continue; + } +#endif if (dontfix) { /* the strategy here is to not insist on dependencies @@ -2005,9 +2057,9 @@ solv->recommends_index = 0; solv->decisionmap = (Id *)sat_calloc(pool->nsolvables, sizeof(Id)); - solv->rules = (Rule *)sat_malloc((solv->nrules + (RULES_BLOCK + 1)) * sizeof(Rule)); - memset(solv->rules, 0, sizeof(Rule)); solv->nrules = 1; + solv->rules = sat_extend_resize(solv->rules, solv->nrules, sizeof(Rule), RULES_BLOCK); + memset(solv->rules, 0, sizeof(Rule)); return solv; } @@ -2701,7 +2753,7 @@ */ Id * -create_obsoletesmap(Solver *solv) +create_decisions_obsoletesmap(Solver *solv) { Pool *pool = solv->pool; Repo *installed = solv->installed; @@ -2774,7 +2826,7 @@ { Pool *pool = solv->pool; Repo *installed = solv->installed; - Id p, *obsoletesmap = create_obsoletesmap( solv ); + Id p, *obsoletesmap = create_decisions_obsoletesmap( solv ); int i; Solvable *s; @@ -3270,7 +3322,9 @@ } -/* for each installed solvable find which packages with *different* names +/* create reverse obsoletes map for installed solvables + * + * for each installed solvable find which packages with *different* names * obsolete the solvable. * this index is used in policy_findupdatepackages if noupdateprovide is set. */ @@ -3286,7 +3340,6 @@ if (!installed || !installed->nsolvables) return; - /* create reverse obsoletes map for installed solvables */ solv->obsoletes = obsoletes = sat_calloc(installed->end - installed->start, sizeof(Id)); for (i = 1; i < pool->nsolvables; i++) { @@ -3582,10 +3635,22 @@ { /* loop over all installed solvables */ /* we create all update rules, but disable some later on depending on the job */ for (i = installed->start, s = pool->solvables + i; i < installed->end; i++, s++) - if (s->repo == installed) - addupdaterule(solv, s, 0); /* allowall = 0 */ - else - addupdaterule(solv, 0, 0); /* create dummy rule; allowall = 0 */ + { + /* no system rules for patch atoms */ + if (s->freshens && !s->supplements) + { + const char *name = id2str(pool, s->name); + if (name[0] == 'a' && !strncmp(name, "atom:", 5)) + { + addrule(solv, 0, 0); + continue; + } + } + if (s->repo == installed) + addupdaterule(solv, s, 0); /* allowall = 0 */ + else + addrule(solv, 0, 0); /* create dummy rule */ + } /* consistency check: we added a rule for _every_ system solvable */ assert(solv->nrules - solv->systemrules == installed->end - installed->start); } Modified: trunk/sat-solver/src/strpool.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/strpool.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/strpool.c (original) +++ trunk/sat-solver/src/strpool.c Tue Jan 29 17:14:03 2008 @@ -13,17 +13,19 @@ #define STRINGSPACE_BLOCK 65535 void -stringpool_init (Stringpool *ss, const char *strs[]) +stringpool_init(Stringpool *ss, const char *strs[]) { unsigned totalsize = 0; unsigned count; + + memset(ss, 0, sizeof(*ss)); // count number and total size of predefined strings for (count = 0; strs[count]; count++) totalsize += strlen(strs[count]) + 1; // alloc appropriate space - ss->stringspace = sat_malloc((totalsize + STRINGSPACE_BLOCK) & ~STRINGSPACE_BLOCK); - ss->strings = sat_malloc2((count + STRING_BLOCK) & ~STRING_BLOCK, sizeof(Offset)); + ss->stringspace = sat_extend_resize(0, totalsize, 1, STRINGSPACE_BLOCK); + ss->strings = sat_extend_resize(0, count, sizeof(Offset), STRING_BLOCK); // now copy predefined strings into allocated space ss->sstrings = 0; @@ -36,6 +38,29 @@ ss->nstrings = count; } +void +stringpool_init_empty(Stringpool *ss) +{ + const char *emptystrs[] = { + "<NULL>", + "", + 0, + }; + stringpool_init(ss, emptystrs); +} + +void +stringpool_clone(Stringpool *ss, Stringpool *from) +{ + memset(ss, 0, sizeof(*ss)); + ss->strings = sat_extend_resize(0, from->nstrings, sizeof(Offset), STRING_BLOCK); + memcpy(ss->strings, from->strings, from->nstrings * sizeof(Offset)); + ss->stringspace = sat_extend_resize(0, from->sstrings, 1, STRINGSPACE_BLOCK); + memcpy(ss->stringspace, from->stringspace, from->sstrings); + ss->nstrings = from->nstrings; + ss->sstrings = from->sstrings; +} + Id stringpool_strn2id (Stringpool *ss, const char *str, unsigned len, int create) { @@ -96,17 +121,13 @@ id = ss->nstrings++; hashtbl[h] = id; - // - if ((id & STRING_BLOCK) == 0) - ss->strings = sat_realloc2(ss->strings, (ss->nstrings + STRING_BLOCK) & ~STRING_BLOCK, sizeof(Hashval)); + ss->strings = sat_extend(ss->strings, id, 1, sizeof(Offset), STRING_BLOCK); // 'pointer' into stringspace is Offset of next free pos: sstrings ss->strings[id] = ss->sstrings; space_needed = len + 1; - - // resize string buffer if needed - if (((ss->sstrings + space_needed - 1) | STRINGSPACE_BLOCK) != ((ss->sstrings - 1) | STRINGSPACE_BLOCK)) - ss->stringspace = sat_realloc(ss->stringspace, (ss->sstrings + space_needed + STRINGSPACE_BLOCK) & ~STRINGSPACE_BLOCK); + // make room in string buffer + ss->stringspace = sat_extend(ss->stringspace, ss->sstrings, space_needed, 1, STRINGSPACE_BLOCK); // copy new string into buffer memcpy(ss->stringspace + ss->sstrings, str, space_needed - 1); // add the sentinel, we can't rely on it being in the source string (in @@ -128,6 +149,6 @@ void stringpool_shrink (Stringpool *ss) { - ss->stringspace = (char *)sat_realloc(ss->stringspace, (ss->sstrings + STRINGSPACE_BLOCK) & ~STRINGSPACE_BLOCK); - ss->strings = (Offset *)sat_realloc2(ss->strings, (ss->nstrings + STRING_BLOCK) & ~STRING_BLOCK, sizeof(Offset)); + ss->stringspace = sat_extend_resize(ss->stringspace, ss->sstrings, 1, STRINGSPACE_BLOCK); + ss->strings = sat_extend_resize(ss->strings, ss->nstrings, sizeof(Offset), STRING_BLOCK); } Modified: trunk/sat-solver/src/strpool.h URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/strpool.h?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/strpool.h (original) +++ trunk/sat-solver/src/strpool.h Tue Jan 29 17:14:03 2008 @@ -24,11 +24,15 @@ Hashmask stringhashmask; // modulo value for hash table (size of table - 1) }; -void stringpool_init (Stringpool *ss, const char *strs[]); +void stringpool_init(Stringpool *ss, const char *strs[]); +void stringpool_init_empty(Stringpool *ss); +void stringpool_clone(Stringpool *ss, Stringpool *from); + Id stringpool_str2id (Stringpool *ss, const char *str, int create); -Id stringpool_strn2id (Stringpool *ss, const char *str, unsigned len, int create); +Id stringpool_strn2id (Stringpool *ss, const char *str, unsigned int len, int create); void stringpool_shrink (Stringpool *ss); + static inline const char * stringpool_id2str (Stringpool *ss, Id id) { Modified: trunk/sat-solver/src/util.h URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/src/util.h?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/src/util.h (original) +++ trunk/sat-solver/src/util.h Tue Jan 29 17:14:03 2008 @@ -20,4 +20,33 @@ extern void *sat_realloc2(void *, size_t, size_t); extern void *sat_free(void *); +static inline void *sat_extend(void *buf, size_t len, size_t nmemb, size_t size, size_t block) +{ + if (nmemb == 1) + { + if ((len & block) == 0) + buf = sat_realloc2(buf, len + (1 + block), size); + } + else + { + if (((len - 1) | block) != ((len + nmemb - 1) | block)) + buf = sat_realloc2(buf, (len + (nmemb + block)) & ~block, size); + } + return buf; +} + +static inline void *sat_extend_resize(void *buf, size_t len, size_t size, size_t block) +{ + if (len) + buf = sat_realloc2(buf, (len + block) & ~block, size); + return buf; +} + +static inline void *sat_extend_cleanup(void *buf, size_t len, size_t size) +{ + if (len) + buf = sat_realloc2(buf, len, size); + return buf; +} + #endif /* SATSOLVER_UTIL_H */ Modified: trunk/sat-solver/tools/CMakeLists.txt URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/CMakeLists.txt?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/CMakeLists.txt (original) +++ trunk/sat-solver/tools/CMakeLists.txt Tue Jan 29 17:14:03 2008 @@ -43,14 +43,14 @@ ADD_EXECUTABLE( mergesolv ${mergesolv_REPOS} ) TARGET_LINK_LIBRARIES( mergesolv satsolver) -SET(dumpattr_REPOS dumpattr.c) -ADD_EXECUTABLE( dumpattr ${dumpattr_REPOS} ) -TARGET_LINK_LIBRARIES( dumpattr satsolver) +#SET(dumpattr_REPOS dumpattr.c) +#ADD_EXECUTABLE( dumpattr ${dumpattr_REPOS} ) +#TARGET_LINK_LIBRARIES( dumpattr satsolver) -ADD_EXECUTABLE( addstore addstore.c repo_write.c ) -TARGET_LINK_LIBRARIES( addstore satsolver ) +#ADD_EXECUTABLE( addstore addstore.c repo_write.c ) +#TARGET_LINK_LIBRARIES( addstore satsolver ) -install(TARGETS dumpattr +install(TARGETS mergesolv dumpsolv patchxml2solv @@ -58,7 +58,6 @@ helix2solv rpmmd2solv rpmdb2solv - addstore content2solv DESTINATION ${BIN_INSTALL_DIR} ) Modified: trunk/sat-solver/tools/content2solv.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/content2solv.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/content2solv.c (original) +++ trunk/sat-solver/tools/content2solv.c Tue Jan 29 17:14:03 2008 @@ -23,7 +23,7 @@ Pool *pool = pool_create(); Repo *repo = repo_create(pool, "<stdin>"); repo_add_content(repo, stdin); - repo_write(repo, stdout); + repo_write(repo, stdout, 0, 0); pool_free(pool); return 0; } Modified: trunk/sat-solver/tools/dumpsolv.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/dumpsolv.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/dumpsolv.c (original) +++ trunk/sat-solver/tools/dumpsolv.c Tue Jan 29 17:14:03 2008 @@ -12,6 +12,7 @@ #include "pool.h" #include "repo_solv.h" +#if 0 #include "attr_store.h" #include "attr_store_p.h" @@ -104,6 +105,7 @@ } printf("\n"); } +#endif static void printids(Repo *repo, char *kind, Offset ido) @@ -118,6 +120,75 @@ printf(" %s\n", dep2str(pool, id)); } +static void +printdir(Repodata *data, Id dir) +{ + Id comp; + Id parent = dirpool_parent(&data->dirpool, dir); + if (parent) + { + printdir(data, parent); + putchar('/'); + } + comp = dirpool_compid(&data->dirpool, dir); + if (data->localpool) + printf("%s", stringpool_id2str(&data->spool, comp)); + else + printf("%s", id2str(data->repo->pool, comp)); +} + +int +dump_repoattrs_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv) +{ + const char *keyname; + + keyname = id2str(data->repo->pool, key->name); + switch(key->type) + { + case TYPE_ID: + if (data->localpool) + kv->str = stringpool_id2str(&data->spool, kv->id); + else + kv->str = id2str(data->repo->pool, kv->id); + printf("%s: %s\n", keyname, kv->str); + break; + case TYPE_STR: + printf("%s: %s\n", keyname, kv->str); + break; + case TYPE_VOID: + printf("%s\n", keyname); + break; + case TYPE_NUM: + case TYPE_CONSTANT: + printf("%s: %d\n", keyname, kv->num); + break; + case TYPE_DIRNUMNUMARRAY: + printf("%s: ", keyname); + printdir(data, kv->id); + printf(" %d %d\n", kv->num, kv->num2); + break; + default: + printf("%s: ?\n", keyname); + break; + } + return 0; +} + +void +dump_repoattrs(Repo *repo, Id p) +{ + int i; + Repodata *data; + for (i = 0, data = repo->repodata; i < repo->nrepodata; i++, data++) + { + if (data->state == REPODATA_STUB || data->state == REPODATA_ERROR) + continue; + if (p < data->start || p >= data->end) + continue; + repodata_search(data, p - data->start, 0, dump_repoattrs_cb, 0); + } +} + int main(int argc, char **argv) { Repo *repo; @@ -134,9 +205,14 @@ } } pool = pool_create(); + pool_setdebuglevel(pool, 1); + repo = repo_create(pool, argc != 1 ? argv[1] : "<stdin>"); - repo_add_solv(repo, stdin); + if (repo_add_solv(repo, stdin)) + printf("could not read repository\n"); +#if 0 dump_repodata (repo); +#endif printf("repo contains %d solvables\n", repo->nsolvables); for (i = repo->start, n = 1; i < repo->end; i++) { @@ -158,7 +234,10 @@ printids(repo, "supplements", s->supplements); printids(repo, "enhances", s->enhances); printids(repo, "freshens", s->freshens); +#if 0 dump_attrs (repo, n - 1); +#endif + dump_repoattrs(repo, i); n++; } pool_free(pool); Modified: trunk/sat-solver/tools/helix2solv.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/helix2solv.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/helix2solv.c (original) +++ trunk/sat-solver/tools/helix2solv.c Tue Jan 29 17:14:03 2008 @@ -31,7 +31,7 @@ Pool *pool = pool_create(); Repo *repo = repo_create(pool, "<stdin>"); repo_add_helix(repo, stdin); - repo_write(repo, stdout); + repo_write(repo, stdout, 0, 0); pool_free(pool); exit(0); } Modified: trunk/sat-solver/tools/mergesolv.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/mergesolv.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/mergesolv.c (original) +++ trunk/sat-solver/tools/mergesolv.c Tue Jan 29 17:14:03 2008 @@ -103,7 +103,7 @@ pool->repos[0]->idarraydata = new_id; pool->repos[0]->idarraysize = new_id_size; - repo_write(pool->repos[0], stdout); + repo_write(pool->repos[0], stdout, 0, 0); pool_free(pool); return 0; Modified: trunk/sat-solver/tools/patchxml2solv.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/patchxml2solv.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/patchxml2solv.c (original) +++ trunk/sat-solver/tools/patchxml2solv.c Tue Jan 29 17:14:03 2008 @@ -23,7 +23,7 @@ Pool *pool = pool_create(); Repo *repo = repo_create(pool, "<stdin>"); repo_add_patchxml(repo, stdin); - repo_write(repo, stdout); + repo_write(repo, stdout, 0, 0); pool_free(pool); exit(0); } Modified: trunk/sat-solver/tools/repo_susetags.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/repo_susetags.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/repo_susetags.c (original) +++ trunk/sat-solver/tools/repo_susetags.c Tue Jan 29 17:14:03 2008 @@ -14,7 +14,9 @@ #include "pool.h" #include "repo.h" +#if 0 #include "attr_store.h" +#endif #include "repo_susetags.h" static int @@ -40,6 +42,7 @@ struct parsedata { char *kind; Repo *repo; + Repodata *data; char *tmp; int tmpl; char **sources; @@ -140,7 +143,9 @@ return repo_addid_dep(pd->repo, olddeps, id, marker); } +#if 0 Attrstore *attr; +#endif static Id id_authors; static Id id_description; static Id id_diskusage; @@ -162,7 +167,7 @@ static Id id_time; static void -add_location (char *line, Solvable *s, unsigned entry) +add_location(struct parsedata *pd, char *line, Solvable *s, unsigned entry) { Pool *pool = s->repo->pool; char *sp[3]; @@ -182,9 +187,15 @@ { /* medianr filename dir don't optimize this one */ +#if 0 add_attr_special_int (attr, entry, id_medianr, atoi (sp[0])); add_attr_localids_id (attr, entry, id_mediadir, str2localid (attr, sp[2], 1)); add_attr_string (attr, entry, id_mediafile, sp[1]); +#else + repodata_set_constant(pd->data, entry, id_medianr, atoi(sp[0])); + repodata_set_poolstr(pd->data, entry, id_mediadir, sp[2]); + repodata_set_str(pd->data, entry, id_mediafile, sp[1]); +#endif return; } else @@ -213,19 +224,30 @@ break; if (*n2 || strcmp (n1, ".rpm")) goto nontrivial; +#if 0 add_attr_special_int (attr, entry, id_medianr, medianr); add_attr_void (attr, entry, id_mediafile); +#else + repodata_set_constant(pd->data, entry, id_medianr, medianr); + repodata_set_void(pd->data, entry, id_mediafile); +#endif return; nontrivial: +#if 0 add_attr_special_int (attr, entry, id_medianr, medianr); add_attr_string (attr, entry, id_mediafile, sp[1]); +#else + repodata_set_constant(pd->data, entry, id_medianr, medianr); + repodata_set_str(pd->data, entry, id_mediafile, sp[1]); +#endif return; } } +#if 0 static void -add_source (char *line, struct parsedata *pd, Solvable *s, unsigned entry, int first) +add_source(struct parsedata *pd, char *line, Solvable *s, unsigned entry, int first) { Repo *repo = s->repo; Pool *pool = repo->pool; @@ -245,7 +267,9 @@ (src or nosrc), code only that fact. */ if (s->name == name && s->evr == evr && (arch == ARCH_SRC || arch == ARCH_NOSRC)) - add_attr_void (attr, entry, arch == ARCH_SRC ? id_source : id_nosource); + { + add_attr_void (attr, entry, arch == ARCH_SRC ? id_source : id_nosource); + } else if (first) { if (entry >= pd->nsources) @@ -296,6 +320,7 @@ } } } +#endif static void add_dirline (struct parsedata *pd, char *line) @@ -303,18 +328,18 @@ char *sp[6]; if (split (line, sp, 6) != 5) return; - if (!(pd->ndirs & 31)) - { - if (pd->dirs) - pd->dirs = realloc (pd->dirs, (pd->ndirs + 32) * sizeof (pd->dirs[0])); - else - pd->dirs = malloc ((pd->ndirs + 32) * sizeof (pd->dirs[0])); - } + pd->dirs = sat_extend(pd->dirs, pd->ndirs, 1, sizeof(pd->dirs[0]), 31); long filesz = strtol (sp[1], 0, 0); filesz += strtol (sp[2], 0, 0); long filenum = strtol (sp[3], 0, 0); filenum += strtol (sp[4], 0, 0); - unsigned dirid = dir_lookup (attr, sp[0], 1); + /* hack: we know that there's room for a / */ + if (*sp[0] != '/') + *--sp[0] = '/'; + unsigned dirid = repodata_str2dir(pd->data, sp[0], 1); +#if 0 +fprintf(stderr, "%s -> %d\n", sp[0], dirid); +#endif pd->dirs[pd->ndirs][0] = dirid; pd->dirs[pd->ndirs][1] = filesz; pd->dirs[pd->ndirs][2] = filenum; @@ -333,6 +358,7 @@ commit_diskusage (struct parsedata *pd, unsigned entry) { unsigned i; + Dirpool *dp = &pd->data->dirpool; /* Now sort in dirid order. This ensures that parents come before their children. */ if (pd->ndirs > 1) @@ -343,9 +369,9 @@ the array moving to the start, hence seeing leafs before parents. */ for (i = pd->ndirs; i--;) { - unsigned p = dir_parent (attr, pd->dirs[i][0]); + unsigned p = dirpool_parent(dp, pd->dirs[i][0]); unsigned j = i; - for (; p; p = dir_parent (attr, p)) + for (; p; p = dirpool_parent(dp, p)) { for (; j--;) if (pd->dirs[j][0] == p) @@ -382,13 +408,17 @@ for (i = 0; i < pd->ndirs; i++) if (pd->dirs[i][1] || pd->dirs[i][2]) { + repodata_add_dirnumnum(pd->data, entry, id_diskusage, pd->dirs[i][0], pd->dirs[i][1], pd->dirs[i][2]); +#if 0 add_attr_intlist_int (attr, entry, id_diskusage, pd->dirs[i][0]); add_attr_intlist_int (attr, entry, id_diskusage, pd->dirs[i][1]); add_attr_intlist_int (attr, entry, id_diskusage, pd->dirs[i][2]); +#endif } pd->ndirs = 0; } + /* Unfortunately "a"[0] is no constant expression in the C languages, so we need to pass the four characters individually :-/ */ #define CTAG(a,b,c,d) ((unsigned)(((unsigned char)a) << 24) \ @@ -416,10 +446,15 @@ int last_found_pack = 0; char *sp[5]; struct parsedata pd; + Repodata *data = 0; +#if 1 if (with_attr) { +#if 0 attr = new_store(pool); +#endif + data = repo_add_repodata(repo); id_authors = str2id (pool, "authors", 1); id_description = str2id (pool, "description", 1); id_diskusage = str2id (pool, "diskusage", 1); @@ -440,12 +475,14 @@ id_summary = str2id (pool, "summary", 1); id_time = str2id (pool, "time", 1); } +#endif memset(&pd, 0, sizeof(pd)); line = malloc(1024); aline = 1024; pd.repo = repo; + pd.data = data; linep = line; s = 0; @@ -538,6 +575,8 @@ pd.kind = "pattern"; s = pool_id2solvable(pool, repo_add_solvable(repo)); last_found_pack = (s - pool->solvables) - repo->start; + if (data) + repodata_extend(data, s - pool->solvables); if (split(line + 5, sp, 5) != 4) { fprintf(stderr, "Bad line: %s\n", line); @@ -558,7 +597,6 @@ { if (s && pd.ndirs) commit_diskusage (&pd, last_found_pack); - Id name, evr, arch; int n, nn; pd.kind = 0; @@ -654,52 +692,99 @@ switch (tag) { case CTAG('=', 'G', 'r', 'p'): + repodata_set_poolstr(data, last_found_pack, id_group, line + 6); +#if 0 add_attr_localids_id (attr, last_found_pack, id_group, str2localid (attr, line + 6, 1)); +#endif continue; case CTAG('=', 'L', 'i', 'c'): + repodata_set_poolstr(data, last_found_pack, id_license, line + 6); +#if 0 add_attr_localids_id (attr, last_found_pack, id_license, str2localid (attr, line + 6, 1)); +#endif continue; case CTAG('=', 'L', 'o', 'c'): - add_location (line + 6, s, last_found_pack); + add_location(&pd, line + 6, s, last_found_pack); continue; +#if 0 case CTAG('=', 'S', 'r', 'c'): - add_source (line + 6, &pd, s, last_found_pack, 1); + add_source(&pd, line + 6, s, last_found_pack, 1); continue; +#endif case CTAG('=', 'S', 'i', 'z'): if (split (line + 6, sp, 3) == 2) { + repodata_set_num(data, last_found_pack, id_downloadsize, (atoi(sp[0]) + 1023) / 1024); + repodata_set_num(data, last_found_pack, id_installsize, (atoi(sp[1]) + 1023) / 1024); +#if 0 add_attr_int (attr, last_found_pack, id_downloadsize, (atoi (sp[0]) + 1023) / 1024); add_attr_int (attr, last_found_pack, id_installsize, (atoi (sp[1]) + 1023) / 1024); +#endif } continue; case CTAG('=', 'T', 'i', 'm'): { unsigned int t = atoi (line + 6); if (t) - add_attr_int (attr, last_found_pack, id_time, t); + { +#if 0 + add_attr_int (attr, last_found_pack, id_time, t); +#else + repodata_set_num(data, last_found_pack, id_time, t); +#endif + } } continue; case CTAG('=', 'K', 'w', 'd'): +#if 0 add_attr_localids_id (attr, last_found_pack, id_keywords, str2localid (attr, line + 6, 1)); +#else + repodata_set_poolstr(data, last_found_pack, id_keywords, line + 6); +#endif continue; case CTAG('=', 'A', 'u', 't'): +#if 0 add_attr_blob (attr, last_found_pack, id_authors, line + 6, strlen (line + 6) + 1); +#else + repodata_set_str(data, last_found_pack, id_authors, line + 6); +#endif continue; case CTAG('=', 'S', 'u', 'm'): +#if 0 add_attr_string (attr, last_found_pack, id_summary, line + 6); +#else + repodata_set_str(data, last_found_pack, id_summary, line + 6); +#endif continue; case CTAG('=', 'D', 'e', 's'): +#if 0 add_attr_blob (attr, last_found_pack, id_description, line + 6, strlen (line + 6) + 1); +#else + repodata_set_str(data, last_found_pack, id_description, line + 6); +#endif continue; case CTAG('=', 'E', 'u', 'l'): +#if 0 add_attr_blob (attr, last_found_pack, id_eula, line + 6, strlen (line + 6) + 1); +#else + repodata_set_str(data, last_found_pack, id_eula, line + 6); +#endif continue; case CTAG('=', 'I', 'n', 's'): +#if 0 add_attr_blob (attr, last_found_pack, id_messageins, line + 6, strlen (line + 6) + 1); +#else + repodata_set_str(data, last_found_pack, id_messageins, line + 6); +#endif continue; case CTAG('=', 'D', 'e', 'l'): +#if 0 add_attr_blob (attr, last_found_pack, id_messagedel, line + 6, strlen (line + 6) + 1); +#else + repodata_set_str(data, last_found_pack, id_messagedel, line + 6); +#endif continue; +#if 0 case CTAG('=', 'S', 'h', 'r'): if (last_found_pack >= pd.nshare) { @@ -714,6 +799,7 @@ } pd.share_with[last_found_pack] = strdup (line + 6); continue; +#endif case CTAG('=', 'D', 'i', 'r'): add_dirline (&pd, line + 6); continue; @@ -727,16 +813,18 @@ s->provides = repo_addid_dep(repo, s->provides, rel2id(pool, s->name, s->evr, REL_EQ, 1), 0); if (s) s->supplements = repo_fix_legacy(repo, s->provides, s->supplements); + if (s && pd.ndirs) - commit_diskusage (&pd, last_found_pack); + commit_diskusage(&pd, last_found_pack); +#if 0 if (pd.sources) { int i, last_found; for (i = 0; i < pd.nsources; i++) if (pd.sources[i]) { - add_source (pd.sources[i], &pd, pool->solvables + repo->start + i, i, 0); + add_source(&pd, pd.sources[i], pool->solvables + repo->start + i, i, 0); free (pd.sources[i]); } free (pd.sources); @@ -776,6 +864,11 @@ } free (pd.share_with); } +#endif + + if (data) + repodata_internalize(data); + if (pd.tmp) free(pd.tmp); free(line); Modified: trunk/sat-solver/tools/repo_write.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/repo_write.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/repo_write.c (original) +++ trunk/sat-solver/tools/repo_write.c Tue Jan 29 17:14:03 2008 @@ -27,8 +27,6 @@ #include "util.h" #include "repo_write.h" -/* #define IGNORE_NEED_FREQUENCY */ - /*------------------------------------------------------------------*/ /* Id map optimizations */ @@ -38,7 +36,7 @@ } NeedId; -#define RELOFF(id) (pool->ss.nstrings + GETRELID(id)) +#define RELOFF(id) (needid[0].map + GETRELID(id)) /* * increment need Id @@ -49,15 +47,30 @@ * */ -static int -incneedid(Pool *pool, Id *idarray, NeedId *needid) +static void +incneedid(Pool *pool, Id id, NeedId *needid) { - if (!idarray) - return 0; + while (ISRELDEP(id)) + { + Reldep *rd = GETRELDEP(pool, id); + needid[RELOFF(id)].need++; + if (ISRELDEP(rd->evr)) + incneedid(pool, rd->evr, needid); + else + needid[rd->evr].need++; + id = rd->name; + } + needid[id].need++; +} +static int +incneedidarray(Pool *pool, Id *idarray, NeedId *needid) +{ Id id; int n = 0; + if (!idarray) + return 0; while ((id = *idarray++) != 0) { n++; @@ -66,12 +79,7 @@ Reldep *rd = GETRELDEP(pool, id); needid[RELOFF(id)].need++; if (ISRELDEP(rd->evr)) - { - Id ida[2]; - ida[0] = rd->evr; - ida[1] = 0; - incneedid(pool, ida, needid); - } + incneedid(pool, rd->evr, needid); else needid[rd->evr].need++; id = rd->name; @@ -99,34 +107,24 @@ } static Pool *cmp_pool; + static int needid_cmp_need_s(const void *ap, const void *bp) { const NeedId *a = ap; const NeedId *b = bp; - if (a == b) - return 0; -#ifdef IGNORE_NEED_FREQUENCY - if (a->need == 0) - return 1; - else if (b->need == 0) - return -1; -#else int r; r = b->need - a->need; if (r) return r; -#endif - char *as = cmp_pool->ss.stringspace + cmp_pool->ss.strings[a->map]; - char *bs = cmp_pool->ss.stringspace + cmp_pool->ss.strings[b->map]; - size_t alen = strlen (as); - size_t blen = strlen (bs); - return memcmp (as, bs, alen < blen ? alen + 1 : blen + 1); + const char *as = cmp_pool->ss.stringspace + cmp_pool->ss.strings[a->map]; + const char *bs = cmp_pool->ss.stringspace + cmp_pool->ss.strings[b->map]; + return strcmp(as, bs); } /*------------------------------------------------------------------*/ -/* output routines */ +/* output helper routines */ /* * unsigned 32-bit @@ -160,6 +158,19 @@ } } +/* + * data blob + */ + +static void +write_blob(FILE *fp, void *data, int len) +{ + if (fwrite(data, len, 1, fp) != 1) + { + perror("write error"); + exit(1); + } +} /* * Id @@ -185,16 +196,17 @@ } } +#if 0 static void write_str(FILE *fp, const char *str) { - if (fputs (str, fp) == EOF - || putc (0, fp) == EOF) + if (fputs (str, fp) == EOF || putc (0, fp) == EOF) { perror("write error"); exit(1); } } +#endif /* * Array of Ids @@ -239,50 +251,52 @@ write_idarray_sort(FILE *fp, Pool *pool, NeedId *needid, Id *ids) { int len, i; + Id lids[64], *sids; + if (!ids) return; if (!*ids) { - write_u8 (fp, 0); + write_u8(fp, 0); return; } - /* If we ever share idarrays we can't do this in-place. */ - for (len = 0; ids[len]; len++) + for (len = 0; len < 64 && ids[len]; len++) { Id id = ids[len]; if (needid) - ids[len] = needid[ISRELDEP(id) ? RELOFF(id) : id].need; + id = needid[ISRELDEP(id) ? RELOFF(id) : id].need; + lids[len] = id; + } + if (ids[len]) + { + for (i = len + 1; ids[i]; i++) + ; + sids = sat_malloc2(i, sizeof(Id)); + memcpy(sids, lids, 64 * sizeof(Id)); + for (; ids[len]; len++) + { + Id id = ids[len]; + if (needid) + id = needid[ISRELDEP(id) ? RELOFF(id) : id].need; + sids[len] = id; + } } + else + sids = lids; /* That bloody solvable:prereqmarker needs to stay in position :-( */ Id prereq = SOLVABLE_PREREQMARKER; if (needid) prereq = needid[prereq].need; for (i = 0; i < len; i++) - if (ids[i] == prereq) + if (sids[i] == prereq) break; if (i > 1) - qsort (ids, i, sizeof (Id), cmp_ids); + qsort(sids, i, sizeof (Id), cmp_ids); if ((len - i) > 2) - qsort (ids + i + 1, len - i - 1, sizeof (Id), cmp_ids); + qsort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids); - Id old = 0; - for (i = 0; i < len; i++) - /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker, - hence all real differences are offsetted by 1. Otherwise we would - have to handle negative differences, which would cost code space for - the encoding of the sign. We loose the exact mapping of prereq here, - but we know the result, so we can recover from that in the reader. */ - if (ids[i] == prereq) - old = ids[i] = 0; - else - { - ids[i] -= old; - old = ids[i] + old; - /* XXX If difference is zero we have multiple equal elements, - we might want to skip writing them out. */ - ids[i]++; - } + Id id, old = 0; /* The differencing above produces many runs of ones and twos. I tried fairly elaborate schemes to RLE those, but they give only very mediocre @@ -292,15 +306,35 @@ invest some complexity into sharing idarrays, than RLEing. */ for (i = 0; i < len - 1; i++) { - Id id = ids[i]; + id = sids[i]; + /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker, + hence all real differences are offsetted by 1. Otherwise we would + have to handle negative differences, which would cost code space for + the encoding of the sign. We loose the exact mapping of prereq here, + but we know the result, so we can recover from that in the reader. */ + if (id == prereq) + id = old = 0; + else + { + id = id - old + 1; + old = sids[i]; + } + /* XXX If difference is zero we have multiple equal elements, + we might want to skip writing them out. */ if (id >= 64) id = (id & 63) | ((id & ~63) << 1); write_id(fp, id | 64); } - old = ids[i]; - if (old >= 64) - old = (old & 63) | ((old & ~63) << 1); - write_id(fp, old); + id = sids[i]; + if (id == prereq) + id = 0; + else + id = id - old + 1; + if (id >= 64) + id = (id & 63) | ((id & ~63) << 1); + write_id(fp, id); + if (sids != lids) + sat_free(sids); } static inline void @@ -314,171 +348,785 @@ write_id(fp, value | (eof ? 0 : 64)); } -struct schemata { - int nschemata; - Id *schemadata, *schemadatap; - int schemadatafree; - Id lastschema[256]; - Id lastschemakey[256]; +struct extdata { + unsigned char *buf; + int len; +}; + +struct cbdata { + Repo *repo; + + Stringpool *ownspool; + Dirpool *owndirpool; + + Repokey *mykeys; + int nmykeys; + + Id *keymap; + int nkeymap; + Id *keymapstart; + + NeedId *needid; + + Id *schema; /* schema construction space */ + Id *sp; /* pointer in above */ + + Id *myschemata; + int nmyschemata; + + Id *myschemadata; + int myschemadatalen; + + Id schematacache[256]; + + Id *solvschemata; + Id *incorelen; + + struct extdata *extdata; + + Id *dirused; + Id *dirmap; + + Id vstart; }; +#define NEEDED_BLOCK 1023 +#define SCHEMATA_BLOCK 31 +#define SCHEMATADATA_BLOCK 255 +#define EXTDATA_BLOCK 1023 + +void +data_addid(struct extdata *xd, Id x) +{ + unsigned char *dp; + xd->buf = sat_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK); + dp = xd->buf + xd->len; + + if (x >= (1 << 14)) + { + if (x >= (1 << 28)) + *dp++ = (x >> 28) | 128; + if (x >= (1 << 21)) + *dp++ = (x >> 21) | 128; + *dp++ = (x >> 14) | 128; + } + if (x >= (1 << 7)) + *dp++ = (x >> 7) | 128; + *dp++ = x & 127; + xd->len = dp - xd->buf; +} + +void +data_addideof(struct extdata *xd, Id x, int eof) +{ + if (x >= 64) + x = (x & 63) | ((x & ~63) << 1); + data_addid(xd, (eof ? x: x | 64)); +} + +void +data_addblob(struct extdata *xd, unsigned char *blob, int len) +{ + xd->buf = sat_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK); + memcpy(xd->buf + xd->len, blob, len); + xd->len += len; +} + static Id -addschema(struct schemata *schemata, Id *schema) +addschema(struct cbdata *cbdata, Id *schema) { int h, len; - Id *sp, schemaid; + Id *sp, cid; for (sp = schema, len = 0, h = 0; *sp; len++) h = h * 7 + *sp++; h &= 255; len++; - if (schemata->lastschema[h] && !memcmp(schemata->schemadata + schemata->lastschema[h], schema, len * sizeof(Id))) - return schemata->lastschemakey[h]; - schemaid = 0; - for (sp = schemata->schemadata + 1; sp < schemata->schemadatap; ) + cid = cbdata->schematacache[h]; + if (cid) { - if (!memcmp(sp, schemata->schemadatap, len * sizeof(Id))) - return schemaid; - while (*sp++) - ; - schemaid++; + if (!memcmp(cbdata->myschemadata + cbdata->myschemata[cid], schema, len * sizeof(Id))) + return cid; + /* cache conflict */ + for (cid = 1; cid < cbdata->nmyschemata; cid++) + if (!memcmp(cbdata->myschemadata + cbdata->myschemata[cid], schema, len * sizeof(Id))) + return cid; + } + /* a new one. make room. */ + cbdata->myschemadata = sat_extend(cbdata->myschemadata, cbdata->myschemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK); + cbdata->myschemata = sat_extend(cbdata->myschemata, cbdata->nmyschemata, 1, sizeof(Id), SCHEMATA_BLOCK); + if (!cbdata->nmyschemata) + { + cbdata->myschemata[0] = 0; + cbdata->myschemadata[0] = 0; + cbdata->nmyschemata = 1; + cbdata->myschemadatalen = 1; + } + /* add schema */ + memcpy(cbdata->myschemadata + cbdata->myschemadatalen, schema, len * sizeof(Id)); + cbdata->myschemata[cbdata->nmyschemata] = cbdata->myschemadatalen; + cbdata->myschemadatalen += len; + cbdata->schematacache[h] = cbdata->nmyschemata; + return cbdata->nmyschemata++; +} + +static Id +putinownpool(struct cbdata *cbdata, Stringpool *ss, Id id) +{ + const char *str = stringpool_id2str(ss, id); + id = stringpool_str2id(cbdata->ownspool, str, 1); + if (id >= cbdata->needid[0].map) + { + int oldoff = cbdata->needid[0].map; + int newoff = (id + 1 + NEEDED_BLOCK) & ~NEEDED_BLOCK; + int nrels = cbdata->repo->pool->nrels; + fprintf(stderr, "growing needid...\n"); + cbdata->needid = sat_realloc2(cbdata->needid, newoff + nrels, sizeof(NeedId)); + if (nrels) + memmove(cbdata->needid + newoff, cbdata->needid + oldoff, nrels * sizeof(NeedId)); + memset(cbdata->needid + oldoff, 0, (newoff - oldoff) * sizeof(NeedId)); + cbdata->needid[0].map = newoff; } + return id; +} + +Id +putinowndirpool(struct cbdata *cbdata, Repodata *data, Dirpool *dp, Id dir) +{ + Id compid, parent; + + parent = dirpool_parent(dp, dir); + if (parent) + parent = putinowndirpool(cbdata, data, dp, parent); + compid = dp->dirs[dir]; + if (cbdata->ownspool && compid > 1) + compid = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, compid); + return dirpool_add_dir(cbdata->owndirpool, parent, compid, 1); +} - /* a new one */ - if (len > schemata->schemadatafree) +static inline void +setdirused(struct cbdata *cbdata, Dirpool *dp, Id dir) +{ + if (cbdata->dirused[dir]) + return; + cbdata->dirused[dir] = 1; + while ((dir = dirpool_parent(dp, dir)) != 0) { - int l = schemata->schemadatap - schemata->schemadata; - schemata->schemadata = sat_realloc(schemata->schemadata, (schemata->schemadatap - schemata->schemadata + len + 256) * sizeof(Id)); - schemata->schemadatafree = len + 256; - schemata->schemadatap = schemata->schemadata + l; - if (l == 0) - { - /* leave first one free so that our lastschema test works */ - *schemata->schemadatap++ = 0; - schemata->schemadatafree--; - } - } - if (schemaid != schemata->nschemata) - abort(); - schemata->lastschema[h] = schemata->schemadatap - schemata->schemadata; - schemata->lastschemakey[h] = schemaid; - memcpy(schemata->schemadatap, schema, len * sizeof(Id)); - schemata->schemadatafree -= len; - schemata->schemadatap += len; - schemata->nschemata++; - return schemaid; + if (cbdata->dirused[dir] == 2) + return; + if (cbdata->dirused[dir]) + { + cbdata->dirused[dir] = 2; + return; + } + cbdata->dirused[dir] = 2; + } + cbdata->dirused[0] = 2; } +int +repo_write_cb_needed(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv) +{ + struct cbdata *cbdata = vcbdata; + Repo *repo = s ? s->repo : 0; + Id id; + int rm; + +#if 0 + fprintf(stderr, "solvable %d (%s): key %d %d\n", s ? s - s->repo->pool->solvables : 0, s ? id2str(s->repo->pool, s->name) : "", key->name, key->type); +#endif + rm = cbdata->keymap[cbdata->keymapstart[data->repo->repodata - data] + (key - data->keys)]; + if (!rm) + return SEARCH_NEXT_KEY; /* we do not want this one */ + if (cbdata->sp == cbdata->schema || cbdata->sp[-1] != rm) + *cbdata->sp++ = rm; + switch(key->type) + { + case TYPE_ID: + case TYPE_IDARRAY: + id = kv->id; + if (!ISRELDEP(id) && cbdata->ownspool && id > 1) + id = putinownpool(cbdata, data->localpool ? &data->spool : &repo->pool->ss, id); + incneedid(repo->pool, id, cbdata->needid); + break; + case TYPE_DIR: + case TYPE_DIRNUMNUMARRAY: + id = kv->id; + if (cbdata->owndirpool) + putinowndirpool(cbdata, data, &data->dirpool, id); + else + setdirused(cbdata, &data->dirpool, id); + break; + default: + break; + } + return 0; +} + +int +repo_write_cb_sizes(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv) +{ + struct cbdata *cbdata = vcbdata; + int rm; + Id id; + unsigned int u32; + unsigned char v[4]; + struct extdata *xd; + + rm = cbdata->keymap[cbdata->keymapstart[data->repo->repodata - data] + (key - data->keys)]; + if (!rm) + return 0; /* we do not want this one */ + + if (cbdata->mykeys[rm].storage == KEY_STORAGE_VERTICAL_OFFSET) + { + xd = cbdata->extdata + rm; /* vertical buffer */ + if (!cbdata->vstart) + cbdata->vstart = xd->len; + } + else + xd = cbdata->extdata + 0; /* incore buffer */ + switch(key->type) + { + case TYPE_VOID: + case TYPE_CONSTANT: + break; + case TYPE_ID: + id = kv->id; + if (!ISRELDEP(id) && cbdata->ownspool && id > 1) + id = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, id); + id = cbdata->needid[id].need; + data_addid(xd, id); + break; + case TYPE_IDARRAY: + id = kv->id; + if (cbdata->ownspool && id > 1) + id = putinownpool(cbdata, data->localpool ? &data->spool : &data->repo->pool->ss, id); + id = cbdata->needid[id].need; + data_addideof(xd, id, kv->eof); + break; + case TYPE_STR: + data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1); + break; + case TYPE_U32: + u32 = kv->num; + v[0] = u32 >> 24; + v[1] = u32 >> 16; + v[2] = u32 >> 8; + v[3] = u32; + data_addblob(xd, v, 4); + break; + case TYPE_DIR: + id = kv->id; + id = cbdata->dirused[id]; + data_addid(xd, id); + break; + case TYPE_NUM: + data_addid(xd, kv->num); + break; + case TYPE_DIRNUMNUMARRAY: + id = kv->id; + id = cbdata->dirused[id]; + data_addid(xd, id); + data_addid(xd, kv->num); + data_addideof(xd, kv->num2, kv->eof); + break; + case TYPE_DIRSTRARRAY: + id = kv->id; + id = cbdata->dirused[id]; + data_addideof(xd, id, kv->eof); + data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1); + break; + default: + fprintf(stderr, "unknown type for %d: %d\n", key->name, key->type); + exit(1); + } + if (cbdata->mykeys[rm].storage == KEY_STORAGE_VERTICAL_OFFSET && kv->eof) + { + data_addid(cbdata->extdata + 0, cbdata->vstart); /* add offset into incore data */ + data_addid(cbdata->extdata + 0, xd->len - cbdata->vstart); /* add length into incore data */ + cbdata->vstart = 0; + } + return 0; +} + +static int +traverse_dirs(Dirpool *dp, Id *dirmap, Id n, Id dir, Id *used) +{ + Id sib, child; + Id parent, lastn; + + parent = n; + /* special case for '/', which has to come first */ + if (parent == 1) + dirmap[n++] = 1; + for (sib = dir; sib; sib = dirpool_sibling(dp, sib)) + { + if (used && !used[sib]) + continue; + if (sib == 1 && parent == 1) + continue; /* already did that one above */ + dirmap[n++] = sib; + } + lastn = n; + for (; parent < lastn; parent++) + { + sib = dirmap[parent]; + if (used && used[sib] != 2) + continue; + child = dirpool_child(dp, sib); + if (child) + { + dirmap[n++] = -parent; + n = traverse_dirs(dp, dirmap, n, child, used); + } + } + return n; +} + +#define BLOB_PAGEBITS 15 +#define BLOB_PAGESIZE (1 << BLOB_PAGEBITS) + +static void +write_compressed_page(FILE *fp, unsigned char *page, int len) +{ + int clen; + unsigned char cpage[BLOB_PAGESIZE]; + + clen = repodata_compress_page(page, len, cpage, len - 1); + if (!clen) + { + write_u32(fp, len * 2); + write_blob(fp, page, len); + } + else + { + write_u32(fp, clen * 2 + 1); + write_blob(fp, cpage, clen); + } +} /* * Repo */ void -repo_write(Repo *repo, FILE *fp) +repo_write(Repo *repo, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata) { Pool *pool = repo->pool; - int i, n; + int i, j, k, n; Solvable *s; NeedId *needid; - int nstrings, nrels, nkeys; + int nstrings, nrels; unsigned int sizeid; unsigned int solv_flags; Reldep *ran; Id *idarraydata; - int idsizes[ID_NUM_INTERNAL]; - int id2key[ID_NUM_INTERNAL]; - int nsolvables; - - Id schema[ID_NUM_INTERNAL], *sp; - struct schemata schemata; - Id *solvschema; /* schema of our solvables */ - Id repodataschema, repodataschema_internal; + Id id, *sp; - nsolvables = 0; - idarraydata = repo->idarraydata; + Id *dirmap; + int ndirmap; + Id *keyused; + int ext0len; + unsigned char *repodataused; + + struct cbdata cbdata; + int needrels; + Repokey *key; + int poolusage, dirpoolusage, idused, dirused; + int reloff; + unsigned char *incoredata; + + Repodata *data; + Stringpool ownspool, *spool; + Dirpool owndirpool, *dirpool; + + memset(&cbdata, 0, sizeof(cbdata)); + + /* go through all repodata and find the keys we need */ + /* also unify keys */ + /* creates: mykeys - key array, still has global pool ids */ + /* keymapstart - maps repo number to keymap offset */ + /* keymap - maps repo key to my key, 0 -> not used */ - needid = sat_calloc(pool->ss.nstrings + pool->nrels, sizeof(*needid)); - memset(idsizes, 0, sizeof(idsizes)); + /* start with all KEY_STORAGE_SOLVABLE ids */ - repodataschema = repodataschema_internal = 0; + n = ID_NUM_INTERNAL; + for (i = 0; i < repo->nrepodata; i++) + n += repo->repodata[i].nkeys; + cbdata.mykeys = sat_calloc(n, sizeof(Repokey)); + cbdata.keymap = sat_calloc(n, sizeof(Id)); + cbdata.keymapstart = sat_calloc(repo->nrepodata, sizeof(Id)); + repodataused = sat_calloc(repo->nrepodata, 1); + + cbdata.nmykeys = 1; + needrels = 0; + poolusage = 0; + for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++) + { + key = cbdata.mykeys + i; + key->name = i; + if (i < SOLVABLE_PROVIDES) + key->type = TYPE_ID; + else if (i < RPM_RPMDBID) + key->type = TYPE_REL_IDARRAY; + else + key->type = TYPE_U32; + key->size = 0; + key->storage = KEY_STORAGE_SOLVABLE; + if (keyfilter) + { + key->storage = keyfilter(repo, key, kfdata); + if (key->storage == KEY_STORAGE_DROPPED) + continue; + key->storage = KEY_STORAGE_SOLVABLE; + } + poolusage = 1; + if (key->type == TYPE_IDARRAY || key->type == TYPE_REL_IDARRAY) + needrels = 1; + cbdata.keymap[i] = i; + } + cbdata.nmykeys = i; + + dirpoolusage = 0; + + spool = 0; + dirpool = 0; + n = ID_NUM_INTERNAL; for (i = 0; i < repo->nrepodata; i++) { - int j; - idsizes[REPODATA_EXTERNAL] = 1; - idsizes[REPODATA_KEYS]++; - if (repo->repodata[i].location) + data = repo->repodata + i; + cbdata.keymapstart[i] = n; + cbdata.keymap[n++] = 0; /* key 0 */ + idused = 0; + dirused = 0; + for (j = 1; j < data->nkeys; j++) + { + key = data->keys + j; + /* see if we already had this one, should use hash for fast miss */ + for (k = 0; k < cbdata.nmykeys; k++) + { + if (key->name == cbdata.mykeys[k].name && key->type == cbdata.mykeys[k].type) + { + if (key->type == TYPE_CONSTANT && key->size != cbdata.mykeys[k].size) + continue; + break; + } + } + if (k < cbdata.nmykeys) + { + cbdata.keymap[n++] = 0; + continue; + } + cbdata.mykeys[cbdata.nmykeys] = *key; + key = cbdata.mykeys + cbdata.nmykeys; + key->storage = KEY_STORAGE_INCORE; + if (key->type != TYPE_CONSTANT) + key->size = 0; + if (keyfilter) + { + key->storage = keyfilter(repo, key, kfdata); + if (key->storage == KEY_STORAGE_DROPPED) + { + cbdata.keymap[n++] = 0; + continue; + } + } + /* load repodata if not already loaded */ + if (data->state == REPODATA_STUB) + { + if (data->loadcallback) + data->loadcallback(data); + else + data->state = REPODATA_ERROR; + if (data->state != REPODATA_ERROR) + { + /* redo this repodata! */ + j = 0; + n = cbdata.keymapstart[i] + 1; + continue; + } + } + if (data->state == REPODATA_ERROR) + { + /* too bad! */ + cbdata.keymap[n++] = 0; + continue; + } + cbdata.keymap[n++] = cbdata.nmykeys++; + repodataused[i] = 1; + if (key->type != TYPE_STR && key->type != TYPE_U32) + idused = 1; + if (key->type == TYPE_DIR || key->type == TYPE_DIRNUMNUMARRAY) + dirused = 1; + } + if (idused) + { + if (data->localpool) + { + if (poolusage) + poolusage = 3; /* need local pool */ + else + { + poolusage = 2; + spool = &data->spool; + } + } + else + { + if (poolusage == 0) + poolusage = 1; + else if (poolusage != 1) + poolusage = 3; /* need local pool */ + } + } + if (dirused) { - repodataschema = 1; /* mark that we need it */ - idsizes[REPODATA_LOCATION] = 1; + if (dirpoolusage) + dirpoolusage = 3; /* need local dirpool */ + else + { + dirpoolusage = 2; + dirpool = &data->dirpool; + } + } + } + cbdata.nkeymap = n; + + /* 0: no pool needed at all */ + /* 1: use global pool */ + /* 2: use repodata local pool */ + /* 3: need own pool */ + if (poolusage == 3) + { + spool = &ownspool; + if (needrels) + { + /* hack: reuse global pool so we don't have to map rel ids */ + stringpool_clone(spool, &repo->pool->ss); } else - repodataschema_internal = 1; /* mark that we need it */ - for (j = 0; j < repo->repodata[i].nkeys; j++) - needid[repo->repodata[i].keys[j].name].need++; - idsizes[REPODATA_KEYS] += 2 * repo->repodata[i].nkeys; + stringpool_init_empty(spool); + cbdata.ownspool = spool; } + else if (poolusage == 0 || poolusage == 1) + { + poolusage = 1; + spool = &repo->pool->ss; + } + + if (dirpoolusage == 3) + { + dirpool = &owndirpool; + dirpool_create(dirpool); + cbdata.owndirpool = dirpool; + } + else if (dirpool) + cbdata.dirused = sat_calloc(dirpool->ndirs, sizeof(Id)); + + +/********************************************************************/ +#if 0 +fprintf(stderr, "poolusage: %d\n", poolusage); +fprintf(stderr, "dirpoolusage: %d\n", dirpoolusage); +fprintf(stderr, "nmykeys: %d\n", cbdata.nmykeys); +for (i = 1; i < cbdata.nmykeys; i++) + fprintf(stderr, " %2d: %d %d\n", i, cbdata.mykeys[i].name, cbdata.mykeys[i].type); +#endif + +/********************************************************************/ + + /* set needed count of all strings and rels, + * find which keys are used in the solvables + * put all strings in own spool + */ - idsizes[SOLVABLE_NAME] = 1; - idsizes[SOLVABLE_ARCH] = 1; - idsizes[SOLVABLE_EVR] = 1; - if (repo->rpmdbid) - idsizes[RPM_RPMDBID] = 1; - for (i = repo->start, s = pool->solvables + i; i < repo->end; i++, s++) + reloff = spool->nstrings; + if (poolusage == 3) + reloff = (reloff + NEEDED_BLOCK) & ~NEEDED_BLOCK; + + needid = calloc(reloff + pool->nrels, sizeof(*needid)); + needid[0].map = reloff; + + cbdata.needid = needid; + cbdata.schema = sat_calloc(cbdata.nmykeys, sizeof(Id)); + cbdata.sp = cbdata.schema; + cbdata.solvschemata = sat_calloc(repo->nsolvables, sizeof(Id)); + + idarraydata = repo->idarraydata; + + for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++) { if (s->repo != repo) continue; - nsolvables++; - needid[s->name].need++; - needid[s->arch].need++; - needid[s->evr].need++; - if (s->vendor) - { - needid[s->vendor].need++; - idsizes[SOLVABLE_VENDOR] = 1; - } - if (s->provides) - idsizes[SOLVABLE_PROVIDES] += incneedid(pool, idarraydata + s->provides, needid); - if (s->requires) - idsizes[SOLVABLE_REQUIRES] += incneedid(pool, idarraydata + s->requires, needid); - if (s->conflicts) - idsizes[SOLVABLE_CONFLICTS] += incneedid(pool, idarraydata + s->conflicts, needid); - if (s->obsoletes) - idsizes[SOLVABLE_OBSOLETES] += incneedid(pool, idarraydata + s->obsoletes, needid); - if (s->recommends) - idsizes[SOLVABLE_RECOMMENDS] += incneedid(pool, idarraydata + s->recommends, needid); - if (s->suggests) - idsizes[SOLVABLE_SUGGESTS] += incneedid(pool, idarraydata + s->suggests, needid); - if (s->supplements) - idsizes[SOLVABLE_SUPPLEMENTS] += incneedid(pool, idarraydata + s->supplements, needid); - if (s->enhances) - idsizes[SOLVABLE_ENHANCES] += incneedid(pool, idarraydata + s->enhances, needid); - if (s->freshens) - idsizes[SOLVABLE_FRESHENS] += incneedid(pool, idarraydata + s->freshens, needid); + + /* set schema info, keep in sync with further down */ + sp = cbdata.schema; + if (cbdata.keymap[SOLVABLE_NAME]) + { + *sp++ = SOLVABLE_NAME; + needid[s->name].need++; + } + if (cbdata.keymap[SOLVABLE_ARCH]) + { + *sp++ = SOLVABLE_ARCH; + needid[s->arch].need++; + } + if (cbdata.keymap[SOLVABLE_EVR]) + { + *sp++ = SOLVABLE_EVR; + needid[s->arch].need++; + } + if (s->vendor && cbdata.keymap[SOLVABLE_VENDOR]) + { + *sp++ = SOLVABLE_VENDOR; + needid[s->vendor].need++; + } + if (s->provides && cbdata.keymap[SOLVABLE_PROVIDES]) + { + *sp++ = SOLVABLE_PROVIDES; + cbdata.mykeys[SOLVABLE_PROVIDES].size += incneedidarray(pool, idarraydata + s->provides, needid); + } + if (s->obsoletes && cbdata.keymap[SOLVABLE_OBSOLETES]) + { + *sp++ = SOLVABLE_OBSOLETES; + cbdata.mykeys[SOLVABLE_OBSOLETES].size += incneedidarray(pool, idarraydata + s->obsoletes, needid); + } + if (s->conflicts && cbdata.keymap[SOLVABLE_CONFLICTS]) + { + *sp++ = SOLVABLE_CONFLICTS; + cbdata.mykeys[SOLVABLE_CONFLICTS].size += incneedidarray(pool, idarraydata + s->conflicts, needid); + } + if (s->requires && cbdata.keymap[SOLVABLE_REQUIRES]) + { + *sp++ = SOLVABLE_REQUIRES; + cbdata.mykeys[SOLVABLE_REQUIRES].size += incneedidarray(pool, idarraydata + s->requires, needid); + } + if (s->recommends && cbdata.keymap[SOLVABLE_RECOMMENDS]) + { + *sp++ = SOLVABLE_RECOMMENDS; + cbdata.mykeys[SOLVABLE_RECOMMENDS].size += incneedidarray(pool, idarraydata + s->recommends, needid); + } + if (s->suggests && cbdata.keymap[SOLVABLE_SUGGESTS]) + { + *sp++ = SOLVABLE_SUGGESTS; + cbdata.mykeys[SOLVABLE_SUGGESTS].size += incneedidarray(pool, idarraydata + s->suggests, needid); + } + if (s->supplements && cbdata.keymap[SOLVABLE_SUPPLEMENTS]) + { + *sp++ = SOLVABLE_SUPPLEMENTS; + cbdata.mykeys[SOLVABLE_SUPPLEMENTS].size += incneedidarray(pool, idarraydata + s->supplements, needid); + } + if (s->enhances && cbdata.keymap[SOLVABLE_ENHANCES]) + { + *sp++ = SOLVABLE_ENHANCES; + cbdata.mykeys[SOLVABLE_ENHANCES].size += incneedidarray(pool, idarraydata + s->enhances, needid); + } + if (s->freshens && cbdata.keymap[SOLVABLE_FRESHENS]) + { + *sp++ = SOLVABLE_FRESHENS; + cbdata.mykeys[SOLVABLE_FRESHENS].size += incneedidarray(pool, idarraydata + s->freshens, needid); + } + if (repo->rpmdbid && cbdata.keymap[RPM_RPMDBID]) + { + *sp++ = RPM_RPMDBID; + cbdata.mykeys[RPM_RPMDBID].size++; + } + cbdata.sp = sp; + + for (j = 0, data = repo->repodata; j < repo->nrepodata; j++, data++) + { + if (!repodataused[j]) + continue; + if (i < data->start || i >= data->end) + continue; + repodata_search(data, i - data->start, 0, repo_write_cb_needed, &cbdata); + needid = cbdata.needid; + } + *cbdata.sp = 0; + cbdata.solvschemata[n] = addschema(&cbdata, cbdata.schema); + n++; } - if (nsolvables != repo->nsolvables) - abort(); - for (i = SOLVABLE_NAME; i < ID_NUM_INTERNAL; i++) + reloff = needid[0].map; + + +/********************************************************************/ + + /* remove unused keys, also increment needid for key names */ + keyused = sat_calloc(cbdata.nmykeys, sizeof(Id)); + for (i = 0; i < cbdata.myschemadatalen; i++) + keyused[cbdata.myschemadata[i]] = 1; + keyused[0] = 0; + for (n = i = 1; i < cbdata.nmykeys; i++) { - if (idsizes[i]) - needid[i].need++; + if (!keyused[i]) + continue; + keyused[i] = n; + if (i != n) + cbdata.mykeys[n] = cbdata.mykeys[i]; + needid[cbdata.mykeys[n].name].need++; + n++; + } + cbdata.nmykeys = n; + for (i = 0; i < cbdata.myschemadatalen; i++) + cbdata.myschemadata[i] = keyused[cbdata.myschemadata[i]]; + for (i = 0; i < cbdata.nkeymap; i++) + cbdata.keymap[i] = keyused[cbdata.keymap[i]]; + keyused = sat_free(keyused); + +/********************************************************************/ + + /* increment need id for used dir components */ + if (cbdata.dirused && !cbdata.dirused[0]) + { + /* no dirs used at all */ +fprintf(stderr, "no dirs used!\n"); + sat_free(cbdata.dirused); + dirpool = 0; } + if (dirpool) + { + for (i = 1; i < dirpool->ndirs; i++) + { +#if 0 +if (cbdata.dirused) + fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused[i]); +#endif + id = dirpool->dirs[i]; + if (id <= 0) + continue; + if (cbdata.dirused && !cbdata.dirused[i]) + continue; + needid[id].need++; + } + } + +/********************************************************************/ + + /* + * create mapping table, new keys are sorted by needid[].need + * + * needid[key].need : old key -> new key + * needid[key].map : new key -> old key + */ + + /* zero out id 0 and rel 0 just in case */ needid[0].need = 0; - needid[pool->ss.nstrings].need = 0; - for (i = 0; i < pool->ss.nstrings + pool->nrels; i++) + needid[reloff].need = 0; + + for (i = 1; i < reloff + pool->nrels; i++) needid[i].map = i; cmp_pool = pool; - qsort(needid + 1, pool->ss.nstrings - 1, sizeof(*needid), needid_cmp_need_s); - qsort(needid + pool->ss.nstrings, pool->nrels, sizeof(*needid), needid_cmp_need); + qsort(needid + 1, reloff - 1, sizeof(*needid), needid_cmp_need_s); + qsort(needid + reloff, pool->nrels, sizeof(*needid), needid_cmp_need); sizeid = 0; - for (i = 1; i < pool->ss.nstrings; i++) + for (i = 1; i < reloff; i++) { if (!needid[i].need) break; @@ -487,103 +1135,93 @@ } nstrings = i; - for (i = 0; i < nstrings; i++) + for (i = 1; i < nstrings; i++) needid[needid[i].map].need = i; for (i = 0; i < pool->nrels; i++) { - if (!needid[pool->ss.nstrings + i].need) + if (!needid[reloff + i].need) break; else - needid[pool->ss.nstrings + i].need = 0; + needid[reloff + i].need = 0; } nrels = i; for (i = 0; i < nrels; i++) - { - needid[needid[pool->ss.nstrings + i].map].need = nstrings + i; + needid[needid[reloff + i].map].need = nstrings + i; + + +/********************************************************************/ + + /* create dir map */ + ndirmap = 0; + dirmap = 0; + if (dirpool) + { + if (cbdata.dirused && !cbdata.dirused[1]) + cbdata.dirused[1] = 1; /* always want / entry */ + dirmap = sat_calloc(dirpool->ndirs, sizeof(Id)); + dirpool_make_dirtraverse(dirpool); + dirmap[0] = 0; + ndirmap = traverse_dirs(dirpool, dirmap, 1, dirpool_child(dirpool, 0), cbdata.dirused); + if (!cbdata.dirused) + cbdata.dirused = sat_malloc2(dirpool->ndirs, sizeof(Id)); + memset(cbdata.dirused, 0, dirpool->ndirs * sizeof(Id)); + for (i = 1; i < ndirmap; i++) + { + if (dirmap[i] <= 0) + continue; + cbdata.dirused[dirmap[i]] = i; + dirmap[i] = needid[dirpool->dirs[dirmap[i]]].need; + } } - /* find the keys we need */ - nkeys = 1; - memset(id2key, 0, sizeof(id2key)); - for (i = SOLVABLE_NAME; i < ID_NUM_INTERNAL; i++) - if (idsizes[i]) - id2key[i] = nkeys++; - - /* find the schemata we need */ - memset(&schemata, 0, sizeof(schemata)); - solvschema = sat_calloc(repo->nsolvables, sizeof(Id)); +/********************************************************************/ + cbdata.extdata = sat_calloc(cbdata.nmykeys, sizeof(struct extdata)); + cbdata.incorelen = sat_calloc(repo->nsolvables, sizeof(Id)); + /* calculate incore/vertical data and sizes */ + ext0len = 0; - for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++) + for (i = 1; i < cbdata.nmykeys; i++) + if (cbdata.mykeys[i].storage != KEY_STORAGE_SOLVABLE) + break; + if (i < cbdata.nmykeys) { - if (s->repo != repo) - continue; - sp = schema; - *sp++ = SOLVABLE_NAME; - *sp++ = SOLVABLE_ARCH; - *sp++ = SOLVABLE_EVR; - if (s->vendor) - *sp++ = SOLVABLE_VENDOR; - if (s->provides) - *sp++ = SOLVABLE_PROVIDES; - if (s->obsoletes) - *sp++ = SOLVABLE_OBSOLETES; - if (s->conflicts) - *sp++ = SOLVABLE_CONFLICTS; - if (s->requires) - *sp++ = SOLVABLE_REQUIRES; - if (s->recommends) - *sp++ = SOLVABLE_RECOMMENDS; - if (s->suggests) - *sp++ = SOLVABLE_SUGGESTS; - if (s->supplements) - *sp++ = SOLVABLE_SUPPLEMENTS; - if (s->enhances) - *sp++ = SOLVABLE_ENHANCES; - if (s->freshens) - *sp++ = SOLVABLE_FRESHENS; - if (repo->rpmdbid) - *sp++ = RPM_RPMDBID; - *sp = 0; - solvschema[n++] = addschema(&schemata, schema); - } - - if (repodataschema) - { - /* add us a schema for our repodata */ - sp = schema; - *sp++ = REPODATA_EXTERNAL; - *sp++ = REPODATA_KEYS; - *sp++ = REPODATA_LOCATION; - *sp = 0; - repodataschema = addschema(&schemata, schema); - } - if (repodataschema_internal) - { - sp = schema; - *sp++ = REPODATA_EXTERNAL; - *sp++ = REPODATA_KEYS; - *sp = 0; - repodataschema_internal = addschema(&schemata, schema); - } - - /* convert all schemas to local keys */ - if (schemata.nschemata) - for (sp = schemata.schemadata; sp < schemata.schemadatap; sp++) - *sp = id2key[*sp]; + /* we need incore/vertical data */ + for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++) + { + if (s->repo != repo) + continue; + for (j = 0, data = repo->repodata; j < repo->nrepodata; j++, data++) + { + if (!repodataused[j]) + continue; + if (i < data->start || i >= data->end) + continue; + repodata_search(data, i - data->start, 0, repo_write_cb_sizes, &cbdata); + } + cbdata.incorelen[n] = cbdata.extdata[0].len - ext0len; + ext0len = cbdata.extdata[0].len; + n++; + } + } + +/********************************************************************/ + + /* write header */ /* write file header */ write_u32(fp, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V'); - write_u32(fp, SOLV_VERSION_3); + write_u32(fp, SOLV_VERSION_5); /* write counts */ write_u32(fp, nstrings); write_u32(fp, nrels); - write_u32(fp, nsolvables); - write_u32(fp, nkeys); - write_u32(fp, schemata.nschemata); - write_u32(fp, repo->nrepodata); /* info blocks. */ + write_u32(fp, ndirmap); + write_u32(fp, repo->nsolvables); + write_u32(fp, cbdata.nmykeys); + write_u32(fp, cbdata.nmyschemata); + write_u32(fp, 0); /* info blocks. */ solv_flags = 0; solv_flags |= SOLV_FLAG_PREFIX_POOL; #if 0 @@ -597,12 +1235,12 @@ that this actually is an expansion we can't easily reuse the stringspace for this. The max expansion per string is 1 byte, so it will fit into sizeid+nstrings bytes. */ - char *prefix = sat_malloc (sizeid + nstrings); + char *prefix = sat_malloc(sizeid + nstrings); char *pp = prefix; char *old_str = ""; for (i = 1; i < nstrings; i++) { - char *str = pool->ss.stringspace + pool->ss.strings[needid[i].map]; + char *str = spool->stringspace + spool->strings[needid[i].map]; int same; size_t len; for (same = 0; same < 255; same++) @@ -625,58 +1263,55 @@ perror("write error"); exit(1); } - sat_free (prefix); + sat_free(prefix); /* * write RelDeps */ for (i = 0; i < nrels; i++) { - ran = pool->rels + (needid[pool->ss.nstrings + i].map - pool->ss.nstrings); + ran = pool->rels + (needid[reloff + i].map - pool->ss.nstrings); write_id(fp, needid[ISRELDEP(ran->name) ? RELOFF(ran->name) : ran->name].need); write_id(fp, needid[ISRELDEP(ran->evr) ? RELOFF(ran->evr) : ran->evr].need); write_u8(fp, ran->flags); } /* + * write dirs (skip both root and / entry) + */ + for (i = 2; i < ndirmap; i++) + { + if (dirmap[i] > 0) + write_id(fp, dirmap[i]); + else + write_id(fp, nstrings - dirmap[i]); + } + + /* * write keys */ - for (i = SOLVABLE_NAME; i < ID_NUM_INTERNAL; i++) + for (i = 1; i < cbdata.nmykeys; i++) { - if (!idsizes[i]) - continue; - write_id(fp, needid[i].need); - if (i >= SOLVABLE_PROVIDES && i <= SOLVABLE_FRESHENS) - write_id(fp, TYPE_REL_IDARRAY); - else if (i == RPM_RPMDBID) - write_id(fp, TYPE_U32); - else if (i == REPODATA_EXTERNAL) - write_id(fp, TYPE_VOID); - else if (i == REPODATA_KEYS) - write_id(fp, TYPE_IDVALUEARRAY); - else if (i == REPODATA_LOCATION) - write_id(fp, TYPE_STR); + write_id(fp, needid[cbdata.mykeys[i].name].need); + write_id(fp, cbdata.mykeys[i].type); + if (cbdata.mykeys[i].storage != KEY_STORAGE_VERTICAL_OFFSET) + write_id(fp, cbdata.mykeys[i].size); else - write_id(fp, TYPE_ID); - write_id(fp, idsizes[i]); + write_id(fp, cbdata.extdata[i].len); + write_id(fp, cbdata.mykeys[i].storage); } /* * write schemata */ - if (schemata.nschemata) + write_id(fp, cbdata.myschemadatalen); + if (cbdata.nmyschemata) { - write_id(fp, schemata.schemadatap - schemata.schemadata - 1); - for (sp = schemata.schemadata + 1; sp < schemata.schemadatap; ) - { - write_idarray(fp, pool, 0, sp); - while (*sp++) - ; - } + for (i = 1; i < cbdata.nmyschemata; i++) + write_idarray(fp, pool, 0, cbdata.myschemadata + cbdata.myschemata[i]); } - else - write_id(fp, 0); +#if 0 /* * write info block */ @@ -697,46 +1332,120 @@ if (repo->repodata[i].location) write_str(fp, repo->repodata[i].location); } +#endif + + +/********************************************************************/ + /* * write Solvables */ + incoredata = cbdata.extdata[0].buf; for (i = repo->start, s = pool->solvables + i, n = 0; i < repo->end; i++, s++) { if (s->repo != repo) continue; /* keep in sync with schema generation! */ - write_id(fp, solvschema[n++]); - write_id(fp, needid[s->name].need); - write_id(fp, needid[s->arch].need); - write_id(fp, needid[s->evr].need); - if (s->vendor) + write_id(fp, cbdata.solvschemata[n]); +#if 0 +{ + Id *sp; + fprintf(stderr, "write solvable %d (%s): \n", n, id2str(pool, s->name)); + sp = cbdata.myschemadata + cbdata.myschemata[cbdata.solvschemata[n]]; + for (; *sp; sp++) + fprintf(stderr, " (%d,%d)", cbdata.mykeys[*sp].name, cbdata.mykeys[*sp].type); + fprintf(stderr, "\n"); +} +#endif + if (cbdata.keymap[SOLVABLE_NAME]); + write_id(fp, needid[s->name].need); + if (cbdata.keymap[SOLVABLE_ARCH]); + write_id(fp, needid[s->arch].need); + if (cbdata.keymap[SOLVABLE_EVR]); + write_id(fp, needid[s->evr].need); + if (s->vendor && cbdata.keymap[SOLVABLE_VENDOR]) write_id(fp, needid[s->vendor].need); - if (s->provides) + if (s->provides && cbdata.keymap[SOLVABLE_PROVIDES]) write_idarray_sort(fp, pool, needid, idarraydata + s->provides); - if (s->obsoletes) + if (s->obsoletes && cbdata.keymap[SOLVABLE_OBSOLETES]) write_idarray_sort(fp, pool, needid, idarraydata + s->obsoletes); - if (s->conflicts) + if (s->conflicts && cbdata.keymap[SOLVABLE_CONFLICTS]) write_idarray_sort(fp, pool, needid, idarraydata + s->conflicts); - if (s->requires) + if (s->requires && cbdata.keymap[SOLVABLE_REQUIRES]) write_idarray_sort(fp, pool, needid, idarraydata + s->requires); - if (s->recommends) + if (s->recommends && cbdata.keymap[SOLVABLE_RECOMMENDS]) write_idarray_sort(fp, pool, needid, idarraydata + s->recommends); - if (s->suggests) + if (s->suggests && cbdata.keymap[SOLVABLE_SUGGESTS]) write_idarray_sort(fp, pool, needid, idarraydata + s->suggests); - if (s->supplements) + if (s->supplements && cbdata.keymap[SOLVABLE_SUPPLEMENTS]) write_idarray_sort(fp, pool, needid, idarraydata + s->supplements); - if (s->enhances) + if (s->enhances && cbdata.keymap[SOLVABLE_ENHANCES]) write_idarray_sort(fp, pool, needid, idarraydata + s->enhances); - if (s->freshens) + if (s->freshens && cbdata.keymap[SOLVABLE_FRESHENS]) write_idarray_sort(fp, pool, needid, idarraydata + s->freshens); - if (repo->rpmdbid) + if (repo->rpmdbid && cbdata.keymap[RPM_RPMDBID]) write_u32(fp, repo->rpmdbid[i - repo->start]); + if (cbdata.incorelen[n]) + { + write_blob(fp, incoredata, cbdata.incorelen[n]); + incoredata += cbdata.incorelen[n]; + } + n++; + } + sat_free(cbdata.extdata[0].buf); + + /* write vertical data */ + for (i = 1; i < cbdata.nmykeys; i++) + if (cbdata.extdata[i].len) + break; + if (i < cbdata.nmykeys) + { + unsigned char *dp, vpage[BLOB_PAGESIZE]; + int l, ll, lpage = 0; + + write_u32(fp, BLOB_PAGESIZE); + for (i = 1; i < cbdata.nmykeys; i++) + { + if (!cbdata.extdata[i].len) + continue; + l = cbdata.extdata[i].len; + dp = cbdata.extdata[i].buf; + while (l) + { + ll = BLOB_PAGESIZE - lpage; + if (l < ll) + ll = l; + memcpy(vpage + lpage, dp, ll); + dp += ll; + lpage += ll; + l -= ll; + if (lpage == BLOB_PAGESIZE) + { + write_compressed_page(fp, vpage, lpage); + lpage = 0; + } + } + } + if (lpage) + write_compressed_page(fp, vpage, lpage); } +#if 0 + /* write vertical_offset entries */ + write_u32(fp, 0); /* no paging */ + for (i = 1; i < cbdata.nmykeys; i++) + if (cbdata.extdata[i].len) + write_blob(fp, cbdata.extdata[i].buf, cbdata.extdata[i].len); +#endif + + for (i = 1; i < cbdata.nmykeys; i++) + sat_free(cbdata.extdata[i].buf); + sat_free(needid); - sat_free(solvschema); - sat_free(schemata.schemadata); + sat_free(cbdata.solvschemata); + sat_free(cbdata.myschemadata); + sat_free(cbdata.myschemata); } // EOF Modified: trunk/sat-solver/tools/repo_write.h URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/repo_write.h?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/repo_write.h (original) +++ trunk/sat-solver/tools/repo_write.h Tue Jan 29 17:14:03 2008 @@ -18,6 +18,6 @@ #include "pool.h" #include "repo.h" -extern void repo_write(Repo *repo, FILE *fp); +void repo_write(Repo *repo, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata); #endif Modified: trunk/sat-solver/tools/rpmdb2solv.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/rpmdb2solv.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/rpmdb2solv.c (original) +++ trunk/sat-solver/tools/rpmdb2solv.c Tue Jan 29 17:14:03 2008 @@ -54,7 +54,7 @@ ref = 0; } - repo_write(repo, stdout); + repo_write(repo, stdout, 0, 0); pool_free(pool); exit(0); Modified: trunk/sat-solver/tools/rpmmd2solv.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/rpmmd2solv.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/rpmmd2solv.c (original) +++ trunk/sat-solver/tools/rpmmd2solv.c Tue Jan 29 17:14:03 2008 @@ -23,7 +23,7 @@ Pool *pool = pool_create(); Repo *repo = repo_create(pool, "<stdin>"); repo_add_rpmmd(repo, stdin); - repo_write(repo, stdout); + repo_write(repo, stdout, 0, 0); pool_free(pool); exit(0); } Modified: trunk/sat-solver/tools/susetags2solv.c URL: http://svn.opensuse.org/viewcvs/zypp/trunk/sat-solver/tools/susetags2solv.c?rev=8389&r1=8388&r2=8389&view=diff ============================================================================== --- trunk/sat-solver/tools/susetags2solv.c (original) +++ trunk/sat-solver/tools/susetags2solv.c Tue Jan 29 17:14:03 2008 @@ -16,9 +16,50 @@ #include "repo.h" #include "repo_susetags.h" #include "repo_write.h" +#if 0 #include "attr_store.h" extern Attrstore *attr; +#endif + +static char *verticals[] = { + "authors", + "description", + "messagedel", + "messageins", + "eula", + "diskusage", + 0 +}; + +static unsigned char *filter; +static int nfilter; + +static void +create_filter(Pool *pool) +{ + char **s; + Id id; + for (s = verticals; *s; s++) + { + id = str2id(pool, *s, 1); + if (id >= nfilter) + { + filter = sat_realloc(filter, id + 16); + memset(filter + nfilter, 0, id + 16 - nfilter); + nfilter = id + 16; + } + filter[id] = 1; + } +} + +static int +keyfilter(Repo *data, Repokey *key, void *kfdata) +{ + if (key->name < nfilter && filter[key->name]) + return KEY_STORAGE_VERTICAL_OFFSET; + return KEY_STORAGE_INCORE; +} int main(int argc, char **argv) @@ -41,13 +82,16 @@ Pool *pool = pool_create(); Repo *repo = repo_create(pool, "<stdin>"); repo_add_susetags(repo, stdin, 0, with_attr); - repo_write(repo, stdout); + create_filter(pool); + repo_write(repo, stdout, keyfilter, 0); +#if 0 if (with_attr && attr) { FILE *fp = fopen ("test.attr", "w"); write_attr_store (fp, attr); fclose (fp); } +#endif pool_free(pool); exit(0); } -- To unsubscribe, e-mail: zypp-commit+unsubscribe@opensuse.org For additional commands, e-mail: zypp-commit+help@opensuse.org