ref: refs/heads/master
commit 444d9b743fd1825799b7abe138f51d465668f456
Author: Michael Schroeder
Date: Wed Sep 16 17:51:07 2009 +0200
- bring down memory usage by
* reducing REPODATA_ATTRS_BLOCK from 63 to 31
* freeing the string/rel hashes in tool_write
* not creating an in-core prefix compressed string area
---
ext/repo_write.c | 55 +++++++++++++++++++++++++++++++++++++++++--------
src/repodata.c | 2 +-
tools/common_write.c | 19 ++++++++---------
3 files changed, 56 insertions(+), 20 deletions(-)
diff --git a/ext/repo_write.c b/ext/repo_write.c
index 33cdfd2..9cb4f57 100644
--- a/ext/repo_write.c
+++ b/ext/repo_write.c
@@ -207,17 +207,15 @@ write_id_eof(FILE *fp, Id x, int eof)
-#if 0
-static void
+static inline void
write_str(FILE *fp, const char *str)
{
- if (fputs (str, fp) == EOF || putc (0, fp) == EOF)
+ if (fputs(str, fp) == EOF || putc(0, fp) == EOF)
{
- perror("write error");
+ perror("write error str");
exit(1);
}
}
-#endif
/*
* Array of Ids
@@ -251,7 +249,7 @@ write_idarray(FILE *fp, Pool *pool, NeedId *needid, Id *ids)
}
static int
-cmp_ids (const void *pa, const void *pb, void *dp)
+cmp_ids(const void *pa, const void *pb, void *dp)
{
Id a = *(Id *)pa;
Id b = *(Id *)pb;
@@ -303,7 +301,7 @@ write_idarray_sort(FILE *fp, Pool *pool, NeedId *needid, Id *ids, Id marker)
if (sids[i] == marker)
break;
if (i > 1)
- sat_sort(sids, i, sizeof (Id), cmp_ids, 0);
+ sat_sort(sids, i, sizeof(Id), cmp_ids, 0);
if ((len - i) > 2)
sat_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
@@ -478,7 +476,7 @@ data_addidarray_sort(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id
if (sids[i] == marker)
break;
if (i > 1)
- sat_sort(sids, i, sizeof (Id), cmp_ids, 0);
+ sat_sort(sids, i, sizeof(Id), cmp_ids, 0);
if ((len - i) > 2)
sat_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
@@ -1746,6 +1744,44 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
solv_flags |= SOLV_FLAG_PREFIX_POOL;
write_u32(fp, solv_flags);
+ /*
+ * calculate prefix encoding of the strings
+ */
+ unsigned char *prefixcomp = sat_malloc(nstrings);
+ unsigned int compsum = 0;
+ char *old_str = "";
+
+ prefixcomp[0] = 0;
+ for (i = 1; i < nstrings; i++)
+ {
+ char *str = spool->stringspace + spool->strings[needid[i].map];
+ int same;
+ for (same = 0; same < 255; same++)
+ if (!old_str[same] || old_str[same] != str[same])
+ break;
+ prefixcomp[i] = same;
+ compsum += same;
+ old_str = str;
+ }
+
+ /*
+ * write strings
+ */
+ write_u32(fp, sizeid);
+ /* we save compsum bytes but need 1 extra byte for every string */
+ write_u32(fp, sizeid + (nstrings ? nstrings - 1 : 0) - compsum);
+ if (sizeid + (nstrings ? nstrings - 1 : 0) != compsum)
+ {
+ for (i = 1; i < nstrings; i++)
+ {
+ char *str = spool->stringspace + spool->strings[needid[i].map];
+ write_u8(fp, prefixcomp[i]);
+ write_str(fp, str + prefixcomp[i]);
+ }
+ }
+ sat_free(prefixcomp);
+
+#if 0
/* Build the prefix-encoding of the string pool. We need to know
the size of that before writing it to the file, so we have to
build a separate buffer for that. As it's temporarily possible
@@ -1765,7 +1801,7 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
break;
*pp++ = same;
len = strlen(str + same) + 1;
- memcpy (pp, str + same, len);
+ memcpy(pp, str + same, len);
pp += len;
old_str = str;
}
@@ -1784,6 +1820,7 @@ fprintf(stderr, "dir %d used %d\n", i, cbdata.dirused ? cbdata.dirused[i] : 1);
}
}
sat_free(prefix);
+#endif
/*
* write RelDeps
diff --git a/src/repodata.c b/src/repodata.c
index 8844ea7..1b657f7 100644
--- a/src/repodata.c
+++ b/src/repodata.c
@@ -1768,7 +1768,7 @@ repodata_extend_block(Repodata *data, Id start, Id num)
/**********************************************************************/
-#define REPODATA_ATTRS_BLOCK 63
+#define REPODATA_ATTRS_BLOCK 31
#define REPODATA_ATTRDATA_BLOCK 1023
#define REPODATA_ATTRIDDATA_BLOCK 63
diff --git a/tools/common_write.c b/tools/common_write.c
index bb4bd19..73067bd 100644
--- a/tools/common_write.c
+++ b/tools/common_write.c
@@ -92,22 +92,21 @@ keyfilter_attr(Repo *data, Repokey *key, void *kfdata)
static int
keyfilter_language(Repo *repo, Repokey *key, void *kfdata)
{
+ Pool *pool = repo->pool;
const char *name, *p;
- char *lang = kfdata, *bname;
+ char *lang = kfdata;
int i;
- Id id;
name = id2str(repo->pool, key->name);
p = strrchr(name, ':');
if (!p || strcmp(p + 1, lang) != 0)
return KEY_STORAGE_DROPPED;
- /* find base name id */
- bname = strdup(name);
- bname[p - name] = 0;
- id = str2id(repo->pool, bname, 1);
for (i = 0; verticals[i]; i++)
- if (id == verticals[i])
- return KEY_STORAGE_VERTICAL_OFFSET;
+ {
+ const char *vname = id2str(pool, verticals[i]);
+ if (!strncmp(name, vname, p - name) && vname[p - name] == 0)
+ return KEY_STORAGE_VERTICAL_OFFSET;
+ }
return KEY_STORAGE_INCORE;
}
@@ -213,6 +212,8 @@ tool_write(Repo *repo, const char *basename, const char *attrname)
}
sat_free(addedfileprovides);
+ pool_freeidhashes(repo->pool); /* free some mem */
+
if (basename)
{
char fn[4096];
@@ -302,7 +303,6 @@ tool_write(Repo *repo, const char *basename, const char *attrname)
free(languages[i]);
sat_free(languages);
repodata_free(info);
- repo->nrepodata--;
return 0;
}
if (attrname)
@@ -316,6 +316,5 @@ tool_write(Repo *repo, const char *basename, const char *attrname)
repodata_internalize(info);
repo_write(repo, stdout, keyfilter_solv, &kd, 0);
repodata_free(info);
- repo->nrepodata--;
return 0;
}
--
To unsubscribe, e-mail: zypp-commit+unsubscribe@opensuse.org
For additional commands, e-mail: zypp-commit+help@opensuse.org
