Hello community,
here is the log from the commit of package clicfs for openSUSE:Factory
checked in at Tue Aug 18 18:32:07 CEST 2009.
--------
--- clicfs/clicfs.changes 2009-08-12 20:46:41.000000000 +0200
+++ clicfs/clicfs.changes 2009-08-18 17:14:10.000000000 +0200
@@ -1,0 +2,7 @@
+Tue Aug 18 17:13:09 CEST 2009 - coolo@novell.com
+
+- update to 1.3.4
+ - better data structure for cache
+ - limit cache to 100MB
+
+-------------------------------------------------------------------
calling whatdependson for head-i586
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ clicfs.spec ++++++
--- /var/tmp/diff_new_pack.yL06VE/_old 2009-08-18 18:31:00.000000000 +0200
+++ /var/tmp/diff_new_pack.yL06VE/_new 2009-08-18 18:31:00.000000000 +0200
@@ -1,5 +1,5 @@
#
-# spec file for package clicfs (Version 1.3.3)
+# spec file for package clicfs (Version 1.3.4)
#
# Copyright (c) 2009 SUSE LINUX Products GmbH, Nuernberg, Germany.
#
@@ -22,7 +22,7 @@
BuildRequires: cmake e2fsprogs-devel fuse-devel gcc-c++ openssl-devel xz-devel
Requires: fuse
Summary: Compressed Loop Image Container
-Version: 1.3.3
+Version: 1.3.4
Release: 1
License: GPL v2 only
Group: System/Filesystems
++++++ clicfs.tar.bz2 ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/misc/.gitignore new/misc/.gitignore
--- old/misc/.gitignore 1970-01-01 01:00:00.000000000 +0100
+++ new/misc/.gitignore 2009-08-16 17:11:11.000000000 +0200
@@ -0,0 +1 @@
+e2block2file
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/clicfs.c new/src/clicfs.c
--- old/src/clicfs.c 2009-08-12 20:45:41.000000000 +0200
+++ new/src/clicfs.c 2009-08-16 17:11:11.000000000 +0200
@@ -29,6 +29,7 @@
#include
#include
#include
+#include
#define DEBUG 1
@@ -37,6 +38,7 @@
static size_t detached_allocated = 0;
static size_t sparse_memory = 0;
static char *cowfilename = 0;
+static off_t memory_used = 0;
static struct timeval start;
@@ -55,7 +57,7 @@
if (!cowfilename || cowfile_ro == 1 || !detached_allocated)
return 0;
- if (logger) fprintf(logger, "cow detached %dMB\n", (int)(detached_allocated / 1024));
+ //if (logger) fprintf(logger, "cow detached %dMB\n", (int)(detached_allocated / 1024));
uint32_t i;
for (i = 0; i < num_pages; ++i)
@@ -204,16 +206,18 @@
struct buffer_combo {
// the buffer of the part
unsigned char *out_buffer;
+ off_t out_buffer_size;
+ int mmapped;
uint32_t part;
time_t last_used;
- struct buffer_combo *next_by_part;
- struct buffer_combo *prev_by_part;
struct buffer_combo *next_by_use;
struct buffer_combo *prev_by_use;
};
// first
-struct buffer_combo *coms_sort_by_part = 0;
+struct buffer_combo **coms_by_part = 0;
+#define MAX_COMS_SIZE 30000
+int32_t coms_sort_by_part_size = 0;
struct buffer_combo *coms_sort_by_use_first = 0;
struct buffer_combo *coms_sort_by_use_last = 0;
static unsigned int com_count = 0;
@@ -222,16 +226,6 @@
FILE *pack;
-static void clic_insert_after(struct buffer_combo *prev, struct buffer_combo *com)
-{
- assert(prev->part < com->part);
- com->next_by_part = prev->next_by_part;
- prev->next_by_part = com;
- com->prev_by_part = prev;
- if (com->next_by_part)
- com->next_by_part->prev_by_part = com;
-}
-
static void clic_append_by_use(struct buffer_combo *com)
{
if (!coms_sort_by_use_last) {
@@ -246,45 +240,30 @@
}
/** I wrote this while watching TV, I know it sucks */
-static void clic_insert_com(struct buffer_combo *com)
+static void clic_insert_com(struct buffer_combo *com, int32_t after)
{
- if (!coms_sort_by_part) {
+ if (!coms_sort_by_part_size) {
assert(!coms_sort_by_use_first);
assert(!coms_sort_by_use_last);
- coms_sort_by_part = com;
+ coms_by_part[0] = com;
coms_sort_by_use_first = com;
coms_sort_by_use_last = com;
- com->next_by_part = 0;
com->next_by_use = 0;
- com->prev_by_part = 0;
com->prev_by_use = 0;
com_count++;
+ coms_sort_by_part_size++;
return;
}
- struct buffer_combo *first = coms_sort_by_part;
- if (first->part > com->part) {
- com->next_by_part = coms_sort_by_part;
- com->prev_by_part = 0;
- coms_sort_by_part->prev_by_part = com;
- coms_sort_by_part = com;
- first = 0;
- }
- while (first) {
- if (first->part < com->part)
- {
- if (!first->next_by_part) {
- clic_insert_after(first, com);
- break;
- } else {
- if (first->next_by_part->part < com->part)
- first = first->next_by_part;
- else {
- clic_insert_after(first, com);
- break;
- }
- }
- }
+ if (coms_sort_by_part_size == after + 1) { // just append
+ coms_by_part[coms_sort_by_part_size] = com;
+ } else {
+ // I don't like memmove
+ int i;
+ for (i = coms_sort_by_part_size-1; i > after; i--)
+ coms_by_part[i+1] = coms_by_part[i];
+ coms_by_part[after+1] = com;
}
+ coms_sort_by_part_size++;
clic_append_by_use(com);
}
@@ -294,7 +273,7 @@
return;
struct buffer_combo *c = coms_sort_by_use_first;
- fprintf(logger, "dump ");
+ fprintf(logger, "dump %ldMB ", memory_used / 1024 / 1024);
while (c) {
fprintf(logger, "%ld ", (long)c->part);
c = c->next_by_use;
@@ -302,19 +281,30 @@
fprintf(logger, "\n");
}
-static struct buffer_combo *clic_pick_part(uint32_t part)
+/* slightly modified binary_search.
+ If target is not found, return the index of the value that's
+ in the array before it
+*/
+int32_t binary_search(struct buffer_combo **A, size_t size, uint32_t target)
{
- pthread_mutex_lock(&picker);
- struct buffer_combo *com = coms_sort_by_part;
- while (com && com->part < part) {
- com = com->next_by_part;
- if (com && com->part == part)
- break;
+ if (!size)
+ return -1;
+ int lo = 0, hi = size-1;
+ if (target > A[hi]->part)
+ return hi;
+ while (lo <= hi) {
+ int mid = lo + (hi-lo)/2;
+ if (A[mid]->part == target)
+ return mid;
+ else {
+ if (A[mid]->part < target)
+ lo = mid+1;
+ else
+ hi = mid-1;
+ }
}
- if (com && com->part != part)
- com = 0;
- pthread_mutex_unlock(&picker);
- return com;
+
+ return hi;
}
static void clic_remove_com_from_use(struct buffer_combo *com)
@@ -336,53 +326,84 @@
static void clic_free_com(struct buffer_combo *com)
{
clic_remove_com_from_use(com);
- if (coms_sort_by_part == com)
- coms_sort_by_part = com->next_by_part;
-
- struct buffer_combo *n = com->next_by_part;
- struct buffer_combo *p = com->prev_by_part;
- if (n)
- n->prev_by_part = p;
- if (p)
- p->next_by_part = n;
-
- free(com->out_buffer);
+
+ if (com->mmapped == 1) {
+ int ret = munmap(com->out_buffer, com->out_buffer_size);
+ if (ret == -1) {
+ perror("munmap");
+ exit(1);
+ }
+ } else
+ free(com->out_buffer);
+ memory_used -= com->out_buffer_size;
+ int32_t res = binary_search(coms_by_part, coms_sort_by_part_size, com->part);
+ assert(coms_by_part[res] == com);
+ // I don't like memmove
+ while (res < coms_sort_by_part_size - 1)
+ {
+ coms_by_part[res] = coms_by_part[res+1];
+ res++;
+ }
free(com);
+ memory_used -= sizeof(struct buffer_combo);
+ coms_sort_by_part_size--;
}
static const unsigned char *clic_uncompress(uint32_t part)
{
//if (logger) fprintf(logger, "clic_uncompress %d %d\n", part, parts);
time_t now = time(0);
- struct buffer_combo *com = clic_pick_part(part);
-
- if (com)
+
+ if (coms_sort_by_use_first) // clean up
{
- const unsigned char *buf = com->out_buffer;
- if (com->last_used != now)
- {
- com->last_used = now;
- clic_remove_com_from_use(com);
- clic_append_by_use(com);
- if (1)
- clic_dump_use();
- }
- // if the oldest is 30s, drop it
- while (now - coms_sort_by_use_first->last_used > 30) {
+ if (0) clic_dump_use();
+ // if the oldest is 1m, drop it
+ while (coms_sort_by_use_first && (now - coms_sort_by_use_first->last_used > 60 || (memory_used > 1024 * 1024 * 100 && coms_sort_by_use_first->part != part))) {
clic_free_com(coms_sort_by_use_first);
}
+ //clic_dump_use();
+ }
+
+ int32_t res = binary_search(coms_by_part, coms_sort_by_part_size, part);
+ if (res >= 0 && coms_by_part[res] && coms_by_part[res]->part == part)
+ {
+ struct buffer_combo *com = coms_by_part[res];
+ const unsigned char *buf = com->out_buffer;
+ com->last_used = now;
+ clic_remove_com_from_use(com);
+ clic_append_by_use(com);
return buf;
}
- com = malloc(sizeof(struct buffer_combo));
- if (part < largeparts)
- com->out_buffer = malloc(blocksize_large*pagesize);
- else
+ // need room?
+ if (coms_sort_by_part_size == MAX_COMS_SIZE) {
+ // the index moves
+ if (coms_sort_by_use_first->part < part)
+ res--;
+ clic_free_com(coms_sort_by_use_first);
+ }
+
+ struct buffer_combo *com = malloc(sizeof(struct buffer_combo));
+ memory_used += sizeof(struct buffer_combo);
+ if (part < largeparts) {
+ com->out_buffer_size = blocksize_large*pagesize;
+ // TODO: round up to the next PAGE_SIZE (no worry for now)
+ com->out_buffer = mmap(0, com->out_buffer_size, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
+ if (com->out_buffer == MAP_FAILED) {
+ perror("mmap");
+ exit(1);
+ }
+ com->mmapped = 1;
+ } else {
+ com->out_buffer_size = blocksize_small*pagesize;
com->out_buffer = malloc(blocksize_small*pagesize);
+ com->mmapped = 0;
+ }
+ memory_used += com->out_buffer_size;
com->last_used = now;
com->part = part;
- clic_insert_com(com);
+ clic_insert_com(com, res);
pthread_mutex_lock(&seeker);
unsigned char *inbuffer = malloc(sizes[part]);
@@ -444,7 +465,7 @@
clic_read_block(newptr, block);
if (PTR_CLASS(ptr) == CLASS_COW) { // we need to mark the place in the cow obsolete
- if (logger) fprintf(logger, "detach block %ld (was %ld)\n", (long)block, (long)ptr >> 2);
+ //if (logger) fprintf(logger, "detach block %ld (was %ld)\n", (long)block, (long)ptr >> 2);
cows[cows_index++] = (long)ptr >> 2;
cow_pages--;
}
@@ -518,8 +539,6 @@
if (block >= num_pages)
return -EFAULT;
- clic_log_access(block);
-
if (!blockmap[block]) { // sparse block
memset(buf, 0, pagesize);
return pagesize;
@@ -541,6 +560,8 @@
assert(PTR_CLASS(ptr) == CLASS_RO); // in read only part
assert(block < num_pages);
+ clic_log_access(block);
+
off_t mapped_block = clic_map_block(block);
off_t part, off;
@@ -548,7 +569,7 @@
assert(part < parts);
- if (part >= largeparts && logger) { fprintf(logger, "big access %ld+8\n", block*8); }
+ //if (part >= largeparts && logger) { fprintf(logger, "big access %ld+8\n", block*8); }
const unsigned char *partbuf = clic_uncompress(part);
assert(partbuf);
@@ -777,7 +798,9 @@
for (i = 0; i < largeparts; ++i) {
posix_fadvise( fileno(packfile), offs[i], sizes[i], POSIX_FADV_SEQUENTIAL);
}
+ coms_by_part = malloc(sizeof(struct buffer_combo*)*MAX_COMS_SIZE);
gettimeofday(&start, 0);
+ /* MAIN LOOP */
int ret = fuse_main(args.argc, args.argv, &clic_oper, NULL);
clic_write_cow();
close(cowfilefd);
@@ -804,6 +827,7 @@
free(cowfilename);
if (cows)
free(cows);
+ free(coms_by_part);
clic_free_lzma();
fuse_opt_free_args(&args);
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Remember to have fun...
--
To unsubscribe, e-mail: opensuse-commit+unsubscribe@opensuse.org
For additional commands, e-mail: opensuse-commit+help@opensuse.org