Hello community,
here is the log from the commit of package clicfs for openSUSE:Factory
checked in at Tue Aug 11 15:57:25 CEST 2009.
--------
--- clicfs/clicfs.changes 2009-05-19 18:02:31.000000000 +0200
+++ clicfs/clicfs.changes 2009-08-10 10:05:44.000000000 +0200
@@ -1,0 +2,13 @@
+Mon Aug 10 10:05:09 CEST 2009 - coolo@novell.com
+
+- update to 1.3
+ - caching readonly is dynamic now and profiled data
+ is stored in large parts
+
+-------------------------------------------------------------------
+Thu Aug 6 10:34:02 CEST 2009 - coolo@novell.com
+
+- update to 1.2:
+ - implement cow on special devices use case
+
+-------------------------------------------------------------------
calling whatdependson for head-i586
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ clicfs.spec ++++++
--- /var/tmp/diff_new_pack.HCIE8g/_old 2009-08-11 15:57:11.000000000 +0200
+++ /var/tmp/diff_new_pack.HCIE8g/_new 2009-08-11 15:57:11.000000000 +0200
@@ -1,5 +1,5 @@
#
-# spec file for package clicfs (Version 1.1.3)
+# spec file for package clicfs (Version 1.3)
#
# Copyright (c) 2009 SUSE LINUX Products GmbH, Nuernberg, Germany.
#
@@ -22,7 +22,7 @@
BuildRequires: cmake fuse-devel gcc-c++ openssl-devel xz-devel
Requires: fuse
Summary: Compressed Loop Image Container
-Version: 1.1.3
+Version: 1.3
Release: 1
License: GPL v2 only
Group: System/Filesystems
++++++ clicfs.tar.bz2 ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/clicfs.c new/src/clicfs.c
--- old/src/clicfs.c 2009-05-19 18:01:14.000000000 +0200
+++ new/src/clicfs.c 2009-08-08 22:35:42.000000000 +0200
@@ -30,6 +30,8 @@
#include
#include
+#define DEBUG 1
+
FILE *logger = 0;
static size_t detached_allocated = 0;
@@ -40,43 +42,91 @@
static uint32_t clic_find_next_cow()
{
- //fprintf(stderr, "clic_find_next %ld %ld\n", (long)cows_index, (long)cow_pages);
if (cows_index > 0)
return cows[--cows_index];
- return cow_pages++;
+ return cow_pages + cow_index_pages;
}
+static int clic_detach(size_t block);
+static int clic_write_cow();
+
static int clic_write_cow()
{
if (!cowfilename || cowfile_ro == 1 || !detached_allocated)
return 0;
- fprintf(stderr, "cow detached %dMB\n", (int)(detached_allocated / 1024));
+ if (logger) fprintf(logger, "cow detached %dMB\n", (int)(detached_allocated / 1024));
- uint32_t indexlen = 0;
uint32_t i;
for (i = 0; i < num_pages; ++i)
{
long ptr = (long)blockmap[i];
if ( ptr && PTR_CLASS(ptr) == CLASS_MEMORY ) { // detached now
- uint32_t cowindex = clic_find_next_cow();
- lseek(cowfilefd, cowindex * pagesize, SEEK_SET);
+ off_t cowindex = clic_find_next_cow();
+ off_t seeked = lseek(cowfilefd, cowindex * pagesize, SEEK_SET);
+ assert(seeked == (off_t)(cowindex * pagesize));
size_t ret = write(cowfilefd, blockmap[i], pagesize);
assert(ret == pagesize);
free(blockmap[i]);
detached_allocated -= (pagesize / 1024);
blockmap[i] = (unsigned char*)(long)(cowindex << 2) + 2;
+ cow_pages++;
}
}
assert(!detached_allocated);
- lseek(cowfilefd, cow_pages * pagesize, SEEK_SET);
+ off_t seeked = lseek(cowfilefd, 0, SEEK_SET);
+ assert(seeked == 0);
uint64_t stringlen = thefilesize;
- indexlen += write(cowfilefd, (char*)&stringlen, sizeof(uint64_t));
+
+ char head[10];
+ sprintf(head, "CLICCOW%02d", DOENER_MAGIC);
+ uint32_t index_len = write(cowfilefd, head, 9);
+
+ index_len += write(cowfilefd, (char*)&stringlen, sizeof(uint64_t));
stringlen = cow_pages;
- indexlen += write(cowfilefd, (char*)&stringlen, sizeof(uint32_t));
+ index_len += write(cowfilefd, (char*)&stringlen, sizeof(uint32_t));
stringlen = 0;
+
+ index_len += 2 * sizeof(uint32_t) * cow_pages;
+ uint32_t new_cow_index_pages = index_len / pagesize + 1;
+ uint32_t moving;
+ uint32_t moved = 0;
+
+ // should all be out
+ assert(cows_index == 0);
+
+ for (moving = cow_index_pages; moving < new_cow_index_pages; ++moving)
+ {
+ // we only have a map from memory to cow, so we need to
+ // look up in reverse to find the page to move
+ // if this proves to be slow, we need even more memory
+ // to keep the reverse map
+ for (i = 0; i < num_pages; ++i)
+ {
+ long ptr = (long)blockmap[i];
+ if (PTR_CLASS(ptr) == CLASS_COW) { // block
+ if ((ptr >> 2) == moving) {
+ if (logger) fprintf(logger, "moving %ld %ld\n", (long)moving, (long)i);
+ clic_detach(i);
+ moved++;
+ break;
+ }
+ }
+ }
+ }
+
+ assert(moved == cows_index);
+
+ cow_index_pages = new_cow_index_pages;
+
+ /* if we moved, we need to redetach */
+ if (moved) {
+ cows_index = 0;
+ return clic_write_cow();
+ }
+
for (i = 0; i < num_pages; ++i)
{
long ptr = (long)blockmap[i];
@@ -84,16 +134,18 @@
uint32_t key = i, value = ptr >> 2;
write(cowfilefd, (char*)&key, sizeof(uint32_t));
write(cowfilefd, (char*)&value, sizeof(uint32_t));
- indexlen += 2 * sizeof(uint32_t);
stringlen++;
}
}
assert(stringlen == cow_pages);
- write(cowfilefd, (char*)&indexlen, sizeof(uint32_t));
+ write(cowfilefd, (char*)&index_len, sizeof(uint32_t));
return 0;
}
+/**
+ * fuse callback to get stat informations
+ */
static int clic_getattr(const char *path, struct stat *stbuf)
{
//fprintf(logger, "getattr %s\n", path);
@@ -115,6 +167,10 @@
return res;
}
+/**
+ * fuse callback to get directory informations.
+ * We only have one file in one dir
+ */
static int clic_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
off_t offset, struct fuse_file_info *fi)
{
@@ -141,78 +197,216 @@
// if((fi->flags & 3) != O_RDONLY)
// return -EACCES;
- fi->keep_cache = 1;
+ fi->keep_cache = 1;
return 0;
}
struct buffer_combo {
+ // the buffer of the part
unsigned char *out_buffer;
-
uint32_t part;
- pthread_mutex_t lock;
- int free;
- int index;
- int used;
+ time_t last_used;
+ struct buffer_combo *next_by_part;
+ struct buffer_combo *prev_by_part;
+ struct buffer_combo *next_by_use;
+ struct buffer_combo *prev_by_use;
};
-static int used_counter = 0;
-
-struct buffer_combo *coms;
-static unsigned int com_count = 1;
+// first
+struct buffer_combo *coms_sort_by_part = 0;
+struct buffer_combo *coms_sort_by_use_first = 0;
+struct buffer_combo *coms_sort_by_use_last = 0;
+static unsigned int com_count = 0;
pthread_mutex_t picker = PTHREAD_MUTEX_INITIALIZER, seeker = PTHREAD_MUTEX_INITIALIZER;;
FILE *pack;
-static const unsigned char *clic_uncompress(uint32_t part)
+static void clic_insert_after(struct buffer_combo *prev, struct buffer_combo *com)
{
- struct buffer_combo *com;
+ assert(prev->part < com->part);
+ com->next_by_part = prev->next_by_part;
+ prev->next_by_part = com;
+ com->prev_by_part = prev;
+ if (com->next_by_part)
+ com->next_by_part->prev_by_part = com;
+}
+
+static void clic_append_by_use(struct buffer_combo *com)
+{
+ assert(coms_sort_by_use_last);
+ coms_sort_by_use_last->next_by_use = com;
+ com->prev_by_use = coms_sort_by_use_last;
+ com->next_by_use = 0;
+ coms_sort_by_use_last = com;
+}
+
+/** I wrote this while watching TV, I know it sucks */
+static void clic_insert_com(struct buffer_combo *com)
+{
+ if (!coms_sort_by_part) {
+ assert(!coms_sort_by_use_first);
+ assert(!coms_sort_by_use_last);
+ coms_sort_by_part = com;
+ coms_sort_by_use_first = com;
+ coms_sort_by_use_last = com;
+ com->next_by_part = 0;
+ com->next_by_use = 0;
+ com->prev_by_part = 0;
+ com->prev_by_use = 0;
+ com_count++;
+ return;
+ }
+ struct buffer_combo *first = coms_sort_by_part;
+ while (first) {
+ if (first->part < com->part)
+ {
+ if (!first->next_by_part) {
+ clic_insert_after(first, com);
+ break;
+ } else {
+ if (first->next_by_part->part < com->part)
+ first = first->next_by_part;
+ else {
+ clic_insert_after(first, com);
+ break;
+ }
+ }
+ }
+ }
+ clic_append_by_use(com);
+}
- //fprintf(logger, "clic_uncompress %d %d %d\n", part, parts, wparts);
+/**
+ * the use list is sorted from oldest to newest buffer. This function switches two
+ elememnts from P->F->S->N to P->S->F->N (first is F and second is S)
+*/
+static void clic_switch_use(struct buffer_combo *first, struct buffer_combo *second)
+{
+ assert(first->last_used > second->last_used);
+ struct buffer_combo *n = second->next_by_use;
+ struct buffer_combo *p = first->prev_by_use;
+
+ second->next_by_use = first;
+ first->next_by_use = n;
+ if (p)
+ p->next_by_use = second;
+
+ second->prev_by_use = p;
+ if (n)
+ n->prev_by_use = first;
+ first->prev_by_use = second;
+ if (coms_sort_by_use_last == second)
+ coms_sort_by_use_last = first;
+ if (coms_sort_by_use_first == first)
+ coms_sort_by_use_first = second;
+}
- pthread_mutex_lock(&picker);
- int index = -1;
- unsigned int i;
- for (i = 0; i < com_count; ++i)
+static int clic_resort_by_use(struct buffer_combo *c)
+{
+ // the assertion of the list is prev->lu < next->lu
+ if (c->prev_by_use && c->prev_by_use->last_used > c->last_used)
{
- if (coms[i].part == part)
- {
- index = i;
- break;
- }
+ clic_switch_use(c->prev_by_use, c);
+ clic_resort_by_use(c);
+ return 1;
}
- if (index == -1)
+
+ if (c->next_by_use && c->next_by_use->last_used < c->last_used)
{
- index = 0;
- for (i = 0; i < com_count -1; ++i)
- {
- if (coms[i].free) {
- index = i;
- break;
- }
- }
- for (i = index + 1; i < com_count; ++i)
- {
- if (coms[i].free && coms[index].used > coms[i].used)
- index = i;
- }
+ clic_switch_use(c, c->next_by_use);
+ clic_resort_by_use(c);
+ return 1;
+ }
+
+ return 0;
+}
+
+static void clic_dump_use()
+{
+ if (!logger)
+ return;
+
+ struct buffer_combo *c = coms_sort_by_use_first;
+ fprintf(logger, "dump ");
+ while (c) {
+ fprintf(logger, "%ld ", (long)c->part);
+ c = c->next_by_use;
+ }
+ fprintf(logger, "\n");
+}
+
+static struct buffer_combo *clic_pick_part(uint32_t part)
+{
+ pthread_mutex_lock(&picker);
+ struct buffer_combo *com = coms_sort_by_part;
+ while (com && com->part < part) {
+ com = com->next_by_part;
+ if (com && com->part == part)
+ break;
}
- com = coms + index;
- pthread_mutex_lock(&com->lock);
- com->free = 0;
- com->used = used_counter++;
+ if (com && com->part != part)
+ com = 0;
pthread_mutex_unlock(&picker);
+ return com;
+}
+
+static void clic_free_com(struct buffer_combo *com)
+{
+ if (coms_sort_by_use_first == com)
+ coms_sort_by_use_first = com->next_by_use;
+ assert(com != coms_sort_by_use_last);
+ // P->C->N -> P->N
+ struct buffer_combo *n = com->next_by_use;
+ struct buffer_combo *p = com->prev_by_use;
+ if (n)
+ n->prev_by_use = p;
+ if (p)
+ p->next_by_use = n;
+
+ if (coms_sort_by_part == com)
+ coms_sort_by_part = com->next_by_part;
+
+ n = com->next_by_part;
+ p = com->prev_by_part;
+ if (n)
+ n->prev_by_part = p;
+ if (p)
+ p->next_by_part = n;
- if (com->part == part)
+ free(com->out_buffer);
+ free(com);
+}
+
+static const unsigned char *clic_uncompress(uint32_t part)
+{
+ //if (logger) fprintf(logger, "clic_uncompress %d %d\n", part, parts);
+ time_t now = time(0);
+ struct buffer_combo *com = clic_pick_part(part);
+
+ if (com)
{
const unsigned char *buf = com->out_buffer;
- com->free = 1;
- pthread_mutex_unlock(&com->lock);
+ com->last_used = now;
+ if (clic_resort_by_use(com) && 1)
+ clic_dump_use();
+ // if the oldest is 30s, drop it
+ while (now - coms_sort_by_use_first->last_used > 30) {
+ clic_free_com(coms_sort_by_use_first);
+ }
return buf;
}
+ com = malloc(sizeof(struct buffer_combo));
+ if (part < largeparts)
+ com->out_buffer = malloc(blocksize_large*pagesize);
+ else
+ com->out_buffer = malloc(blocksize_small*pagesize);
+ com->last_used = now;
com->part = part;
+ clic_insert_com(com);
+
pthread_mutex_lock(&seeker);
unsigned char *inbuffer = malloc(sizes[part]);
struct timeval begin, end;
@@ -221,7 +415,7 @@
gettimeofday(&end, 0);
#if defined(DEBUG)
- if (logger) fprintf(logger, "uncompress %d %d %ld %ld (read took %ld - started %ld)\n", part, com->index, (long)offs[part], (long)sizes[part], (end.tv_sec - begin.tv_sec) * 1000 + (end.tv_usec - begin.tv_usec) / 1000, (begin.tv_sec - start.tv_sec) * 1000 + (begin.tv_usec - start.tv_usec) / 1000 );
+ if (logger) fprintf(logger, "uncompress %d %ld-%ld %ld (read took %ld - started %ld)\n", part, (long)offs[part], (long)sizes[part], (long)readin, (end.tv_sec - begin.tv_sec) * 1000 + (end.tv_usec - begin.tv_usec) / 1000, (begin.tv_sec - start.tv_sec) * 1000 + (begin.tv_usec - start.tv_usec) / 1000 );
#endif
if (!readin)
return 0;
@@ -230,32 +424,27 @@
clic_decompress_part(com->out_buffer, inbuffer, readin);
free(inbuffer);
- com->part = part;
- com->free = 1;
-
- pthread_mutex_unlock(&com->lock);
-
return com->out_buffer;
}
static void clic_log_access(size_t block)
{
- if (!logger) return;
+ if (!logger) return;
- static size_t firstblock = 0;
- static ssize_t lastblock = -1;
+ static size_t firstblock = 0;
+ static ssize_t lastblock = -1;
- if (lastblock >= 0 && block != (size_t)(lastblock + 1))
- {
- fprintf(logger, "access %ld+%ld\n", (long)firstblock, (long)lastblock-firstblock);
- firstblock = block;
- }
- lastblock = block;
- if (block > firstblock + 30)
- {
- fprintf(logger, "access %ld+%ld\n", (long)firstblock, (long)lastblock-firstblock);
- firstblock = block;
- }
+ if (lastblock >= 0 && block != (size_t)(lastblock + 1))
+ {
+ fprintf(logger, "access %ld+%ld\n", (long)firstblock*8, (long)(lastblock-firstblock+1)*8);
+ firstblock = block;
+ }
+ lastblock = block;
+ if (block > firstblock + 30)
+ {
+ fprintf(logger, "access %ld+%ld\n", (long)firstblock*8, (long)(lastblock-firstblock+1)*8);
+ firstblock = block;
+ }
}
static size_t clic_read_block(char *buf, size_t block);
@@ -277,8 +466,11 @@
if (logger && detached_allocated % 1024 == 0 ) fprintf(logger, "detached %dMB\n", (int)(detached_allocated / 1024));
clic_read_block(newptr, block);
- if (PTR_CLASS(ptr) == CLASS_COW) // we need to mark the place in the cow obsolete
+ if (PTR_CLASS(ptr) == CLASS_COW) { // we need to mark the place in the cow obsolete
+ if (logger) fprintf(logger, "detach block %ld (was %ld)\n", (long)block, (long)ptr >> 2);
cows[cows_index++] = (long)ptr >> 2;
+ cow_pages--;
+ }
blockmap[block] = (unsigned char*)newptr;
return 1;
@@ -364,7 +556,8 @@
}
if (PTR_CLASS(ptr) == CLASS_COW) {
- lseek(cowfilefd, (ptr >> 2) * pagesize, SEEK_SET);
+ off_t target = ptr >> 2;
+ lseek(cowfilefd, target * pagesize, SEEK_SET);
return read(cowfilefd, buf, pagesize);
}
@@ -372,13 +565,14 @@
assert(block < num_pages);
off_t mapped_block = clic_map_block(block);
-
- size_t part = (size_t)(mapped_block / bsize);
+
+ off_t part, off;
+ clic_find_block( mapped_block, &part, &off);
assert(part < parts);
const unsigned char *partbuf = clic_uncompress(part);
assert(partbuf);
- memcpy(buf, partbuf + pagesize * (mapped_block % bsize), pagesize);
+ memcpy(buf, partbuf + pagesize * off, pagesize);
return pagesize;
}
@@ -444,16 +638,6 @@
.fsync = clic_fsync
};
-static void clic_init_buffer(int i)
-{
- coms[i].part = -1;
- coms[i].used = 0;
- coms[i].index = i + 1;
- coms[i].free = 1;
- coms[i].out_buffer = malloc(bsize*pagesize);
- pthread_mutex_init(&coms[i].lock, 0);
-}
-
char *packfilename = 0;
char *logfile = 0;
int ignore_cow_errors = 0;
@@ -495,6 +679,7 @@
break;
case FUSE_OPT_IGNORE_COW_ERRORS:
ignore_cow_errors = 1;
+ return 0;
break;
}
@@ -503,21 +688,32 @@
static int init_cow()
{
- FILE *cow = fopen(cowfilename, "w");
- if (!cow) {
- perror("opening cow");
- return 1;
- }
- uint64_t stringlen64 = (thefilesize / pagesize * pagesize) + sparse_memory * 1024 * 1024;
- fwrite((char*)&stringlen64, 1, sizeof(uint64_t), cow);
- uint32_t stringlen = 0;
- // there are 0 blocks
- fwrite((char*)&stringlen, 1, sizeof(uint32_t), cow);
- // the whole index is 12 bytes long
- stringlen = sizeof(uint32_t) + sizeof(uint64_t);
- fwrite((char*)&stringlen, 1, sizeof(uint32_t), cow);
- fclose(cow);
- return 0;
+ FILE *cow = fopen(cowfilename, "w");
+ if (!cow) {
+ perror("opening cow");
+ return 1;
+ }
+ uint64_t bigfilesize = (thefilesize / pagesize * pagesize);
+ if (bigfilesize < thefilesize)
+ thefilesize += pagesize;
+ bigfilesize += sparse_memory * 1024 * 1024;
+
+ assert( DOENER_MAGIC < 100 );
+ int index_len = fprintf(cow, "CLICCOW%02d", DOENER_MAGIC );
+
+ index_len += fwrite((char*)&bigfilesize, 1, sizeof(uint64_t), cow);
+ uint32_t stringlen = 0;
+ // there are 0 blocks
+ index_len += fwrite((char*)&stringlen, 1, sizeof(uint32_t), cow);
+ // the whole index is 12 bytes long
+ stringlen = index_len + sizeof(uint32_t);
+ index_len += fwrite((char*)&stringlen, 1, sizeof(uint32_t), cow);
+ fclose(cow);
+
+ cow_index_pages = index_len / pagesize + 1;
+ cow_pages = 0;
+
+ return 0;
}
int main(int argc, char *argv[])
@@ -525,20 +721,20 @@
struct fuse_args args = FUSE_ARGS_INIT(argc, argv);
if (fuse_opt_parse(&args, NULL, clic_opt, clic_opt_proc) == -1) {
- perror("fuse_opt_part");
- return 1;
+ perror("fuse_opt_part");
+ return 1;
}
if (logfile) {
- if (!strcmp(logfile, "-"))
- logger = stderr;
- else
- logger = fopen(logfile, "w");
- if (!logger) {
- perror("open");
- return 1;
- }
- free(logfile);
+ if (!strcmp(logfile, "-"))
+ logger = stderr;
+ else
+ logger = fopen(logfile, "w");
+ if (!logger) {
+ perror("open");
+ return 1;
+ }
+ free(logfile);
}
// not sure why but multiple threads make it slower
@@ -550,46 +746,39 @@
}
if (clicfs_read_pack(packfilename)) {
- perror("read_pack");
- return 1;
+ perror("read_pack");
+ return 1;
}
free(packfilename);
if (cowfilename) {
- if (access(cowfilename, R_OK))
- init_cow();
+ if (access(cowfilename, R_OK))
+ init_cow();
- if (clicfs_read_cow(cowfilename)) {
- if (!ignore_cow_errors)
- return 1;
+ if (clicfs_read_cow(cowfilename)) {
+ if (!ignore_cow_errors)
+ return 1;
- init_cow();
- if (clicfs_read_cow(cowfilename))
- return 1;
- }
- sparse_memory = 0; // ignore the option if we have a cow
+ init_cow();
+ if (clicfs_read_cow(cowfilename))
+ return 1;
+ }
+ sparse_memory = 0; // ignore the option if we have a cow
}
// fake for write
if (sparse_memory) {
- thefilesize = (thefilesize / pagesize * pagesize) + sparse_memory * 1024 * 1024;
- size_t write_pages = thefilesize / pagesize;
- blockmap = realloc(blockmap, sizeof(unsigned char*)*write_pages);
- uint32_t i;
- for (i = num_pages; i < write_pages; ++i)
- blockmap[i] = 0;
- num_pages = write_pages;
+ thefilesize = (thefilesize / pagesize * pagesize) + sparse_memory * 1024 * 1024;
+ size_t write_pages = thefilesize / pagesize;
+ blockmap = realloc(blockmap, sizeof(unsigned char*)*write_pages);
+ uint32_t i;
+ for (i = num_pages; i < write_pages; ++i)
+ blockmap[i] = 0;
+ num_pages = write_pages;
}
- uint32_t i;
-
- com_count = 6000000 / (bsize*pagesize); // get 6MB of cache
- coms = malloc(sizeof(struct buffer_combo) * com_count);
- for (i = 0; i < com_count; ++i)
- clic_init_buffer(i);
-
gettimeofday(&start, 0);
int ret = fuse_main(args.argc, args.argv, &clic_oper, NULL);
clic_write_cow();
@@ -598,9 +787,6 @@
if (logger) fclose(logger);
free(blockmap);
- for (i = 0; i < com_count; ++i)
- free(coms[i].out_buffer);
- free(coms);
free(sizes);
free(offs);
fclose(packfile);
@@ -609,6 +795,7 @@
free(cowfilename);
if (cows)
free(cows);
+ clic_free_lzma();
fuse_opt_free_args(&args);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/clicfs.h new/src/clicfs.h
--- old/src/clicfs.h 2009-05-19 18:01:14.000000000 +0200
+++ new/src/clicfs.h 2009-08-08 22:35:42.000000000 +0200
@@ -26,7 +26,8 @@
extern FILE *packfile;
extern int cowfilefd;
-#define DOENER_MAGIC 1
+// magic 2 added large parts
+#define DOENER_MAGIC 2
#define PTR_CLASS(x) ((long)x & 0x3)
@@ -39,12 +40,21 @@
extern size_t pagesize;
extern uint64_t *sizes;
extern uint64_t *offs;
+// the number of parts all in all
extern uint32_t parts;
+// how many parts contain many blocks, the rest is small
+extern uint32_t largeparts;
extern uint32_t pindex;
-extern size_t bsize;
+// the number of pages in a part (~32)
+extern size_t blocksize_small;
+// the number of pages in a large part (~3200)
+extern size_t blocksize_large;
+// the number of pages in total (full image)
extern uint32_t num_pages;
-// the number of pages in the cow index
+// the number of pages marked as CLASS_COW
extern uint32_t cow_pages;
+// the number of pages in the cow for index
+extern unsigned int cow_index_pages;
// support temporary changes on ro medium
extern int cowfile_ro;
@@ -62,3 +72,6 @@
extern size_t clic_readpart(unsigned char *buffer, int part);
extern off_t clic_map_block(off_t block);
extern uint32_t clic_readindex_fd(int fd );
+extern void clic_free_lzma();
+
+extern void clic_find_block( off_t block, off_t *part, off_t *offset );
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/clicfs_common.c new/src/clicfs_common.c
--- old/src/clicfs_common.c 2009-05-19 18:01:14.000000000 +0200
+++ new/src/clicfs_common.c 2009-08-08 22:35:42.000000000 +0200
@@ -37,11 +37,14 @@
uint64_t *sizes = 0;
uint64_t *offs = 0;
uint32_t parts = 0;
+uint32_t largeparts = 0;
uint32_t pindex = 0;
-size_t bsize = 0;
+size_t blocksize_large = 0;
+size_t blocksize_small = 0;
unsigned char **blockmap;
uint32_t num_pages = 0;
uint32_t cow_pages = 0;
+uint32_t cow_index_pages = 0;
uint32_t *cows = 0;
unsigned int cows_index = 0;
int cowfile_ro = 0;
@@ -90,13 +93,21 @@
}
} else
cowfile_ro = 0;
-
- struct stat st;
- fstat(cowfilefd, &st);
- lseek(cowfilefd, st.st_size - sizeof(uint32_t), SEEK_SET);
- uint32_t indexlen = clic_readindex_fd(cowfilefd) + sizeof(uint32_t);
- if (lseek(cowfilefd, st.st_size - indexlen, SEEK_SET ) == -1)
- perror("seek");
+
+ char head[10];
+ char expected[10];
+ if (read(cowfilefd, head, 9) != 9) {
+ fprintf(stderr, "can't read from %s\n", cowfilename);
+ return 1;
+ }
+
+ head[9] = 0;
+ sprintf(expected, "CLICCOW%02d", DOENER_MAGIC);
+ if (strcmp(head,expected)) {
+ fprintf(stderr, "wrong magic: %s vs %s\n", head, expected);
+ return 1;
+ }
+
thefilesize = clic_readindex_fd64(cowfilefd);
uint32_t newpages = thefilesize / pagesize;
blockmap = realloc(blockmap, sizeof(unsigned char*)*newpages);
@@ -114,6 +125,10 @@
}
cows = malloc(sizeof(uint32_t) * CLICFS_COW_COUNT);
cows_index = 0;
+
+ uint32_t index_len = clic_readindex_fd(cowfilefd);
+ cow_index_pages = index_len / pagesize + 1;
+
return 0;
}
@@ -146,9 +161,10 @@
thefile[stringlen] = 0;
uint64_t oparts = clic_readindex_file(packfile);
- bsize = clic_readindex_file(packfile);
+ blocksize_small = clic_readindex_file(packfile);
+ blocksize_large = clic_readindex_file(packfile);
pagesize = clic_readindex_file(packfile);
- thefilesize = oparts * bsize * pagesize;
+ thefilesize = oparts * blocksize_small * pagesize;
preset = clic_readindex_file(packfile);
num_pages = clic_readindex_file(packfile);
blockmap = malloc(sizeof(unsigned char*)*num_pages);
@@ -160,6 +176,7 @@
}
parts = clic_readindex_file(packfile);
+ largeparts = clic_readindex_file(packfile);
sizes = malloc(sizeof(uint64_t)*parts);
offs = malloc(sizeof(uint64_t)*parts);
@@ -182,7 +199,7 @@
fprintf(stderr, "unreasonable part number 0\n");
return 1;
}
- fseek(packfile, (oparts-parts)*sizeof(uint64_t)*2, SEEK_CUR);
+ fseeko(packfile, (oparts-parts)*sizeof(uint64_t)*2, SEEK_CUR);
const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED;
// C sucks
@@ -226,7 +243,8 @@
strm.next_in = in;
strm.avail_in = readin;
strm.next_out = out;
- strm.avail_out = bsize*pagesize;
+ // TODO: this doesn't need to be large all the time
+ strm.avail_out = blocksize_large*pagesize;
strm.total_in = 0;
strm.total_out = 0;
@@ -240,6 +258,29 @@
break;
}
+ if (ret == LZMA_DATA_ERROR) {
+ fprintf(stderr, "lzma data corrupt!\n");
+ exit(1);
+ }
assert (ret == LZMA_OK);
/* don't use lzma_end (will free buffers) or LZMA_FINISH (will forbid any new use) */
}
+
+void clic_free_lzma()
+{
+ lzma_end(&strm);
+}
+
+void clic_find_block( off_t block, off_t *part, off_t *offset )
+{
+ // we have X blocks in large parts and Y blocks in appendix
+ if (block > (off_t)(largeparts * blocksize_large) )
+ {
+ *part = (block - largeparts * blocksize_large) / blocksize_small + largeparts;
+ *offset = (block - largeparts * blocksize_large) % blocksize_small;
+ } else {
+ *part = block / blocksize_large;
+ *offset = block % blocksize_large;
+ }
+ //fprintf(stderr, "clic_find_block %ld => %ld %ld\n", block, *part, *offset);
+}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/mkclicfs.cpp new/src/mkclicfs.cpp
--- old/src/mkclicfs.cpp 2009-05-19 18:01:14.000000000 +0200
+++ new/src/mkclicfs.cpp 2009-08-08 22:35:42.000000000 +0200
@@ -87,10 +87,12 @@
assert (ret == LZMA_STREAM_END);
lzma_end(&strm);
+ //fprintf( stderr, "compress %ld %ld %ld\n", insize, outsize, strm.total_out );
return strm.total_out;
}
int blocksize = 32;
+int blocksizelarge = 3200;
int infd = -1;
uint32_t *ublocks = 0;
uint32_t *found = 0;
@@ -140,11 +142,11 @@
return queue;
}
-int queue_length(struct queue *queue)
+int queue_length(struct queue *queue)
{
pthread_mutex_lock(&queue->mutex);
int ret = (queue->writep - queue->readp + queue->size) % queue->size;
- pthread_mutex_unlock(&queue->mutex);
+ pthread_mutex_unlock(&queue->mutex);
return ret;
}
@@ -182,6 +184,7 @@
// the number of really saved parts
uint32_t parts = 0;
+uint32_t largeparts = 0;
struct inbuf_struct {
size_t readin, totalin;
@@ -212,17 +215,26 @@
int currentblocks = 0;
inbuf_struct *in = new inbuf_struct();
- in->inbuf = new unsigned char[blocksize*pagesize];
+ in->inbuf = new unsigned char[blocksizelarge*pagesize];
in->readin = 0;
in->totalin = 0;
in->lastblock = false;
- while ( currentblocks < blocksize ) {
+ size_t currentblocksize = blocksize;
+
+ if (rindex + 1 < pindex) {
+ currentblocksize = blocksizelarge;
+ largeparts++;
+ }
+
+ //fprintf( stderr, "cbl %ld %ld %ld\n", currentblocksize, rindex, pindex );
+
+ while ( currentblocks < currentblocksize ) {
off_t cindex = 0;
if (rindex < pindex) {
cindex = ublocks[rindex];
} else {
- while (found[uindex] && uindex < num_pages) uindex++;
+ while (found[uindex] && uindex < num_pages) uindex++;
assert( uindex < num_pages );
if ( uindex < num_pages ) {
cindex = uindex;
@@ -291,16 +303,16 @@
inbuf_struct *in = (inbuf_struct*)queue_get(from_reader);
outbuf_struct *out = new outbuf_struct();
- out->outbuf = new unsigned char[blocksize*pagesize + 300];
+ out->outbuf = new unsigned char[blocksizelarge*pagesize + 300];
// fprintf( stderr, "compress start %ld %d %x %ld\n", pthread_self(), in->part, in->inbuf, in->readin );
- out->outsize = compress(preset, in->inbuf, in->readin, out->outbuf, blocksize*pagesize + 300);
+ out->outsize = compress(preset, in->inbuf, in->readin, out->outbuf, blocksizelarge*pagesize + 300);
out->part = in->part;
out->insize = in->readin;
out->bpp = in->bpp;
out->lastblock = in->lastblock;
out->totalin = in->totalin;
-// fprintf( stderr, "compress %ld %d %x %ld -> %ld\n", pthread_self(), in->part, in->inbuf, in->readin, out->outsize );
+ //fprintf( stderr, "compress %ld %d %x %ld -> %ld\n", pthread_self(), in->part, in->inbuf, in->readin, out->outsize );
delete [] in->inbuf;
delete in;
@@ -388,6 +400,7 @@
while ( comps[lastpart + 1] ) {
comp = comps[++lastpart];
+ //fprintf( stderr, "comp %ld %ld %ld\n", comp->part, ( long )comp->totalin, ( long )comp->outsize );
sizes[comp->part] = comp->outsize;
offs[comp->part] = total_out + index_off;
total_in += comp->totalin;
@@ -403,13 +416,13 @@
lastpercentage+1, (long)comp->part,
(int)(total_out * 100 / total_in), (current.tv_sec - start.tv_sec) * 1000 + ((current.tv_usec - start.tv_usec) / 1000 ));
start.tv_sec = current.tv_sec;
- start.tv_usec = current.tv_usec;
+ start.tv_usec = current.tv_usec;
lastpercentage++;
}
if ( comp->lastblock ) {
delete comp;
- goto out;
+ goto out;
}
comps[comp->part] = 0;
delete comp;
@@ -426,7 +439,7 @@
bool usage = false;
int opt;
- while ((opt = getopt(argc, argv, "dp:b:l:c:n:")) != -1) {
+ while ((opt = getopt(argc, argv, "dp:b:l:c:n:h:")) != -1) {
switch (opt) {
case 'd':
check_dups = false;
@@ -439,6 +452,11 @@
if (blocksize <= 0)
usage = true;
break;
+ case 'h':
+ blocksizelarge = atoi(optarg);
+ if (blocksizelarge <= 0)
+ usage = true;
+ break;
case 'p':
pagesize = atoi(optarg);
if (pagesize <= 0)
@@ -504,6 +522,10 @@
if (strncmp(line, "access ", 5))
continue;
if (sscanf(line, "access %ld+%ld", &offset, &size) == 2) {
+ // the access data is in blocks - so we can compare it with
+ // blktrace, but we need pages here
+ offset /= 8;
+ size /= 8;
for (i = 0; i <= size; i++) {
if (offset + i < num_pages && found[offset+i] == 0) {
ublocks[pindex++] = offset + i;
@@ -547,6 +569,9 @@
if (!writeindex(out, blocksize )) return 1;
index_off += sizeof(uint32_t);
+ if (!writeindex(out, 100 * blocksize )) return 1;
+ index_off += sizeof(uint32_t);
+
if (!writeindex(out, pagesize )) return 1;
index_off += sizeof(uint32_t);
@@ -561,14 +586,15 @@
blockindex = new uint32_t[num_pages];
off_t index_part = index_off;
- index_off += 2 * oparts * sizeof(uint64_t) + sizeof(uint32_t);
- fseek(out, index_off, SEEK_SET);
+ // oparts it the size of the table and we need place for parts and largeparts
+ index_off += 2 * oparts * sizeof(uint64_t) + sizeof(uint32_t) * 2;
+ fseeko(out, index_off, SEEK_SET);
initialise_threads();
if ( writer(oparts, index_off, out, sizes, offs, st.st_size) )
return 1;
- if (fseek(out, index_blocks, SEEK_SET) < 0) {
+ if (fseeko(out, index_blocks, SEEK_SET) < 0) {
perror("seek"); return 1;
}
@@ -576,11 +602,12 @@
if (!writeindex(out, blockindex[i]))
return 1;
- if (fseek(out, index_part, SEEK_SET) < 0) {
+ if (fseeko(out, index_part, SEEK_SET) < 0) {
perror("seek"); return 1;
}
if (!writeindex(out, parts)) return 1;
+ if (!writeindex(out, largeparts )) return 1;
for (i = 0; i < parts; ++i) {
if (fwrite((char*)(sizes + i), sizeof(uint64_t), 1, out) != 1 ||
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/unclicfs.c new/src/unclicfs.c
--- old/src/unclicfs.c 2009-05-19 18:01:14.000000000 +0200
+++ new/src/unclicfs.c 2009-08-08 22:35:42.000000000 +0200
@@ -25,31 +25,9 @@
#include
#include
-static off_t lastpart = (off_t)-1;
static unsigned char *inbuf = 0;
static unsigned char *outbuf = 0;
-static size_t clic_read_block(unsigned char *buf, off_t block)
-{
- off_t mapped_block = clic_map_block(block);
- assert(mapped_block < (off_t)num_pages);
-
- off_t part = (off_t)(mapped_block / bsize);
- assert(part < parts);
- if ( part != lastpart) {
- size_t readin = clic_readpart(inbuf, part);
- if (readin == 0) {
- return 0;
- }
- clic_decompress_part(outbuf, inbuf, readin);
- lastpart = part;
- }
-
- memcpy(buf, outbuf + pagesize * (mapped_block % bsize), pagesize);
-
- return pagesize;
-}
-
int main(int argc, char *argv[])
{
if (argc != 2) {
@@ -60,27 +38,50 @@
if (clicfs_read_pack(packfilename))
return 1;
- inbuf = malloc(bsize*pagesize + 300);
- outbuf = malloc(bsize*pagesize);
+ inbuf = malloc(blocksize_large*pagesize + 300);
+ outbuf = malloc(blocksize_large*pagesize);
FILE *outfile = fopen(thefile, "w");
size_t delta = num_pages / 100;
- size_t i;
- unsigned char tbuf[pagesize];
- for (i = 0; i < num_pages; ++i)
+ size_t page;
+ off_t cpart;
+ size_t written_pages = 0;
+
+ for (cpart = 0; cpart < parts; ++cpart)
{
- size_t diff = clic_read_block(tbuf, i);
- if (i % delta == 0)
+ size_t readin = clic_readpart(inbuf, cpart);
+ if (readin == 0) {
+ return 1;
+ }
+ clic_decompress_part(outbuf, inbuf, readin);
+
+ for (page = 0; page < num_pages; ++page)
{
- fprintf(stderr, "read %d%%\n", (int)(i * 100 / num_pages));
+ off_t mapped_block = clic_map_block(page);
+ assert(mapped_block < (off_t)num_pages);
+
+ off_t part, off;
+ clic_find_block( mapped_block, &part, &off);
+ assert(part < parts);
+
+ if (part != cpart) continue;
+
+ if (++written_pages % delta == 0)
+ {
+ fprintf(stderr, "read %d%%\n", (int)(written_pages * 100 / num_pages));
+ }
+ if (fseeko(outfile, page * pagesize, SEEK_SET)) {
+ perror("seek");
+ return 1;
+ }
+
+ if (fwrite(outbuf + pagesize * off, 1, pagesize, outfile) != pagesize) {
+ perror("write");
+ break;
+ }
}
- assert(diff == pagesize);
- if (fwrite(tbuf, 1, pagesize, outfile) != pagesize) {
- perror("write");
- break;
- }
}
fclose(outfile);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/update.sh new/update.sh
--- old/update.sh 2009-05-19 18:01:14.000000000 +0200
+++ new/update.sh 2009-08-08 22:35:42.000000000 +0200
@@ -1,7 +1,7 @@
-rm -rf openSUSE:Factory:Live
-osc co openSUSE:Factory:Live/clicfs
-git archive --format tar HEAD | bzip2 > openSUSE:Factory:Live/clicfs/clicfs.tar.bz2
+rm -rf filesystems
+osc co filesystems/clicfs
+git archive --format tar HEAD | bzip2 > filesystems/clicfs/clicfs.tar.bz2
commit=`git log -n 1 HEAD | head -n 1` ;\
-( cd openSUSE:Factory:Live/clicfs/ ;\
+( cd filesystems/clicfs/ ;\
osc commit -m "$$commit" . )
-rm -rf openSUSE:Factory:Live
+rm -rf filesystems
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Remember to have fun...
--
To unsubscribe, e-mail: opensuse-commit+unsubscribe@opensuse.org
For additional commands, e-mail: opensuse-commit+help@opensuse.org