Hello community,
here is the log from the commit of package clicfs for openSUSE:Factory
checked in at Mon Mar 21 15:32:46 CET 2011.
--------
--- clicfs/clicfs.changes 2010-11-30 13:38:39.000000000 +0100
+++ clicfs/clicfs.changes 2011-03-21 14:58:39.000000000 +0100
@@ -1,0 +2,8 @@
+Mon Mar 21 13:57:39 UTC 2011 - coolo@novell.com
+
+- update to 1.4.0
+ - rework COW file format (packed file format unchanged)
+ - make use of more threads again
+ - make COW syncing much more robust and faster too
+
+-------------------------------------------------------------------
calling whatdependson for head-i586
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ clicfs.spec ++++++
--- /var/tmp/diff_new_pack.ppkIJl/_old 2011-03-21 15:29:36.000000000 +0100
+++ /var/tmp/diff_new_pack.ppkIJl/_new 2011-03-21 15:29:36.000000000 +0100
@@ -1,7 +1,7 @@
#
-# spec file for package clicfs (Version 1.3.10)
+# spec file for package clicfs
#
-# Copyright (c) 2010 SUSE LINUX Products GmbH, Nuernberg, Germany.
+# Copyright (c) 2011 SUSE LINUX Products GmbH, Nuernberg, Germany.
#
# All modifications and additions to the file contributed by third parties
# remain the property of their copyright owners, unless otherwise agreed
@@ -22,7 +22,7 @@
BuildRequires: cmake e2fsprogs-devel fuse-devel gcc-c++ openssl-devel xz-devel
Requires: fuse
Summary: Compressed Loop Image Container
-Version: 1.3.10
+Version: 1.4.0
Release: 1
License: GPLv2
Group: System/Filesystems
@@ -36,7 +36,6 @@
creating a copy on write behaviour.
-
%prep
%setup -c %name
++++++ clicfs.tar.bz2 ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/README new/README
--- old/README 2010-11-30 13:37:15.000000000 +0100
+++ new/README 2011-03-21 14:55:37.000000000 +0100
@@ -1 +1,9 @@
This is not really worth the read (yet).
+
+COWFILE FORMAT:
+ 9 bytes: CLICCOW%02d
+ 8 bytes: file size (including sparse)
+
+ num_pages*:
+ - 4 bytes: page index in cow (if > 0)
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/misc/makeiso.sh new/misc/makeiso.sh
--- old/misc/makeiso.sh 1970-01-01 01:00:00.000000000 +0100
+++ new/misc/makeiso.sh 2011-03-21 14:55:37.000000000 +0100
@@ -0,0 +1 @@
+make && cp src/clicfs initrd/usr/bin/clicfs && (cd initrd ; find . | cpio --create --format=newc --quiet | gzip -9 -f > ../CD1/boot/x86_64/loader/initrd) && genisoimage -R -J -f -pad -joliet-long -no-emul-boot -boot-load-size 4 -boot-info-table -b boot/x86_64/loader/isolinux.bin -o kde2.iso CD1/; isohybrid -id $(cat CD1/boot/grub/mbrid) kde2.iso && dd if=/dev/zero seek=1000 count=1 bs=1M of=kde2.iso
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/clicfs.c new/src/clicfs.c
--- old/src/clicfs.c 2010-11-30 13:37:15.000000000 +0100
+++ new/src/clicfs.c 2011-03-21 14:55:37.000000000 +0100
@@ -17,6 +17,7 @@
*/
#define FUSE_USE_VERSION 26
+#define _GNU_SOURCE
#include
#include "clicfs.h"
@@ -31,7 +32,7 @@
#include
#include
-#define DEBUG 0
+//#define DEBUG 1
FILE *logger = 0;
@@ -46,30 +47,31 @@
static uint32_t clic_find_next_cow()
{
- if (cows_index > 0)
- return cows[--cows_index];
- return cow_pages + cow_index_pages;
+ if (cows_index > 0) {
+ //if (logger) fprintf(logger, "find_next (old): %d\n", cows[cows_index-1]);
+ return cows[--cows_index];
+ }
+ //if (logger) fprintf(logger, "find_next (new): %d\n", cow_pages + 1);
+ return cow_pages + 1;
}
-static int clic_detach(size_t block, int islock);
+static int clic_detach(size_t block);
static int clic_write_cow();
pthread_mutex_t cowfile_mutex = PTHREAD_MUTEX_INITIALIZER;
pthread_mutex_t cowfile_mutex_writer = PTHREAD_MUTEX_INITIALIZER;
+pthread_mutex_t coms_by_part_mutex = PTHREAD_MUTEX_INITIALIZER;
-static int clic_write_cow(int islocked)
+static int clic_write_cow()
{
if (!cowfilename || cowfile_ro == 1 || !detached_allocated)
return 0;
int ret = 0;
- //if (logger) fprintf(logger, "cow detached %dMB\n", (int)(detached_allocated / 1024));
+ if (logger) fprintf(logger, "cow detached %dMB\n", (int)(detached_allocated / 1024));
if (logger) fprintf(logger, "clic_write_cow %ld\n", pthread_self());
- if (!islocked) {
- pthread_mutex_lock(&cowfile_mutex_writer);
- }
pthread_mutex_lock(&cowfile_mutex);
uint32_t i;
@@ -78,95 +80,38 @@
long ptr = (long)blockmap[i];
if ( ptr && PTR_CLASS(ptr) == CLASS_MEMORY ) { // detached now
off_t cowindex = clic_find_next_cow();
- off_t seeked = lseek(cowfilefd, cowindex * pagesize, SEEK_SET);
- assert(seeked == (off_t)(cowindex * pagesize));
- size_t ret = write(cowfilefd, blockmap[i], pagesize);
- assert(ret == pagesize);
+ ssize_t pret = pwrite(cowfilefd, blockmap[i], pagesize, cowindex * pagesize + cow_pages_start);
+ //if (logger) fprintf(logger, "pwrote %ld %ld -> %ld\n", pagesize, cowindex * pagesize + cow_pages_start, pret);
+ if (pret < 0) {
+ perror("pwrite cow");
+ if (logger) fprintf(logger, "failed %s\n", strerror(errno));
+ ret = -errno;
+ cowfile_ro = 1;
+ goto exit;
+ }
+ assert(pret == (ssize_t)pagesize);
free(blockmap[i]);
detached_allocated -= (pagesize / 1024);
- blockmap[i] = (unsigned char*)(long)(cowindex << 2) + 2;
- cow_pages++;
- }
- }
-
- assert(!detached_allocated);
-
- off_t seeked = lseek(cowfilefd, 0, SEEK_SET);
- assert(seeked == 0);
- uint64_t stringlen = thefilesize;
-
- char head[10];
- sprintf(head, "CLICCOW%02d", DOENER_MAGIC);
- uint32_t index_len = write(cowfilefd, head, 9);
-
- index_len += write(cowfilefd, (char*)&stringlen, sizeof(uint64_t));
- stringlen = cow_pages;
- index_len += write(cowfilefd, (char*)&stringlen, sizeof(uint32_t));
- stringlen = 0;
-
- index_len += 2 * sizeof(uint32_t) * cow_pages;
- uint32_t new_cow_index_pages = index_len / pagesize + 1;
- uint32_t moving;
- uint32_t moved = 0;
-
- // should all be out
- assert(cows_index == 0);
-
- pthread_mutex_unlock(&cowfile_mutex);
-
- for (moving = cow_index_pages; moving < new_cow_index_pages; ++moving)
- {
- // we only have a map from memory to cow, so we need to
- // look up in reverse to find the page to move
- // if this proves to be slow, we need even more memory
- // to keep the reverse map
- for (i = 0; i < num_pages; ++i)
- {
- long ptr = (long)blockmap[i];
- if (PTR_CLASS(ptr) == CLASS_COW) { // block
- if ((uint32_t)(ptr >> 2) == moving) {
- if (logger) fprintf(logger, "moving %ld %ld\n", (long)moving, (long)i);
- clic_detach(i, 1);
- moved++;
- break;
- }
+ blockmap[i] = (unsigned char*)(long)(cowindex << 2) + CLASS_COW;
+ uint32_t value = cowindex;
+ off_t offset = cow_index_start + i * sizeof(uint32_t);
+ pret = pwrite(cowfilefd, (char*)&value, sizeof(uint32_t), offset);
+ if (pret < 0) {
+ perror("pwrite2 cow");
+ ret = -errno;
}
+ //if (logger) fprintf(logger, "pwrote2 %d %d %ld -> %ld\n", i, cowindex, offset, pret);
+ cow_pages++;
}
}
- assert(moved == cows_index);
-
- cow_index_pages = new_cow_index_pages;
-
- /* if we moved, we need to redetach */
- if (moved) {
- cows_index = 0;
- ret = clic_write_cow(1);
- goto exit;
- }
-
- pthread_mutex_lock(&cowfile_mutex);
-
- for (i = 0; i < num_pages; ++i)
- {
- long ptr = (long)blockmap[i];
- if (PTR_CLASS(ptr) == CLASS_COW) { // block
- uint32_t key = i, value = ptr >> 2;
- write(cowfilefd, (char*)&key, sizeof(uint32_t));
- write(cowfilefd, (char*)&value, sizeof(uint32_t));
- stringlen++;
- }
- }
-
- assert(stringlen == cow_pages);
- write(cowfilefd, (char*)&index_len, sizeof(uint32_t));
+ fdatasync(cowfilefd);
+ last_sync = time(0);
+ // not true for threads assert(!detached_allocated);
- pthread_mutex_unlock(&cowfile_mutex);
-
exit:
- if (!islocked)
- pthread_mutex_unlock(&cowfile_mutex_writer);
-
+ pthread_mutex_unlock(&cowfile_mutex);
+ if (logger) fprintf(logger, "clic_write_cow %ld done %d\n", pthread_self(), ret);
return ret;
}
@@ -247,8 +192,6 @@
struct buffer_combo *coms_sort_by_use_last = 0;
static unsigned int com_count = 0;
-pthread_mutex_t picker = PTHREAD_MUTEX_INITIALIZER, seeker = PTHREAD_MUTEX_INITIALIZER;;
-
FILE *pack;
static void clic_append_by_use(struct buffer_combo *com)
@@ -280,6 +223,7 @@
return;
}
if (coms_sort_by_part_size == after + 1) { // just append
+ assert(coms_sort_by_part_size < MAX_COMS_SIZE);
coms_by_part[coms_sort_by_part_size] = com;
} else {
// I don't like memmove
@@ -287,6 +231,7 @@
for (i = coms_sort_by_part_size-1; i > after; i--)
coms_by_part[i+1] = coms_by_part[i];
coms_by_part[after+1] = com;
+ assert(after + 1 < MAX_COMS_SIZE);
}
coms_sort_by_part_size++;
clic_append_by_use(com);
@@ -360,6 +305,7 @@
}
} else
free(com->out_buffer);
+ if (logger) fprintf(logger, "free block %d\n", com->part);
memory_used -= com->out_buffer_size;
int32_t res = binary_search(coms_by_part, coms_sort_by_part_size, com->part);
assert(coms_by_part[res] == com);
@@ -379,11 +325,13 @@
//if (logger) fprintf(logger, "clic_uncompress %d %d\n", part, parts);
time_t now = time(0);
+ pthread_mutex_lock(&coms_by_part_mutex);
+
if (coms_sort_by_use_first) // clean up
{
if (0) clic_dump_use();
// if the oldest is 1m, drop it
- while (coms_sort_by_use_first && (now - coms_sort_by_use_first->last_used > 30 || (memory_used > 1024 * 1024 * 10 && coms_sort_by_use_first->part != part))) {
+ while (coms_sort_by_use_first && (now - coms_sort_by_use_first->last_used > 40 || (memory_used > 1024 * 1024 * 40 && coms_sort_by_use_first->part != part))) {
clic_free_com(coms_sort_by_use_first);
}
//clic_dump_use();
@@ -397,6 +345,7 @@
com->last_used = now;
clic_remove_com_from_use(com);
clic_append_by_use(com);
+ pthread_mutex_unlock(&coms_by_part_mutex);
return buf;
}
@@ -409,6 +358,7 @@
}
struct buffer_combo *com = malloc(sizeof(struct buffer_combo));
+ assert(com);
memory_used += sizeof(struct buffer_combo);
if (part < largeparts) {
com->out_buffer_size = blocksize_large*pagesize;
@@ -422,6 +372,7 @@
} else {
com->out_buffer_size = blocksize_small*pagesize;
com->out_buffer = malloc(blocksize_small*pagesize);
+ assert(com->out_buffer);
com->mmapped = 0;
}
memory_used += com->out_buffer_size;
@@ -430,12 +381,12 @@
clic_insert_com(com, res);
- pthread_mutex_lock(&seeker);
+ pthread_mutex_unlock(&coms_by_part_mutex);
unsigned char *inbuffer = malloc(sizes[part]);
+ assert(inbuffer);
struct timeval begin, end;
gettimeofday(&begin, 0);
size_t readin = clic_readpart(inbuffer, part);
- pthread_mutex_unlock(&seeker);
gettimeofday(&end, 0);
if (!readin) {
free(inbuffer);
@@ -446,7 +397,7 @@
if (logger) fprintf(logger, "uncompress %d %ld-%ld %ld (read took %ld - started %ld)\n", part, (long)offs[part], (long)sizes[part], (long)readin, (end.tv_sec - begin.tv_sec) * 1000 + (end.tv_usec - begin.tv_usec) / 1000, (begin.tv_sec - start.tv_sec) * 1000 + (begin.tv_usec - start.tv_usec) / 1000 );
#endif
if (!clic_decompress_part(com->out_buffer, inbuffer, readin)) {
- if (logger) fprintf(logger, "uncompess of part %d failed - ignoring", part);
+ if (logger) fprintf(logger, "uncompess of part %d failed - ignoring\n", part);
}
free(inbuffer);
@@ -475,81 +426,105 @@
static ssize_t clic_read_block(char *buf, size_t block);
-static int clic_detach(size_t block, int islocked)
+static int clic_detach(size_t block)
{
assert(block < num_pages);
- if (!islocked)
- pthread_mutex_lock(&cowfile_mutex_writer);
-
int ret = 0;
+ //if (logger) fprintf(logger, "clic_detach\n");
unsigned char *ptr = blockmap[block];
+ //if (logger) fprintf(logger, "clic_detach1 %ld\n", PTR_CLASS(ptr));
if ((PTR_CLASS(ptr) == CLASS_RO ) || (PTR_CLASS(ptr) == CLASS_COW))
{
if (PTR_CLASS(ptr) == CLASS_COW) {
- if (cows_index == CLICFS_COW_COUNT - 1)
- clic_write_cow(1);
+ if (logger) fprintf(logger, "detach2 cow %d index\n", cows_index);
+ if (cows_index == CLICFS_COW_COUNT - 1) {
+ ret = clic_write_cow();
+ if (logger) fprintf(logger, "detach cow %d\n", ret);
+ }
+ }
+
+ if (cowfilename && cowfile_ro == 1) {
+ ret = -EROFS;
+ goto exit;
}
char *newptr = malloc(pagesize);
+ assert(newptr);
+ //if (logger) fprintf(logger, "clic_detach3 %ld\n", PTR_CLASS(newptr));
detached_allocated += (pagesize / 1024);
if (logger && detached_allocated % 1024 == 0 ) fprintf(logger, "detached %dMB\n", (int)(detached_allocated / 1024));
clic_read_block(newptr, block);
- if (PTR_CLASS(ptr) == CLASS_COW) { // we need to mark the place in the cow obsolete
- //if (logger) fprintf(logger, "detach block %ld (was %ld)\n", (long)block, (long)ptr >> 2);
+ if (PTR_CLASS(ptr) == CLASS_COW && !cowfile_ro) { // we need to mark the place in the cow obsolete
+ if (logger) fprintf(logger, "detach block %ld (was %ld)\n", (long)block, (long)ptr >> 2);
+ assert(cows_index < CLICFS_COW_COUNT);
cows[cows_index++] = (long)ptr >> 2;
cow_pages--;
}
blockmap[block] = (unsigned char*)newptr;
- ret = 1;
goto exit;
}
+ //if (logger) fprintf(logger, "clic_detach2 %p\n", blockmap[block]);
if (!blockmap[block])
{
- blockmap[block] = malloc(pagesize);
- assert(PTR_CLASS(ptr) == CLASS_MEMORY);
+ if (cowfilename && cowfile_ro == 1) {
+ ret = -EROFS;
+ goto exit;
+ }
+
+ blockmap[block] = malloc(pagesize);
+ //if (logger) fprintf(logger, "clic_detach4 %p\n", blockmap[block]);
+ assert(blockmap[block]);
+ //assert(PTR_CLASS(ptr) == CLASS_MEMORY);
detached_allocated += (pagesize / 1024);
if (logger && detached_allocated % 1024 == 0 ) fprintf(logger, "detached %dMB\n", (int)(detached_allocated / 1024));
memset(blockmap[block],0,pagesize);
- ret = 1;
}
exit:
- if (!islocked)
- pthread_mutex_unlock(&cowfile_mutex_writer);
-
+ //if (logger) fprintf(logger, "clic_detach done %d\n", ret);
return ret;
}
static size_t clic_write_block(const char *buf, off_t block, off_t ioff, size_t size)
{
- clic_detach(block, 1);
+ //if (logger) fprintf(logger, "clic_write_block %ld\n", detached_allocated);
+ if (clic_detach(block)) {
+ if (logger) fprintf(logger, "clic_detach FAILED\n");
+ return -ENOSPC;
+ }
memcpy(blockmap[block]+ioff, buf, size);
+
+ if (detached_allocated > 40000)
+ clic_write_cow();
+
return size;
}
static int clic_write(const char *path, const char *buf, size_t size, off_t offset,
struct fuse_file_info *fi)
{
- //if (logger) fprintf(logger, "write %s %ld %ld\n", path, offset, size);
+ if (logger) fprintf(logger, "%lx write0 %s %ld %ld %lx\n", pthread_self(), path, offset, size, pthread_self());
(void) fi;
if(path[0] == '/' && strcmp(path + 1, thefile) != 0)
return -ENOENT;
- if (offset >= (off_t)thefilesize)
+ if (offset >= (off_t)thefilesize) {
+ if (logger) fprintf(logger, "%lx writeF %s %ld %ld -> 0!! %lx\n", pthread_self(), path, offset, size, pthread_self());
return 0;
+ }
- if (offset+size > (off_t)thefilesize)
+ if ((off_t)(offset+size) > (off_t)thefilesize)
size = thefilesize-offset;
- if (!size)
+ if (!size) {
+ if (logger) fprintf(logger, "%lx write %s %ld %ld -> 0!!\n", pthread_self(), path, offset, size);
return 0;
-
- pthread_mutex_lock(&cowfile_mutex_writer);
+ }
off_t block = offset / pagesize;
off_t ioff = offset - block * pagesize;
@@ -561,12 +536,16 @@
int ret = 0;
if (size <= pagesize) {
+ if (logger) fprintf(logger, "%lx write2 %s %ld %ld\n", pthread_self(), path, offset, size);
ret = clic_write_block(buf, block, ioff, size);
+ if (logger) fprintf(logger, "%lx write3 %s %ld %ld -> %d\n", pthread_self(), path, offset, size, ret);
} else {
size_t wrote = 0;
do
{
+ if (logger) fprintf(logger, "%lx write4 %s %ld %ld %lx\n", pthread_self(), path, offset, size, pthread_self());
size_t diff = clic_write_block(buf, block, ioff, size > pagesize ? pagesize : size);
+ if (logger) fprintf(logger, "%lx write5 %s %ld %ld -> %ld\n", pthread_self(), path, offset, size, diff);
ioff = 0;
size -= diff;
buf += diff;
@@ -576,13 +555,13 @@
ret = wrote;
}
- pthread_mutex_unlock(&cowfile_mutex_writer);
+ if (logger) fprintf(logger, "%lx writeD %s %ld %ld -> %d\n", pthread_self(), path, offset, size, ret);
return ret;
}
static ssize_t clic_read_block(char *buf, size_t block)
{
- if (block >= num_pages)
+ if (block >= num_pages)
return -EFAULT;
if (!blockmap[block]) { // sparse block
@@ -600,8 +579,7 @@
if (PTR_CLASS(ptr) == CLASS_COW) {
off_t target = ptr >> 2;
pthread_mutex_lock(&cowfile_mutex);
- lseek(cowfilefd, target * pagesize, SEEK_SET);
- ssize_t haveread = read(cowfilefd, buf, pagesize);
+ ssize_t haveread = pread(cowfilefd, buf, pagesize, target * pagesize + cow_pages_start);
pthread_mutex_unlock(&cowfile_mutex);
return haveread;
}
@@ -635,7 +613,7 @@
static int clic_read(const char *path, char *buf, size_t size, off_t offset,
struct fuse_file_info *fi)
{
- //if (logger) fprintf(logger, "read %ld %ld %ld\n", offset, size, thefilesize);
+ // if (logger) fprintf(logger, "read %ld %ld %ld\n", offset, size, thefilesize);
(void) fi;
if(path[0] == '/' && strcmp(path + 1, thefile) != 0)
return -ENOENT;
@@ -645,8 +623,6 @@
assert(size % pagesize == 0);
assert(offset % pagesize == 0);
- pthread_mutex_lock(&cowfile_mutex_writer);
-
do
{
if (offset >= (off_t)thefilesize)
@@ -654,7 +630,8 @@
off_t block = offset / pagesize;
ssize_t diff = clic_read_block(buf, block);
if (diff < 0) {
- return diff;
+ //if (logger) fprintf(logger, "read %ld %ld %ld -> %ld!!\n", offset, size, thefilesize, diff);
+ return diff;
}
//if (logger) fprintf(logger, "read block %ld: %ld bytes\n", (long)block, (long)diff);
if (!diff)
@@ -665,8 +642,7 @@
readtotal += diff;
} while (size > 0);
- pthread_mutex_unlock(&cowfile_mutex_writer);
-
+ //if (logger) fprintf(logger, "read %ld %ld %ld -> %ld\n", offset, size, thefilesize, readtotal);
return readtotal;
}
@@ -675,9 +651,8 @@
(void)path;
(void)fi;
// TODO write out cow
- if (logger) fflush(logger);
- clic_write_cow(0);
- return 0;
+ if (logger) { fprintf(logger, "flush\n"); fflush(logger); }
+ return clic_write_cow();
}
static int clic_fsync(const char *path, int datasync, struct fuse_file_info *fi)
@@ -686,13 +661,10 @@
(void)fi;
(void)datasync;
// TODO write out cow
- if (logger) fflush(logger);
- clic_write_cow(0);
- last_sync = time(0);
- pthread_mutex_lock(&cowfile_mutex);
- if (cowfilefd >= 0) fsync(cowfilefd);
- pthread_mutex_unlock(&cowfile_mutex);
- return 0;
+ if (logger) { fprintf(logger, "sync\n"); fflush(logger); }
+
+ int ret = clic_write_cow();
+ return ret;
}
static void *clic_sync_thread(void *arg)
@@ -720,6 +692,7 @@
{
// avoid random reads or our profiling will be destroyed
conn->max_readahead = 0;
+ clic_sync_tid = 0;
pthread_create(&clic_sync_tid, NULL, clic_sync_thread, 0);
@@ -730,9 +703,11 @@
{
(void)arg;
if (logger) fprintf(logger, "destroy\n");
- pthread_cancel(clic_sync_tid);
- void *res;
- pthread_join(clic_sync_tid, &res);
+ if (clic_sync_tid > 0) {
+ pthread_cancel(clic_sync_tid);
+ void *res;
+ pthread_join(clic_sync_tid, &res);
+ }
}
@@ -810,18 +785,21 @@
assert( DOENER_MAGIC < 100 );
int index_len = fprintf(cow, "CLICCOW%02d", DOENER_MAGIC );
-
index_len += fwrite((char*)&bigfilesize, 1, sizeof(uint64_t), cow);
- uint32_t stringlen = 0;
- // there are 0 blocks
- index_len += fwrite((char*)&stringlen, 1, sizeof(uint32_t), cow);
- // the whole index is 12 bytes long
- stringlen = index_len + sizeof(uint32_t);
- index_len += fwrite((char*)&stringlen, 1, sizeof(uint32_t), cow);
- fclose(cow);
- cow_index_pages = index_len / pagesize + 1;
- cow_pages = 0;
+ char zeros[sizeof(uint32_t)];
+ memset(zeros, 0, sizeof(uint32_t));
+
+ size_t write_pages = bigfilesize / pagesize;
+ blockmap = realloc(blockmap, sizeof(unsigned char*)*write_pages);
+ unsigned int i;
+ for (i = num_pages; i < write_pages; ++i)
+ blockmap[i] = 0;
+ num_pages = write_pages;
+
+ for (i = 0; i < num_pages; ++i)
+ fwrite(zeros, 1, sizeof(uint32_t), cow);
+ fclose(cow);
return 0;
}
@@ -848,7 +826,8 @@
}
// not sure why but multiple threads make it slower
- fuse_opt_add_arg(&args, "-s");
+ //fuse_opt_add_arg(&args, "-s");
+ fuse_opt_add_arg(&args, "-obig_writes");
if (!packfilename) {
fprintf(stderr, "usage: [-m <mb>] [-l ] [-c <cowfile>] <packfile> <mntpoint>\n");
@@ -862,6 +841,12 @@
free(packfilename);
+ int procfile = open("/proc/self/om_score_adj", O_WRONLY);
+ if (procfile > 0) {
+ write(procfile, "-1000", sizeof("-1000"));
+ close(procfile);
+ }
+
if (cowfilename) {
if (access(cowfilename, R_OK))
@@ -891,16 +876,18 @@
}
for (i = 0; i < largeparts; ++i) {
- posix_fadvise( fileno(packfile), offs[i], sizes[i], POSIX_FADV_SEQUENTIAL);
+ posix_fadvise( packfilefd, offs[i], sizes[i], POSIX_FADV_SEQUENTIAL);
}
coms_by_part = malloc(sizeof(struct buffer_combo*)*MAX_COMS_SIZE);
+ assert(coms_by_part);
gettimeofday(&start, 0);
/* MAIN LOOP */
int ret = fuse_main(args.argc, args.argv, &clic_oper, NULL);
- clic_write_cow(0);
+ clic_write_cow(); // ignored
if (cowfilefd >= 0) close(cowfilefd);
if (logger) fclose(logger);
+ logger=0;
while (coms_sort_by_use_first)
clic_free_com(coms_sort_by_use_first);
@@ -916,7 +903,7 @@
free(blockmap);
free(sizes);
free(offs);
- fclose(packfile);
+ close(packfilefd);
if (cowfilename)
free(cowfilename);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/clicfs.h new/src/clicfs.h
--- old/src/clicfs.h 2010-11-30 13:37:15.000000000 +0100
+++ new/src/clicfs.h 2011-03-21 14:55:37.000000000 +0100
@@ -23,7 +23,7 @@
#include
extern int preset;
-extern FILE *packfile;
+extern int packfilefd;
extern int cowfilefd;
// magic 2 added large parts
@@ -53,8 +53,10 @@
extern uint32_t num_pages;
// the number of pages marked as CLASS_COW
extern uint32_t cow_pages;
-// the number of pages in the cow for index
-extern unsigned int cow_index_pages;
+// the offset in the cow file where the page index starts
+extern off_t cow_index_start;
+// the offset in the cow file where the pages start
+extern off_t cow_pages_start;
// support temporary changes on ro medium
extern int cowfile_ro;
@@ -71,7 +73,7 @@
extern int clic_decompress_part(unsigned char *out, const unsigned char *in, size_t size);
extern size_t clic_readpart(unsigned char *buffer, int part);
extern off_t clic_map_block(off_t block);
-extern uint32_t clic_readindex_fd(int fd );
+extern uint32_t clic_readindex_fd( int fd );
extern void clic_free_lzma();
extern void clic_find_block( off_t block, off_t *part, off_t *offset );
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/src/clicfs_common.c new/src/clicfs_common.c
--- old/src/clicfs_common.c 2010-11-30 13:37:15.000000000 +0100
+++ new/src/clicfs_common.c 2011-03-21 14:55:37.000000000 +0100
@@ -16,6 +16,9 @@
02110-1301, USA
*/
+#define _GNU_SOURCE
+//#define DEBUG 1
+
#include "clicfs.h"
#include
#include
@@ -26,9 +29,10 @@
#include
#include
#include
+#include
int preset = 0;
-FILE *packfile = 0;
+int packfilefd = -1;
int cowfilefd = -1;
char thefile[PATH_MAX];
@@ -44,12 +48,14 @@
unsigned char **blockmap;
uint32_t num_pages = 0;
uint32_t cow_pages = 0;
-uint32_t cow_index_pages = 0;
+off_t cow_index_start = 0;
+off_t cow_pages_start = 0;
uint32_t *cows = 0;
unsigned int cows_index = 0;
int cowfile_ro = 0;
static lzma_stream strm;
+static pthread_mutex_t lzma_mutex = PTHREAD_MUTEX_INITIALIZER;
uint32_t clic_readindex_fd(int fd)
{
@@ -69,15 +75,6 @@
return stringlen;
}
-uint32_t clic_readindex_file(FILE * f)
-{
- uint32_t stringlen = 0;
- if (fread(&stringlen, 1, sizeof(uint32_t), f) != sizeof(uint32_t)) {
- return 0;
- }
- return stringlen;
-}
-
int clicfs_read_cow(const char *cowfilename)
{
cowfilefd = open(cowfilename, O_RDWR);
@@ -115,33 +112,39 @@
for (i = num_pages; i < newpages; ++i)
blockmap[i] = 0;
num_pages = newpages;
- cow_pages = clic_readindex_fd(cowfilefd);
- for (i = 0; i < cow_pages; ++i)
+ cow_pages = 0;
+ for (i = 0; i < num_pages; ++i)
{
- uint32_t pageindex = clic_readindex_fd(cowfilefd);
uint32_t page = clic_readindex_fd(cowfilefd);
- assert(pageindex < num_pages);
- blockmap[pageindex] = (unsigned char*)(long)(page << 2) + 2;
+ if (page) {
+ blockmap[i] = (unsigned char*)(long)(page << 2) + CLASS_COW;
+ cow_pages++;
+ }
}
cows = malloc(sizeof(uint32_t) * CLICFS_COW_COUNT);
cows_index = 0;
- uint32_t index_len = clic_readindex_fd(cowfilefd);
- cow_index_pages = index_len / pagesize + 1;
+ cow_index_start = 9 + sizeof(uint64_t);
+ cow_pages_start = cow_index_start + num_pages * sizeof(uint32_t);
+ // we do not round up but down here as cow pages start with 1
+ cow_pages_start = ( cow_pages_start / pagesize + 0) * pagesize;
return 0;
}
int clicfs_read_pack(const char *packfilename)
{
- packfile = fopen(packfilename, "r");
- if (!packfile) {
+ packfilefd = open(packfilename, O_LARGEFILE|O_RDONLY);
+ if (packfilefd < 0) {
fprintf(stderr, "packfile %s can't be opened\n", packfilename);
return 1;
}
char head[7];
char expected[7];
- fread(head, 1, 6, packfile);
+ if (read(packfilefd, head, 6) != 6) {
+ perror("can't read from packfile\n");
+ return 1;
+ }
head[6] = 0;
sprintf(expected, "CLIC%02d", DOENER_MAGIC);
if (strcmp(head,expected)) {
@@ -149,46 +152,46 @@
return 1;
}
- uint32_t stringlen = clic_readindex_file(packfile);
+ uint32_t stringlen = clic_readindex_fd(packfilefd);
if (stringlen == 0 || stringlen >= PATH_MAX) {
fprintf(stderr, "abnormal len %lx\n", (long)stringlen);
return 1;
}
- if (fread(thefile, 1, stringlen, packfile) != stringlen) {
+ if (read(packfilefd, thefile, stringlen) != stringlen) {
fprintf(stderr, "short read %ld\n", (long)stringlen);
return 1;
}
thefile[stringlen] = 0;
- uint64_t oparts = clic_readindex_file(packfile);
- blocksize_small = clic_readindex_file(packfile);
- blocksize_large = clic_readindex_file(packfile);
- pagesize = clic_readindex_file(packfile);
+ uint64_t oparts = clic_readindex_fd(packfilefd);
+ blocksize_small = clic_readindex_fd(packfilefd);
+ blocksize_large = clic_readindex_fd(packfilefd);
+ pagesize = clic_readindex_fd(packfilefd);
thefilesize = oparts * blocksize_small * pagesize;
- preset = clic_readindex_file(packfile);
- num_pages = clic_readindex_file(packfile);
+ preset = clic_readindex_fd(packfilefd);
+ num_pages = clic_readindex_fd(packfilefd);
blockmap = malloc(sizeof(unsigned char*)*num_pages);
uint32_t i;
for (i = 0; i < num_pages; ++i) {
// make sure it's odd to diff between pointer and block
- blockmap[i] = (unsigned char*)(long)((clic_readindex_file(packfile) << 2) + 1);
+ blockmap[i] = (unsigned char*)(long)((clic_readindex_fd(packfilefd) << 2) + 1);
}
- parts = clic_readindex_file(packfile);
- largeparts = clic_readindex_file(packfile);
+ parts = clic_readindex_fd(packfilefd);
+ largeparts = clic_readindex_fd(packfilefd);
sizes = malloc(sizeof(uint64_t)*parts);
offs = malloc(sizeof(uint64_t)*parts);
for (i = 0; i < parts; ++i)
{
- if (fread((char*)(sizes + i), sizeof(uint64_t), 1, packfile) != 1)
+ if (read(packfilefd, (char*)(sizes + i), sizeof(uint64_t)) != sizeof(uint64_t))
parts = 0;
if (!sizes[i]) {
fprintf(stderr, "unreasonable size 0 for part %d\n", i);
return 1;
}
- if (fread((char*)(offs + i), sizeof(uint64_t), 1, packfile) != 1)
+ if (read(packfilefd, (char*)(offs + i), sizeof(uint64_t)) != sizeof(uint64_t))
parts = 0;
if (i > 0 && offs[i] <= offs[i-1]) {
fprintf(stderr, "the offset for i is not larger than i-1: %ld\n", (long)i);
@@ -199,7 +202,7 @@
fprintf(stderr, "unreasonable part number 0\n");
return 1;
}
- fseeko(packfile, (oparts-parts)*sizeof(uint64_t)*2, SEEK_CUR);
+ lseek(packfilefd, (oparts-parts)*sizeof(uint64_t)*2, SEEK_CUR);
const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED;
// C sucks
@@ -224,15 +227,11 @@
size_t clic_readpart(unsigned char *buffer, int part)
{
- if (fseeko(packfile, offs[part], SEEK_SET)) {
- fprintf(stderr, "seek failed %d %lld\n", part, (long long)offs[part]);
- return 0;
- }
#if defined(DEBUG)
fprintf(stderr, "uncompress part=%d/%d off=%ld size=%ld\n", part, parts, offs[part], sizes[part] );
#endif
- size_t readin = fread(buffer, 1, sizes[part], packfile);
- if (readin != sizes[part]) {
+ ssize_t readin = pread(packfilefd, buffer, sizes[part], offs[part]);
+ if (readin != (ssize_t)sizes[part]) {
fprintf(stderr, "short read: %d %ld %ld %ld\n", part, (long)offs[part], (long)sizes[part], (long)readin);
// returning half read blocks won't help lzma
return 0;
@@ -242,6 +241,7 @@
int clic_decompress_part(unsigned char *out, const unsigned char *in, size_t readin)
{
+ pthread_mutex_lock(&lzma_mutex);
strm.next_in = in;
strm.avail_in = readin;
strm.next_out = out;
@@ -264,12 +264,14 @@
if (ret == LZMA_DATA_ERROR) {
fprintf(stderr, "lzma data corrupt!\n");
+ pthread_mutex_unlock(&lzma_mutex);
return 0;
}
#if defined(DEBUG)
fprintf(stderr, "ret %d\n", ret);
#endif
assert (ret == LZMA_OK);
+ pthread_mutex_unlock(&lzma_mutex);
/* don't use lzma_end (will free buffers) or LZMA_FINISH (will forbid any new use) */
return 1;
}
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Remember to have fun...
--
To unsubscribe, e-mail: opensuse-commit+unsubscribe@opensuse.org
For additional commands, e-mail: opensuse-commit+help@opensuse.org